diff --git a/.gitignore b/.gitignore index 1bfc08ab..e78e17ee 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,10 @@ coverage.html /galactic /galactic-router /galactic-cni +/galactic-tap-cni +/galactic-ipam +/galactic-bgp +/galactic-route # Go workspace go.work diff --git a/Taskfile.yaml b/Taskfile.yaml index 8f45d766..0dbe9d88 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -133,6 +133,10 @@ tasks: -X go.datum.net/galactic/internal/metadata.GitURL={{.GIT_URL}} cmds: - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-cni ./cmd/galactic-cni + - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-tap-cni ./cmd/galactic-tap-cni + - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-ipam ./cmd/galactic-ipam + - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-bgp ./cmd/galactic-bgp + - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-route ./cmd/galactic-route - go build -ldflags "{{.LDFLAGS}}" -o bin/galactic-router ./cmd/galactic-router - go build -ldflags "{{.LDFLAGS}}" -o bin/vmtap-cni ./cmd/vmtap-cni - GOBIN={{.LOCALBIN}} go install github.com/containernetworking/plugins/plugins/main/host-device@v1.9.1 diff --git a/cmd/galactic-bgp/main.go b/cmd/galactic-bgp/main.go new file mode 100644 index 00000000..c96d2c97 --- /dev/null +++ b/cmd/galactic-bgp/main.go @@ -0,0 +1,91 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "fmt" + "log" + "os" + "strings" + + "github.com/containernetworking/cni/pkg/version" + "github.com/spf13/cobra" + "golang.org/x/term" + + "go.datum.net/galactic/internal/cnibgp" + "go.datum.net/galactic/internal/metadata" +) + +const ( + appName = "galactic-bgp" + + appDesc = `Galactic BGP CNI Plugin + + The BGP/SRv6/eBPF publish plugin in the galactic CNI chain — chained after + galactic-cni/galactic-tap-cni (and, when present, galactic-route) per + conflist order, never run standalone. Has zero kernel-interface + dependency: every address it advertises comes from prevResult, not from a + runtime call into an interface it doesn't own. + + Find more information at: https://www.datum.net/docs` +) + +func newRootCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: appName, + Short: strings.Split(appDesc, "\n")[0], + Long: appDesc, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + cnibgp.InitCNIConfig() + confFile, _ := cmd.Flags().GetString("conf-file") + if confFile != "" { + cnibgp.ConfFile = confFile + } + return nil + }, + RunE: func(cmd *cobra.Command, _ []string) error { + if ok, _ := cmd.Flags().GetBool("build-info"); ok { + fmt.Println(metadata.BuildInfo(appName)) + return nil + } + if ok, _ := cmd.Flags().GetBool("version"); ok { + fmt.Printf("%s version %s\n", appName, metadata.Version) + return nil + } + if os.Getenv("CNI_COMMAND") == "VERSION" { + return version.All.Encode(os.Stdout) + } + + // Real CNI runtimes always pipe the network config JSON on + // stdin and close it. If stdin is an interactive terminal + // instead, no config will ever arrive and skel's blocking + // stdin read would hang forever — print version info instead. + if term.IsTerminal(int(os.Stdin.Fd())) { + fmt.Printf("%s version %s\n", appName, metadata.Version) + fmt.Printf("CNI protocol versions supported: %s\n", strings.Join(version.All.SupportedVersions(), ", ")) + return nil + } + + // Unlike galactic-cni/galactic-tap-cni, this plugin never enters + // any network namespace at all (it only makes k8s API calls), + // so it needs neither the stdin peek-and-repipe dance nor + // CNI_NETNS_OVERRIDE those two use to detect and handle + // tap-mode's host-netns invocation. + cnibgp.RunPlugin() + return nil + }, + } + + cmd.PersistentFlags().String("conf-file", cnibgp.ConfFile, "Path to CNI conflist file") + cmd.Flags().Bool("build-info", false, "Print build information and exit") + cmd.Flags().BoolP("version", "V", false, "Print version and exit") + return cmd +} + +func main() { + if err := newRootCommand().Execute(); err != nil { + log.Fatalf("error: %v", err) + } +} diff --git a/cmd/galactic-cni/main.go b/cmd/galactic-cni/main.go index f5968664..ded54112 100644 --- a/cmd/galactic-cni/main.go +++ b/cmd/galactic-cni/main.go @@ -5,10 +5,8 @@ package main import ( - "encoding/json" "errors" "fmt" - "io" "log" "os" "strings" @@ -108,27 +106,14 @@ func newRootCommand() *cobra.Command { return nil } - // Read stdin once so we can inspect the CNI config before the - // library runs its netns validation. We pipe the buffered bytes - // back as os.Stdin so the CNI library can still read them. - stdinData, _ := io.ReadAll(os.Stdin) - r, w, _ := os.Pipe() - go func() { - _, _ = w.Write(stdinData) - _ = w.Close() - }() - oldStdin := os.Stdin - os.Stdin = r - - // Tap mode never enters a network namespace — all operations are - // host-side. Set the override so the CNI library skips its same- - // netns rejection check, which would otherwise reject kraftlet - // workloads that pass the host netns. - if isTapMode(stdinData) { - _ = os.Setenv("CNI_NETNS_OVERRIDE", "true") - } - - defer func() { os.Stdin = oldStdin }() + // galactic-cni is veth-only: it always moves an interface into + // the container's own netns, so it always needs the CNI + // library's normal same-netns rejection check — unlike + // galactic-tap-cni (which unconditionally sets + // CNI_NETNS_OVERRIDE, since tap workloads never enter a netns + // at all), there is no stdin-peeking tap-mode detection here + // anymore. Interface kind is which binary you invoke now, not a + // config field this process branches on. cni.RunPlugin() return nil }, @@ -142,17 +127,6 @@ func newRootCommand() *cobra.Command { return cmd } -// isTapMode returns true when the CNI config requests tap interface type. -// Only a minimal JSON parse is needed — full validation happens later in -// parseConf inside cmdAdd. -func isTapMode(stdinData []byte) bool { - var cfg struct { - InterfaceType string `json:"interface_type"` - } - _ = json.Unmarshal(stdinData, &cfg) - return cfg.InterfaceType == "tap" -} - func main() { if err := newRootCommand().Execute(); err != nil { log.Fatalf("error: %v", err) diff --git a/cmd/galactic-ipam/main.go b/cmd/galactic-ipam/main.go new file mode 100644 index 00000000..0d61a691 --- /dev/null +++ b/cmd/galactic-ipam/main.go @@ -0,0 +1,79 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "fmt" + "log" + "os" + "strings" + + "github.com/containernetworking/cni/pkg/version" + "github.com/spf13/cobra" + "golang.org/x/term" + + "go.datum.net/galactic/internal/cniipam" + "go.datum.net/galactic/internal/metadata" +) + +const ( + appName = "galactic-ipam" + + appDesc = `Galactic IPAM CNI Plugin + + The delegated CNI IPAM plugin in the galactic CNI chain — invoked by + galactic-cni/galactic-tap-cni's own "ipam" block via the CNI IPAM + delegation protocol (github.com/containernetworking/cni/pkg/ipam), never + run directly from a conflist. Has no Kubernetes dependency at all: + allocation state persists in on-disk marker files under this node's own + filesystem. + + Find more information at: https://www.datum.net/docs` +) + +func newRootCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: appName, + Short: strings.Split(appDesc, "\n")[0], + Long: appDesc, + RunE: func(cmd *cobra.Command, _ []string) error { + if ok, _ := cmd.Flags().GetBool("build-info"); ok { + fmt.Println(metadata.BuildInfo(appName)) + return nil + } + if ok, _ := cmd.Flags().GetBool("version"); ok { + fmt.Printf("%s version %s\n", appName, metadata.Version) + return nil + } + if os.Getenv("CNI_COMMAND") == "VERSION" { + return version.All.Encode(os.Stdout) + } + + // Real CNI runtimes (via IPAM delegation's ExecAdd/ExecDel/ + // ExecCheck) always pipe the netconf JSON on stdin and close + // it. If stdin is an interactive terminal instead, no config + // will ever arrive and skel's blocking stdin read would hang + // forever — print version info instead. + if term.IsTerminal(int(os.Stdin.Fd())) { + fmt.Printf("%s version %s\n", appName, metadata.Version) + fmt.Printf("CNI protocol versions supported: %s\n", strings.Join(version.All.SupportedVersions(), ", ")) + return nil + } + + cniipam.RunPlugin() + return nil + }, + } + + cmd.Flags().Bool("build-info", false, "Print build information and exit") + cmd.Flags().BoolP("version", "V", false, "Print version and exit") + return cmd +} + +func main() { + if err := newRootCommand().Execute(); err != nil { + log.Fatalf("error: %v", err) + } +} diff --git a/cmd/galactic-route/main.go b/cmd/galactic-route/main.go new file mode 100644 index 00000000..cbd49068 --- /dev/null +++ b/cmd/galactic-route/main.go @@ -0,0 +1,126 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "os" + "strings" + + "github.com/containernetworking/cni/pkg/version" + "github.com/spf13/cobra" + "golang.org/x/term" + + "go.datum.net/galactic/internal/cniroute" + "go.datum.net/galactic/internal/metadata" +) + +const ( + appName = "galactic-route" + + appDesc = `Galactic Route CNI Plugin + + The termination-route plugin in the galactic CNI chain — chained after + galactic-cni/galactic-tap-cni and before galactic-bgp per conflist order, + never run standalone, and optional (only present for attachments with + terminations to install). Has no Kubernetes dependency at all: it only + installs kernel routes into the VRF routing table the master plugin + already created. + + Find more information at: https://www.datum.net/docs` +) + +func newRootCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: appName, + Short: strings.Split(appDesc, "\n")[0], + Long: appDesc, + PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { + cniroute.InitCNIConfig() + confFile, _ := cmd.Flags().GetString("conf-file") + if confFile != "" { + cniroute.ConfFile = confFile + } + return nil + }, + RunE: func(cmd *cobra.Command, _ []string) error { + if ok, _ := cmd.Flags().GetBool("build-info"); ok { + fmt.Println(metadata.BuildInfo(appName)) + return nil + } + if ok, _ := cmd.Flags().GetBool("version"); ok { + fmt.Printf("%s version %s\n", appName, metadata.Version) + return nil + } + if os.Getenv("CNI_COMMAND") == "VERSION" { + return version.All.Encode(os.Stdout) + } + + // Real CNI runtimes always pipe the network config JSON on + // stdin and close it. If stdin is an interactive terminal + // instead, no config will ever arrive and skel's blocking + // stdin read would hang forever — print version info instead. + if term.IsTerminal(int(os.Stdin.Fd())) { + fmt.Printf("%s version %s\n", appName, metadata.Version) + fmt.Printf("CNI protocol versions supported: %s\n", strings.Join(version.All.SupportedVersions(), ", ")) + return nil + } + + // Unlike galactic-cni/galactic-tap-cni, this plugin never talks + // to the API server. It does, however, run natively in whatever + // netns CNI_NETNS points at rather than entering it — for a + // veth-mode attachment CNI_NETNS is the container's netns, which + // differs from this process's own ambient (host) netns, so the + // CNI library's same-netns rejection check never fires. For a + // tap-mode attachment, though, CNI_NETNS is deliberately set to + // the host's own root netns (there's no per-VM netns to enter), + // which does equal this process's ambient netns — so the same + // peek-and-repipe dance and CNI_NETNS_OVERRIDE galactic-cni uses + // for tap mode are needed here too, or the library rejects every + // tap-mode ADD/DEL after the route is already installed. + stdinData, _ := io.ReadAll(os.Stdin) + r, w, _ := os.Pipe() + go func() { + _, _ = w.Write(stdinData) + _ = w.Close() + }() + oldStdin := os.Stdin + os.Stdin = r + defer func() { os.Stdin = oldStdin }() + + if isTapMode(stdinData) { + _ = os.Setenv("CNI_NETNS_OVERRIDE", "true") + } + + cniroute.RunPlugin() + return nil + }, + } + + cmd.PersistentFlags().String("conf-file", cniroute.ConfFile, "Path to CNI conflist file") + cmd.Flags().Bool("build-info", false, "Print build information and exit") + cmd.Flags().BoolP("version", "V", false, "Print version and exit") + return cmd +} + +// isTapMode returns true when the CNI config requests tap interface type. +// Only a minimal JSON parse is needed — full validation happens later in +// parseConf inside cmdAdd/cmdDel. +func isTapMode(stdinData []byte) bool { + var cfg struct { + InterfaceType string `json:"interface_type"` + } + _ = json.Unmarshal(stdinData, &cfg) + return cfg.InterfaceType == "tap" +} + +func main() { + if err := newRootCommand().Execute(); err != nil { + log.Fatalf("error: %v", err) + } +} diff --git a/cmd/galactic-tap-cni/main.go b/cmd/galactic-tap-cni/main.go new file mode 100644 index 00000000..023741bd --- /dev/null +++ b/cmd/galactic-tap-cni/main.go @@ -0,0 +1,95 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "fmt" + "log" + "os" + "strings" + + "github.com/containernetworking/cni/pkg/version" + "github.com/spf13/cobra" + "golang.org/x/term" + + "go.datum.net/galactic/internal/cnitap" + "go.datum.net/galactic/internal/metadata" +) + +const ( + appName = "galactic-tap-cni" + + appDesc = `Galactic tap CNI Plugin + + The tap master plugin in the galactic CNI chain, for VM-based workloads + (Kata, Firecracker, kraftlet/Unikraft) attaching directly to a galactic VPC + network. Unrelated to vmtap-cni, which is chained after Cilium's own CNI + plugin for a different purpose entirely (see its own doc comment). + + Find more information at: https://www.datum.net/docs` +) + +func newRootCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: appName, + Short: strings.Split(appDesc, "\n")[0], + Long: appDesc, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + cnitap.InitCNIConfig() + confFile, _ := cmd.Flags().GetString("conf-file") + if confFile != "" { + cnitap.ConfFile = confFile + } + return nil + }, + RunE: func(cmd *cobra.Command, _ []string) error { + if ok, _ := cmd.Flags().GetBool("build-info"); ok { + fmt.Println(metadata.BuildInfo(appName)) + return nil + } + if ok, _ := cmd.Flags().GetBool("version"); ok { + fmt.Printf("%s version %s\n", appName, metadata.Version) + return nil + } + if os.Getenv("CNI_COMMAND") == "VERSION" { + return version.All.Encode(os.Stdout) + } + + // Real CNI runtimes always pipe the network config JSON on + // stdin and close it. If stdin is an interactive terminal + // instead, no config will ever arrive and skel's blocking + // stdin read would hang forever — print version info instead. + if term.IsTerminal(int(os.Stdin.Fd())) { + fmt.Printf("%s version %s\n", appName, metadata.Version) + fmt.Printf("CNI protocol versions supported: %s\n", strings.Join(version.All.SupportedVersions(), ", ")) + return nil + } + + // Tap mode never enters a network namespace — all operations + // are host-side. Set the override so the CNI library skips its + // same-netns rejection check, which would otherwise reject + // kraftlet workloads that pass the host netns. Unconditional + // here (unlike galactic-cni, which has no override logic at + // all): every invocation of this binary is tap mode, so there + // is no config content to peek at first. + _ = os.Setenv("CNI_NETNS_OVERRIDE", "true") + + cnitap.RunPlugin() + return nil + }, + } + + cmd.PersistentFlags().String("conf-file", cnitap.ConfFile, "Path to CNI conflist file") + cmd.Flags().Bool("build-info", false, "Print build information and exit") + cmd.Flags().BoolP("version", "V", false, "Print version and exit") + + return cmd +} + +func main() { + if err := newRootCommand().Execute(); err != nil { + log.Fatalf("error: %v", err) + } +} diff --git a/containers/galactic-cni/Dockerfile b/containers/galactic-cni/Dockerfile index c949f093..c8c9fe17 100644 --- a/containers/galactic-cni/Dockerfile +++ b/containers/galactic-cni/Dockerfile @@ -48,6 +48,60 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ -X go.datum.net/galactic/internal/metadata.GitURL=${GIT_URL}" \ -o galactic-cni cmd/galactic-cni/main.go +# Build galactic-tap-cni, the tap master plugin in the galactic CNI chain +# (VM workloads: Kata, Firecracker, kraftlet/Unikraft). Ships in this same +# image/binary set since every plugin in the chain is staged onto the host +# by the same galactic-cni init container (installer.Bootstrap). +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ + -ldflags "-s -w \ + -X go.datum.net/galactic/internal/metadata.Version=${VERSION} \ + -X go.datum.net/galactic/internal/metadata.GitCommit=${GIT_COMMIT} \ + -X go.datum.net/galactic/internal/metadata.GitTreeState=${GIT_TREE_STATE} \ + -X go.datum.net/galactic/internal/metadata.BuildDate=${BUILD_DATE} \ + -X go.datum.net/galactic/internal/metadata.SPDXLicense=${SPDX_LICENSE} \ + -X go.datum.net/galactic/internal/metadata.GitURL=${GIT_URL}" \ + -o galactic-tap-cni cmd/galactic-tap-cni/main.go + +# Build galactic-ipam, the delegated CNI IPAM plugin in the galactic CNI +# chain. Ships in this same image/binary set for the same reason +# galactic-tap-cni does — see its own comment above. +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ + -ldflags "-s -w \ + -X go.datum.net/galactic/internal/metadata.Version=${VERSION} \ + -X go.datum.net/galactic/internal/metadata.GitCommit=${GIT_COMMIT} \ + -X go.datum.net/galactic/internal/metadata.GitTreeState=${GIT_TREE_STATE} \ + -X go.datum.net/galactic/internal/metadata.BuildDate=${BUILD_DATE} \ + -X go.datum.net/galactic/internal/metadata.SPDXLicense=${SPDX_LICENSE} \ + -X go.datum.net/galactic/internal/metadata.GitURL=${GIT_URL}" \ + -o galactic-ipam cmd/galactic-ipam/main.go + +# Build galactic-bgp, the BGP/SRv6/eBPF publish plugin in the galactic CNI +# chain. Ships in this same image/binary set for the same reason +# galactic-tap-cni/galactic-ipam do — see their own comments above. +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ + -ldflags "-s -w \ + -X go.datum.net/galactic/internal/metadata.Version=${VERSION} \ + -X go.datum.net/galactic/internal/metadata.GitCommit=${GIT_COMMIT} \ + -X go.datum.net/galactic/internal/metadata.GitTreeState=${GIT_TREE_STATE} \ + -X go.datum.net/galactic/internal/metadata.BuildDate=${BUILD_DATE} \ + -X go.datum.net/galactic/internal/metadata.SPDXLicense=${SPDX_LICENSE} \ + -X go.datum.net/galactic/internal/metadata.GitURL=${GIT_URL}" \ + -o galactic-bgp cmd/galactic-bgp/main.go + +# Build galactic-route, the termination-route plugin in the galactic CNI +# chain. Ships in this same image/binary set for the same reason +# galactic-tap-cni/galactic-ipam/galactic-bgp do — see their own comments +# above. +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build \ + -ldflags "-s -w \ + -X go.datum.net/galactic/internal/metadata.Version=${VERSION} \ + -X go.datum.net/galactic/internal/metadata.GitCommit=${GIT_COMMIT} \ + -X go.datum.net/galactic/internal/metadata.GitTreeState=${GIT_TREE_STATE} \ + -X go.datum.net/galactic/internal/metadata.BuildDate=${BUILD_DATE} \ + -X go.datum.net/galactic/internal/metadata.SPDXLicense=${SPDX_LICENSE} \ + -X go.datum.net/galactic/internal/metadata.GitURL=${GIT_URL}" \ + -o galactic-route cmd/galactic-route/main.go + # Build vmtap-cni. It ships in this image rather than one of its own so the # vmtap DaemonSet (config/vmtap/) can reference the same published # ghcr.io/datum-cloud/galactic-cni image instead of a second, separately @@ -82,6 +136,10 @@ FROM gcr.io/distroless/static:nonroot AS production # Copy binaries from builder into the distroless production image COPY --from=builder /workspace/galactic-cni /galactic-cni +COPY --from=builder /workspace/galactic-tap-cni /galactic-tap-cni +COPY --from=builder /workspace/galactic-ipam /galactic-ipam +COPY --from=builder /workspace/galactic-bgp /galactic-bgp +COPY --from=builder /workspace/galactic-route /galactic-route COPY --from=builder /workspace/vmtap-cni /vmtap-cni COPY --from=builder /workspace/host-device /host-device COPY --from=builder /var/run/galactic-cni /var/run/galactic-cni @@ -94,6 +152,10 @@ FROM docker.io/library/alpine:latest # Copy binaries from the production (distroless) image COPY --from=production /galactic-cni /galactic-cni +COPY --from=production /galactic-tap-cni /galactic-tap-cni +COPY --from=production /galactic-ipam /galactic-ipam +COPY --from=production /galactic-bgp /galactic-bgp +COPY --from=production /galactic-route /galactic-route COPY --from=production /vmtap-cni /vmtap-cni COPY --from=production /host-device /host-device COPY --from=production /var/run/galactic-cni /var/run/galactic-cni diff --git a/deploy/containerlab/docs/tenants.md b/deploy/containerlab/docs/tenants.md index 8ecb0915..cfdda621 100644 --- a/deploy/containerlab/docs/tenants.md +++ b/deploy/containerlab/docs/tenants.md @@ -9,11 +9,13 @@ pod-to-pod connectivity. Every one of them follows the same mechanism: Multus attaches each `netshoot` pod to its VPC's `private` NetworkAttachmentDefinition (via the `v1.multus-cni.io/default-network` annotation, which makes the VPC interface the pod's `eth0` rather than an additional `net1` — there is no -`k8s.v1.cni.cncf.io/networks` annotation in play here), which invokes -`galactic-cni` to create a VRF, veth pair, SRv6 encapsulation route, and a -`BGPAdvertisement` CRD. The `galactic-router` controller then advertises each -pod's EVPN route to the route reflector, distributing reachability across -sites. +`k8s.v1.cni.cncf.io/networks` annotation in play here), which invokes the +galactic CNI plugin chain: `galactic-cni` creates a VRF and veth pair, then +`galactic-bgp` registers the attachment against the eBPF uSID datapath and +writes a `BGPAdvertisement` CRD (see [docs/cni-cmd-sequence.md](../../../docs/cni-cmd-sequence.md) +for the full per-binary ADD sequence). The `galactic-router` controller then +advertises each pod's EVPN route to the route reflector, distributing +reachability across sites. They differ only in scope and addressing: @@ -37,7 +39,7 @@ fabric. The low hextet of a pod's USID is `(Function << 12) | Argument` `0xE` (`FunctionEndDT46`) for every plain L3 VRF attachment, and `Argument` is a 12-bit value `galactic-router` allocates per-node as the lowest unused slot in `[0x001, 0xFFF]` among that node's existing `BGPVRFInstance` CRDs -(`allocateArgument`, `internal/cni/bgp.go`) — **not** a decode of the NAD's +(`allocateArgument`, `internal/cnibgp/bgp.go`) — **not** a decode of the NAD's `vpc`/`vpcattachment` values. Concretely, expect hextets in the `0xe001`–`0xefff` range; the exact value depends on allocation order (`ns50` is provisioned first in `task deploy`, then `ns10`, `ns20`, `ns30`, `ns40` in diff --git a/deploy/containerlab/resources/tenants/ns10/dfw/nad.yaml b/deploy/containerlab/resources/tenants/ns10/dfw/nad.yaml index 5cdd682b..a57005bf 100644 --- a/deploy/containerlab/resources/tenants/ns10/dfw/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns10/dfw/nad.yaml @@ -5,14 +5,27 @@ metadata: name: private namespace: ns10 spec: - config: | + config: |- { "cniVersion": "1.0.0", "name": "private", - "type": "galactic-cni", - "vpc": "10", - "vpcattachment": "10", - "namespace": "galactic-system", - "ipv6_subnet": "fd20:10:ff01::/48", - "address_families": ["ipv6"] + "plugins": [ + { + "type": "galactic-cni", + "vpc": "10", + "vpcattachment": "10", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv6_subnet": "fd20:10:ff01::/48", + "address_families": ["ipv6"] + } + }, + { + "type": "galactic-bgp", + "vpc": "10", + "vpcattachment": "10", + "namespace": "galactic-system" + } + ] } diff --git a/deploy/containerlab/resources/tenants/ns10/iad/nad.yaml b/deploy/containerlab/resources/tenants/ns10/iad/nad.yaml index 9955b1ec..a8a80968 100644 --- a/deploy/containerlab/resources/tenants/ns10/iad/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns10/iad/nad.yaml @@ -5,14 +5,27 @@ metadata: name: private namespace: ns10 spec: - config: | + config: |- { "cniVersion": "1.0.0", "name": "private", - "type": "galactic-cni", - "vpc": "10", - "vpcattachment": "10", - "namespace": "galactic-system", - "ipv6_subnet": "fd20:10:ff03::/48", - "address_families": ["ipv6"] + "plugins": [ + { + "type": "galactic-cni", + "vpc": "10", + "vpcattachment": "10", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv6_subnet": "fd20:10:ff03::/48", + "address_families": ["ipv6"] + } + }, + { + "type": "galactic-bgp", + "vpc": "10", + "vpcattachment": "10", + "namespace": "galactic-system" + } + ] } diff --git a/deploy/containerlab/resources/tenants/ns10/sjc/nad.yaml b/deploy/containerlab/resources/tenants/ns10/sjc/nad.yaml index fdd6439b..2bdccbe3 100644 --- a/deploy/containerlab/resources/tenants/ns10/sjc/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns10/sjc/nad.yaml @@ -5,14 +5,27 @@ metadata: name: private namespace: ns10 spec: - config: | + config: |- { "cniVersion": "1.0.0", "name": "private", - "type": "galactic-cni", - "vpc": "10", - "vpcattachment": "10", - "namespace": "galactic-system", - "ipv6_subnet": "fd20:10:ff02::/48", - "address_families": ["ipv6"] + "plugins": [ + { + "type": "galactic-cni", + "vpc": "10", + "vpcattachment": "10", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv6_subnet": "fd20:10:ff02::/48", + "address_families": ["ipv6"] + } + }, + { + "type": "galactic-bgp", + "vpc": "10", + "vpcattachment": "10", + "namespace": "galactic-system" + } + ] } diff --git a/deploy/containerlab/resources/tenants/ns20/dfw/nad.yaml b/deploy/containerlab/resources/tenants/ns20/dfw/nad.yaml index 0dec53aa..58e96428 100644 --- a/deploy/containerlab/resources/tenants/ns20/dfw/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns20/dfw/nad.yaml @@ -5,15 +5,28 @@ metadata: name: private namespace: ns20 spec: - config: | + config: |- { "cniVersion": "1.0.0", "name": "private", - "type": "galactic-cni", - "vpc": "20", - "vpcattachment": "20", - "namespace": "galactic-system", - "ipv6_subnet": "fd20:20:ff01::/48", - "ipv4_subnet": "172.21.1.0/24", - "address_families": ["ipv6", "ipv4"] + "plugins": [ + { + "type": "galactic-cni", + "vpc": "20", + "vpcattachment": "20", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv6_subnet": "fd20:20:ff01::/48", + "ipv4_subnet": "172.21.1.0/24", + "address_families": ["ipv6", "ipv4"] + } + }, + { + "type": "galactic-bgp", + "vpc": "20", + "vpcattachment": "20", + "namespace": "galactic-system" + } + ] } diff --git a/deploy/containerlab/resources/tenants/ns20/iad/nad.yaml b/deploy/containerlab/resources/tenants/ns20/iad/nad.yaml index bfa8f449..1ec302ec 100644 --- a/deploy/containerlab/resources/tenants/ns20/iad/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns20/iad/nad.yaml @@ -5,15 +5,28 @@ metadata: name: private namespace: ns20 spec: - config: | + config: |- { "cniVersion": "1.0.0", "name": "private", - "type": "galactic-cni", - "vpc": "20", - "vpcattachment": "20", - "namespace": "galactic-system", - "ipv6_subnet": "fd20:20:ff03::/48", - "ipv4_subnet": "172.21.10.0/24", - "address_families": ["ipv6", "ipv4"] + "plugins": [ + { + "type": "galactic-cni", + "vpc": "20", + "vpcattachment": "20", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv6_subnet": "fd20:20:ff03::/48", + "ipv4_subnet": "172.21.10.0/24", + "address_families": ["ipv6", "ipv4"] + } + }, + { + "type": "galactic-bgp", + "vpc": "20", + "vpcattachment": "20", + "namespace": "galactic-system" + } + ] } diff --git a/deploy/containerlab/resources/tenants/ns20/sjc/nad.yaml b/deploy/containerlab/resources/tenants/ns20/sjc/nad.yaml index 13e93bcd..ecdf720a 100644 --- a/deploy/containerlab/resources/tenants/ns20/sjc/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns20/sjc/nad.yaml @@ -5,15 +5,28 @@ metadata: name: private namespace: ns20 spec: - config: | + config: |- { "cniVersion": "1.0.0", "name": "private", - "type": "galactic-cni", - "vpc": "20", - "vpcattachment": "20", - "namespace": "galactic-system", - "ipv6_subnet": "fd20:20:ff02::/48", - "ipv4_subnet": "172.21.20.0/24", - "address_families": ["ipv6", "ipv4"] + "plugins": [ + { + "type": "galactic-cni", + "vpc": "20", + "vpcattachment": "20", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv6_subnet": "fd20:20:ff02::/48", + "ipv4_subnet": "172.21.20.0/24", + "address_families": ["ipv6", "ipv4"] + } + }, + { + "type": "galactic-bgp", + "vpc": "20", + "vpcattachment": "20", + "namespace": "galactic-system" + } + ] } diff --git a/deploy/containerlab/resources/tenants/ns30/dfw/nad.yaml b/deploy/containerlab/resources/tenants/ns30/dfw/nad.yaml index 0f0bb879..ef3fba15 100644 --- a/deploy/containerlab/resources/tenants/ns30/dfw/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns30/dfw/nad.yaml @@ -5,14 +5,27 @@ metadata: name: private namespace: ns30 spec: - config: | + config: |- { "cniVersion": "1.0.0", "name": "private", - "type": "galactic-cni", - "vpc": "30", - "vpcattachment": "30", - "namespace": "galactic-system", - "ipv6_subnet": "fd20:30:ff01::/48", - "address_families": ["ipv6"] + "plugins": [ + { + "type": "galactic-cni", + "vpc": "30", + "vpcattachment": "30", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv6_subnet": "fd20:30:ff01::/48", + "address_families": ["ipv6"] + } + }, + { + "type": "galactic-bgp", + "vpc": "30", + "vpcattachment": "30", + "namespace": "galactic-system" + } + ] } diff --git a/deploy/containerlab/resources/tenants/ns40/iad/nad.yaml b/deploy/containerlab/resources/tenants/ns40/iad/nad.yaml index 2b341ee3..fd610037 100644 --- a/deploy/containerlab/resources/tenants/ns40/iad/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns40/iad/nad.yaml @@ -5,14 +5,27 @@ metadata: name: private namespace: ns40 spec: - config: | + config: |- { "cniVersion": "1.0.0", "name": "private", - "type": "galactic-cni", - "vpc": "40", - "vpcattachment": "40", - "namespace": "galactic-system", - "ipv4_subnet": "172.40.10.0/24", - "address_families": ["ipv4"] + "plugins": [ + { + "type": "galactic-cni", + "vpc": "40", + "vpcattachment": "40", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv4_subnet": "172.40.10.0/24", + "address_families": ["ipv4"] + } + }, + { + "type": "galactic-bgp", + "vpc": "40", + "vpcattachment": "40", + "namespace": "galactic-system" + } + ] } diff --git a/deploy/containerlab/resources/tenants/ns50/dfw/nad.yaml b/deploy/containerlab/resources/tenants/ns50/dfw/nad.yaml index 949a90da..6a458e23 100644 --- a/deploy/containerlab/resources/tenants/ns50/dfw/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns50/dfw/nad.yaml @@ -5,16 +5,29 @@ metadata: name: private namespace: ns50 spec: - config: | + config: |- { "cniVersion": "1.0.0", "name": "private", - "type": "galactic-cni", - "vpc": "50", - "vpcattachment": "50", - "namespace": "galactic-system", - "ipv4_subnet": "172.20.1.0/24", - "address_families": ["ipv4"] + "plugins": [ + { + "type": "galactic-cni", + "vpc": "50", + "vpcattachment": "50", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv4_subnet": "172.20.1.0/24", + "address_families": ["ipv4"] + } + }, + { + "type": "galactic-bgp", + "vpc": "50", + "vpcattachment": "50", + "namespace": "galactic-system" + } + ] } --- @@ -24,7 +37,7 @@ metadata: name: igw namespace: ns50 spec: - config: | + config: |- { "cniVersion": "1.0.0", "name": "public", diff --git a/deploy/containerlab/resources/tenants/ns50/iad/nad.yaml b/deploy/containerlab/resources/tenants/ns50/iad/nad.yaml index ad08218b..a13dc6b5 100644 --- a/deploy/containerlab/resources/tenants/ns50/iad/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns50/iad/nad.yaml @@ -5,14 +5,27 @@ metadata: name: private namespace: ns50 spec: - config: | + config: |- { "cniVersion": "1.0.0", "name": "private", - "type": "galactic-cni", - "vpc": "50", - "vpcattachment": "50", - "namespace": "galactic-system", - "ipv4_subnet": "172.20.10.0/24", - "address_families": ["ipv4"] + "plugins": [ + { + "type": "galactic-cni", + "vpc": "50", + "vpcattachment": "50", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv4_subnet": "172.20.10.0/24", + "address_families": ["ipv4"] + } + }, + { + "type": "galactic-bgp", + "vpc": "50", + "vpcattachment": "50", + "namespace": "galactic-system" + } + ] } diff --git a/deploy/containerlab/resources/tenants/ns50/sjc/nad.yaml b/deploy/containerlab/resources/tenants/ns50/sjc/nad.yaml index 25ea8b37..b3d88dd8 100644 --- a/deploy/containerlab/resources/tenants/ns50/sjc/nad.yaml +++ b/deploy/containerlab/resources/tenants/ns50/sjc/nad.yaml @@ -5,14 +5,27 @@ metadata: name: private namespace: ns50 spec: - config: | + config: |- { "cniVersion": "1.0.0", "name": "private", - "type": "galactic-cni", - "vpc": "50", - "vpcattachment": "50", - "namespace": "galactic-system", - "ipv4_subnet": "172.20.20.0/24", - "address_families": ["ipv4"] + "plugins": [ + { + "type": "galactic-cni", + "vpc": "50", + "vpcattachment": "50", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv4_subnet": "172.20.20.0/24", + "address_families": ["ipv4"] + } + }, + { + "type": "galactic-bgp", + "vpc": "50", + "vpcattachment": "50", + "namespace": "galactic-system" + } + ] } diff --git a/docs/agents/ARCHITECTURE.md b/docs/agents/ARCHITECTURE.md index 27c2bf36..eea223d0 100644 --- a/docs/agents/ARCHITECTURE.md +++ b/docs/agents/ARCHITECTURE.md @@ -1,22 +1,34 @@ # Architecture -> Galactic is the SRv6 data plane for multi-cloud VPC networking, deployed as two -> binaries on each Kubernetes node: a CNI plugin that attaches containers to VPC -> networks, and a router that reconciles BGP CRDs and drives an embedded +> Galactic is the SRv6 data plane for multi-cloud VPC networking, deployed on +> each Kubernetes node as a CNI plugin chain that attaches containers/VMs to +> VPC networks, and a router that reconciles BGP CRDs and drives an embedded > GoBGP server to distribute EVPN (L2VPN/EVPN AFI/SAFI) paths between nodes. -_Last updated: 2026-07-14_ +_Last updated: 2026-08-08_ --- ## Overview Galactic implements VPC isolation and cross-cluster reachability using Linux SRv6. -When a pod is attached to a VPC, the CNI plugin creates the required kernel state -(VRF, veth pair, SRv6 ingress route) and writes a `BGPAdvertisement` CRD. -`galactic-router` watches that CRD and injects the EVPN path into the node-local -GoBGP server. GoBGP distributes the path to a BGP route reflector, enabling pods -on different nodes or clusters to reach each other via SRv6-encapsulated traffic. +When a pod or VM is attached to a VPC, a chain of CNI plugins creates the required +kernel state (VRF, veth pair or tap device, host-side routes) and writes a +`BGPAdvertisement` CRD. `galactic-router` watches that CRD and injects the EVPN +path into the node-local GoBGP server. GoBGP distributes the path to a BGP route +reflector, enabling pods/VMs on different nodes or clusters to reach each other +via SRv6-encapsulated traffic. + +The CNI attach side is a **chain of small binaries**, not one monolithic +plugin: a master plugin (`galactic-cni` for containers, `galactic-tap-cni` for +VM workloads) creates the VRF and host-side interface; an optional +`galactic-route` installs static termination routes; `galactic-bgp` publishes +the BGP/SRv6/eBPF state. IPAM is delegated to a separate `galactic-ipam` +binary via the standard CNI IPAM delegation protocol, not chained. All five +binaries (plus `vmtap-cni`, `host-device`) ship in the same +`ghcr.io/datum-cloud/galactic-cni` image and are staged onto the host by the +same init container — see [Repository Layout](#repository-layout) and +[Module / Package Reference](#module--package-reference) below. VPC and VPCAttachment CRDs are owned by a separate companion operator (`go.datum.net/cloud`). Galactic receives pre-populated identifiers through the @@ -26,22 +38,35 @@ CNI config and acts on them. `galactic-router` reconciles BGP CRDs ### SRv6 SID encoding - -Each container endpoint is assigned a /128 USID (Unique Local SID, RFC 8986 Section 3.2). -There is no longer a companion-operator-injected `srv6_sid` NAD/config field: the CNI -itself computes the SID in `resolveSRv6SID` (`internal/cni/bgp.go`) from the node's -`BGPRouter.spec.srv6Locator` + `spec.nodeID` plus this attachment's VRFID (`srv6.ComputeSID`, -`internal/plumbing/srv6/usid.go`), using the End.DT46 function. If the router lacks either -`srv6Locator` or `nodeID`, SID resolution — and SRv6 ingress setup — is skipped entirely for -that attachment. The CNI installs an END.DT46 decap route for the computed /128 and -advertises it in a BGP Prefix-SID path attribute (RFC 9252 SRv6 L3 Service TLV, -`internal/runtime/gobgp/paths.go`'s `prefixSIDAttr`) — not the EVPN Type 5 route's own -Gateway IP field, which RFC 9136 requires to share the prefix's own address family and so -cannot carry an IPv6 SID for an IPv4 VPC prefix. + (SID encoding/allocation, base62 interface naming, eBPF uSID datapath, + EVPN Type 5 path construction, worked ContainerLab example), does not + exist in this repository. Either write it, point this section + elsewhere, or remove this note — flagged for a human decision. --> + +Each attachment endpoint is assigned a /128 USID (Unique Local SID, RFC 8986 +Section 3.2). There is no companion-operator-injected `srv6_sid` NAD/config +field: `galactic-bgp` (`internal/cnibgp/bgp.go`'s `registerEBPFDatapath`) +derives the uSID `Block` from the node's `BGPRouter.spec.srv6Locator` via +`uformat.Block`, and registers `locator_table`/`function_table`/`vrf_table` +entries keyed on that `Block` plus this attachment's `Argument` (its +`BGPVRFInstance`'s VRFID) in the eBPF uSID datapath's pinned maps +(`internal/plumbing/ebpf/usidmap`) — no kernel seg6local route is installed +per attachment anymore; the TC-BPF program (`internal/plumbing/ebpf/prog/usid.c`) +is the only ingress/decap path (see Known Constraints below for the cutover +history). If the router lacks either `srv6Locator` or `nodeID`, eBPF +registration is skipped entirely for that attachment (`registerEBPFDatapath` +returns `registered=false`, not an error). + +`galactic-router`'s own reconciler independently derives the *same* SID value +from the same inputs (`srv6.ComputeSID`, `internal/plumbing/srv6/usid.go`, +called from `internal/reconcile/reconcile.go`) for the BGP control-plane +side: advertising it in a BGP Prefix-SID path attribute (RFC 9252 SRv6 L3 +Service TLV, `internal/runtime/gobgp/paths.go`'s `prefixSIDAttr`) — not the +EVPN Type 5 route's own Gateway IP field, which RFC 9136 requires to share +the prefix's own address family and so cannot carry an IPv6 SID for an IPv4 +VPC prefix. The CNI side and the router side compute the same bit layout via +separate code paths (`uformat` is the single source of truth both build on), +by design — see `internal/plumbing/ebpf/doc.go`. All nodes in the same VPC derive the same BGP Route Target by truncating the 48-bit hex VPC identifier to its low 32 bits (`uint32(v)`), formatted as @@ -56,8 +81,13 @@ Distinguisher and import/export Route Target. ``` galactic/ ├── cmd/ -│ ├── galactic-cni/ # CNI binary -│ └── galactic-router/ # Router binary (controller-runtime reconciler) +│ ├── galactic-cni/ # veth master plugin binary +│ ├── galactic-tap-cni/ # tap master plugin binary (VM workloads) +│ ├── galactic-ipam/ # delegated CNI IPAM plugin binary +│ ├── galactic-bgp/ # BGP/SRv6/eBPF publish plugin binary +│ ├── galactic-route/ # termination-route plugin binary (optional chain stage) +│ ├── galactic-router/ # Router binary (controller-runtime reconciler) +│ └── vmtap-cni/ # Cilium chain-conflist patcher for VM tap interfaces ├── internal/ │ ├── controller/ # controller-runtime reconcilers (BGPRouter, BGPPeer, │ │ # BGPAdvertisement, BGPVRFInstance, BGPPolicy, Secret, @@ -70,39 +100,73 @@ galactic/ │ ├── model/ # DesiredRouter and family; re-exports BGP API enums │ ├── hash/ # SHA-256 change detection over DesiredRouter │ ├── metadata/ # Build-time version info (Version, GitCommit, etc.) -│ ├── gc/ # Orphaned BGPAdvertisement/BGPVRFInstance CRD and -│ │ # stale kernel VRF cleanup, driven by the GC controller -│ ├── cni/ # CNI cmdAdd / cmdDel / cmdCheck, PluginConf parsing, -│ │ # BGP CRD publish, built-in IPAM wiring -│ │ ├── ipam/ # Built-in IPv6 pool + static IP allocators +│ ├── gc/ # Orphaned BGPAdvertisement/BGPVRFInstance CRD, stale +│ │ # kernel VRF, and eBPF vrf_table entry cleanup, +│ │ # driven by the GC controller +│ ├── cni/ # galactic-cni: veth master plugin (cmdAdd/cmdDel/ +│ │ │ # cmdCheck/cmdStatus, PluginConf parsing, NAD +│ │ │ # annotation, host-device delegation) +│ │ ├── hostconf/ # Shared static-conflist HostConf schema/loader, +│ │ │ # read by every binary in the chain +│ │ ├── hostgw/ # Host-side gateway address/route configuration, +│ │ │ # called directly by both master plugins +│ │ ├── crdnames/ # Deterministic BGPVRFInstance/BGPAdvertisement +│ │ │ # CRD name derivation, shared by cnibgp/gc +│ │ ├── nadpatch/ # NAD annotation patch, shared by cni/cnitap +│ │ ├── ipam/ # Built-in IPv6/IPv4 pool + static IP allocators, +│ │ │ # on-disk marker-file persistence │ │ ├── route/ # Host-side static routes via netlink │ │ ├── tap/ # Tap interface management (VM workloads) │ │ └── veth/ # veth pair management +│ ├── cnitap/ # galactic-tap-cni: tap master plugin (mirrors +│ │ # internal/cni; no host-device delegation, no +│ │ # guest netns) +│ ├── cniipam/ # galactic-ipam: CNI IPAM delegation protocol +│ │ # (cmdAdd/cmdDel/cmdCheck/cmdStatus); no k8s +│ │ # dependency +│ ├── cnibgp/ # galactic-bgp: BGP/SRv6/eBPF publish plugin; +│ │ # learns interface kind + addresses from +│ │ # prevResult alone, zero kernel-interface access +│ ├── cniroute/ # galactic-route: termination-route plugin; +│ │ # zero Kubernetes dependency +│ ├── vmtap/ # vmtap-cni: patches Cilium's own chain conflist +│ │ # to add a tap-interface stage for VM workloads │ ├── installer/ # galactic-cni DaemonSet init/run logic: binary -│ │ # staging, conflist templating, kubeconfig -│ │ # refresh, gRPC health server +│ │ # staging (all binaries above, one init +│ │ # container/image), conflist templating, +│ │ # kubeconfig refresh, gRPC health server │ └── plumbing/ # Low-level kernel and network primitives │ ├── intf/ # Interface naming, base62↔hex encoding -│ ├── srv6/ # SRv6 ingress route add/del (END.DT46) +│ ├── srv6/ # SID computation (ComputeSID) for the router's +│ │ # own BGP Prefix-SID path attribute +│ ├── ebpf/ # TC-BPF uSID datapath: preflight, uformat, +│ │ # prog (usid.c), attach, usidmap, metrics — +│ │ # see internal/plumbing/ebpf/doc.go │ ├── sysctl/ # Interface sysctl helpers │ └── vrf/ # Linux VRF create/delete/lookup ├── config/ # Kustomize-composed; `kubectl apply -k config/` deploys everything -│ ├── system/ # galactic-system namespace (shared by both components) +│ ├── system/ # galactic-system namespace (shared by all components) │ ├── router/ # Shared RBAC/ServiceAccount, plus: │ │ ├── base/ # common DaemonSet spec │ │ ├── tenant/ # per-node role: base + node affinity excluding control-plane │ │ │ # and tenant-control nodes │ │ └── tenant-control/ # route-reflector role: base + GALACTIC_ROUTER_REFLECTOR=true, │ │ # opt-in via the galactic.datumapis.com/node=control node label -│ └── cni/ # hostNetwork DaemonSet: `init` container stages -│ # galactic-cni/host-device into /opt/cni/bin -│ # and writes the conflist + kubeconfig; `run` -│ # container refreshes credentials and serves -│ # gRPC health checks +│ ├── cni/ # hostNetwork DaemonSet: `init` container stages +│ │ # every chain binary + host-device into +│ │ # /opt/cni/bin and writes the conflist + +│ │ # kubeconfig; `run` container refreshes +│ │ # credentials, manages the eBPF datapath, and +│ │ # serves gRPC health checks +│ ├── vmtap/ # vmtap-cni DaemonSet: stages vmtap-cni and patches +│ │ # Cilium's chain conflist +│ └── fabric/ # FRR underlay eBGP DaemonSet (not part of +│ # `kubectl apply -k config/` — see below) ├── deploy/ │ └── containerlab/ # ContainerLab lab topology and scripts └── containers/ - ├── galactic-cni/ # galactic-cni + host-device image (e2e test and production publish) + ├── galactic-cni/ # galactic-cni/-tap-cni/-ipam/-bgp/-route + vmtap-cni + │ # + host-device image (e2e test and production publish) └── galactic-router/ # galactic-router production image ``` @@ -128,16 +192,20 @@ See [docs/agent-startup.md](../agent-startup.md) for the router startup sequence | `internal/runtime/frr` | `galactic-router` | FRR stub (`--mode=fabric`) — returns "not implemented" for every method | | `internal/model` | `galactic-router` | Internal BGP model types | | `internal/hash` | `galactic-router` | Change detection | -| `internal/metadata` | both | Build-time version info stamped via `-ldflags` | -| `internal/gc` | `galactic-router` | Orphaned CRD/VRF cleanup, driven by the GC controller's ticker | -| `internal/cni` | `galactic-cni` | CNI cmdAdd / cmdDel / cmdCheck; BGP CRD publish | -| `internal/cni/ipam` | `galactic-cni` | Built-in IPv6 pool + static allocators | -| `internal/cni/tap` | `galactic-cni` | Tap interface create/delete (VM workloads) | -| `internal/installer` | `galactic-cni` | DaemonSet `init`/`run` logic: binary staging, conflist/kubeconfig templating, credential refresh, gRPC health server | -| `internal/plumbing/intf` | both | Interface naming, base62↔hex encoding | -| `internal/plumbing/srv6` | both | SRv6 ingress route add/del (END.DT46) | -| `internal/plumbing/vrf` | both | Linux VRF create/delete/lookup | -| `internal/plumbing/sysctl` | both | Interface sysctl helpers | +| `internal/metadata` | every binary | Build-time version info stamped via `-ldflags` | +| `internal/gc` | `galactic-router` | Orphaned CRD/VRF/eBPF-entry cleanup, driven by the GC controller's ticker | +| `internal/cni` | `galactic-cni` | Veth master plugin: cmdAdd/cmdDel/cmdCheck/cmdStatus, PluginConf parsing, NAD annotation, host-device delegation | +| `internal/cnitap` | `galactic-tap-cni` | Tap master plugin (mirrors `internal/cni`, no guest netns) | +| `internal/cniipam` | `galactic-ipam` | Delegated CNI IPAM plugin (no k8s dependency) | +| `internal/cnibgp` | `galactic-bgp` | BGP/SRv6/eBPF publish plugin (zero kernel-interface dependency) | +| `internal/cniroute` | `galactic-route` | Termination-route plugin (no k8s dependency) | +| `internal/vmtap` | `vmtap-cni` | Cilium chain-conflist patcher for VM tap interfaces | +| `internal/installer` | `galactic-cni` | DaemonSet `init`/`run` logic: binary staging (every chain binary), conflist/kubeconfig templating, credential refresh, gRPC health server | +| `internal/plumbing/intf` | every CNI-chain binary + router | Interface naming, base62↔hex encoding | +| `internal/plumbing/srv6` | `galactic-router` | SID computation (`ComputeSID`) for the BGP Prefix-SID path attribute | +| `internal/plumbing/ebpf` | `galactic-cni` (attach/metrics via `run`), `galactic-bgp` (registration), `galactic-router` (GC sweep) | TC-BPF uSID datapath: preflight, uformat, prog, attach, usidmap, metrics | +| `internal/plumbing/vrf` | every CNI-chain binary + router | Linux VRF create/delete/lookup | +| `internal/plumbing/sysctl` | `galactic-cni`, `galactic-tap-cni` | Interface sysctl helpers | --- @@ -164,18 +232,65 @@ wrap `internal/installer`. `parseConf()` resolves node name, kubeconfig, namespace, and log file on every invocation (conflist → env vars → API auto-detect, in that precedence). +`galactic-cni`'s own ADD creates the VRF, veth pair, and (if `"ipam"` is +present) delegates IPAM and configures the host gateway — it prints its own +CNI result and returns; BGP/SRv6/eBPF publish is `galactic-bgp`'s job, +invoked next by the CNI runtime per conflist order, not by this process. + Two subcommands support the DaemonSet (see Known Constraints below for the manifest): - `init` — `--node-name`/`-n` flag (or `GALACTIC_CNI_NODE_NAME`/`NODE_NAME` env), - calls `installer.Bootstrap(ctx, nodeName)`: stages the `galactic-cni`/`host-device` - binaries onto the host, does a one-shot dual-stack node-identity check against the - Kubernetes API, and writes `ca.crt`/kubeconfig plus the static conflist. + calls `installer.Bootstrap(ctx, nodeName)`: stages every binary in the CNI chain + (`galactic-cni`, `galactic-tap-cni`, `galactic-ipam`, `galactic-bgp`, + `galactic-route`, `host-device`) onto the host, does a one-shot dual-stack + node-identity check against the Kubernetes API, and writes `ca.crt`/kubeconfig + plus the static conflist. - `run` — `--grpc-health-port` flag (default `5180`), calls `installer.Run(ctx, - grpcHealthPort)`: serves gRPC health checks and periodically refreshes the - kubeconfig token and rotates the CNI log file. + grpcHealthPort)`: serves gRPC health checks, manages the eBPF uSID datapath's + load/attach lifecycle, and periodically refreshes the kubeconfig token and + rotates the CNI log file. See [docs/cni-cmd-sequence.md](../cni-cmd-sequence.md) for the full ADD/DEL sequence. +### `cmd/galactic-tap-cni/main.go` — tap master plugin + +Mirrors `cmd/galactic-cni/main.go`'s CNI-invocation role exactly (`internal/cnitap.RunPlugin()`), +minus the `init`/`run` DaemonSet subcommands — those live only on `galactic-cni`, since +there's exactly one init container per node regardless of how many workload types it serves. +`internal/cnitap` mirrors `internal/cni` (VRF + tap creation, NAD annotation, IPAM +delegation, host gateway configuration) but never delegates to host-device and never +configures a guest netns — the VM hypervisor manages the tap fd directly. + +### `cmd/galactic-ipam/main.go` — delegated IPAM plugin + +The simplest binary in the chain: no DaemonSet subcommands, no Kubernetes client, no +node-name/kubeconfig resolution. Invoked only via the CNI IPAM delegation protocol +(`github.com/containernetworking/plugins/pkg/ipam.ExecAdd`/`ExecDel`/`ExecCheck`), never +directly from a conflist. `internal/cniipam.RunPlugin()` implements `cmdAdd`/`cmdDel`/ +`cmdCheck`/`cmdStatus`; allocation state persists in on-disk marker files under this +node's own filesystem (`internal/cni/ipam.DefaultLockDir`), so `cmdDel` never needs to +read anything back from a CRD. + +### `cmd/galactic-bgp/main.go` — BGP/SRv6/eBPF publish plugin + +Chained after the master plugin (and, when present, `galactic-route`) per conflist +order, never run standalone. `internal/cnibgp.RunPlugin()` learns everything it needs +— which interface kind was created, what addresses were allocated — from `prevResult` +alone (`RawPrevResult`, not the never-populated typed `PrevResult` field — see +`internal/cnibgp/prevresult.go`), so it has zero kernel-interface dependency of its +own. Unlike `galactic-ipam`/`galactic-route`, it does resolve node name/kubeconfig +(reusing `internal/config.CNIConfig`, the same `GALACTIC_CNI_*` env vars every other +k8s-talking binary in the chain uses) since it talks to the API server for +`BGPVRFInstance`/`BGPAdvertisement` CRUD. + +### `cmd/galactic-route/main.go` — termination-route plugin + +Optional chain stage — present only for attachments with `terminations` to install. +`internal/cniroute.RunPlugin()` installs each termination as a VRF-table route via +`internal/cni/route`, deriving the host device name from `(vpc, vpcAttachment)` alone +(identical for a veth or tap master's own host interface, so no interface-kind +inference is needed). No Kubernetes dependency at all. + ### `cmd/galactic-router/main.go` / `root.go` — Router daemon `main.go` is a 3-line wrapper around `newRootCommand().Execute()`; all startup logic @@ -222,47 +337,61 @@ lives in `root.go`'s `runCmd`: See [docs/router/configuration.md](../router/configuration.md) for the full reference, including CLI flags and precedence. -### galactic-cni CNI config fields (`PluginConf`) - -| Field | Type | Description | -|-----------------|----------|-------------------------------------------------------------------------| -| `vpc` | string | Base62-encoded 48-bit VPC identifier | -| `vpcattachment` | string | Base62-encoded 16-bit VPCAttachment identifier | -| `interface_type`| string | `veth` (default) or `tap`; tap mode omits guest-side/host-device config but still runs IPAM and SRv6/BGP publish (see the ADD result section below) | -| `namespace` | string | Kubernetes namespace for BGP CRDs; resolution order is this field → `GALACTIC_CNI_NAMESPACE` → `HostConf.Namespace` (from the conflist) → `DefaultNamespace` (`galactic-system`) | -| `mtu` | int | MTU for the host-side interface (veth pair or tap); 0 uses kernel default | -| `terminations` | array | Static routes to install on the host-side interface (`network`, `via`) | -| `ipam` | object | Built-in IPv6 pool/static allocator config (Galactic has no external IPAM delegation); used identically in `veth` and `tap` mode — `tap`'s `cmdAdd` calls `allocateIPAM()` unconditionally, so omitting this without `GALACTIC_CNI_ENABLE_LOCAL_IPAM` set is not safely tolerated in tap mode. See [docs/cni/configuration.md](../cni/configuration.md). | - -### galactic-cni environment variables - -There is no longer a `--node-name`/`--enable-local-ipam` CLI flag on the plugin -invocation path (those flags now only exist on the `init`/`run` installer -subcommands, and only `init`'s `--node-name` overlaps in purpose). `parseConf()` -(`internal/cni/config.go`) resolves each setting below on every ADD/DEL/CHECK/STATUS -call, in the listed precedence, and re-exports the result as a process env var for -the rest of the invocation: - -| Variable | Resolution precedence (highest first) | Default | -|------------------------------------|--------------------------------------------------------------------------------------------------------|---------| -| Node name (`NODE_NAME`) | `GALACTIC_CNI_NODE_NAME` → `NODE_NAME` → `HostConf.NodeName` (conflist) → `detectNodeNameFromAPI()` (matches local interface addrs against Node `InternalIP`) | _(error if still empty)_ | -| Kubeconfig (`KUBECONFIG`) | `GALACTIC_CNI_KUBECONFIG` → `HostConf.Kubeconfig` (conflist) | `/var/lib/galactic/kubeconfig` | -| Namespace | `conf.Namespace` (CNI config JSON) → `GALACTIC_CNI_NAMESPACE` → `HostConf.Namespace` (conflist) | `galactic-system` | -| Log file | `GALACTIC_CNI_LOG_FILE` → `HostConf.LogFile` (conflist) | `/var/log/galactic/galactic-cni.log` | -| Log level | `GALACTIC_CNI_LOG_LEVEL` → `HostConf.LogLevel` (conflist) | `info` | -| `GALACTIC_CNI_ENABLE_LOCAL_IPAM` | Read directly as an env var in `parseConf()` (no conflist or CLI-flag equivalent) | `false` | +### CNI chain config fields + +There is no single `PluginConf` shape anymore — each binary in the chain reads +only the fields its own JSON stanza carries. See +[docs/cni/configuration.md](../cni/configuration.md) for the full per-binary +field tables and seven example conflists; summary: + +| Field | Read by | Description | +|-----------------|-------------------------------------------------------|--------------------------------------------------------------------------------------------------------------| +| `vpc` | every binary | Base62-encoded 48-bit VPC identifier | +| `vpcattachment` | every binary | Base62-encoded 16-bit VPCAttachment identifier | +| `namespace` | `galactic-cni`/`galactic-tap-cni`/`galactic-bgp` | Kubernetes namespace for NAD/BGP CRD lookup; resolution order is this field → `GALACTIC_CNI_NAMESPACE` → `HostConf.Namespace` → `galactic-system` | +| `mtu` | `galactic-cni`/`galactic-tap-cni` | MTU for the host-side interface (veth pair or tap); 0 uses kernel default | +| `ipam` | `galactic-cni`/`galactic-tap-cni` (decides whether to delegate), `galactic-ipam` (reads the block's own sub-fields) | IPAM delegation block; `type` names the delegate binary (`galactic-ipam`). See [docs/cni/configuration.md](../cni/configuration.md#ipam-fields). | +| `terminations` | `galactic-route` only | Static routes to install on the host-side interface (`network`, `via`) | + +There is no `interface_type` field anymore — which binary you invoke *is* the +interface type (`galactic-cni` → veth, `galactic-tap-cni` → tap). + +### galactic-cni / galactic-tap-cni / galactic-bgp environment variables + +There is no `--node-name`/`--enable-local-ipam` CLI flag on any plugin invocation +path (`--node-name` exists only on `galactic-cni`'s own `init` installer +subcommand). Each binary's own `parseConf()` resolves the settings below on every +ADD/DEL/CHECK/STATUS call, in the listed precedence, re-exporting the result as a +process env var. `galactic-ipam` and `galactic-route` skip this table entirely — +see [docs/cni/configuration.md](../cni/configuration.md#runtime-configuration). + +| Variable | Resolution precedence (highest first) | Default | Resolved by | +|------------------------------------|--------------------------------------------------------------------------------------------------------|---------|-------------| +| Node name (`NODE_NAME`) | `GALACTIC_CNI_NODE_NAME` → `NODE_NAME` → `HostConf.NodeName` (conflist) → `detectNodeNameFromAPI()` (matches local interface addrs against Node `InternalIP`) | _(error if still empty)_ | `galactic-cni`, `galactic-tap-cni`, `galactic-bgp` | +| Kubeconfig (`KUBECONFIG`) | `GALACTIC_CNI_KUBECONFIG` → `HostConf.Kubeconfig` (conflist) | `/var/lib/galactic/kubeconfig` | `galactic-cni`, `galactic-tap-cni`, `galactic-bgp` | +| Namespace | `conf.Namespace` (CNI config JSON) → `GALACTIC_CNI_NAMESPACE` → `HostConf.Namespace` (conflist) | `galactic-system` | `galactic-cni`, `galactic-tap-cni`, `galactic-bgp` | +| Log file | `GALACTIC_CNI_LOG_FILE` → `HostConf.LogFile` (conflist) | `/var/log/galactic/galactic-cni.log` | every binary in the chain | +| Log level | `GALACTIC_CNI_LOG_LEVEL` → `HostConf.LogLevel` (conflist) | `info` | every binary in the chain | +| `GALACTIC_IPAM_ENABLE_LOCAL_IPAM` | Read directly as an env var by `galactic-ipam` only (`internal/config/ipam.go`); no conflist/CLI-flag equivalent, and it can no longer manufacture an `"ipam"` block that isn't already present | `false` | `galactic-ipam` only | + +`GALACTIC_CNI_ENABLE_LOCAL_IPAM` (the old, master-plugin-side predecessor of the +row above) no longer exists at all — removed along with the dead +`config.CNIGetEnableLocalIPAM()` it backed once IPAM's own env-var handling +moved entirely into `galactic-ipam`. `HostConf` (`node_name`, `kubeconfig`, `namespace`, `log_file`, `log_level`) is the JSON -shape the `init` installer subcommand writes into the `galactic-cni`-typed plugin entry -of the conflist at `--conf-file` (see `internal/installer/installer.go` and -Entry Points above). `log_level` (`debug`/`info`/`warn`/`error`) controls how much detail +shape the `init` installer subcommand writes into the static conflist at +`--conf-file` — the same file every binary in the chain reads (see +`internal/installer/installer.go` and Entry Points above). `log_level` +(`debug`/`info`/`warn`/`error`) controls how much detail each binary's own `setupLogging()` emits — `info` (the default) logs one line per operation for start/outcome plus all warnings/errors; `debug` adds per-resource milestones. See [docs/cni/configuration.md#log-verbosity](../cni/configuration.md#log-verbosity). -### galactic-cni ADD result +### CNI chain ADD result -On a successful ADD, the plugin returns a CNI spec v1.0.0 result with the following structure: +On a successful ADD, the master plugin (`galactic-cni`/`galactic-tap-cni`) +returns a CNI spec v1.0.0 result with the following structure (veth shown): ```json { @@ -289,30 +418,37 @@ On a successful ADD, the plugin returns a CNI spec v1.0.0 result with the follow The VRF dummy interface (`G{vpc}{att}V`) is **not** reported — it is pre-existing infrastructure created by the `vrf.Add()` plumbing function, not by the CNI attachment itself. -This is the `veth`-mode result. In `tap` mode the result has a single interface (the -host-side tap, empty sandbox, index `0`) — there is no guest interface entry since the fd -is handed off to the VM hypervisor, not moved into a container netns. Tap mode is **not** -"no IPAM, no BGP": `cmdAdd` (`internal/cni/ops_add.go`) calls `allocateIPAM()` to allocate -a subnet/gateway and `configureHostGateway()` to assign it on the host tap, includes the -resulting `ips`/`routes` in the tap result (`buildTapResult`, interface index `0`), and then -calls `publishBGPStateK8s()` to create the SRv6 ingress route and -`BGPVRFInstance`/`BGPAdvertisement` CRDs — the same BGP publish step veth mode uses. The -only things tap mode skips are host-device delegation and guest-netns configuration, since -there is no container network namespace to move an interface into. - -`configureHostGateway()` assigns the IPv4 gateway as a `/25` on the host tap (vs. `/32` -everywhere else) so it looks like a real subnet to the VM guest, adding it with -`IFA_F_NOPREFIXROUTE` to suppress the kernel's auto-created connected route for the wider -mask — otherwise this would reintroduce the subnet-router-anycast hazard that `/32` avoids -elsewhere. See [docs/cni/configuration.md](../cni/configuration.md) for details. - -The result is printed to Multus **before** SRv6 ingress setup and BGP CRD publish run -(see [docs/cni-cmd-sequence.md](../cni-cmd-sequence.md)) — a successful ADD response does not -guarantee the BGPAdvertisement/BGPVRFInstance CRDs exist yet. - -On DEL, the result contains only `cniVersion` (empty result; DEL only deallocates the -pod's IPAM bookkeeping and does not attempt to unwind kernel/CRD state — see the -`cmdDel` note in [docs/cni-cmd-sequence.md](../cni-cmd-sequence.md) and Known Constraints below). +This is the veth-master result. `galactic-tap-cni`'s own result has a single +interface (the host-side tap, empty sandbox, index `0`) — there is no guest +interface entry since the fd is handed off to the VM hypervisor, not moved +into a container netns. Both masters run IPAM identically (if `"ipam"` is +present) and both configure the host gateway (`internal/cni/hostgw`) before +printing their own result — see +[Interface Types](../cni/configuration.md#master-plugin-fields-galactic-cni--galactic-tap-cni) +in the CNI config doc. + +The host gateway's IPv4 address is assigned as a `/25` on the host tap (vs. +`/32` everywhere else) so it looks like a real subnet to the VM guest, adding +it with `IFA_F_NOPREFIXROUTE` to suppress the kernel's auto-created connected +route for the wider mask — otherwise this would reintroduce the +subnet-router-anycast hazard that `/32` avoids elsewhere. See +[docs/cni/configuration.md](../cni/configuration.md) for details. + +**Every stage after the master plugin passes `prevResult` straight through as +its own result, unchanged** — `galactic-route` and `galactic-bgp` add no +interfaces or IPs of their own. This means the master's own printed result is +the runtime's authoritative CNI result for the whole chain; a successful ADD +response does not by itself guarantee `galactic-bgp` has even run yet, let +alone that the `BGPAdvertisement`/`BGPVRFInstance` CRDs exist (see +[docs/cni-cmd-sequence.md](../cni-cmd-sequence.md)). + +On DEL, every binary's result contains only `cniVersion` (empty result). Each +binary's own DEL only cleans up what it itself created *and* is safe to +release immediately for that specific container — IPAM deallocation +(`galactic-ipam`, via its own on-disk marker file) and the guest-netns +flush/host-device DEL (`galactic-cni` only). It does not attempt to unwind +any shared, per-attachment kernel/CRD state — see the `cmdDel` note in +[docs/cni-cmd-sequence.md](../cni-cmd-sequence.md) and Known Constraints below. --- @@ -327,18 +463,28 @@ pod's IPAM bookkeeping and does not attempt to unwind kernel/CRD state — see t | `internal/runtime/frr` | galactic-router | FRR stub — returns "not implemented" for every method | No | | `internal/model` | both | `DesiredRouter`, `DesiredPeer`, `DesiredAdvertisement`, `DesiredPolicy`, `DesiredVRFInstance`, `RuntimeStatus`; re-exports BGP API enums | No | | `internal/hash` | galactic-router | SHA-256 fingerprint of `DesiredRouter` for no-op suppression | No | -| `internal/metadata` | both | Build-time vars (`Version`, `GitCommit`, `GitTreeState`, `BuildDate`) stamped via `-ldflags` | No | -| `internal/gc` | galactic-router | Collects orphaned `BGPAdvertisement`/`BGPVRFInstance` CRDs and stale kernel VRFs; invoked by the GC controller's ticker | No | -| `internal/cni` | galactic-cni | `cmdAdd` / `cmdDel` / `cmdCheck`; CNI PluginConf parsing; BGPVRFInstance/BGPAdvertisement lifecycle; delegates kernel work to plumbing | No | -| `internal/cni/ipam` | galactic-cni | Built-in IPv6 pool allocator (in-memory, ephemeral) and static IP allocator | Yes (pool allocations) | -| `internal/cni/route` | galactic-cni | Host-side static route add/delete via netlink | No | -| `internal/cni/tap` | galactic-cni | Tap interface create/delete for VM workloads (Kata, Firecracker, QEMU) | No | +| `internal/metadata` | every binary | Build-time vars (`Version`, `GitCommit`, `GitTreeState`, `BuildDate`) stamped via `-ldflags` | No | +| `internal/gc` | galactic-router | Collects orphaned `BGPAdvertisement`/`BGPVRFInstance` CRDs, stale kernel VRFs, and stale eBPF `vrf_table` entries; invoked by the GC controller's ticker | No | +| `internal/cni` | galactic-cni | Veth master plugin: `cmdAdd`/`cmdDel`/`cmdCheck`/`cmdStatus`; PluginConf parsing; NAD annotation; host-device delegation; delegates kernel work to plumbing | No | +| `internal/hostconf` | every CNI-chain binary | Shared `HostConf` schema + static-conflist loader, plus API-based node-name auto-detect | No | +| `internal/cni/hostgw` | galactic-cni, galactic-tap-cni | Host-side gateway address/route configuration for a VPC attachment's allocated IPAM addresses | No | +| `internal/crdnames` | galactic-cni, galactic-bgp, galactic-router (gc) | Deterministic `BGPVRFInstance`/`BGPAdvertisement` CRD name + annotation-key derivation | No | +| `internal/nadpatch` | galactic-cni, galactic-tap-cni | NAD annotation patch (host interface name) + pod-namespace parsing from `CNI_ARGS` | No | +| `internal/cni/ipam` | galactic-ipam | IPv6/IPv4 pool allocators + static IP allocator; on-disk marker-file persistence (flock-guarded, keyed by containerID) | Yes (pool allocations + marker files) | +| `internal/cni/route` | galactic-route | Host-side static route add/delete via netlink | No | +| `internal/cni/tap` | galactic-tap-cni | Tap interface create/delete for VM workloads (Kata, Firecracker, kraftlet/Unikraft) | No | | `internal/cni/veth` | galactic-cni | veth pair create/delete | No | -| `internal/installer` | galactic-cni | DaemonSet `init`/`run` support: binary staging, node-identity check, conflist/kubeconfig templating, credential refresh ticker, log rotation, gRPC health server | No | -| `internal/plumbing/intf` | both | Deterministic interface naming (`G{vpc9}{att3}V/H/G`); base62↔hex encoding | No | -| `internal/plumbing/srv6` | galactic-cni | SRv6 END.DT46 ingress route add/delete via netlink | No | -| `internal/plumbing/vrf` | galactic-cni | Linux VRF create/delete/lookup via netlink | No | -| `internal/plumbing/sysctl` | galactic-cni | Per-interface sysctl helpers | No | +| `internal/cnitap` | galactic-tap-cni | Tap master plugin (mirrors `internal/cni`; no host-device delegation, no guest netns) | No | +| `internal/cniipam` | galactic-ipam | CNI IPAM delegation protocol (`cmdAdd`/`cmdDel`/`cmdCheck`/`cmdStatus`); explicit `"ipam"`-block contract; no k8s dependency | No | +| `internal/cnibgp` | galactic-bgp | BGP/SRv6/eBPF publish: SID/Argument allocation + collision detection, `registerEBPFDatapath`/`unregisterEBPFDatapath`, `BGPVRFInstance`/`BGPAdvertisement` CRUD with retry; learns everything from `prevResult` | No | +| `internal/cniroute` | galactic-route | Termination-route plugin: installs/rolls-back VRF-table routes; no k8s dependency | No | +| `internal/vmtap` | vmtap-cni | Patches Cilium's own chain conflist to add a tap-interface stage for VM workloads | No | +| `internal/installer` | galactic-cni | DaemonSet `init`/`run` support: binary staging (every chain binary), node-identity check, conflist/kubeconfig templating, credential refresh ticker, log rotation, eBPF datapath lifecycle, gRPC health server | No | +| `internal/plumbing/intf` | every CNI-chain binary + router | Deterministic interface naming (`G{vpc9}{att3}V/H/G`); base62↔hex encoding | No | +| `internal/plumbing/srv6` | galactic-router | SID computation (`ComputeSID`) for the router's own BGP Prefix-SID path attribute | No | +| `internal/plumbing/ebpf` | galactic-cni (attach/metrics via `run`), galactic-bgp (registration), galactic-router (gc sweep) | TC-BPF uSID datapath: kernel preflight, uFMT bit-layout codec, compiled program + bindings, load/pin/attach lifecycle, map read/write API, Prometheus metrics | Yes (pinned BPF maps) | +| `internal/plumbing/vrf` | every CNI-chain binary + router | Linux VRF create/delete/lookup via netlink | No | +| `internal/plumbing/sysctl` | galactic-cni, galactic-tap-cni | Per-interface sysctl helpers | No | --- @@ -349,10 +495,10 @@ pod's IPAM bookkeeping and does not attempt to unwind kernel/CRD state — see t | `github.com/osrg/gobgp/v4` | v4.7.0 | Embedded BGP server (tenant mode) | | `go.datum.net/network` | bumped frequently | BGP CRD API types (BGPRouter, BGPPeer, BGPAdvertisement, BGPPolicy, BGPVRFInstance) | | `sigs.k8s.io/controller-runtime` | v0.24.1 | Reconciler framework, manager, field indexes | -| `github.com/spf13/cobra` | v1.10.2 | CLI command/flag handling for both binaries | -| `github.com/spf13/viper` | v1.21.0 | Config resolution (flags/env/defaults) for `galactic-router` only; `galactic-cni` resolves config itself (conflist/env/API auto-detect in `internal/cni/config.go`) and does not import viper | +| `github.com/spf13/cobra` | v1.10.2 | CLI command/flag handling for every binary | +| `github.com/spf13/viper` | v1.21.0 | Config resolution (flags/env/defaults) for `galactic-router` only; every CNI-chain binary resolves config itself (conflist/env/API auto-detect) and does not import viper | | `github.com/containernetworking/cni` | v1.3.0 | CNI plugin spec, skel, invoke | -| `github.com/containernetworking/plugins` | v1.9.1 | `host-device` plugin, delegated to for moving the guest veth into the pod netns | +| `github.com/containernetworking/plugins` | v1.9.1 | `pkg/ipam.ExecAdd`/`ExecDel`/`ExecCheck` (real IPAM delegation to `galactic-ipam`, used by `galactic-cni`/`galactic-tap-cni`); `host-device` plugin, delegated to by `galactic-cni` for moving the guest veth into the pod netns | | `github.com/vishvananda/netlink` | pinned pseudo-version | Linux netlink: VRF, veth, SRv6 routes | | `github.com/kenshaw/baseconv` | v0.1.1 | Base62↔hex conversion for interface names | | `github.com/lorenzosaino/go-sysctl` | v0.3.1 | Interface sysctl helpers | @@ -364,15 +510,15 @@ pod's IPAM bookkeeping and does not attempt to unwind kernel/CRD state — see t ## Key Design Decisions -- **USID per endpoint, router-side computation.** Each (VPC, VPCAttachment) pair is assigned a unique /128 USID computed entirely by the CNI (`resolveSRv6SID`/`srv6.ComputeSID`) from the owning `BGPRouter`'s `srv6Locator` + `nodeID` plus this attachment's VRFID — there is no config-supplied SID field. The CNI installs an END.DT46 decap route for that /128. VPC identity is not encoded in the SID itself — VPC scoping comes from the BGPVRFInstance's route target instead. +- **USID per endpoint, computed independently on both sides.** Each (VPC, VPCAttachment) pair is assigned a unique /128 USID from the owning `BGPRouter`'s `srv6Locator` + `nodeID` plus this attachment's VRFID — there is no config-supplied SID field. `galactic-bgp` registers the eBPF uSID datapath's map entries for it (`internal/cnibgp/bgp.go`'s `registerEBPFDatapath`); `galactic-router` independently derives the same value (`srv6.ComputeSID`) for the BGP Prefix-SID path attribute. See [SRv6 SID encoding](#srv6-sid-encoding) above. VPC identity is not encoded in the SID itself — VPC scoping comes from the BGPVRFInstance's route target instead. - **Base62 interface names.** Kernel interface names use the format `G{9-char-vpc-base62}{3-char-att-base62}{suffix}` (suffix: `V` = VRF, `H` = host veth/tap, `G` = guest veth pre-move), fitting in the 15-character kernel limit. The hex form is used for BGP route targets; base62 for kernel interfaces. - **GoBGP embedded, lazy-started.** GoBGP runs in-process (`--mode=tenant` only) and starts only when the first `BGPRouter` is reconciled for that router; `Apply` re-runs on every subsequent reconcile too (subject to hash-based no-op suppression), re-applying peers/VRFs/EVPN/policies each time. `listenPort` defaults to `179`; `-1` (outbound-only) is an operator choice for specific deployments, not the codebase default. ASN or RouterID changes trigger a full `Reconfigure` (fresh `BgpServer` — `StopBgp` is not called because it permanently terminates the v4 Serve loop). - **Overlay BGP port.** galactic-router peers connect outbound on port `1790` by default (configurable per-peer via `BGPPeer.spec.remotePort`). Port `179` is occupied by the underlay FRR `bgpd` on every node, so the overlay uses a non-conflicting port. The `BGPPeer` CRD defaults `remotePort` to `179` (the IANA BGP port); galactic-router overrides this to `1790` when the field is unset, so existing CRDs without an explicit value continue to work. Set `remotePort: 179` explicitly when peering with external BGP speakers that listen on the standard port. -- **VRF/route-target model via BGPVRFInstance.** The CNI creates a `BGPVRFInstance` (RouteDistinguisher + import/export Route Targets, all set to the derived RT) before the `BGPAdvertisement`; `galactic-router`'s GoBGP runtime applies VRFs (`applyVRFs`) before originating EVPN paths (`applyEVPN`). -- **CRD-driven config, no sidecar gRPC.** `galactic-router` watches BGP CRDs directly via controller-runtime. The CNI writes `BGPVRFInstance`/`BGPAdvertisement` CRDs; the router reconciler picks them up. No in-node gRPC calls between the two binaries. +- **VRF/route-target model via BGPVRFInstance.** `galactic-bgp` creates a `BGPVRFInstance` (RouteDistinguisher + import/export Route Targets, all set to the derived RT) before the `BGPAdvertisement`; `galactic-router`'s GoBGP runtime applies VRFs (`applyVRFs`) before originating EVPN paths (`applyEVPN`). +- **CRD-driven config, no sidecar gRPC.** `galactic-router` watches BGP CRDs directly via controller-runtime. `galactic-bgp` writes `BGPVRFInstance`/`BGPAdvertisement` CRDs; the router reconciler picks them up. No in-node gRPC calls between any of the CNI-chain binaries and `galactic-router`. - **Hash-based no-op suppression.** SHA-256 over the sorted `DesiredRouter` prevents redundant GoBGP Apply calls on every CRD event. - **RuntimeFactory pattern.** `--mode=tenant` (`GALACTIC_ROUTER_ROUTER_MODE=tenant`) selects GoBGP; `--mode=fabric` selects the FRR stub; `--mode=transit` is accepted by validation but returns an error at startup (not yet implemented). The mode is selected at startup; no controller changes are needed to add a new mode. -- **DEL is intentionally minimal; GC reclaims shared state asynchronously.** `cmdDel` only deallocates the pod's IPAM bookkeeping — it does not delete the VRF, veth/tap, routes, SRv6 ingress route, or `BGPAdvertisement`/`BGPVRFInstance` CRDs, because those are keyed by `(vpc, vpcAttachment)` and may be shared/reused by another pod (deleting them in DEL would race with a concurrent ADD during pod restarts). `galactic-router`'s GC controller (ticker-driven, default every 5m) reclaims orphaned CRDs and stale kernel VRFs once no live container still references them. +- **DEL is intentionally minimal everywhere in the CNI chain; GC reclaims shared state asynchronously.** Every binary's own `cmdDel` only cleans up what it itself created *and* is safe to release immediately per-container (IPAM deallocation via `galactic-ipam`'s own on-disk marker file; guest-netns flush/host-device DEL in `galactic-cni`). None of them delete the VRF, veth/tap, routes, the eBPF `vrf_table` entry, or `BGPAdvertisement`/`BGPVRFInstance` CRDs — those are keyed by `(vpc, vpcAttachment)` and may be shared/reused by another pod/VM (deleting them in DEL would race with a concurrent ADD during restarts). `galactic-router`'s GC controller (ticker-driven, default every 5m) reclaims orphaned CRDs, stale kernel VRFs, and stale eBPF entries once no live container still references them. - **gRPC health, configurable port.** Liveness and readiness probes use the gRPC health protocol (`google.golang.org/grpc/health`) on a configurable port (default `5000`). No HTTP health endpoint. --- @@ -381,11 +527,11 @@ pod's IPAM bookkeeping and does not attempt to unwind kernel/CRD state — see t | Layer | Command | Framework | Scope | |------------|------------------|---------------------|------------------------------------------------------------------------| -| Unit | `task test:unit` | `go test -race` | `internal/cni` (`cni_test.go`, `bgp_test.go`, `netns_test.go` — `buildResult`, `parseConf`, `routeTarget`, `lookupBGPRouter`), `internal/cni/{ipam,tap,veth}`, `internal/installer` (`installer_test.go` — `Bootstrap`/`Run` with mocked k8s client and netlink/host paths), `internal/plumbing/srv6`, `internal/gc`, `internal/reconcile`, `internal/controller`, `internal/plumbing/intf`, `internal/metadata`, `internal/runtime/gobgp` (partial), `internal/runtime/frr` | -| E2E | `task test:e2e` | Kind + `go test` | Full BGPRouter lifecycle in a Kind cluster; builds and loads image | +| Unit | `task test:unit` | `go test -race` | `internal/cni`, `internal/cnitap`, `internal/cniipam`, `internal/cnibgp`, `internal/cniroute` (`buildResult`/`buildVethResult`, `parseConf`, `routeTarget`, `lookupBGPRouter`, `inferFromPrevResult` — each package's own `cmdAdd`/`cmdDel`/`cmdCheck`/`cmdStatus`), `internal/cni/{hostconf,hostgw,crdnames,nadpatch,ipam,route,tap,veth}`, `internal/installer` (`installer_test.go` — `Bootstrap`/`Run` with mocked k8s client and netlink/host paths), `internal/plumbing/{srv6,ebpf}`, `internal/gc`, `internal/reconcile`, `internal/controller`, `internal/plumbing/intf`, `internal/metadata`, `internal/runtime/gobgp` (partial), `internal/runtime/frr` | +| E2E | `task test:e2e` | Kind + `go test` | `galactic-tap-cni`'s own ADD (VRF + tap + IPAM delegation), kernel capability checks, CNI VERSION report. Does **not** exercise `galactic-route` or `galactic-bgp` (no BGPRouter fixture in the e2e suite) — see Known Constraints below. Full BGPRouter lifecycle coverage for `galactic-router` comes from this same Kind cluster's separate reconciler tests. | | CI full | `task ci` | all of the above | lint → build → test:unit → test:e2e | -`internal/plumbing/vrf` has no unit tests — it requires `CAP_NET_ADMIN` and a real kernel. `internal/cni` and `internal/plumbing/srv6` now have unit coverage for their pure-logic paths (this used to not be the case). `internal/plumbing/intf` is pure-function and fully unit-testable. +`internal/plumbing/vrf` has no unit tests — it requires `CAP_NET_ADMIN` and a real kernel. `internal/cni/route` (wrapped by `internal/cniroute`) also has no unit tests of its own. `internal/plumbing/intf` is pure-function and fully unit-testable. --- @@ -412,10 +558,12 @@ Runs on every PR and push to `main`. Two tiers: - **GoBGP RIB is ephemeral.** All BGP state is in-process memory. On restart, sessions and paths must be re-established from CRD state; controller-runtime's reconcile loop handles this automatically. - **EVPN Type 5 is implemented, not deferred.** `internal/runtime/gobgp/paths.go`'s `buildEVPNPaths` builds real `EVPNIPPrefixRoute` NLRIs, deriving the Route Distinguisher from `routerID + ":0"` (not from the CRD). The `BGPVRFInstance` CRD carries its own explicit `RouteDistinguisher` and import/export Route Targets (see Key Design Decisions above), applied via `internal/runtime/gobgp/runtime.go`'s `applyVRFs`. There is no `ErrMissingRouteDistinguisher` or similar rejection path in the current code. -- **`cmdDel` does not tear down shared kernel/CRD state.** By design (see Key Design Decisions above) — cleanup of VRF, veth/tap, routes, SRv6 ingress, and BGP CRDs is deferred to `galactic-router`'s asynchronous GC controller, not performed synchronously in `cmdDel`. -- **`internal/plumbing/vrf` has no unit tests.** It requires `CAP_NET_ADMIN` and a real kernel. `internal/cni` and `internal/plumbing/srv6` do now have unit coverage for their pure-logic paths. `internal/plumbing/intf` is fully unit-testable (pure functions only). Kernel-path coverage otherwise comes from the e2e suite (`task test:e2e`). +- **No binary's `cmdDel` tears down shared kernel/CRD state.** By design (see Key Design Decisions above) — cleanup of VRF, veth/tap, routes, the eBPF `vrf_table` entry, and BGP CRDs is deferred to `galactic-router`'s asynchronous GC controller, not performed synchronously in any chain binary's `cmdDel`. +- **`internal/plumbing/vrf` and `internal/cni/route` have no unit tests.** `vrf` requires `CAP_NET_ADMIN` and a real kernel; `route` (wrapped by `internal/cniroute`, which does have its own tests) was never backfilled with tests of its own when the CNI plugin-chain split moved its caller out of `internal/cni`. `internal/plumbing/intf` is fully unit-testable (pure functions only). Kernel-path coverage otherwise comes from the e2e suite (`task test:e2e`). +- **The e2e suite doesn't cover `galactic-route` or `galactic-bgp`.** `TestCNITapInterface` (`tests/e2e/e2e_test.go`) only drives `galactic-tap-cni`'s own ADD — verifying BGP/SRv6/eBPF publish end-to-end would need a `BGPRouter` CRD fixture and additional RBAC the test doesn't set up. This gap predates the CNI plugin-chain split too: the monolithic `galactic-cni` this replaced was never e2e-verified past its own CNI result shape either. +- **`docs/agents/ARCHITECTURE.md` and `docs/cni-cmd-sequence.md` describe the CNI chain; `vmtap-cni`/`internal/vmtap` (a separate, Cilium-chain-conflist-patching binary for VM tap interfaces, unrelated to the `galactic-cni`/`galactic-tap-cni`/`galactic-ipam`/`galactic-bgp`/`galactic-route` chain) has its own doc at `docs/vmtap-cni/configuration.md`** — cross-referenced from [Repository Layout](#repository-layout) and the [Module / Package Reference](#module--package-reference) table above, but not otherwise elaborated on in this document. - **`--mode=transit` is unimplemented.** Accepted by CLI/env validation, but `runCmd` returns an error at startup ("mode=transit is not yet supported"). -- **`galactic-cni`'s install DaemonSet is a Go installer, not a shell script.** `config/cni/configmap.yaml`/`install.sh` were deleted; `config/cni/daemonset.yaml` now runs `hostNetwork: true` with an `install-cni` init container (`command: ["/galactic-cni", "init"]`, calling `installer.Bootstrap`) and a `credential-refresh` main container (`command: ["/galactic-cni", "run"]`, calling `installer.Run`), both on the same image (see CI/CD above). `Bootstrap` writes the CNI binaries to `/opt/cni/bin`, the static conflist to `/etc/cni/net.d/10-galactic.conflist`, and `ca.crt`/kubeconfig to `/var/lib/galactic` (chosen over `/etc/galactic` specifically so it lands under `/var`, the one path immutable-root distros like Talos allow hostPath writes to without a host-level `extraMounts` entry); `Run` refreshes the kubeconfig token every 300s and rotates the CNI log once it exceeds 10MB. `/opt/cni/bin` is fixed by the CNI/kubelet plugin-discovery convention and can't be relocated by this DaemonSet alone — on Talos it needs its own `extraMounts` entry in the machine config if it isn't writable by default. The `run` container also serves gRPC health checks on port `5180` (`livenessProbe`/`readinessProbe` in the DaemonSet spec), and `config/cni/rbac.yaml` grants `get` on `nodes` for `Bootstrap`'s node-identity check. +- **`galactic-cni`'s install DaemonSet is a Go installer, not a shell script.** `config/cni/configmap.yaml`/`install.sh` were deleted; `config/cni/daemonset.yaml` now runs `hostNetwork: true` with an `install-cni` init container (`command: ["/galactic-cni", "init"]`, calling `installer.Bootstrap`) and a `credential-refresh` main container (`command: ["/galactic-cni", "run"]`, calling `installer.Run`), both on the same image (see CI/CD above). `Bootstrap` writes every binary in the CNI chain to `/opt/cni/bin`, the static conflist to `/etc/cni/net.d/10-galactic.conflist`, and `ca.crt`/kubeconfig to `/var/lib/galactic` (chosen over `/etc/galactic` specifically so it lands under `/var`, the one path immutable-root distros like Talos allow hostPath writes to without a host-level `extraMounts` entry); `Run` refreshes the kubeconfig token every 300s and rotates the CNI log once it exceeds 10MB. `/opt/cni/bin` is fixed by the CNI/kubelet plugin-discovery convention and can't be relocated by this DaemonSet alone — on Talos it needs its own `extraMounts` entry in the machine config if it isn't writable by default. The `run` container also serves gRPC health checks on port `5180` (`livenessProbe`/`readinessProbe` in the DaemonSet spec), and `config/cni/rbac.yaml` grants `get` on `nodes` for `Bootstrap`'s node-identity check. - **The uSID TC-BPF datapath (`internal/plumbing/ebpf/prog/usid.c`) doesn't generate PMTUD ICMPv6 errors.** When `bpf_fib_lookup()` returns `BPF_FIB_LKUP_RET_FRAG_NEEDED` (egress route's MTU is smaller than the inner packet), the program counts `DROP_REASON_FIB_FRAG_NEEDED` and silently drops (`TC_ACT_SHOT`) rather than sending an ICMPv6 Packet Too Big back to the original sender — unlike the static-route `SEG6_LOCAL_ACTION_END_DT46` path this datapath replaces, where the kernel's own IPv6 stack emits that ICMP message. Accepted as a known cost of the TC-BPF cutover for this milestone; generating ICMPv6 PTB from the datapath itself is unscheduled future work, not planned for a specific milestone yet. --- @@ -426,9 +574,15 @@ Runs on every PR and push to `main`. Two tiers: | Concern | Start here | |--------------------------------------------|--------------------------------------------------------------| -| CNI attach/detach flow | `internal/cni/ops_add.go:cmdAdd`, `internal/cni/ops_del.go:cmdDel` (`internal/cni/cni.go` only holds `RunPlugin`) | -| CNI runtime config resolution (conflist/env/API auto-detect) | `internal/cni/config.go:parseConf`, `loadHostConf`, `detectNodeNameFromAPI` | -| BGP CRD publish (VRF + advertisement) | `internal/cni/bgp.go:publishBGPState` | +| CNI master-plugin attach/detach flow (veth) | `internal/cni/ops_add.go:cmdAdd`, `internal/cni/ops_del.go:cmdDel` (`internal/cni/cni.go` only holds `RunPlugin`) | +| CNI master-plugin attach/detach flow (tap) | `internal/cnitap/ops_add.go:cmdAdd`, `internal/cnitap/ops_del.go:cmdDel` (mirrors `internal/cni`) | +| CNI runtime config resolution (conflist/env/API auto-detect) | `internal/cni/config.go:parseConf`, `loadHostConf`, `internal/hostconf.DetectNodeNameFromAPI` | +| IPAM delegation (master plugin side) | `internal/cni/result.go:configureIPAM` (`ipam.ExecAdd`), `internal/cni/ops_del.go:cmdDel` (`ipam.ExecDel`) | +| IPAM delegation protocol (delegate side) | `internal/cniipam/ops.go:cmdAdd`/`cmdDel`, `internal/cniipam/allocate.go` | +| Termination-route chain stage | `internal/cniroute/ops_add.go:cmdAdd`, `internal/cni/route/route.go` | +| BGP CRD publish (VRF + advertisement) + eBPF registration | `internal/cnibgp/bgp.go:publishBGPState`, `registerEBPFDatapath`; entry point `internal/cnibgp/ops_add.go:cmdAdd` | +| How `galactic-bgp`/`galactic-route` learn state without touching the kernel | `internal/cnibgp/prevresult.go:inferFromPrevResult` (reads `RawPrevResult`, not the dead `PrevResult` field) | +| Host gateway address/route configuration (shared by both master plugins) | `internal/cni/hostgw/hostgw.go:ConfigureHostGateway` | | CNI DaemonSet install/refresh | `internal/installer/installer.go:Bootstrap` (init container), `internal/installer/installer.go:Run` (long-running container) | | CRD → BGP translation | `internal/reconcile/reconcile.go:BuildDesiredRouter` | | BGP runtime application (GoBGP) | `internal/runtime/gobgp/runtime.go:Apply` | @@ -453,6 +607,7 @@ Runs on every PR and push to `main`. Two tiers: - `peerStatusRequeue = 30s` periodic requeue keeps BGPPeer session state current because BGP FSM transitions are not Kubernetes events. - `annotationConfigHash` is persisted on the BGPRouter object (not just in memory) so no-op detection survives pod restarts without re-applying GoBGP config. - GoBGP `Reconfigure()` calls `old.Stop()` then creates a fresh `BgpServer` — it does NOT call the BGP-level `StopBgp`/`StartBgp` on the old server, avoiding the v4 "Serve loop permanently dead" problem. -- The CNI ADD result is printed to Multus **before** SRv6 ingress setup and BGP CRD publish run (`publishBGPState` is called after `PrintResult` inside `cmdAdd`) — a successful ADD response does not by itself guarantee the BGP CRDs exist yet. -- `cmdDel` never deletes the VRF, veth/tap, routes, SRv6 ingress route, or `BGPAdvertisement`/`BGPVRFInstance` CRDs — only IPAM bookkeeping. Shared-resource cleanup is entirely the GC controller's job (`internal/gc`), to avoid racing a concurrent ADD during pod restarts. -- Production images are published by `.github/workflows/publish.yaml` as two separate per-binary images (`galactic-cni`, `galactic-router`), not one shared image — see CI/CD above. +- The master plugin's ADD result is the runtime's authoritative CNI result for the whole chain — `galactic-route`/`galactic-bgp` (chained after it) both pass `prevResult` straight through unchanged. A successful ADD response does not by itself guarantee `galactic-bgp` has even run yet, let alone that the BGP CRDs exist. +- `types.PluginConf.PrevResult` (from `containernetworking/cni/pkg/types`) has JSON tag `"-"` and is **never populated** by plain `json.Unmarshal`; only the sibling `RawPrevResult map[string]interface{}` field actually receives the previous plugin's result. A pre-existing quirk of that library, not specific to this codebase — every CNI-chain package that reads prevResult (`internal/cni/ops_check.go`, `internal/cnibgp/prevresult.go`, `internal/cniroute/ops_add.go`) reads `RawPrevResult` for this reason. +- No binary's `cmdDel` deletes the VRF, veth/tap, routes, the eBPF `vrf_table` entry, or `BGPAdvertisement`/`BGPVRFInstance` CRDs — each binary's own DEL only handles its own per-container bookkeeping (IPAM deallocation, guest-netns/host-device cleanup). Shared-resource cleanup is entirely the GC controller's job (`internal/gc`), to avoid racing a concurrent ADD during pod restarts. +- Production images are published by `.github/workflows/publish.yaml` as two separate per-binary images (`galactic-cni`, `galactic-router`), not one shared image — see CI/CD above. `galactic-cni`'s own image carries all five CNI-chain binaries plus `vmtap-cni`/`host-device`, not just `galactic-cni` itself. diff --git a/docs/cni-cmd-sequence.md b/docs/cni-cmd-sequence.md index e810b623..3be37f74 100644 --- a/docs/cni-cmd-sequence.md +++ b/docs/cni-cmd-sequence.md @@ -1,8 +1,38 @@ # CNI cmdAdd / cmdDel Sequence Diagrams -Per-interface-type sequence diagrams for the galactic-cni ADD and DEL paths. - -## cmdAdd — veth +Per-binary sequence diagrams for the galactic CNI plugin chain's ADD and DEL +paths. + +## Chain overview + +The chain is one master plugin, plus up to two optional plugins invoked +after it in conflist order: + +1. **Master** — `galactic-cni` (veth, for containers) or `galactic-tap-cni` + (tap, for VM workloads: Kata, Firecracker, kraftlet/Unikraft). Creates + the VRF and host-side interface, annotates the NAD, delegates IPAM (if + an `"ipam"` block is present) and — for `galactic-cni` only — + host-device (to move the guest veth into the container netns), then + configures the host gateway address/route and prints the CNI result. +2. **`galactic-route`** (optional — present only when the attachment has + `terminations`) — installs each termination as a route into the VRF + table, then passes `prevResult` through unchanged. +3. **`galactic-bgp`** — publishes BGP/SRv6/eBPF state: `BGPVRFInstance`, + `BGPAdvertisement`, and (when this node's `BGPRouter` has SRv6 + configured) the eBPF uSID datapath's `vrf_table` registration. Learns + everything it needs (interface kind, allocated addresses) from + `prevResult` alone — it never touches a kernel interface. Passes + `prevResult` through unchanged as the final CNI result. + +Every binary's own `cmdDel` is a no-op beyond binary-local, per-container +cleanup (IPAM deallocation, guest netns flush, host-device DEL — all +`galactic-cni`/`galactic-tap-cni` only). Shared node-level state (VRF, +host interface, routes, SRv6/eBPF registration, `BGPVRFInstance`, +`BGPAdvertisement`) is kept because it may still be in use by another +pod/VM on the same `(vpc, vpcAttachment)`; `galactic-router`'s GC +controller reclaims it once nothing references it anymore. + +## cmdAdd — veth (galactic-cni → galactic-route → galactic-bgp) ```mermaid sequenceDiagram @@ -11,7 +41,7 @@ sequenceDiagram activate CNI CNI->>CNI: parseConf() - CNI->>CNI: resourceTracker{vpc, attachment, veth} + CNI->>CNI: resourceTracker{vpc, attachment} CNI->>VRF: Add(vpc, attachment) activate VRF @@ -28,16 +58,8 @@ sequenceDiagram Veth-->>CNI: ok deactivate Veth - loop terminations - CNI->>Route: Add(vpc, attachment, network, via, dev) - activate Route - Route->>Route: netlink.RouteAdd in VRF table - Route-->>CNI: ok - deactivate Route - end - - CNI->>CNI: buildVethResult() - activate CNI + CNI->>K8s: newK8sClient() + CNI->>K8s: AnnotateNAD(name, podNamespace, hostName) Note over CNI: host-device delegation CNI->>HostDevice: ADD (move guest veth to container netns) @@ -45,47 +67,72 @@ sequenceDiagram HostDevice-->>CNI: ok deactivate HostDevice - CNI->>CNI: configureIPAM() - CNI->>CNI: allocateIPAM() -> subnet + gateway - CNI->>NetNS: configureInterfaceInNetns(guest, subnet, gw) - activate NetNS - NetNS->>NetNS: AddrAdd(subnet), LinkSetUp, RouteAdd(default via gw) - NetNS-->>CNI: ok - deactivate NetNS + opt "ipam" block present + CNI->>IPAM: ExecAdd(ipam.type, stdin) + activate IPAM + IPAM->>IPAM: allocate subnet/address (pool or static) + IPAM-->>CNI: CNI IPAM result + deactivate IPAM + CNI->>NetNS: configureInterfaceInNetns(guest, subnet, gw) + activate NetNS + NetNS->>NetNS: AddrAdd(subnet), LinkSetUp, RouteAdd(default via gw) + NetNS-->>CNI: ok + deactivate NetNS + end CNI->>CNI: readGuestInterface(MAC, MTU) + CNI->>HostGW: ConfigureHostGateway(vpc, attachment, ipamResult, guestMAC) + activate HostGW + HostGW->>HostGW: AddrAdd(gateway) on host veth + HostGW->>HostGW: RouteAdd(pod subnet) in VRF table + HostGW->>HostGW: NeighSet(pod IP -> guest MAC), permanent + HostGW-->>CNI: ok + deactivate HostGW + CNI->>CNI: buildResult() + PrintResult() + CNI-->>Runtime: CNI result (JSON) — becomes next plugin's prevResult deactivate CNI - CNI->>CNI: publishBGPState() - activate CNI - - CNI->>CNI: configureHostGateway(vpc, attachment, ipamResult) - CNI->>CNI: AddrAdd(gateway/128) on host veth - CNI->>CNI: RouteAdd(subnet to host veth) in VRF table + opt attachment has terminations + Runtime->>Route: ADD (stdin config, prevResult) + activate Route + Route->>Route: parseConf(); require prevResult + loop terminations + Route->>Route: route.Add(vpc, attachment, network, via, dev) + end + Route-->>Runtime: prevResult, unchanged + deactivate Route + end - CNI->>CNI: decode VPC hex + VRFID - CNI->>K8s: newK8sClient() + Runtime->>BGP: ADD (stdin config, prevResult) + activate BGP + BGP->>BGP: parseConf(); inferFromPrevResult(prevResult) + Note over BGP: interface kind + IPAM result inferred from
prevResult shape alone — no kernel access + BGP->>K8s: newK8sClient() + + BGP->>BGP: publishBGPState() (retry loop) + activate BGP + BGP->>K8s: lookupBGPRouter(node) + BGP->>K8s: allocateArgument() -> VRFID + BGP->>K8s: CreateOrUpdate BGPVRFInstance + BGP->>K8s: checkArgumentCollision() + opt router has srv6Locator + nodeID + BGP->>eBPF: registerEBPFDatapath(block, argument, vrfTableID, egressKind) + activate eBPF + eBPF->>eBPF: locator_table / function_table / vrf_table entries + eBPF-->>BGP: ok + deactivate eBPF + end + BGP->>K8s: CreateOrUpdate BGPAdvertisement(prefixes, annotations) + deactivate BGP - CNI->>CNI: publishBGPStateK8s() (retry loop) - activate CNI - CNI->>K8s: lookupBGPRouter(node) - CNI->>CNI: resolveSRv6SID(locator, nodeID, vrfID) - CNI->>SRv6: RouteIngressAdd(sid, vpc, attachment) - activate SRv6 - SRv6->>SRv6: seg6local End.DT46 route - SRv6-->>CNI: ok - deactivate SRv6 - CNI->>K8s: CreateOrUpdate BGPVRFInstance - CNI->>K8s: CreateOrUpdate BGPAdvertisement(prefix, annotations) - CNI-->>Runtime: ok - deactivate CNI + BGP-->>Runtime: prevResult, unchanged + deactivate BGP Runtime-->>Runtime: CNI result (JSON) - deactivate CNI ``` -## cmdAdd — tap +## cmdAdd — tap (galactic-tap-cni → galactic-route → galactic-bgp) ```mermaid sequenceDiagram @@ -94,7 +141,7 @@ sequenceDiagram activate CNI CNI->>CNI: parseConf() - CNI->>CNI: resourceTracker{vpc, attachment, tap} + CNI->>CNI: resourceTracker{vpc, attachment} CNI->>VRF: Add(vpc, attachment) activate VRF @@ -111,48 +158,76 @@ sequenceDiagram Tap-->>CNI: ok deactivate Tap - loop terminations - CNI->>Route: Add(vpc, attachment, network, via, dev) - activate Route - Route->>Route: netlink.RouteAdd in VRF table - Route-->>CNI: ok - deactivate Route - end + CNI->>K8s: newK8sClient() + CNI->>K8s: AnnotateNAD(name, podNamespace, hostName) - Note over CNI: tap branch - no host-device, no guest netns + Note over CNI: tap branch — no host-device, no guest netns - CNI->>CNI: allocateIPAM() -> subnet + gateway - CNI->>CNI: configureHostGateway(vpc, attachment, ipamResult) - CNI->>CNI: AddrAdd(gateway/128) on host tap - CNI->>CNI: RouteAdd(subnet to host tap) in VRF table + opt "ipam" block present + CNI->>IPAM: ExecAdd(ipam.type, stdin) + activate IPAM + IPAM->>IPAM: allocate subnet/address (pool or static) + IPAM-->>CNI: CNI IPAM result + deactivate IPAM + end + + CNI->>HostGW: ConfigureHostGateway(vpc, attachment, ipamResult, nil) + activate HostGW + HostGW->>HostGW: AddrAdd(gateway, /25 + NOPREFIXROUTE) on host tap + HostGW->>HostGW: RouteAdd(pod subnet) in VRF table + Note over HostGW: no guest MAC in tap mode — no neighbor entry installed + HostGW-->>CNI: ok + deactivate HostGW CNI->>CNI: buildTapResult(ipamResult) + PrintResult() + CNI-->>Runtime: CNI result (JSON) — becomes next plugin's prevResult + deactivate CNI - CNI->>CNI: decode VPC hex + VRFID - CNI->>K8s: newK8sClient() + opt attachment has terminations + Runtime->>Route: ADD (stdin config, prevResult) + activate Route + Route->>Route: parseConf(); require prevResult + loop terminations + Route->>Route: route.Add(vpc, attachment, network, via, dev) + end + Route-->>Runtime: prevResult, unchanged + deactivate Route + end - CNI->>CNI: publishBGPStateK8s() (retry loop) - activate CNI - CNI->>K8s: lookupBGPRouter(node) - CNI->>CNI: resolveSRv6SID(locator, nodeID, vrfID) - CNI->>SRv6: RouteIngressAdd(sid, vpc, attachment) - activate SRv6 - SRv6->>SRv6: seg6local End.DT46 route - SRv6-->>CNI: ok - deactivate SRv6 - CNI->>K8s: CreateOrUpdate BGPVRFInstance - CNI->>K8s: CreateOrUpdate BGPAdvertisement(prefix, annotations) - CNI-->>Runtime: ok - deactivate CNI + Runtime->>BGP: ADD (stdin config, prevResult) + activate BGP + BGP->>BGP: parseConf(); inferFromPrevResult(prevResult) + Note over BGP: single interface, empty sandbox -> ifaceType = tap + BGP->>K8s: newK8sClient() + + BGP->>BGP: publishBGPState() (retry loop) + activate BGP + BGP->>K8s: lookupBGPRouter(node) + BGP->>K8s: allocateArgument() -> VRFID + BGP->>K8s: CreateOrUpdate BGPVRFInstance + BGP->>K8s: checkArgumentCollision() + opt router has srv6Locator + nodeID + BGP->>eBPF: registerEBPFDatapath(block, argument, vrfTableID, egressKind) + activate eBPF + eBPF->>eBPF: locator_table / function_table / vrf_table entries + eBPF-->>BGP: ok + deactivate eBPF + end + BGP->>K8s: CreateOrUpdate BGPAdvertisement(prefixes, annotations) + deactivate BGP + + BGP-->>Runtime: prevResult, unchanged + deactivate BGP Runtime-->>Runtime: CNI result (JSON) - deactivate CNI ``` -## cmdDel — veth / tap (shared) +## cmdDel — every binary in the chain -Both interface types share the same DEL path. Per the CNI spec, DEL is -idempotent — missing resources are never errors. +Per the CNI spec, DEL is idempotent — missing resources are never errors. +The runtime calls DEL on every chain entry that had a successful ADD, in +reverse order; every one of those calls is independently idempotent, so +the order doesn't matter for correctness. ```mermaid sequenceDiagram @@ -166,16 +241,17 @@ sequenceDiagram CNI-->>Runtime: nil deactivate CNI else parse succeeds - alt hasIPAM - CNI->>K8s: newK8sClient() - alt k8s client OK - CNI->>CNI: deallocateIPAM() - CNI->>K8s: Get BGPAdvertisement -> read subnet annotation - CNI->>IPAM: PoolAllocator.Deallocate(subnet) - end + alt "ipam" block present (galactic-cni/galactic-tap-cni only) + CNI->>IPAM: ExecDel(ipam.type, stdin) + activate IPAM + IPAM->>IPAM: look up this containerID's own marker file, delete it + IPAM-->>CNI: ok + deactivate IPAM end - Note over CNI: Shared resources (VRF, interface, routes, SRv6,
BGPAdvertisement, BGPVRFInstance) are NOT deleted here.
They may be in use by another pod on the same (vpc, attachment).
The GC controller collects orphans periodically. + Note over CNI: galactic-cni only: flush the guest netns'
address/route, then forward DEL to host-device + + Note over CNI,BGP: Shared resources (VRF, host interface, routes,
eBPF vrf_table entry, BGPAdvertisement, BGPVRFInstance) are
NOT deleted by any binary's DEL — they may be in use by
another pod/VM on the same (vpc, attachment).
galactic-router's GC controller collects orphans periodically. CNI->>CNI: slog.Info("DEL: skipping shared resource cleanup (handled by GC)") CNI->>CNI: print empty result diff --git a/docs/cni/configuration.md b/docs/cni/configuration.md index 7abd5e9c..5313bbd9 100644 --- a/docs/cni/configuration.md +++ b/docs/cni/configuration.md @@ -1,24 +1,74 @@ # CNI Configuration -`galactic-cni` is configured through the CNI JSON configuration passed by Multus -(or any CNI manager), plus node-local settings resolved at runtime from the -conflist, environment variables, and (as a last resort) the Kubernetes API. +The galactic CNI plugin chain is configured through the CNI JSON conflist passed +by Multus (or any CNI manager), plus node-local settings resolved at runtime from +the static conflist, environment variables, and (for `galactic-cni`/`galactic-tap-cni`/ +`galactic-bgp`, as a last resort) the Kubernetes API. -> Last verified: 2026-07-28 against the current working tree of `internal/cni/config.go`, -> `internal/cni/ipam_ops.go`, and `internal/installer/installer.go`. +> Last verified: 2026-08-08 against the current working tree of `internal/cni/`, +> `internal/cnitap/`, `internal/cniipam/`, `internal/cnibgp/`, `internal/cniroute/`, +> and `internal/installer/installer.go`. + +## Chain structure + +Each `NetworkAttachmentDefinition` (or other Multus-driven config) supplies a +full CNI conflist — a `"plugins"` array, not a single plugin object — because +BGP/SRv6/eBPF publish now runs as its own chained binary rather than inline +inside the master plugin. A real-world attachment's conflist has this shape: + +```json +{ + "cniVersion": "1.0.0", + "name": "private", + "plugins": [ + { "type": "galactic-cni", "...": "..." }, + { "type": "galactic-route", "...": "..." }, + { "type": "galactic-bgp", "...": "..." } + ] +} +``` + +- **`galactic-cni`** (veth, containers) or **`galactic-tap-cni`** (tap, VM + workloads) — always first. Creates the VRF and host-side interface, + annotates the NAD, delegates IPAM (if an `"ipam"` block is present) and + — `galactic-cni` only — host-device (to move the guest veth into the + container netns), configures the host gateway, and prints the CNI + result. +- **`galactic-route`** — optional; include only when the attachment has + `terminations` to install. Installs each as a VRF-table route, then + passes `prevResult` through unchanged. +- **`galactic-bgp`** — publishes `BGPVRFInstance`/`BGPAdvertisement` and + (when this node's `BGPRouter` has SRv6 configured) the eBPF uSID + datapath's `vrf_table` registration. Learns everything it needs + (interface kind, allocated addresses) from `prevResult` alone. In + practice this stage is not optional — without it the attachment is + never BGP-advertised and stays unreachable from other nodes — but + nothing enforces its presence at the CNI-config level; that's the + conflist author's responsibility (the companion operator, cross-repo, + out of scope here). + +Every binary's own JSON stanza carries only the fields that binary itself +reads (`vpc`/`vpcattachment` are duplicated across every stanza; nothing +else is). See [docs/cni-cmd-sequence.md](../cni-cmd-sequence.md) for the +full ADD/DEL sequence across all three stages. ## Runtime Configuration -There is no `--node-name` or `--enable-local-ipam` CLI flag on the `galactic-cni` -plugin invocation itself. Instead, `parseConf()` (`internal/cni/config.go`) resolves -node name, kubeconfig, namespace, log file, and log level on every ADD/DEL/CHECK/STATUS -call, reading (in order) the CNI config JSON, environment variables, and a `HostConf` -block parsed out of the conflist at `--conf-file` (default -`/etc/cni/net.d/10-galactic.conflist`). `HostConf` is written by the `galactic-cni init` -subcommand (`internal/installer.Bootstrap`), which runs as the CNI DaemonSet's init -container — see [docs/agents/ARCHITECTURE.md](../agents/ARCHITECTURE.md#known-constraints) +There is no `--node-name` or similar CLI flag on any plugin's own invocation. +Instead, each binary's own `parseConf()` resolves node-level settings on every +ADD/DEL/CHECK/STATUS call, reading (in order) the CNI config JSON, environment +variables, and a `HostConf` block parsed out of the conflist at `--conf-file` +(default `/etc/cni/net.d/10-galactic.conflist` — this is the one *static*, +node-level conflist every binary shares; not the per-attachment `plugins[]` +conflist described above). `HostConf` is written by the `galactic-cni init` +subcommand (`internal/installer.Bootstrap`), which runs as the CNI DaemonSet's +init container — see [docs/agents/ARCHITECTURE.md](../agents/ARCHITECTURE.md#known-constraints) for how the DaemonSet stages it. +`galactic-ipam` and `galactic-route` have no Kubernetes dependency at all, so +they resolve only `LogFile`/`LogLevel` from `HostConf` — never `NodeName` or +`Kubeconfig`, and never fall back to the Kubernetes API. + ### `HostConf` fields (written into the conflist by `galactic-cni init`) | Field | Description | @@ -26,116 +76,125 @@ for how the DaemonSet stages it. | `NodeName` | The Kubernetes node name the installer bootstrapped on. | | `Kubeconfig` | Path to the kubeconfig `Bootstrap`/`Run` maintain (`/var/lib/galactic/kubeconfig` by default). | | `Namespace` | Kubernetes namespace for BGP CRDs (`galactic-system` by default). | -| `LogFile` | Path the plugin logs to (`/var/log/galactic/galactic-cni.log` by default). | +| `LogFile` | Path the plugin logs to (`/var/log/galactic/galactic-cni.log` by default, shared across every binary in the chain). | | `LogLevel` | Verbosity of plugin logging: `debug`, `info`, `warn`, or `error` (`info` by default). See [Log verbosity](#log-verbosity) below. | ### Resolution precedence -| Setting | Precedence (highest first) | Default (if nothing resolves) | -| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | -| Node name | `GALACTIC_CNI_NODE_NAME` env → `NODE_NAME` env → `HostConf.NodeName` → auto-detect via the Kubernetes API (`detectNodeNameFromAPI`: lists Nodes, matches local interface addresses against `status.addresses[].type=InternalIP`) | _(error: "node name is required")_ | -| Kubeconfig | `GALACTIC_CNI_KUBECONFIG` env → `HostConf.Kubeconfig` | `/var/lib/galactic/kubeconfig` | -| Namespace | `namespace` field in the CNI config JSON → `GALACTIC_CNI_NAMESPACE` env → `HostConf.Namespace` | `galactic-system` | -| Log file | `GALACTIC_CNI_LOG_FILE` env → `HostConf.LogFile` | `/var/log/galactic/galactic-cni.log` | -| Log level | `GALACTIC_CNI_LOG_LEVEL` env → `HostConf.LogLevel` | `info` | -| Enable local IPAM | `GALACTIC_CNI_ENABLE_LOCAL_IPAM` env only (no conflist field, no CLI flag) | `false` | - -The resolved node name is re-exported as the `NODE_NAME` process environment variable -and the resolved kubeconfig as `KUBECONFIG`, since other code in `internal/cni` reads -those directly. Auto-detection exists to tolerate environments (e.g. Kind-based e2e) -where the conflist's hostPath mount isn't populated yet. +| Setting | Precedence (highest first) | Default (if nothing resolves) | Resolved by | +| ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------- | -------------------------------------------------- | +| Node name | `GALACTIC_CNI_NODE_NAME` env → `NODE_NAME` env → `HostConf.NodeName` → auto-detect via the Kubernetes API (`detectNodeNameFromAPI`: lists Nodes, matches local interface addresses against `status.addresses[].type=InternalIP`) | _(error: "node name is required")_ | `galactic-cni`, `galactic-tap-cni`, `galactic-bgp` | +| Kubeconfig | `GALACTIC_CNI_KUBECONFIG` env → `HostConf.Kubeconfig` | `/var/lib/galactic/kubeconfig` | `galactic-cni`, `galactic-tap-cni`, `galactic-bgp` | +| Namespace | `namespace` field in the CNI config JSON → `GALACTIC_CNI_NAMESPACE` env → `HostConf.Namespace` | `galactic-system` | `galactic-cni`, `galactic-tap-cni`, `galactic-bgp` | +| Log file | `GALACTIC_CNI_LOG_FILE` env → `HostConf.LogFile` | `/var/log/galactic/galactic-cni.log` | every binary in the chain | +| Log level | `GALACTIC_CNI_LOG_LEVEL` env → `HostConf.LogLevel` | `info` | every binary in the chain | + +`GALACTIC_CNI_*` env var names are shared as-is across every binary that +resolves node-level settings — there's no per-binary prefix for these, since +they're the same physical node's settings regardless of which chain binary +reads them (unlike `GALACTIC_IPAM_ENABLE_LOCAL_IPAM` below, which is a +domain-specific knob belonging entirely to `galactic-ipam`). + +The resolved node name is re-exported as the `NODE_NAME` process environment +variable and the resolved kubeconfig as `KUBECONFIG` (`galactic-cni`/ +`galactic-tap-cni`/`galactic-bgp` only), since other code in those packages +reads those directly. Auto-detection exists to tolerate environments (e.g. +Kind-based e2e) where the conflist's hostPath mount isn't populated yet. ### Log verbosity -`setupLogging()` (`internal/cni/config.go`) builds a JSON `slog` handler at the -resolved level. Since each CNI invocation is a fresh, short-lived process, this -level is re-resolved on every ADD/DEL/CHECK/STATUS call — there's no persistent -daemon to reconfigure at runtime. +Each binary's own `setupLogging()` builds a JSON `slog` handler at the +resolved level. Since every CNI invocation is a fresh, short-lived process, +this level is re-resolved on every ADD/DEL/CHECK/STATUS call, in every +binary — there's no persistent daemon to reconfigure at runtime, and every +binary shares the same log file by default so a single chain invocation's +log lines interleave in call order. -| Level | What's logged | -| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `debug` | Everything: per-resource milestones (VRF/interface/route/IPAM ready, BGP CRDs applied, kernel-level veth/tap/route operations) in addition to `info` and above. Use this when troubleshooting a specific ADD/DEL failure. | -| `info` (default) | One line per operation marking start and outcome (`ADD: starting` / `ADD: BGP state published`, `DEL: starting` / `DEL: skipping shared resource cleanup`, `CHECK: starting` / `CHECK: passed`/`failed`, `STATUS: probing API server reachability` / `STATUS: ready`), plus all `warn`/`error` events. | -| `warn` | Recoverable anomalies only: stale-state repairs (leftover veth/tap from a prior failed ADD), iptables-missing fallback, k8s API retries. | -| `error` | Failures only. | +| Level | What's logged | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `debug` | Everything: per-resource milestones (VRF/interface/route/IPAM ready, BGP CRDs applied, kernel-level operations) in addition to `info` and above. Use this when troubleshooting a specific ADD/DEL failure. | +| `info` (default) | One line per operation marking start and outcome (`ADD: starting` / `ADD: BGP state published`, `DEL: starting` / `DEL: skipping shared resource cleanup`, `CHECK: starting` / `CHECK: passed`/`failed`, `STATUS: ready`), plus all `warn`/`error` events. | +| `warn` | Recoverable anomalies only: stale-state repairs, k8s API retries. | +| `error` | Failures only. | An unrecognized `log_level` value does not fail the CNI operation — it logs a warning and falls back to `info`. -### `GALACTIC_CNI_ENABLE_LOCAL_IPAM` - -When enabled, the plugin performs IP allocation using a built-in IPv6 pool -allocator even when no explicit `ipam` block is present in the CNI config. -This is useful for simple deployments that do not need an external IPAM -plugin. +### `GALACTIC_IPAM_ENABLE_LOCAL_IPAM` -When local IPAM is active but the config does not specify pool parameters, -the following defaults are used: +Read only by `galactic-ipam` (`internal/config/ipam.go`) — renamed from the +historical `GALACTIC_CNI_ENABLE_LOCAL_IPAM`, which no longer exists at all. +The old name could manufacture an `"ipam"` block out of thin air even when +the master plugin's own config had none; the new one can't; it only fills +in a default IPv6 pool CIDR when an `"ipam"` block is present but specifies +neither `static_ip` nor a subnet: | Parameter | Default | -| ------------- | ---------------------------------------- | -| Pool CIDR | `fd00:10:ff01::/48` | -| Subnet length | `/96` | -| Gateway | First usable address in the pool (`::1`) | +| ------------- | ----------------------------------------- | +| Pool CIDR | `fd00:10:ff01::/64` | +| Subnet length | `/96` | +| Gateway | First usable address in the pool (`::1`) | -If an explicit `ipam` block is present in the CNI config, it takes precedence -and this environment variable has no effect on the allocation behavior. +Whether IPAM runs at all is decided **solely** by whether `"ipam"` is present +in the master plugin's own stanza — no environment variable, on either side +of this rename, can trigger or suppress that decision. See [IPAM Fields](#ipam-fields) +below and `internal/cniipam`'s package doc comment for the full explicit +contract. **Type:** bool **Default:** `false` -## CNI Configuration JSON - -The CNI configuration is a JSON object passed at pod creation time. It extends -the standard CNI `PluginConf` with Galactic-specific fields. +## Master Plugin Fields (`galactic-cni` / `galactic-tap-cni`) -### Top-Level Fields - -| Field | Required | Type | Description | -| ---------------- | -------- | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `vpc` | **Yes** | `string` | Base62-encoded VPC identifier (48-bit value). Used to derive VRF names, interface names, and BGP route targets. | -| `vpcattachment` | **Yes** | `string` | Base62-encoded VPC attachment identifier (16-bit value). Paired with `vpc` for deterministic VRF/BGP naming. | -| `interface_type` | No | `string` | Interface mode: `"veth"` (default, for containers) or `"tap"` (for VMs such as Kata, Firecracker, QEMU). Both modes run IPAM and SRv6/BGP publish; `tap` mode only skips host-device delegation and guest-netns configuration (see the Tap mode section below). | -| `mtu` | No | `int` | MTU for the host-side interface. For `veth` mode this applies to both veth endpoints; for `tap` mode it applies to the tap interface. | -| `terminations` | No | `[]Termination` | Array of static routes to add on the host side (see Termination sub-fields below). | -| `namespace` | No | `string` | Kubernetes namespace used to look up the `BGPRouter` CRD. Resolution order: this field → `GALACTIC_CNI_NAMESPACE` env → `HostConf.Namespace` (conflist) → `galactic-system`. See [Runtime Configuration](#runtime-configuration) above. | -| `ipam` | No* | `IPAM` | Legacy static-IP / local-IPAM configuration block (see IPAM sub-fields below). Only `type: "static"` still drives its own allocation path; `type: "pool"` is otherwise superseded by `ipv6_subnet`/`ipv4_subnet` below. *Required unless `GALACTIC_CNI_ENABLE_LOCAL_IPAM`, `ipv6_subnet`, or `ipv4_subnet` is set — applies identically in `veth` and `tap` mode. In `tap` mode `cmdAdd` (`internal/cni/ops_add.go`) calls `allocateIPAM` unconditionally (unlike `veth` mode, which checks first), so a config satisfying none of those currently produces a nil-pointer panic in `tap` mode rather than a clean validation error — always set one of them for tap. | -| `ipv6_subnet` | No* | `string` | Region IPv6 pool CIDR for the NAD-driven pool-IPAM path; endpoints allocate a `/96` from it by default. Setting this field or `ipv4_subnet` (or both) opts a config into pool IPAM directly — no `ipam` block needed. See [Pool IPAM via `ipv6_subnet`/`ipv4_subnet`](#pool-ipam-via-ipv6_subnetipv4_subnet) below. | -| `ipv4_subnet` | No | `string` | Optional site IPv4 pool CIDR; endpoints allocate a `/32` host address from it. May be set alone (IPv4-only), alongside `ipv6_subnet` (dual-stack), or omitted entirely (IPv6-only, given `ipv6_subnet` is set). | -| `address_families` | No | `[]string` | Families to record as in-use: any of `"ipv6"`, `"ipv4"`. Defaults to `["ipv6"]` when omitted. Validated at parse time, but the families actually allocated are driven by which of `ipv6_subnet`/`ipv4_subnet` are set — keep this field consistent with those. | +| Field | Required | Type | Description | +| --------------- | -------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vpc` | **Yes** | `string` | Base62-encoded VPC identifier (48-bit value). Used to derive VRF names, interface names, and BGP route targets. | +| `vpcattachment` | **Yes** | `string` | Base62-encoded VPC attachment identifier (16-bit value). Paired with `vpc` for deterministic VRF/BGP naming. | +| `mtu` | No | `int` | MTU for the host-side interface. For `galactic-cni` this applies to both veth endpoints; for `galactic-tap-cni` it applies to the tap interface. | +| `namespace` | No | `string` | Kubernetes namespace used for NAD lookup (and, for `galactic-bgp`'s own stanza, `BGPRouter`/BGP CRD lookup). Resolution order: this field → `GALACTIC_CNI_NAMESPACE` env → `HostConf.Namespace` → `galactic-system`. | +| `ipam` | No | `IPAM` | IPAM delegation block (see [IPAM Fields](#ipam-fields) below). Presence alone decides whether IPAM runs at all — no env var or sibling field can trigger or suppress it. | Standard CNI fields (`cniVersion`, `name`, `dns`, `runtimeConfig`) are also -supported via the embedded `types.PluginConf`. `galactic-cni` declares support +supported via the embedded `types.PluginConf`. Both binaries declare support for the full CNI spec range (`version.All`, from `github.com/containernetworking/cni` -v1.3.0 in `go.mod`) and returns CNI Result `1.0.0` (`type100`); generated -configs (the installer's default conflist, Multus `NetworkAttachmentDefinition` -manifests) use `"cniVersion": "1.0.0"`. - -### Interface Types - -#### `veth` (default) +v1.3.0 in `go.mod`) and return CNI Result `1.0.0` (`type100`). + +Despite that broad declared range, `cniVersion` must be `"1.0.0"` or `"1.1.0"` +in practice: `galactic-bgp`, chained after the master plugin, reconstructs +`prevResult` via `type100.NewResult` (`internal/cnibgp/prevresult.go`), which +only accepts a Result whose own `cniVersion` field is exactly one of those two +values — the master plugin echoes the conflist's `cniVersion` straight into +its printed Result, so an older value (e.g. `"0.4.0"`) makes `galactic-bgp`'s +ADD fail for every attachment in the chain. Every config in this doc already +uses `"1.0.0"`; keep it that way for any config authored outside these +examples. + +There is no `interface_type` field anymore: which binary you invoke *is* the +interface type. `galactic-cni` always creates a veth pair; `galactic-tap-cni` +always creates a tap device. There is likewise no `terminations` field on +either master plugin's own stanza anymore — that field now lives entirely on +`galactic-route`'s own stanza (see [Termination Fields](#termination-fields) +below). + +### `galactic-cni` (veth) Creates a veth pair: one endpoint stays in the host namespace (named `GH`, e.g. `G0000000010010H`) and the other is moved into the container via the host-device CNI plugin (renamed to the `CNI_IFNAME` value, typically `eth0`). The guest interface -receives an IP address from IPAM and a default route via the pool gateway. +receives an IP address from IPAM (if configured) and a default route via the +pool gateway. -#### `tap` +### `galactic-tap-cni` (tap) Creates a tap interface in the host namespace (same naming pattern as the veth host endpoint: `GH`) and enslaves it to the VRF. No -interface is moved into the container — the tap fd is managed directly by the -guest VM hypervisor, so `tap` mode skips host-device delegation and guest-netns -configuration. Unlike an earlier version of this plugin, `tap` mode is **not** -"no IPAM, no BGP": `cmdAdd` calls `allocateIPAM` to allocate a subnet/gateway, -`configureHostGateway` to assign the gateway on the host tap and install the pod -subnet route into the VRF table, includes the resulting `ips`/`routes` in the CNI -result (interface index `0`, the host tap — there is no guest interface entry), -and then `publishBGPStateK8s` to create the SRv6 ingress route and -`BGPVRFInstance`/`BGPAdvertisement` CRDs, exactly as `veth` mode does. The guest -VM still configures its own IP addresses independently (the CNI-allocated -subnet/gateway describe only the host-side BGP-advertised state). +interface is moved into a container — the tap fd is managed directly by the +guest VM hypervisor (Kata, Firecracker, kraftlet/Unikraft), so this binary +never delegates to host-device and never configures a guest netns. It still +runs IPAM (if `"ipam"` is present) and configures the host gateway exactly as +`galactic-cni` does; the CNI result carries a single interface (the host tap, +empty sandbox) since there's no guest-side interface entry. The IPv4 gateway address on the host tap is a `/25`, not the `/32` used everywhere else (veth's host/guest gateways, and the pod's own address in both @@ -144,34 +203,51 @@ bare host route. Because a wider mask would normally make the kernel auto-install a connected route for the whole `/25` in the VRF table — exactly the subnet-router-anycast hazard the `/32` choice exists to avoid elsewhere — the address is added with `IFA_F_NOPREFIXROUTE`, which suppresses that -auto-created route. The explicit pod-subnet `/32` route `configureHostGateway` +auto-created route. The explicit pod-subnet `/32` route `hostgw.ConfigureHostGateway` installs remains the only route governing delivery to the VM's address. -> **Note:** Tap mode is intended for VM-based workloads (Kata, Firecracker, -> QEMU) where the hypervisor opens the tap fd and handles guest networking. - -### IPAM Fields - -| Field | Required | Type | Description | -| ------------ | ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `type` | Conditionally required | `string` | `"pool"` or `"static"`. Required whenever an `ipam` block is present — an `ipam` block with an empty `type` is a hard error. `"static"` drives its own allocation path (below); `"pool"` is otherwise vestigial now that pool allocation is driven by `ipv6_subnet`/`ipv4_subnet` (see below) — its former `pool`/`gateway`/`subnet_len` sub-fields have been retired. | -| `static_ip` | Conditionally required | `string` | A single IPv6 address to assign when `type` is `"static"`. | - -#### IPAM `type=static` - -Validates and assigns a single IPv6 address with a `/64` mask. No deallocation -needed, and no IPv4 address is ever allocated alongside it — static IPAM is a -single fixed address, not a dual-stack pool. +## IPAM Fields + +`"ipam"`'s `type` names the **delegated binary** (per the CNI IPAM delegation +protocol, `github.com/containernetworking/plugins/pkg/ipam.ExecAdd`/`ExecDel`/ +`ExecCheck`) — currently only `galactic-ipam` exists, so this is always +`"galactic-ipam"` in practice. It is not a pool-vs-static mode selector: mode +is decided entirely by which of the fields below are present. + +| Field | Required | Type | Description | +| ------------------ | -------- | ---------- | ---------------------------------------------------------------------------------------------------------------------- | +| `type` | **Yes** | `string` | Names the delegated CNI IPAM binary. Always `"galactic-ipam"` today. | +| `static_ip` | No | `string` | A single IPv6 address to assign. Presence selects the static allocation path (below); mutually exclusive in practice with the subnet fields. | +| `ipv6_subnet` | No | `string` | Region IPv6 pool CIDR; endpoints allocate a `/96` from it by default. | +| `ipv4_subnet` | No | `string` | Site IPv4 pool CIDR; endpoints allocate a `/32` host address from it. | +| `address_families` | No | `[]string` | Families to record as in-use: any of `"ipv6"`, `"ipv4"`. Defaults to `["ipv6"]`. Validated at parse time — keep this consistent with which of `ipv6_subnet`/`ipv4_subnet` are set. | +| `routes` | No | `[]Route` | Declared on the `IPAM` struct (`dst`, `gw`) but not read by any current allocation path — vestigial. | +| `addresses` | No | `[]Address`| Declared on the `IPAM` struct (`address`) but not read by any current allocation path — vestigial. | + +Whether IPAM runs at all is decided **solely** by `"ipam"` block presence in +the master plugin's own stanza — no environment variable can trigger or +suppress that (see [`GALACTIC_IPAM_ENABLE_LOCAL_IPAM`](#galactic_ipam_enable_local_ipam) +above, which only fills a default when the block is present but +under-specified). Once delegated to, `static_ip` presence selects the static +path; otherwise `ipv6_subnet`/`ipv4_subnet` (either alone, or both) select the +pool path. See `internal/cniipam`'s package doc comment for the full explicit +contract. + +### IPAM `static_ip` + +Validates and assigns a single IPv6 address with a `/64` mask. No IPv4 address +is ever allocated alongside it — static IPAM is a single fixed address, not a +dual-stack pool. ### Pool IPAM via `ipv6_subnet`/`ipv4_subnet` -This is the NAD-driven path most VPC attachments use (`wantsIPAM`/`allocatePoolIPAM` -in `internal/cni/ipam_ops.go`). Either `ipv6_subnet` or `ipv4_subnet` alone is -sufficient to opt a config into pool IPAM — an `ipam` block is not required, and -neither field depends on the other being set: +This is the NAD-driven path most VPC attachments use (`allocatePool` in +`internal/cniipam/allocate.go`, backed by `internal/cni/ipam`'s pool +allocators). Either field alone is sufficient — neither depends on the other +being set: -- **IPv6-only:** set `ipv6_subnet`, omit `ipv4_subnet`. Allocates a `/96` subnet - from the region pool; no IPv4 address is allocated. +- **IPv6-only:** set `ipv6_subnet`, omit `ipv4_subnet`. Allocates a `/96` + subnet from the region pool; no IPv4 address is allocated. - **IPv4-only:** set `ipv4_subnet`, omit `ipv6_subnet`. Allocates a `/32` host address from the site pool; no IPv6 subnet is allocated, and the resulting `BGPAdvertisement` carries only the IPv4 `/32` prefix. @@ -179,19 +255,28 @@ neither field depends on the other being set: `BGPAdvertisement` carries both prefixes and the CNI result carries both `IPConfig`/route entries. -Allocation is in-memory and thread-safe per pool; allocations are ephemeral -(lost on process restart). `cmdDel` looks up each family's allocated address -independently from its own `BGPAdvertisement` CRD annotation, so cleanup of -one family never depends on the other having been allocated. +Allocation state persists in on-disk marker files under `galactic-ipam`'s own +lock directory (`internal/cni/ipam.DefaultLockDir`, flock-guarded, keyed by +containerID — both address families). `galactic-ipam` never needs a +Kubernetes client for this: `cmdDel` looks up and removes its own +containerID's marker file directly, with no dependency on a `BGPAdvertisement` +CRD annotation (that coupling existed before this allocator gained its own +persistence and has since been removed). -When neither `ipv6_subnet` nor `ipv4_subnet` is set and `GALACTIC_CNI_ENABLE_LOCAL_IPAM` +When neither `ipv6_subnet` nor `ipv4_subnet` is set and `GALACTIC_IPAM_ENABLE_LOCAL_IPAM` is enabled, allocation falls back to the built-in default IPv6 pool CIDR (see -[`GALACTIC_CNI_ENABLE_LOCAL_IPAM`](#galactic_cni_enable_local_ipam) above) — +[`GALACTIC_IPAM_ENABLE_LOCAL_IPAM`](#galactic_ipam_enable_local_ipam) above) — this fallback is IPv6-only; there is no default IPv4 pool. -### Termination Fields +## Termination Fields (`galactic-route`) -Each entry in the `terminations` array has the following fields: +`galactic-route`'s own stanza carries only `vpc`, `vpcattachment`, and +`terminations` — no `namespace` field, since this binary has no Kubernetes +dependency at all. Include this stanza in the chain only for attachments that +actually need static routes; it's the one stage in the chain that's genuinely +optional. + +Each entry in `terminations` has: | Field | Required | Type | Description | | --------- | -------- | -------- | ------------------------------------------------------------------------------------------ | @@ -199,26 +284,42 @@ Each entry in the `terminations` array has the following fields: | `via` | No | `string` | Next-hop gateway IP. If omitted, a link-local route is installed via the host-side device. | Used in `cmdAdd` to install routes into the VRF table for each termination -entry. Deleted in `cmdDel` in reverse order. +entry, via the host-side interface name derived from `(vpc, vpcAttachment)` +alone — identical whether the preceding master plugin was `galactic-cni` or +`galactic-tap-cni`. `cmdDel` is a no-op: like every other shared, per-attachment +resource in the chain, termination routes may still be in use by another pod/VM +on the same attachment, so cleanup is left to `galactic-router`'s GC controller. + +## BGP Publish Fields (`galactic-bgp`) + +`galactic-bgp`'s own stanza carries only `vpc`, `vpcattachment`, and +`namespace` — nothing else. It learns which interface kind was created and +what addresses were allocated entirely from `prevResult` (the accumulated +result of every preceding plugin in the chain), never from its own config or +a kernel call. ## Example Configurations +Every example below is a full conflist (a `NetworkAttachmentDefinition`'s +`spec.config`, or an equivalent standalone conflist file) — not a single +plugin object — per [Chain structure](#chain-structure) above. + ### Minimal configuration (overlay) ```json { "cniVersion": "1.0.0", "name": "galactic", - "type": "galactic-cni", - "vpc": "1", - "vpcattachment": "1" + "plugins": [ + { "type": "galactic-cni", "vpc": "1", "vpcattachment": "1" }, + { "type": "galactic-bgp", "vpc": "1", "vpcattachment": "1" } + ] } ``` -Omits `namespace` (defaults to `galactic-system`), `ipam`, and `terminations`. -Without `GALACTIC_CNI_ENABLE_LOCAL_IPAM` set, no IP address is assigned to the -guest interface. With `GALACTIC_CNI_ENABLE_LOCAL_IPAM` set, a subnet is allocated -from the built-in pool. +Omits `namespace` (defaults to `galactic-system`), `ipam`, and a +`galactic-route` stage. Without `GALACTIC_IPAM_ENABLE_LOCAL_IPAM` set, no IP +address is assigned to the guest interface. ### Pool IPAM, IPv6-only (testvpc) @@ -226,55 +327,75 @@ from the built-in pool. { "cniVersion": "1.0.0", "name": "testvpc", - "type": "galactic-cni", - "vpc": "10", - "vpcattachment": "10", - "namespace": "galactic-system", - "ipv6_subnet": "fd00:10:ff02::/48", - "address_families": ["ipv6"] + "plugins": [ + { + "type": "galactic-cni", + "vpc": "10", + "vpcattachment": "10", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv6_subnet": "fd00:10:ff02::/48", + "address_families": ["ipv6"] + } + }, + { "type": "galactic-bgp", "vpc": "10", "vpcattachment": "10", "namespace": "galactic-system" } + ] } ``` -No `ipam` block needed — `ipv6_subnet` alone opts the config into pool IPAM. - ### Pool IPAM, dual-stack ```json { "cniVersion": "1.0.0", "name": "vpc21", - "type": "galactic-cni", - "vpc": "21", - "vpcattachment": "21", - "namespace": "galactic-system", - "ipv6_subnet": "fd00:10:ff03::/48", - "ipv4_subnet": "172.21.1.0/24", - "address_families": ["ipv6", "ipv4"] + "plugins": [ + { + "type": "galactic-cni", + "vpc": "21", + "vpcattachment": "21", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv6_subnet": "fd00:10:ff03::/48", + "ipv4_subnet": "172.21.1.0/24", + "address_families": ["ipv6", "ipv4"] + } + }, + { "type": "galactic-bgp", "vpc": "21", "vpcattachment": "21", "namespace": "galactic-system" } + ] } ``` Allocates from both pools independently; the `BGPAdvertisement` carries both the IPv6 `/96` and IPv4 `/32` prefixes. -### Pool IPAM, IPv4-only +### Pool IPAM, IPv4-only (tap) ```json { "cniVersion": "1.0.0", "name": "vpc20", - "type": "galactic-cni", - "vpc": "20", - "vpcattachment": "20", - "interface_type": "tap", - "namespace": "galactic-system", - "ipv4_subnet": "172.20.1.0/24", - "address_families": ["ipv4"] + "plugins": [ + { + "type": "galactic-tap-cni", + "vpc": "20", + "vpcattachment": "20", + "namespace": "galactic-system", + "ipam": { + "type": "galactic-ipam", + "ipv4_subnet": "172.20.1.0/24", + "address_families": ["ipv4"] + } + }, + { "type": "galactic-bgp", "vpc": "20", "vpcattachment": "20", "namespace": "galactic-system" } + ] } ``` `ipv4_subnet` alone opts the config into pool IPAM with no IPv6 allocation at -all — no `ipv6_subnet` is required, and the resulting `BGPAdvertisement` -carries only the IPv4 `/32` prefix. +all; the resulting `BGPAdvertisement` carries only the IPv4 `/32` prefix. ### Static IP configuration @@ -282,38 +403,51 @@ carries only the IPv4 `/32` prefix. { "cniVersion": "1.0.0", "name": "galactic", - "type": "galactic-cni", - "vpc": "1", - "vpcattachment": "1", - "ipam": { - "type": "static", - "static_ip": "fd00:1::1" - } + "plugins": [ + { + "type": "galactic-cni", + "vpc": "1", + "vpcattachment": "1", + "ipam": { "type": "galactic-ipam", "static_ip": "fd00:1::1" } + }, + { "type": "galactic-bgp", "vpc": "1", "vpcattachment": "1" } + ] } ``` ### Configuration with terminations +`terminations` goes in `galactic-route`'s own stanza of the conflist's +`plugins` array, not the master plugin's: + ```json { "cniVersion": "1.0.0", "name": "galactic", - "type": "galactic-cni", - "vpc": "1", - "vpcattachment": "1", - "terminations": [ - { "network": "fd00::/48", "via": "fe80::1" }, - { "network": "fd01::/48" } - ], - "ipam": { - "type": "pool", - "pool": "fd00:1:ff01::/48" - } + "plugins": [ + { + "type": "galactic-cni", + "vpc": "1", + "vpcattachment": "1", + "ipam": { "type": "galactic-ipam", "ipv6_subnet": "fd00:1:ff01::/48" } + }, + { + "type": "galactic-route", + "vpc": "1", + "vpcattachment": "1", + "terminations": [ + { "network": "fd00::/48", "via": "fe80::1" }, + { "network": "fd01::/48" } + ] + }, + { "type": "galactic-bgp", "vpc": "1", "vpcattachment": "1" } + ] } ``` The first termination installs a specific next-hop route; the second installs -a link-local route via the host-side device. +an on-link route via the host-side device. `galactic-route` runs between the +master plugin and `galactic-bgp`. ### Tap interface configuration (VM workloads) @@ -321,23 +455,23 @@ a link-local route via the host-side device. { "cniVersion": "1.0.0", "name": "galactic-tap", - "type": "galactic-cni", - "vpc": "1", - "vpcattachment": "1", - "interface_type": "tap", - "mtu": 9000, - "ipam": { - "type": "pool", - "pool": "fd00:10:ff03::/48" - } + "plugins": [ + { + "type": "galactic-tap-cni", + "vpc": "1", + "vpcattachment": "1", + "mtu": 9000, + "ipam": { "type": "galactic-ipam", "ipv6_subnet": "fd00:10:ff03::/48" } + }, + { "type": "galactic-bgp", "vpc": "1", "vpcattachment": "1" } + ] } ``` -Tap mode creates a tap interface in the host namespace, enslaves it to the -VRF, and applies forwarding sysctls. It then runs IPAM (allocating the subnet -shown above) and SRv6/BGP publish exactly as `veth` mode does — see the `tap` -description under Interface Types above. Only host-device delegation and -guest-netns configuration are skipped; the guest VM still configures its own -IP addresses independently once the hypervisor (Kata, Firecracker, QEMU) opens -the tap fd at runtime. The `ipam` block (or `GALACTIC_CNI_ENABLE_LOCAL_IPAM`) -is required here for the same reason it is in `veth` mode. +`galactic-tap-cni` creates a tap interface in the host namespace, enslaves it +to the VRF, and applies forwarding sysctls. It then runs IPAM (allocating the +subnet shown above) and configures the host gateway exactly as `galactic-cni` +does — see [`galactic-tap-cni` (tap)](#galactic-tap-cni-tap) above. Only +host-device delegation and guest-netns configuration are skipped; the guest VM +still configures its own IP addresses independently once the hypervisor +(Kata, Firecracker, kraftlet/Unikraft) opens the tap fd at runtime. diff --git a/go.mod b/go.mod index 3b678efb..d2e95759 100644 --- a/go.mod +++ b/go.mod @@ -59,9 +59,11 @@ require ( github.com/onsi/gomega v1.39.1 // indirect github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect + github.com/safchain/ethtool v0.6.2 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect @@ -96,6 +98,7 @@ require ( k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/knftables v0.0.18 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect sigs.k8s.io/yaml v1.6.0 // indirect diff --git a/go.sum b/go.sum index eb1c522b..3967c824 100644 --- a/go.sum +++ b/go.sum @@ -134,6 +134,8 @@ github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05Zp github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/safchain/ethtool v0.6.2 h1:O3ZPFAKEUEfbtE6J/feEe2Ft7dIJ2Sy8t4SdMRiIMHY= +github.com/safchain/ethtool v0.6.2/go.mod h1:VS7cn+bP3Px3rIq55xImBiZGHVLNyBh5dqG6dDQy8+I= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= @@ -291,6 +293,8 @@ sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9 sigs.k8s.io/controller-runtime v0.24.1/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/knftables v0.0.18 h1:6Duvmu0s/HwGifKrtl6G3AyAPYlWiZqTgS8bkVMiyaE= +sigs.k8s.io/knftables v0.0.18/go.mod h1:f/5ZLKYEUPUhVjUCg6l80ACdL7CIIyeL0DxfgojGRTk= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= diff --git a/internal/cni/bgp.go b/internal/cni/bgp.go deleted file mode 100644 index 1e43af95..00000000 --- a/internal/cni/bgp.go +++ /dev/null @@ -1,890 +0,0 @@ -// Copyright 2025 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package cni - -import ( - "context" - "errors" - "fmt" - "log/slog" - "net" - "net/netip" - "sort" - "strconv" - "strings" - "syscall" - "time" - - "github.com/containernetworking/cni/pkg/skel" - "github.com/vishvananda/netlink" - "golang.org/x/sys/unix" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - "go.datum.net/galactic/internal/plumbing/ebpf/attach" - "go.datum.net/galactic/internal/plumbing/ebpf/uformat" - "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" - "go.datum.net/galactic/internal/plumbing/intf" - "go.datum.net/galactic/internal/plumbing/vrf" - bgpv1alpha1 "go.datum.net/network/api/v1alpha1" -) - -// maxRetries is the maximum number of retry attempts for transient k8s API -// errors during the BGP state publish phase. The total number of attempts -// is maxRetries+1 (initial + retries). -const maxRetries = 2 - -// isTransientError reports whether err is a transient failure that may -// resolve itself on retry (API server unavailable, timeout, network blip). -// Returns false for validation errors, not-found, and other permanent -// failures that should not be retried. -func isTransientError(err error) bool { - if err == nil { - return false - } - // Context-level failures (deadline exceeded, cancelled) are transient - // because they usually indicate the API server was slow/unavailable. - if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { - return true - } - // Unwrap to handle wrapped errors (e.g. from controllerutil.CreateOrUpdate). - unwrapped := errors.Unwrap(err) - if unwrapped != nil { - if errors.Is(unwrapped, context.DeadlineExceeded) || errors.Is(unwrapped, context.Canceled) { - return true - } - } - // Kubernetes API errors: 503 Service Unavailable, 500 Internal Server - // Error, 504 Server Timeout, and 429 Too Many Requests. - if apierrors.IsServiceUnavailable(err) || - apierrors.IsInternalError(err) || - apierrors.IsServerTimeout(err) || - apierrors.IsTooManyRequests(err) { - return true - } - // Network-level transient errors (connection refused/reset, unreachable). - if netErr, ok := unwrapped.(interface{ Temporary() bool }); ok && netErr.Temporary() { - return true - } - return false -} - -// retryK8sOps runs fn with up to maxRetries+1 attempts, retrying on transient -// k8s API errors with exponential backoff. The context passed to fn has a -// timeout derived from timeout (respecting the original ctx deadline when set). -// Non-transient errors are returned immediately without retry. -func retryK8sOps(timeout time.Duration, fn func(ctx context.Context) error) error { - var lastErr error - for attempt := 0; attempt <= maxRetries; attempt++ { - if attempt > 0 { - backoff := time.Duration(1< create -> checkArgumentCollision runs -// sequentially within one call, so if A's create happens before B's create, -// then either B's own check (which always runs after B's own create) sees -// A's already-committed CRD and B reports a collision, or it doesn't only if -// A's check already ran (and hence already reported the collision itself) -// before B created its CRD. At least one side always detects it this way; a -// tie-breaker that lets exactly one side "win" without that guarantee (as a -// prior version of this function did) can let both sides pass when the two -// creates and checks interleave, leaving two BGPVRFInstances -- and, -// consequently, two vrf_table registrations -- permanently sharing the same -// VRFID. Both sides erroring out is harmless: the caller's non-transient -// error triggers the failed-ADD rollback (resourceTracker.cleanup), which -// deletes each side's own BGPVRFInstance, and the CNI runtime retries ADD. -func checkArgumentCollision( - ctx context.Context, k8s client.Client, namespace, routerName, vrfName string, vrfID int32, -) error { - list := &bgpv1alpha1.BGPVRFInstanceList{} - if err := k8s.List(ctx, list, client.InNamespace(namespace)); err != nil { - return fmt.Errorf("list BGPVRFInstances to verify argument uniqueness: %w", err) - } - for _, inst := range list.Items { - if inst.Spec.RouterRef == nil || inst.Spec.RouterRef.Name != routerName { - continue - } - if inst.Name != vrfName && inst.Spec.VRFID == vrfID { - return fmt.Errorf("argument collision: VRFID %d claimed by both %s and %s, retrying", vrfID, inst.Name, vrfName) - } - } - return nil -} - -// lookupBGPRouter finds the BGPRouter targeting this node in the given namespace. -// Returns an error if none is found or if multiple are found (ambiguous). -func lookupBGPRouter(ctx context.Context, k8s client.Client, nodeName, namespace string) (bgpConfig, error) { - routerList := &bgpv1alpha1.BGPRouterList{} - if err := k8s.List(ctx, routerList, client.InNamespace(namespace)); err != nil { - return bgpConfig{}, fmt.Errorf("list BGPRouters in namespace %s: %w", namespace, err) - } - - var matches []bgpv1alpha1.BGPRouter - for _, r := range routerList.Items { - if r.Spec.TargetRef.Name == nodeName { - matches = append(matches, r) - } - } - - switch len(matches) { - case 0: - return bgpConfig{}, fmt.Errorf("no BGPRouter found for node %s in namespace %s", nodeName, namespace) - case 1: - // expected - default: - return bgpConfig{}, fmt.Errorf("ambiguous BGP config: %d BGPRouters target node %s in namespace %s", - len(matches), nodeName, namespace) - } - - slog.Debug("BGP: router matched", "nodeName", nodeName, "router", matches[0].Name, - "asNumber", matches[0].Spec.LocalASN, "srv6Locator", matches[0].Spec.SRv6Locator, "nodeID", matches[0].Spec.NodeID) - - return bgpConfig{ - asNumber: uint32(matches[0].Spec.LocalASN), - routerName: matches[0].Name, - srv6Locator: matches[0].Spec.SRv6Locator, - nodeID: matches[0].Spec.NodeID, - }, nil -} - -// buildVRFInstanceSpec constructs the BGPVRFInstanceSpec for a VPC attachment. -// The route distinguisher is no longer stored on the CRD; it's derived -// downstream from the router's ID and vrfID. -func buildVRFInstanceSpec(routerName, rtValue string, vrfID int32) bgpv1alpha1.BGPVRFInstanceSpec { - return bgpv1alpha1.BGPVRFInstanceSpec{ - RouterTarget: bgpv1alpha1.RouterTarget{ - RouterRef: &bgpv1alpha1.RouterRef{Name: routerName}, - }, - VRFID: vrfID, - ImportRouteTargets: []bgpv1alpha1.RouteTarget{{Value: rtValue}}, - ExportRouteTargets: []bgpv1alpha1.RouteTarget{{Value: rtValue}}, - } -} - -// buildAdvertisementSpec constructs the BGPAdvertisementSpec for a VPC -// attachment's pod subnet(s) — one IPv6 prefix, plus an IPv4 prefix when the -// attachment is dual-stack. RFC 9136's Type-5 route is self-describing per -// NLRI, so a single BGPAdvertisement carrying both families is valid; see -// galactic-router's buildEVPNPaths for the corresponding per-family gateway -// handling. VRFID and Function record structurally what used to live in the -// legacy galactic.datum.net/srv6-sid annotation: which VRF this advertisement -// belongs to, and which SRv6 endpoint behavior the eBPF uSID datapath -// resolves (always End.DT46, regardless of pod-subnet address family — see -// registerEBPFDatapath). -func buildAdvertisementSpec( - routerName, rtValue string, prefixes []string, vrfID int32, -) bgpv1alpha1.BGPAdvertisementSpec { - function := bgpv1alpha1.SRv6FunctionEndDT46 - bgpPrefixes := make([]bgpv1alpha1.Prefix, len(prefixes)) - for i, p := range prefixes { - bgpPrefixes[i] = bgpv1alpha1.Prefix(p) - } - return bgpv1alpha1.BGPAdvertisementSpec{ - RouterRef: bgpv1alpha1.RouterRef{Name: routerName}, - AddressFamily: bgpv1alpha1.AddressFamily{AFI: bgpv1alpha1.AFIL2VPN, SAFI: bgpv1alpha1.SAFIEVPN}, - Prefixes: bgpPrefixes, - Communities: []bgpv1alpha1.Community{bgpv1alpha1.Community(rtValue)}, - VRFID: &vrfID, - Function: &function, - } -} - -// newK8sClient creates a new Kubernetes client using the in-cluster config. -func newK8sClient() (client.Client, error) { - restCfg, err := ctrl.GetConfig() - if err != nil { - return nil, fmt.Errorf("get kubeconfig: %w", err) - } - c, err := client.New(restCfg, client.Options{Scheme: cniScheme}) - if err != nil { - return nil, fmt.Errorf("create k8s client: %w", err) - } - return c, nil -} - -// publishBGPState configures the host gateway, sets up the SRv6 ingress route, -// and creates the BGPVRFInstance and BGPAdvertisement CRDs. The host gateway -// configuration is interface-agnostic (works for both veth and tap). -// -// K8s API operations are retried with exponential backoff on transient errors -// (503, timeout, network blip). Non-k8s operations (kernel networking) run -// once before the retry loop. Non-transient errors (validation, not-found) -// fail immediately without retry. -func publishBGPState( - args *skel.CmdArgs, pluginConf *PluginConf, nodeName, namespace string, ipamResult *ipamResult, - guestHWAddr net.HardwareAddr, tracker *resourceTracker, -) error { - // ---- non-k8s operations (run once) ---- - if err := configureHostGateway(pluginConf.VPC, pluginConf.VPCAttachment, ipamResult, guestHWAddr); err != nil { - return err - } - - vpcHex, err := intf.Base62ToHex(pluginConf.VPC) - if err != nil { - return fmt.Errorf("decode VPC: %w", err) - } - - if tracker.k8s == nil { - return errors.New("k8s client not set in tracker") - } - - // ---- k8s operations (retry on transient errors) ---- - // The SID Argument is allocated inside publishBGPStateK8s's retry - // closure, not here: it depends on this node's BGPRouter (looked up - // there) and must itself be a k8s-retried operation, since it lists - // BGPVRFInstance CRDs. - return publishBGPStateK8s(args, pluginConf, nodeName, namespace, ipamResult, vpcHex, tracker.k8s, tracker) -} - -// ipamAdvertisementPrefixes derives the BGPAdvertisement prefixes to -// originate, plus the per-family values to record in the annotations, from -// ipamResult. ipamResult is nil when the attachment has no IPAM allocation -// (e.g. a tap workload that manages its own addressing), in which case -// prefixes is empty. Either family alone yields a single-entry prefixes -// slice; ipv6Subnet/ipv4Addr are empty when that family wasn't allocated. -func ipamAdvertisementPrefixes(ipamResult *ipamResult) (prefixes []string, ipv6Subnet, ipv4Addr string) { - if ipamResult == nil { - return nil, "", "" - } - if ipamResult.ipv6Subnet != nil { - ipv6Subnet = ipamResult.ipv6Subnet.String() - prefixes = append(prefixes, ipv6Subnet) - } - if ipamResult.ipv4Address != nil { - // The annotation stores the bare address (matching - // IPv4PoolAllocator's marker-file naming, so cmdDel's Deallocate - // call finds it — see ipam_ops.go); the advertised prefix needs the - // explicit /32 CIDR form. - ipv4Addr = ipamResult.ipv4Address.String() - prefixes = append(prefixes, ipv4Addr+"/32") - } - return prefixes, ipv6Subnet, ipv4Addr -} - -// allAdvertisedPrefixes derives the full set of BGP-advertised prefixes for -// a BGPAdvertisement CRD from every subnet annotation currently present on -// it, rather than from just the container currently being processed. -// -// A single BGPAdvertisement is keyed by (vpc, vpcAttachment) alone -// (bgpAdvertisementName), so multiple containers attaching under the same -// VPCAttachment on this node — a second pod, or a second interface with its -// own vpcattachment reusing this one — all share one CRD. Each one's own -// CNI ADD must not clobber another still-live container's already-published -// prefix: annotations is the durable per-container record (subnetAnnotationKeyIPv6/IPv4), -// so recomputing Spec.Prefixes from all of them on every ADD keeps every -// live container's prefix present regardless of ADD order. cmdDel -// deliberately leaves this annotation (and thus this prefix) in place even -// after that container exits — see ops_del.go's "skipping shared resource -// cleanup (handled by GC)" — so a stale entry for an exited container can -// briefly outlive it until gc.CollectOrphanedCRDs removes the whole CRD -// once every container sharing it is gone; that's a pre-existing tradeoff -// this function doesn't change. -func allAdvertisedPrefixes(annotations map[string]string) []string { - var prefixes []string - for key, value := range annotations { - switch { - case strings.HasPrefix(key, annotationAllocatedSubnetIPv6+"."): - prefixes = append(prefixes, value) - case strings.HasPrefix(key, annotationAllocatedSubnetIPv4+"."): - // Annotation stores the bare address (see ipamAdvertisementPrefixes); - // the advertised prefix needs the explicit /32 CIDR form. - prefixes = append(prefixes, value+"/32") - } - } - // Deterministic ordering: map iteration is randomized, and an - // unstable Spec.Prefixes order across otherwise-identical ADDs would - // look like a spurious spec change to anything diffing this CRD. - sort.Strings(prefixes) - return prefixes -} - -// publishBGPStateK8s creates the BGPVRFInstance and BGPAdvertisement CRDs with -// retry on transient k8s API errors. The host gateway must be configured before -// calling this (via configureHostGateway). This is interface-agnostic and can be -// used by both veth and tap code paths. -func publishBGPStateK8s( - args *skel.CmdArgs, pluginConf *PluginConf, nodeName, namespace string, ipamResult *ipamResult, - vpcHex string, k8s client.Client, tracker *resourceTracker, -) error { - return retryK8sOps(cniTimeout, func(ctx context.Context) error { - bgp, err := lookupBGPRouter(ctx, k8s, nodeName, namespace) - if err != nil { - return err - } - - vrfID, err := allocateArgument( - ctx, k8s, namespace, bgp.routerName, bgpVRFInstanceName(pluginConf.VPC, pluginConf.VPCAttachment)) - if err != nil { - return err - } - - rtValue, err := routeTarget(int64(bgp.asNumber), vpcHex) - if err != nil { - return fmt.Errorf("compute route target: %w", err) - } - - // Create the BGPVRFInstance to configure the VRF with its VRFID and - // import/export route targets. This must be created before advertisements - // so the BGP runtime has the VRF context when originating EVPN paths. - vrfName := bgpVRFInstanceName(pluginConf.VPC, pluginConf.VPCAttachment) - vrfInst := &bgpv1alpha1.BGPVRFInstance{ - ObjectMeta: metav1.ObjectMeta{ - Name: vrfName, - Namespace: namespace, - }, - } - _, err = controllerutil.CreateOrUpdate(ctx, k8s, vrfInst, func() error { - vrfInst.Spec = buildVRFInstanceSpec(bgp.routerName, rtValue, vrfID) - return nil - }) - if err != nil { - return fmt.Errorf("apply BGPVRFInstance: %w", err) - } - tracker.vrfInstanceCreated = true - slog.Debug("BGP: BGPVRFInstance applied", "name", vrfName, "namespace", namespace, - "vrfID", vrfID, "routeTarget", rtValue, "router", bgp.routerName) - - if err := checkArgumentCollision(ctx, k8s, namespace, bgp.routerName, vrfName, vrfID); err != nil { - return err - } - - // eBPF uSID datapath registration -- the only forwarding path - // (the legacy seg6local static-route path was removed once this - // datapath covered both veth and tap attachments). registered is - // false, with no error, only when the router has no - // srv6Locator/nodeID configured at all -- SRv6 is intentionally - // not set up for this attachment. Any other failure is fatal: - // with no legacy path to fall back to, an attachment with no - // registered datapath entry has no forwarding path at all. - // registerEBPFDatapath is itself idempotent (Register - // overwrites), so re-running it on a k8s-op retry is safe. - registered, ebpfBlock, err := registerEBPFDatapath( - bgp, pluginConf.VPC, pluginConf.VPCAttachment, pluginConf.InterfaceType, uint16(vrfID), attach.PinDir) - if err != nil { - return fmt.Errorf("register eBPF uSID datapath: %w", err) - } - if registered { - // Recorded so a failed-ADD rollback (resourceTracker.cleanup) - // can unregister this exact (block, argument) pair -- see - // Milestone 7.2. - tracker.ebpfRegistered = true - tracker.ebpfBlock = ebpfBlock - tracker.ebpfArgument = uint16(vrfID) - } - - // Create the BGPAdvertisement to originate the pod's subnet prefix(es). - adv := &bgpv1alpha1.BGPAdvertisement{ - ObjectMeta: metav1.ObjectMeta{ - Name: bgpAdvertisementName(pluginConf.VPC, pluginConf.VPCAttachment), - Namespace: namespace, - }, - } - prefixes, ipv6Subnet, ipv4Addr := ipamAdvertisementPrefixes(ipamResult) - var mergedPrefixes []string - _, err = controllerutil.CreateOrUpdate(ctx, k8s, adv, func() error { - if adv.Annotations == nil { - adv.Annotations = make(map[string]string) - } - // Record the netns path this container attached with, so the GC - // controller can check whether it still exists rather than - // guessing a name from the container ID (see - // gc.ContainerNetNSExistsByPath). - adv.Annotations[netnsAnnotationKey(args.ContainerID)] = args.Netns - // Store the allocated addresses keyed by container ID so cmdDel can - // look them up, one annotation per family so DEL can deallocate - // each independently. - if ipv6Subnet != "" { - adv.Annotations[subnetAnnotationKeyIPv6(args.ContainerID)] = ipv6Subnet - } - if ipv4Addr != "" { - adv.Annotations[subnetAnnotationKeyIPv4(args.ContainerID)] = ipv4Addr - } - // Recompute Spec.Prefixes from every container's annotations, not - // just this one's own — see allAdvertisedPrefixes. Must run after - // this container's own annotations are set above, and be read - // back into mergedPrefixes for the log line below since this - // closure may run more than once (RetryOnConflict). - mergedPrefixes = allAdvertisedPrefixes(adv.Annotations) - adv.Spec = buildAdvertisementSpec(bgp.routerName, rtValue, mergedPrefixes, vrfID) - return nil - }) - if err != nil { - return fmt.Errorf("apply BGPAdvertisement: %w", err) - } - tracker.advCreated = true - slog.Debug("BGP: BGPAdvertisement applied", "name", adv.Name, "namespace", namespace, - "prefixes", mergedPrefixes, "addedPrefixes", prefixes, "containerID", args.ContainerID) - - slog.Info("ADD: BGP state published", "containerID", args.ContainerID, - "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) - return nil - }) -} - -// routeConflicts reports whether an existing route conflicts with the desired -// pod-subnet route. A conflict occurs when the destination matches but the -// gateway or link index differs. -func routeConflicts(existing, desired *netlink.Route) bool { - if existing.Dst == nil || desired.Dst == nil { - return false - } - if existing.Dst.String() != desired.Dst.String() { - return false - } - if (existing.Gw != nil) != (desired.Gw != nil) { - return true - } - if existing.Gw != nil && !existing.Gw.Equal(desired.Gw) { - return true - } - if existing.LinkIndex != 0 && desired.LinkIndex != 0 && existing.LinkIndex != desired.LinkIndex { - return true - } - return false -} - -// configureHostGateway assigns each configured family's gateway address as a -// host address (/128 for IPv6, /32 for IPv4 on veth) on the host-side -// interface (veth or tap) and installs an explicit pod-subnet route for that -// family into the VRF table. IPv4 is skipped entirely when the attachment is -// IPv6-only. -// -// Using a full-length host address (not the pod subnet mask) prevents the -// kernel from auto-creating a subnet-router anycast entry in the VRF local -// table. When the pod address equals the subnet network address the anycast -// absorbs seg6local-decapped inner packets before they reach the guest -// interface. The explicit subnet route replaces the one the kernel would -// have created from the wider mask. -// -// For tap interfaces, the IPv4 gateway is instead assigned as a /25 so the -// address reported on the interface reflects a real subnet (VM guests expect -// this). That reintroduces the wider-mask hazard described above, so the -// address is added with IFA_F_NOPREFIXROUTE: the kernel skips auto-creating -// the connected /25 route entirely, leaving the explicit pod-subnet route -// below as the only thing that governs delivery to this VM's address. -// -// guestHWAddr is the guest-side veth's MAC address, used to prime a -// permanent neighbor table entry for the pod's own address (see -// installGatewayNeighbor). It is nil for tap attachments, which have no -// separate guest-side link in this netns to resolve a MAC from -- tap's -// neighbor resolution, if it turns out to need the same fix, is out of -// scope here since this fix targets the veth-only bug it was found from. -func configureHostGateway(vpc, vpcAttachment string, res *ipamResult, guestHWAddr net.HardwareAddr) error { - if res == nil { - return nil - } - hostName := intf.GenerateInterfaceNameHost(vpc, vpcAttachment) - hostLink, err := netlink.LinkByName(hostName) - if err != nil { - return fmt.Errorf("get host interface %q: %w", hostName, err) - } - tableID, err := vrf.TableID(vpc, vpcAttachment) - if err != nil { - return fmt.Errorf("get VRF table ID for pod subnet route: %w", err) - } - - if res.ipv6Gateway != nil { - gwNet := &net.IPNet{IP: res.ipv6Gateway, Mask: net.CIDRMask(128, 128)} - if err := installGatewayRoute(hostLink, gwNet, res.ipv6Subnet, netlink.FAMILY_V6, int(tableID), 0); err != nil { - return err - } - if guestHWAddr != nil { - if err := installGatewayNeighbor(hostLink, res.ipv6Subnet.IP, netlink.FAMILY_V6, guestHWAddr); err != nil { - return err - } - } - } - if res.ipv4Gateway != nil { - ipv4Mask, addrFlags := ipv4GatewayAddrParams(hostLink) - gwNet := &net.IPNet{IP: res.ipv4Gateway, Mask: ipv4Mask} - ipv4Subnet := &net.IPNet{IP: res.ipv4Address, Mask: net.CIDRMask(32, 32)} - if err := installGatewayRoute(hostLink, gwNet, ipv4Subnet, netlink.FAMILY_V4, int(tableID), addrFlags); err != nil { - return err - } - if guestHWAddr != nil { - if err := installGatewayNeighbor(hostLink, res.ipv4Address, netlink.FAMILY_V4, guestHWAddr); err != nil { - return err - } - } - } - return nil -} - -// installGatewayNeighbor installs a permanent neighbor table entry mapping -// podIP to guestHWAddr on hostLink. -// -// The eBPF uSID ingress datapath (internal/plumbing/ebpf/prog/usid.c) -// decapsulates SRv6 traffic and calls bpf_fib_lookup() to resolve the -// egress path for the inner packet, then redirects it straight to the -// resolved neighbor -- entirely in-kernel, never touching the normal -// forwarding stack. bpf_fib_lookup() does not itself trigger ARP/NDP -// resolution the way ordinary kernel packet forwarding does (that -// resolution happens as a side effect of the slow-path forwarding this -// datapath deliberately bypasses), so without a pre-existing neighbor table -// entry it fails with BPF_FIB_LKUP_RET_NO_NEIGH and the datapath counts and -// drops the packet (DROP_REASON_FIB_LOOKUP_FAILED) -- confirmed live: every -// cross-region packet to a pod that had never otherwise triggered NDP for -// its own address was silently and permanently blackholed, since nothing -// else in this attach path ever resolves it. A permanent entry (installed -// once, at CNI ADD, using the guest veth's own known MAC) means this -// resolution never depends on dynamic ARP/NDP at all. -func installGatewayNeighbor(hostLink netlink.Link, podIP net.IP, family int, guestHWAddr net.HardwareAddr) error { - neigh := &netlink.Neigh{ - LinkIndex: hostLink.Attrs().Index, - Family: family, - State: netlink.NUD_PERMANENT, - IP: podIP, - HardwareAddr: guestHWAddr, - } - if err := netlink.NeighSet(neigh); err != nil { - return fmt.Errorf("add permanent neighbor %s -> %s on host interface %q: %w", - podIP, guestHWAddr, hostLink.Attrs().Name, err) - } - return nil -} - -// ipv4GatewayAddrParams returns the IPv4 gateway mask and netlink address -// flags to use for hostLink. Tap interfaces get a /25 (so the address -// reported on the interface reflects a real subnet) with -// IFA_F_NOPREFIXROUTE, which stops the kernel from auto-creating a connected -// route for the wider mask — see the anycast-avoidance note on -// configureHostGateway for why that route must not exist. Veth interfaces -// keep the plain /32 host address with no flags. -func ipv4GatewayAddrParams(hostLink netlink.Link) (net.IPMask, int) { - if _, isTap := hostLink.(*netlink.Tuntap); isTap { - return net.CIDRMask(25, 32), unix.IFA_F_NOPREFIXROUTE - } - return net.CIDRMask(32, 32), 0 -} - -// installGatewayRoute assigns gwNet as a host address on hostLink and -// installs an explicit route to subnet into the given VRF table, for one -// address family. Idempotent: existing matching routes/addresses are left -// alone, and conflicting ones return an error rather than being overwritten. -// addrFlags is passed through to the netlink address (e.g. IFA_F_NOPREFIXROUTE -// to suppress the kernel's auto-created connected route for a wider mask). -func installGatewayRoute(hostLink netlink.Link, gwNet, subnet *net.IPNet, family, tableID, addrFlags int) error { - hostName := hostLink.Attrs().Name - if err := netlink.AddrAdd(hostLink, &netlink.Addr{IPNet: gwNet, Flags: addrFlags}); err != nil { - if !errors.Is(err, syscall.EEXIST) { - return fmt.Errorf("add gateway address %s to host interface %q: %w", gwNet, hostName, err) - } - } - - desiredRoute := &netlink.Route{ - Dst: subnet, - LinkIndex: hostLink.Attrs().Index, - Table: tableID, - } - - // Check for existing routes with the same destination before installing. - existingRoutes, err := netlink.RouteListFiltered( - family, - &netlink.Route{Table: tableID}, - netlink.RT_FILTER_TABLE, - ) - if err != nil { - return fmt.Errorf("list routes in VRF table: %w", err) - } - for _, r := range existingRoutes { - if r.Dst == nil { - continue - } - if r.Dst.String() != desiredRoute.Dst.String() { - continue - } - if routeConflicts(&r, desiredRoute) { - return fmt.Errorf( - "existing route %v to %s conflicts with desired route %v", - r, desiredRoute.Dst, desiredRoute, - ) - } - // Route already exists with matching attributes — idempotent, skip. - return nil - } - - if err := netlink.RouteAdd(desiredRoute); err != nil { - if errors.Is(err, syscall.EEXIST) { - return nil // already installed by a concurrent caller - } - return fmt.Errorf("add pod subnet route to VRF table: %w", err) - } - return nil -} - -// registerEBPFDatapath registers this attachment against the eBPF uSID -// datapath's pinned maps (design plan §5.1) -- the only forwarding path -// (the legacy seg6local static-route path was removed once this covered -// both veth and tap attachments, Milestone 6.1's tap-mode redirect fix). -// -// Design plan §4.4 assigns locator_table/function_table population to "the -// control daemon, at startup + on locator change." The actual control -// daemon (galactic-cni's "run" subcommand) does not read BGPRouter/watch -// for locator changes -- it only loads/attaches/pins the program -- so -// those two maps would otherwise sit permanently empty and every packet -// would locator_table-miss and pass through unchanged. This function -// registers all three tables (locator_table, function_table, vrf_table) -// from here instead, since the CNI ADD path already independently -// resolves bgp.srv6Locator/bgp.nodeID via lookupBGPRouter on every -// invocation -- an intentional deviation from the design plan's literal -// placement, not an oversight, tracked for revisiting once a real -// control-daemon-side CRD watch exists. -// -// argument is the same real, allocated 12-bit value (Milestone 6.1's -// allocateArgument) the router independently recomputes the BGP-advertised -// SID from (internal/reconcile) -- both must agree on the same value or a -// remote node's encapsulated traffic decodes into the wrong VRF. -// -// registerEBPFDatapath's return values let the caller record exactly what -// (if anything) was registered, so a later failed-ADD rollback -// (resourceTracker.cleanup, Milestone 7.2) can unregister the same -// (block, argument) pair without having to recompute or guess it. -// registered is false, with a nil error, only when this router has no -// srv6Locator/nodeID configured at all -- SRv6 is intentionally not set up -// for this attachment. Any other failure is returned as an error: with no -// legacy path to fall back to, the caller must treat that as fatal. -func registerEBPFDatapath( - bgp bgpConfig, vpc, vpcAttachment, ifaceType string, argument uint16, pinDir string, -) (registered bool, block uint64, err error) { - if bgp.srv6Locator == "" || bgp.nodeID == 0 { - return false, 0, nil - } - - // Validate the raw int32 nodeID against uformat's actual encode-time - // range *before* the uint16 narrowing below, mirroring srv6.ComputeSID's - // own bounds check on this exact value (internal/plumbing/srv6/usid.go). - // registry.Locator.Register below validates its uint16 argument via - // uformat.ValidateNodeID too, but only after this narrowing has already - // happened -- an out-of-[uint16] nodeID (e.g. 65537) wraps to some - // other, often perfectly in-range uint16 (1, here) that check can't - // tell apart from a legitimately-registered node's real Node-ID. Left - // unchecked here, that silently registers a locator_table entry for a - // Node-ID this router was never actually assigned, while ComputeSID - // (used independently by the router to build the SID it advertises) - // rejects the same raw value outright -- so nothing ever advertises - // reachability for the bogus entry this node just committed to - // forwarding, and it may collide with a different node's genuine one. - if bgp.nodeID < uformat.NodeIDMin || bgp.nodeID > uformat.NodeIDMax { - return false, 0, fmt.Errorf("eBPF registration: nodeID %d out of range [%#x,%#x]", - bgp.nodeID, uint16(uformat.NodeIDMin), uint16(uformat.NodeIDMax)) - } - - egressKind, err := egressKindForInterfaceType(ifaceType) - if err != nil { - return false, 0, fmt.Errorf("determine eBPF egress kind: %w", err) - } - - prefix, err := netip.ParsePrefix(bgp.srv6Locator) - if err != nil { - return false, 0, fmt.Errorf("parse SRv6 locator %q for eBPF registration: %w", bgp.srv6Locator, err) - } - block, err = uformat.Block(prefix.Addr()) - if err != nil { - return false, 0, fmt.Errorf("derive eBPF uSID Block from locator %q: %w", bgp.srv6Locator, err) - } - - vrfTableID, err := vrf.TableID(vpc, vpcAttachment) - if err != nil { - return false, 0, fmt.Errorf("look up VRF table id for eBPF registration: %w", err) - } - - registry, closer, err := usidmap.OpenPinnedRegistry(pinDir) - if err != nil { - return false, 0, fmt.Errorf("open pinned eBPF uSID maps: %w", err) - } - defer func() { _ = closer.Close() }() - - if err := registry.Locator.Register(block, uint16(bgp.nodeID)); err != nil { - return false, 0, fmt.Errorf("register eBPF locator_table entry: %w", err) - } - if err := registry.Function.Register(block, uformat.FunctionEndDT46); err != nil { - return false, 0, fmt.Errorf("register eBPF function_table entry: %w", err) - } - - if err := registry.VRF.Register(block, argument, vrfTableID, egressKind); err != nil { - return false, 0, fmt.Errorf("register eBPF vrf_table entry: %w", err) - } - return true, block, nil -} - -// egressKindForInterfaceType maps the CNI's InterfaceType field to the -// vrf_table egress_kind value usid.c's step 9 uses to pick between -// bpf_redirect_peer (veth, crosses into the container's netns) and plain -// bpf_redirect (tap, which never leaves this netns -- internal/cni/tap -// creates it here and never moves it). This is what closes the tap-mode -// redirect_failed gap (Milestone 6.1's fix, design plan §4.2 step 9). -func egressKindForInterfaceType(ifaceType string) (uint32, error) { - switch ifaceType { - case interfaceTypeVeth, "": - // Empty matches config.go's own default-to-veth behavior for an - // omitted interface_type field. - return usidmap.EgressKindVeth, nil - case interfaceTypeTap: - return usidmap.EgressKindTap, nil - default: - return 0, fmt.Errorf("unknown interface type %q", ifaceType) - } -} - -// unregisterEBPFDatapath removes the vrf_table entry registerEBPFDatapath -// wrote for this (block, argument) pair, from the failed-ADD rollback path -// (resourceTracker.cleanup, Milestone 7.2). Unlike registerEBPFDatapath, -// this has no flag/config short-circuit of its own -- callers only invoke -// it when resourceTracker recorded a real registration -// (resourceTracker.ebpfRegistered), so by construction the flag was on and -// the maps were reachable at Register time. Idempotent: not an error if -// the entry is already gone (VRFTable.Unregister's own documented -// behavior). -// -// expectedVRFTableID must be this attachment's own VRF table id (recomputed -// by the caller via vrf.TableID, not read back from the tracker, since it's -// cheap and deterministic to recompute and the whole point here is not to -// trust stale state). A retried k8s-op attempt (retryK8sOps) can re-run the -// same publishBGPStateK8s closure without re-registering the eBPF entry -// (registerEBPFDatapath only runs again if that attempt gets far enough), -// so by the time a later attempt's checkArgumentCollision failure triggers -// this rollback, the (block, argument) slot this attachment originally -// wrote may have since been overwritten by the very other attachment the -// collision was detected against (vrf_table's key is just (block, -// argument); Register always overwrites). Unregistering unconditionally in -// that case would delete a live attachment's forwarding entry instead of -// this rolled-back one's own -- so this only deletes the entry when it -// still resolves to expectedVRFTableID, and leaves it alone otherwise. -func unregisterEBPFDatapath(block uint64, argument uint16, expectedVRFTableID uint32, pinDir string) error { - registry, closer, err := usidmap.OpenPinnedRegistry(pinDir) - if err != nil { - return fmt.Errorf("open pinned eBPF uSID maps: %w", err) - } - defer func() { _ = closer.Close() }() - - entry, ok, err := registry.VRF.Get(block, argument) - if err != nil { - return fmt.Errorf("read eBPF vrf_table entry before unregister: %w", err) - } - if !ok { - return nil // already gone - } - if entry.VRFTableID != expectedVRFTableID { - slog.Warn("Rollback: eBPF vrf_table entry no longer belongs to this attachment, leaving it in place", - "block", block, "argument", argument, - "expectedVRFTableID", expectedVRFTableID, "currentVRFTableID", entry.VRFTableID) - return nil - } - - if err := registry.VRF.Unregister(block, argument); err != nil { - return fmt.Errorf("unregister eBPF vrf_table entry: %w", err) - } - return nil -} diff --git a/internal/cni/cni.go b/internal/cni/cni.go index 33171596..b70d1183 100644 --- a/internal/cni/cni.go +++ b/internal/cni/cni.go @@ -15,55 +15,6 @@ import ( const cniTimeout = 10 * time.Second -// ipamTypeStatic is the ipam type for a single pre-assigned static address. -// Any other (or empty) IPAM.Type value takes the pool-based dual-stack path -// — see wantsIPAM/allocateIPAM in ipam_ops.go. -const ipamTypeStatic = "static" - -// localIPAMDefaultPool is the IPv6 CIDR pool used when local IPAM is enabled -// but IPv6Subnet is unset in the CNI config. Allocations from it use -// ipam.DefaultSubnetLen (/96). -const localIPAMDefaultPool = "fd00:10:ff01::/64" - -const ( - // annotationAllocatedSubnetIPv6 is the BGPAdvertisement annotation key - // prefix holding the allocated IPv6 pod subnet CIDR (the /96) for a - // container ID. The full key appends a truncated container ID; see - // subnetAnnotationKeyIPv6. - annotationAllocatedSubnetIPv6 = "galactic.datum.net/allocated-subnet-ipv6" - - // annotationAllocatedSubnetIPv4 is the BGPAdvertisement annotation key - // prefix holding the allocated IPv4 pod address (the /32) for a - // container ID, when the attachment is dual-stack. The full key appends - // a truncated container ID; see subnetAnnotationKeyIPv4. - annotationAllocatedSubnetIPv4 = "galactic.datum.net/allocated-subnet-ipv4" - - // annotationNetNS is the BGPAdvertisement annotation key prefix holding - // the CNI-provided network namespace path for a container ID. The GC - // controller checks whether this exact path still exists to decide if - // the container is still live — it cannot reconstruct the path from the - // container ID alone, since netns bind-mounts are named by the - // runtime's own convention (e.g. containerd's "cni-"), which is - // unrelated to the container ID. The full key appends a truncated - // container ID; see netnsAnnotationKey. - annotationNetNS = "galactic.datum.net/netns" - - // annotationContainerIDLen is the number of characters used from a - // container ID in annotation keys. Kubernetes limits the name part of an - // annotation key to 63 bytes. The longest prefix sharing this constant is - // "allocated-subnet-ipv6." (or "-ipv4."), both 22 bytes, leaving 41 bytes - // for the container ID suffix — shorter prefixes ("netns.") just leave - // more room than they need. - annotationContainerIDLen = 41 -) - -const ( - // interfaceTypeVeth is the default interface type: veth pair for containers. - interfaceTypeVeth = "veth" - // interfaceTypeTap is the tap interface type: L2 fd for VMs (Kata, Firecracker). - interfaceTypeTap = "tap" -) - // RunPlugin starts the CNI plugin, handling ADD, DEL, CHECK, and STATUS operations. func RunPlugin() { skel.PluginMainFuncs( diff --git a/internal/cni/cni_test.go b/internal/cni/cni_test.go index a4eeb9dc..5de53e3a 100644 --- a/internal/cni/cni_test.go +++ b/internal/cni/cni_test.go @@ -5,29 +5,18 @@ package cni import ( - "context" "errors" "fmt" - "log/slog" "net" "os" - "path/filepath" - "reflect" "strings" "testing" - "time" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" - type100 "github.com/containernetworking/cni/pkg/types/100" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "go.datum.net/galactic/internal/config" - bgpv1alpha1 "go.datum.net/network/api/v1alpha1" + + "go.datum.net/galactic/internal/cniipam" + "go.datum.net/galactic/internal/cnimaster" ) func TestMain(m *testing.M) { @@ -39,17 +28,12 @@ func TestMain(m *testing.M) { const ( testVPC = "abc" testAttachment = "def" - testVPCHex1234 = "0000000004d2" // decimal 1234 - testRD65000_1 = "65000:1" // RD/RT for ASN 65000, NN 1 testContainerID = "test-container" testInvalidBase62 = "abc-def" // shared invalid base62 string for tests testNetns = "/proc/1/ns/net" testMac = "aa:bb:cc:dd:ee:ff" testIfName = "eth0" - testRouterName = "overlay-router" - testSID128 = "2001:db8::1/128" testCNIVersion = "1.0.0" - testIPv4Subnet = "10.128.0.0/20" // testPrevResult is a valid CNI v1.0.0 result used in prevResult tests. testPrevResult = `{"cniVersion":"1.0.0",` + @@ -58,32 +42,6 @@ const ( `"ips":[{"version":"6","address":"fd00:1::1/64"}]}` ) -func fakeClient(objs ...client.Object) client.Client { - return fake.NewClientBuilder().WithScheme(cniScheme).WithObjects(objs...).Build() -} - -// routerForNode builds a BGPRouter with spec.targetRef.name set to nodeName. -func routerForNode(name, nodeName, namespace string, asn int64) *bgpv1alpha1.BGPRouter { - return &bgpv1alpha1.BGPRouter{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: bgpv1alpha1.BGPRouterSpec{ - TargetRef: bgpv1alpha1.TargetRef{ - Kind: "Node", - Name: nodeName, - }, - LocalASN: asn, - RouterID: "10.0.0.1", - Roles: []bgpv1alpha1.RouterRole{bgpv1alpha1.RouterRoleTenant}, - AddressFamilies: []bgpv1alpha1.AddressFamily{ - {AFI: bgpv1alpha1.AFIL2VPN, SAFI: bgpv1alpha1.SAFIEVPN}, - }, - }, - } -} - // assertCNIError verifies that err is a *types.Error with the expected Code // and that its Msg contains wantMsg (substring match). Pass wantMsg == "" to // skip the message check. @@ -101,772 +59,12 @@ func assertCNIError(t *testing.T, err error, wantCode uint, wantMsg string) { } } -// ---- parseConf ----------------------------------------------------------- - -func TestParseConf(t *testing.T) { - tests := []struct { - name string - input string - wantVPC string - wantIfType string - wantAddressFamilies []string // nil means "don't check" - wantErr string - wantCode uint // CNI error code; 0 means "don't check" - }{ - { - name: "valid config", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","srv6_sid":"2001:db8::1/128"}`, - testVPC, testAttachment, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeVeth, - }, - { - name: "invalid JSON", - input: "not json", - wantErr: "invalid CNI config", - wantCode: 7, - }, - { - name: "empty input", - input: "", - wantErr: "invalid CNI config", - wantCode: 7, - }, - { - name: "interface_type=veth", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","interface_type":"veth"}`, - testVPC, testAttachment, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeVeth, - }, - { - name: "interface_type=tap", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","interface_type":"tap"}`, - testVPC, testAttachment, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeTap, - }, - { - name: "interface_type empty defaults to veth", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","interface_type":""}`, - testVPC, testAttachment, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeVeth, - }, - { - name: "interface_type=unknown", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","interface_type":"unknown"}`, - testVPC, testAttachment, - ), - wantErr: `invalid interface_type "unknown": must be "veth" or "tap"`, - wantCode: 7, - }, - { - name: "missing vpc", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpcattachment":"%s"}`, - testAttachment, - ), - wantErr: "vpc is required and must be a non-empty base62 string", - wantCode: 7, - }, - { - name: "empty vpc", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"",`+ - `"vpcattachment":"%s"}`, - testAttachment, - ), - wantErr: "vpc is required and must be a non-empty base62 string", - wantCode: 7, - }, - { - name: "vpc with invalid char hyphen", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s"}`, - testInvalidBase62, testAttachment, - ), - wantErr: fmt.Sprintf("invalid base62 value for field 'vpc': %q", testInvalidBase62), - wantCode: 7, - }, - { - name: "vpc with invalid char underscore", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"abc_def",`+ - `"vpcattachment":"%s"}`, - testAttachment, - ), - wantErr: `invalid base62 value for field 'vpc': "abc_def"`, - wantCode: 7, - }, - { - name: "missing vpcattachment", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s"}`, - testVPC, - ), - wantErr: "vpcattachment is required and must be a non-empty base62 string", - wantCode: 7, - }, - { - name: "empty vpcattachment", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":""}`, - testVPC, - ), - wantErr: "vpcattachment is required and must be a non-empty base62 string", - wantCode: 7, - }, - { - name: "vpcattachment with invalid char space", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"def ghi"}`, - testVPC, - ), - wantErr: `invalid base62 value for field 'vpcattachment': "def ghi"`, - wantCode: 7, - }, - { - name: "valid vpc and vpcattachment with mixed case base62", - input: `{"cniVersion":"1.0.0","name":"test",` + - `"type":"galactic-cni","vpc":"Abc123XYZ",` + - `"vpcattachment":"DeF456"}`, - wantVPC: "Abc123XYZ", - wantIfType: interfaceTypeVeth, - }, - { - name: "valid srv6_sid with /128", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","srv6_sid":"2001:db8::1/128"}`, - testVPC, testAttachment, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeVeth, - }, - { - name: "valid srv6_sid bare IPv6 address", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","srv6_sid":"2001:db8::1"}`, - testVPC, testAttachment, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeVeth, - }, - { - name: "srv6_sid empty is allowed", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","srv6_sid":""}`, - testVPC, testAttachment, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeVeth, - }, - { - name: "srv6_sid missing is allowed", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s"}`, - testVPC, testAttachment, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeVeth, - }, - - { - name: "prevResult valid JSON result is accepted", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s",`+ - `"prevResult":%s}`, - testVPC, testAttachment, testPrevResult, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeVeth, - }, - - // ---- dual-stack addressing fields (ipv6_subnet, ipv4_subnet, address_families) ---- - - { - name: "dual-stack fields omitted parses successfully", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s"}`, - testVPC, testAttachment, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeVeth, - wantAddressFamilies: []string{addressFamilyIPv6}, - }, - { - name: "valid ipv6_subnet accepted", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","ipv6_subnet":"fd00:10:ff01::/48"}`, - testVPC, testAttachment, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeVeth, - }, - { - name: "invalid ipv6_subnet CIDR rejected", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","ipv6_subnet":"not-a-cidr"}`, - testVPC, testAttachment, - ), - wantErr: "invalid CIDR value for field 'ipv6_subnet'", - wantCode: 7, - }, - { - name: "ipv4 CIDR given where ipv6_subnet expected rejected", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","ipv6_subnet":"10.0.0.0/24"}`, - testVPC, testAttachment, - ), - wantErr: "ipv6_subnet must be an IPv6 CIDR, got IPv4", - wantCode: 7, - }, - { - name: "ipv6_subnet prefix length over 96 rejected", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","ipv6_subnet":"fd00:10:ff01::/112"}`, - testVPC, testAttachment, - ), - wantErr: "ipv6_subnet prefix length 112 exceeds maximum of 96", - wantCode: 7, - }, - { - name: "valid ipv4_subnet accepted", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","ipv4_subnet":"10.0.0.0/20"}`, - testVPC, testAttachment, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeVeth, - }, - { - // A standard dotted-decimal CIDR can never carry a mask longer - // than /32 (net.ParseCIDR itself rejects e.g. "10.0.0.0/33"), so - // this exercises the prefix-length guard via an IPv4-mapped IPv6 - // literal, which Go parses with a 128-bit mask space. - name: "ipv4_subnet prefix length over 32 rejected", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","ipv4_subnet":"::ffff:10.0.0.0/40"}`, - testVPC, testAttachment, - ), - wantErr: "ipv4_subnet prefix length 40 exceeds maximum of 32", - wantCode: 7, - }, - { - name: "ipv6 CIDR given where ipv4_subnet expected rejected", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","ipv4_subnet":"2001:db8::/64"}`, - testVPC, testAttachment, - ), - wantErr: "ipv4_subnet must be an IPv4 CIDR, got IPv6", - wantCode: 7, - }, - { - name: "address_families defaults to ipv6 when omitted", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s"}`, - testVPC, testAttachment, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeVeth, - wantAddressFamilies: []string{addressFamilyIPv6}, - }, - { - name: "address_families explicit dual-stack accepted", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","address_families":["ipv6","ipv4"]}`, - testVPC, testAttachment, - ), - wantVPC: testVPC, - wantIfType: interfaceTypeVeth, - wantAddressFamilies: []string{addressFamilyIPv6, addressFamilyIPv4}, - }, - { - name: "invalid address_families entry rejected", - input: fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","address_families":["ipv6","bogus"]}`, - testVPC, testAttachment, - ), - wantErr: `invalid address_families entry "bogus": must be "ipv6" or "ipv4"`, - wantCode: 7, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - conf, err := parseConf([]byte(tt.input)) - if tt.wantErr != "" { - if err == nil { - t.Fatalf("expected error containing %q, got nil", tt.wantErr) - } - if !strings.Contains(err.Error(), tt.wantErr) { - t.Fatalf("error %q does not contain %q", err, tt.wantErr) - } - if tt.wantCode > 0 { - assertCNIError(t, err, tt.wantCode, tt.wantErr) - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if conf.VPC != tt.wantVPC { - t.Errorf("VPC = %q, want %q", conf.VPC, tt.wantVPC) - } - if conf.InterfaceType != tt.wantIfType { - t.Errorf("InterfaceType = %q, want %q", conf.InterfaceType, tt.wantIfType) - } - if tt.wantAddressFamilies != nil && !reflect.DeepEqual(conf.AddressFamilies, tt.wantAddressFamilies) { - t.Errorf("AddressFamilies = %v, want %v", conf.AddressFamilies, tt.wantAddressFamilies) - } - }) - } -} - -// ---- isValidBase62 ------------------------------------------------------- - -func TestIsValidBase62(t *testing.T) { - tests := []struct { - name string - input string - want bool - }{ - {"empty", "", false}, - {"digits only", "1234567890", true}, - {"lowercase only", "abcdefghij", true}, - {"uppercase only", "ABCDEFGHIJ", true}, - {"mixed case", "aBcDeFgHiJ", true}, - {"mixed digits and letters", "abc123XYZ", true}, - {"hyphen", testInvalidBase62, false}, - {"underscore", "abc_def", false}, - {"space", "abc def", false}, - {"dot", "abc.def", false}, - {"slash", "abc/def", false}, - {"plus", "abc+def", false}, - {"equals", "abc=def", false}, - {"single digit", "0", true}, - {"single lowercase", "a", true}, - {"single uppercase", "Z", true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := isValidBase62(tt.input) - if got != tt.want { - t.Errorf("isValidBase62(%q) = %v, want %v", tt.input, got, tt.want) - } - }) - } -} - -func TestSanitizeForError(t *testing.T) { - printable := "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!@#$%^&*()" - tests := []struct { - name string - input string - want string - }{ - {"normal string", testInvalidBase62, testInvalidBase62}, - {"empty", "", ""}, - {"newline", "abc\ndef", sanitizeForErrorBinary}, - {"null byte", "abc\x00def", sanitizeForErrorBinary}, - {"del char", "abc\x7fdef", sanitizeForErrorBinary}, - {"printable range", printable, printable}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := sanitizeForError(tt.input) - if got != tt.want { - t.Errorf("sanitizeForError(%q) = %q, want %q", tt.input, got, tt.want) - } - }) - } -} - -// ---- validatePrevResult -------------------------------------------------- - -func TestValidatePrevResult(t *testing.T) { - validResult := &type100.Result{ - CNIVersion: testCNIVersion, - Interfaces: []*type100.Interface{ - {Name: testIfName, Mac: testMac, Sandbox: testNetns}, - }, - IPs: []*type100.IPConfig{ - {Address: *mustParseCIDR(t, "fd00:1::1/64")}, - }, - } - - tests := []struct { - name string - input types.Result - wantErr bool - }{ - {"nil result allowed", nil, false}, - {"valid CNI result", validResult, false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := validatePrevResult(tt.input) - if tt.wantErr { - if err == nil { - t.Fatal("expected error, got nil") - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - }) - } -} - -func TestValidatePrevResultAdd(t *testing.T) { - validWithInterface := &type100.Result{ - CNIVersion: testCNIVersion, - Interfaces: []*type100.Interface{ - {Name: testIfName, Mac: testMac, Sandbox: testNetns}, - }, - IPs: []*type100.IPConfig{ - {Address: *mustParseCIDR(t, "fd00:1::1/64")}, - }, - } - validWithIPsOnly := &type100.Result{ - CNIVersion: testCNIVersion, - IPs: []*type100.IPConfig{ - {Address: *mustParseCIDR(t, "fd00:1::1/64")}, - }, - } - emptyResult := &type100.Result{ - CNIVersion: testCNIVersion, - // No interfaces, no IPs — should fail content validation. - } - - tests := []struct { - name string - input types.Result - wantErr bool - }{ - {"nil result allowed", nil, false}, - {"valid result with interface", validWithInterface, false}, - {"valid result with IPs only", validWithIPsOnly, false}, - {"empty result (no interfaces or IPs)", emptyResult, true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := validatePrevResultAdd(tt.input) - if tt.wantErr { - if err == nil { - t.Fatal("expected error, got nil") - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - }) - } -} - -// ---- bgpVRFInstanceName -------------------------------------------------- - -func TestBGPVRFInstanceName(t *testing.T) { - tests := []struct{ vpc, attachment, want string }{ - {testVPC, testAttachment, testVPC + "-" + testAttachment}, - {"0000000jU", "00G", "0000000jU-00G"}, - } - for _, tt := range tests { - got := bgpVRFInstanceName(tt.vpc, tt.attachment) - if got != tt.want { - t.Errorf("bgpVRFInstanceName(%q, %q) = %q, want %q", tt.vpc, tt.attachment, got, tt.want) - } - } -} - -// ---- bgpAdvertisementName ------------------------------------------------ - -func TestBGPAdvertisementName(t *testing.T) { - tests := []struct{ vpc, attachment, want string }{ - {testVPC, testAttachment, testVPC + "-" + testAttachment}, - {"0000000jU", "00G", "0000000jU-00G"}, - } - for _, tt := range tests { - got := bgpAdvertisementName(tt.vpc, tt.attachment) - if got != tt.want { - t.Errorf("bgpAdvertisementName(%q, %q) = %q, want %q", tt.vpc, tt.attachment, got, tt.want) - } - } -} - -// ---- routeTarget --------------------------------------------------------- - -func TestRouteTarget(t *testing.T) { - tests := []struct { - name string - asNumber int64 - vpcHex string - want string - wantErr bool - }{ - { - name: "VPC value fits in 32 bits", - asNumber: 65000, - vpcHex: testVPCHex1234, - want: "65000:1234", - }, - { - name: "upper bits beyond 32 stripped", - asNumber: 65000, - vpcHex: "000100000001", // 0x000100000001; low32 = 1 - want: testRD65000_1, - }, - { - name: "low 32 bits all set", - asNumber: 65000, - vpcHex: "0000ffffffff", - want: "65000:4294967295", - }, - { - name: "different ASN", - asNumber: 4200000000, - vpcHex: testVPCHex1234, - want: "4200000000:1234", - }, - { - name: "invalid hex string", - vpcHex: "zzzzzz", - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := routeTarget(tt.asNumber, tt.vpcHex) - if tt.wantErr { - if err == nil { - t.Fatal("expected error, got nil") - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if got != tt.want { - t.Errorf("routeTarget(%d, %q) = %q, want %q", tt.asNumber, tt.vpcHex, got, tt.want) - } - }) - } -} - -// ---- SetEnableLocalIPAM -------------------------------------------------- - -func TestSetEnableLocalIPAM(t *testing.T) { - // Save and restore original state. - original := enableLocalIPAM - defer func() { enableLocalIPAM = original }() - - // Default should be false. - if enableLocalIPAM { - t.Error("enableLocalIPAM default = true, want false") - } - - // Setting to true should work. - SetEnableLocalIPAM(true) - if !enableLocalIPAM { - t.Error("enableLocalIPAM after SetEnableLocalIPAM(true) = false, want true") - } - - // Setting back to false should work. - SetEnableLocalIPAM(false) - if enableLocalIPAM { - t.Error("enableLocalIPAM after SetEnableLocalIPAM(false) = true, want false") - } -} - -// ---- lookupBGPRouter ----------------------------------------------------- - -func TestLookupBGPRouter(t *testing.T) { - ctx := context.Background() - const ( - nodeName = "node1" - namespace = "default" - ) - - matchingRouter := routerForNode(testRouterName, nodeName, namespace, 65000) - - tests := []struct { - name string - objects []client.Object - wantErr string - check func(t *testing.T, cfg bgpConfig) - }{ - { - name: "no router for node", - objects: nil, - wantErr: "no BGPRouter found", - }, - { - name: "single matching router returns correct config", - objects: []client.Object{matchingRouter}, - check: func(t *testing.T, cfg bgpConfig) { - t.Helper() - if cfg.asNumber != 65000 { - t.Errorf("asNumber = %d, want 65000", cfg.asNumber) - } - if cfg.routerName != testRouterName { - t.Errorf("routerName = %q, want %q", cfg.routerName, testRouterName) - } - if cfg.srv6Locator != "" { - t.Errorf("srv6Locator = %q, want empty (not configured on fixture)", cfg.srv6Locator) - } - if cfg.nodeID != 0 { - t.Errorf("nodeID = %d, want 0 (not configured on fixture)", cfg.nodeID) - } - }, - }, - { - name: "router with SRv6Locator and NodeID configured", - objects: []client.Object{ - func() *bgpv1alpha1.BGPRouter { - r := routerForNode("srv6-router", nodeName, namespace, 65000) - r.Spec.SRv6Locator = "fd00:10::/48" - r.Spec.NodeID = 7 - return r - }(), - }, - check: func(t *testing.T, cfg bgpConfig) { - t.Helper() - if cfg.srv6Locator != "fd00:10::/48" { - t.Errorf("srv6Locator = %q, want %q", cfg.srv6Locator, "fd00:10::/48") - } - if cfg.nodeID != 7 { - t.Errorf("nodeID = %d, want 7", cfg.nodeID) - } - }, - }, - { - name: "router in different namespace is ignored", - objects: []client.Object{ - routerForNode("other-ns-router", nodeName, "other-ns", 65001), - }, - wantErr: "no BGPRouter found", - }, - { - name: "non-matching node router is ignored", - objects: []client.Object{ - routerForNode("other-node-router", "node2", namespace, 65001), - matchingRouter, - }, - check: func(t *testing.T, cfg bgpConfig) { - t.Helper() - if cfg.routerName != testRouterName { - t.Errorf("routerName = %q, want %q", cfg.routerName, testRouterName) - } - }, - }, - { - name: "ambiguous: two routers target same node", - objects: []client.Object{ - routerForNode("router-a", nodeName, namespace, 65000), - routerForNode("router-b", nodeName, namespace, 65001), - }, - wantErr: "ambiguous", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - k8s := fakeClient(tt.objects...) - - cfg, err := lookupBGPRouter(ctx, k8s, nodeName, namespace) - if tt.wantErr != "" { - if err == nil { - t.Fatalf("expected error containing %q, got nil", tt.wantErr) - } - if !strings.Contains(err.Error(), tt.wantErr) { - t.Fatalf("error %q does not contain %q", err, tt.wantErr) - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if tt.check != nil { - tt.check(t, cfg) - } - }) - } -} - // ---- buildResult --------------------------------------------------------- func TestBuildResult(t *testing.T) { subnet := mustParseCIDR(t, "fd00:10:ff01::1234/80") gateway := net.ParseIP("fd00:10:ff01::1") - route := mustParseCIDR(t, "::/0") + defaultRoute := mustParseCIDR(t, "::/0") netns := "/proc/1234/ns/net" conf := &PluginConf{ @@ -877,7 +75,7 @@ func TestBuildResult(t *testing.T) { tests := []struct { name string - ipRes *ipamResult + ipRes *cniipam.IPAMResult wantInts int wantIPs int wantRoutes int @@ -885,7 +83,7 @@ func TestBuildResult(t *testing.T) { }{ { name: "with IPAM config", - ipRes: &ipamResult{ipv6Subnet: subnet, ipv6Gateway: gateway, routes: []*net.IPNet{route}}, + ipRes: &cniipam.IPAMResult{IPv6Subnet: subnet, IPv6Gateway: gateway, Routes: []*net.IPNet{defaultRoute}}, wantInts: 2, wantIPs: 1, wantRoutes: 1, @@ -995,12 +193,12 @@ func TestBuildResultDualStack(t *testing.T) { VPC: testVPC, VPCAttachment: testAttachment, } - ipRes := &ipamResult{ - ipv6Subnet: ipv6Subnet, - ipv6Gateway: ipv6Gateway, - ipv4Address: ipv4Address, - ipv4Gateway: ipv4Gateway, - routes: []*net.IPNet{ipv6Route, ipv4Route}, + ipRes := &cniipam.IPAMResult{ + IPv6Subnet: ipv6Subnet, + IPv6Gateway: ipv6Gateway, + IPv4Address: ipv4Address, + IPv4Gateway: ipv4Gateway, + Routes: []*net.IPNet{ipv6Route, ipv4Route}, } result := buildResult(conf, ipRes, "G09-vpc03-vpcAttH", "eth0", @@ -1013,151 +211,30 @@ func TestBuildResultDualStack(t *testing.T) { t.Errorf("IPs[0].Address = %v, want %v", result.IPs[0].Address, ipv6Subnet) } if !result.IPs[0].Gateway.Equal(ipv6Gateway) { - t.Errorf("IPs[0].Gateway = %v, want %v", result.IPs[0].Gateway, ipv6Gateway) - } - wantIPv4Mask := net.CIDRMask(32, 32).String() - if result.IPs[1].Address.IP.String() != ipv4Address.String() || result.IPs[1].Address.Mask.String() != wantIPv4Mask { - t.Errorf("IPs[1].Address = %v, want %s/32", result.IPs[1].Address, ipv4Address) - } - if !result.IPs[1].Gateway.Equal(ipv4Gateway) { - t.Errorf("IPs[1].Gateway = %v, want %v", result.IPs[1].Gateway, ipv4Gateway) - } - for i, r := range result.IPs { - if r.Interface == nil || *r.Interface != 1 { - t.Errorf("IPs[%d].Interface = %v, want 1 (guest)", i, r.Interface) - } - } - if len(result.Routes) != 2 { - t.Errorf("Routes count = %d, want 2", len(result.Routes)) - } -} - -// TestBuildResultIPv4Only verifies that buildResult emits a single IPv4 -// IPConfig (no IPv6 entry, no panic) when ipamResult carries an IPv4-only -// allocation — the NAD config from the reported bug (ipv4_subnet set, no -// ipv6_subnet). -func TestBuildResultIPv4Only(t *testing.T) { - ipv4Address := net.ParseIP("172.20.1.5") - ipv4Gateway := net.ParseIP("172.20.1.1") - ipv4Route := mustParseCIDR(t, "0.0.0.0/0") - - conf := &PluginConf{ - PluginConf: types.PluginConf{CNIVersion: testCNIVersion}, - VPC: testVPC, - VPCAttachment: testAttachment, - } - ipRes := &ipamResult{ - ipv4Address: ipv4Address, - ipv4Gateway: ipv4Gateway, - routes: []*net.IPNet{ipv4Route}, - } - - result := buildResult(conf, ipRes, "G09-vpc03-vpcAttH", "eth0", - "aa:bb:cc:dd:ee:ff", "aa:bb:cc:dd:ee:11", 1500, 1500, "/proc/1234/ns/net") - - if len(result.IPs) != 1 { - t.Fatalf("IPs count = %d, want 1", len(result.IPs)) - } - wantIPv4Mask := net.CIDRMask(32, 32).String() - if result.IPs[0].Address.IP.String() != ipv4Address.String() || result.IPs[0].Address.Mask.String() != wantIPv4Mask { - t.Errorf("IPs[0].Address = %v, want %s/32", result.IPs[0].Address, ipv4Address) - } - if !result.IPs[0].Gateway.Equal(ipv4Gateway) { - t.Errorf("IPs[0].Gateway = %v, want %v", result.IPs[0].Gateway, ipv4Gateway) - } - if result.IPs[0].Interface == nil || *result.IPs[0].Interface != 1 { - t.Errorf("IPs[0].Interface = %v, want 1 (guest)", result.IPs[0].Interface) - } - if len(result.Routes) != 1 { - t.Errorf("Routes count = %d, want 1", len(result.Routes)) - } -} - -// ---- buildTapResult ------------------------------------------------------ - -func TestBuildTapResult(t *testing.T) { - subnet := mustParseCIDR(t, "fd00:10:ff01::1234/80") - gateway := net.ParseIP("fd00:10:ff01::1") - route := mustParseCIDR(t, "::/0") - - conf := &PluginConf{ - PluginConf: types.PluginConf{CNIVersion: testCNIVersion}, - VPC: testVPC, - VPCAttachment: testAttachment, - } - - tests := []struct { - name string - ipRes *ipamResult - wantIPs int - wantRoutes int - }{ - { - name: "with IPAM config", - ipRes: &ipamResult{ipv6Subnet: subnet, ipv6Gateway: gateway, routes: []*net.IPNet{route}}, - wantIPs: 1, - wantRoutes: 1, - }, - { - name: "without IPAM config", - ipRes: nil, - wantIPs: 0, - wantRoutes: 0, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := buildTapResult(conf, tt.ipRes, "H0abc123", "aa:bb:cc:dd:ee:ff", 1500) - - if result.CNIVersion != testCNIVersion { - t.Errorf("CNIVersion = %q, want %q", result.CNIVersion, testCNIVersion) - } - - if len(result.Interfaces) != 1 { - t.Fatalf("Interfaces count = %d, want 1", len(result.Interfaces)) - } - - if result.Interfaces[0].Name != "H0abc123" { - t.Errorf("Interfaces[0].Name = %q, want %q", result.Interfaces[0].Name, "H0abc123") - } - if result.Interfaces[0].Mac != "aa:bb:cc:dd:ee:ff" { - t.Errorf("Interfaces[0].Mac = %q, want %q", result.Interfaces[0].Mac, "aa:bb:cc:dd:ee:ff") - } - if result.Interfaces[0].Mtu != 1500 { - t.Errorf("Interfaces[0].Mtu = %d, want 1500", result.Interfaces[0].Mtu) - } - if result.Interfaces[0].Sandbox != "" { - t.Errorf("Interfaces[0].Sandbox = %q, want empty", result.Interfaces[0].Sandbox) - } - - if len(result.IPs) != tt.wantIPs { - t.Errorf("IPs count = %d, want %d", len(result.IPs), tt.wantIPs) - } - if tt.wantIPs > 0 { - if result.IPs[0].Address.String() != subnet.String() { - t.Errorf("IPs[0].Address = %q, want %q", result.IPs[0].Address, subnet) - } - if !result.IPs[0].Gateway.Equal(gateway) { - t.Errorf("IPs[0].Gateway = %v, want %v", result.IPs[0].Gateway, gateway) - } - if result.IPs[0].Interface == nil || *result.IPs[0].Interface != 0 { - t.Errorf("IPs[0].Interface = %v, want 0", result.IPs[0].Interface) - } - } - - if len(result.Routes) != tt.wantRoutes { - t.Errorf("Routes count = %d, want %d", len(result.Routes), tt.wantRoutes) - } - }) + t.Errorf("IPs[0].Gateway = %v, want %v", result.IPs[0].Gateway, ipv6Gateway) + } + wantIPv4Mask := net.CIDRMask(32, 32).String() + if result.IPs[1].Address.IP.String() != ipv4Address.String() || result.IPs[1].Address.Mask.String() != wantIPv4Mask { + t.Errorf("IPs[1].Address = %v, want %s/32", result.IPs[1].Address, ipv4Address) + } + if !result.IPs[1].Gateway.Equal(ipv4Gateway) { + t.Errorf("IPs[1].Gateway = %v, want %v", result.IPs[1].Gateway, ipv4Gateway) + } + for i, r := range result.IPs { + if r.Interface == nil || *r.Interface != 1 { + t.Errorf("IPs[%d].Interface = %v, want 1 (guest)", i, r.Interface) + } + } + if len(result.Routes) != 2 { + t.Errorf("Routes count = %d, want 2", len(result.Routes)) } } -// TestBuildTapResultIPv4Mask verifies that buildTapResult reports the IPv4 -// address with a /25 mask (matching the host gateway mask -// ipv4GatewayAddrParams installs on the tap interface), not the /32 used for -// veth. -func TestBuildTapResultIPv4Mask(t *testing.T) { +// TestBuildResultIPv4Only verifies that buildResult emits a single IPv4 +// IPConfig (no IPv6 entry, no panic) when ipamResult carries an IPv4-only +// allocation — the NAD config from the reported bug (ipv4_subnet set, no +// ipv6_subnet). +func TestBuildResultIPv4Only(t *testing.T) { ipv4Address := net.ParseIP("172.20.1.5") ipv4Gateway := net.ParseIP("172.20.1.1") ipv4Route := mustParseCIDR(t, "0.0.0.0/0") @@ -1167,65 +244,30 @@ func TestBuildTapResultIPv4Mask(t *testing.T) { VPC: testVPC, VPCAttachment: testAttachment, } - ipRes := &ipamResult{ - ipv4Address: ipv4Address, - ipv4Gateway: ipv4Gateway, - routes: []*net.IPNet{ipv4Route}, + ipRes := &cniipam.IPAMResult{ + IPv4Address: ipv4Address, + IPv4Gateway: ipv4Gateway, + Routes: []*net.IPNet{ipv4Route}, } - result := buildTapResult(conf, ipRes, "H0abc123", "aa:bb:cc:dd:ee:ff", 1500) + result := buildResult(conf, ipRes, "G09-vpc03-vpcAttH", "eth0", + "aa:bb:cc:dd:ee:ff", "aa:bb:cc:dd:ee:11", 1500, 1500, "/proc/1234/ns/net") if len(result.IPs) != 1 { t.Fatalf("IPs count = %d, want 1", len(result.IPs)) } - wantIPv4Mask := net.CIDRMask(25, 32).String() + wantIPv4Mask := net.CIDRMask(32, 32).String() if result.IPs[0].Address.IP.String() != ipv4Address.String() || result.IPs[0].Address.Mask.String() != wantIPv4Mask { - t.Errorf("IPs[0].Address = %v, want %s/25", result.IPs[0].Address, ipv4Address) + t.Errorf("IPs[0].Address = %v, want %s/32", result.IPs[0].Address, ipv4Address) } if !result.IPs[0].Gateway.Equal(ipv4Gateway) { t.Errorf("IPs[0].Gateway = %v, want %v", result.IPs[0].Gateway, ipv4Gateway) } - if result.IPs[0].Interface == nil || *result.IPs[0].Interface != 0 { - t.Errorf("IPs[0].Interface = %v, want 0 (host tap)", result.IPs[0].Interface) - } -} - -// TestBuildTapResultHostNetns verifies that the tap path produces a valid -// CNI result when args.Netns is the host network namespace. Kraftlet/unikraft -// workloads pass the host netns because they don't have a Linux network -// namespace. The main.go entry point detects interface_type=tap and sets -// CNI_NETNS_OVERRIDE to bypass the CNI library's same-netns rejection check. -// The tap result must not reference a sandbox. -func TestBuildTapResultHostNetns(t *testing.T) { - subnet := mustParseCIDR(t, "fd00:10:ff01::1234/80") - gateway := net.ParseIP("fd00:10:ff01::1") - route := mustParseCIDR(t, "::/0") - - conf := &PluginConf{ - PluginConf: types.PluginConf{CNIVersion: testCNIVersion}, - VPC: testVPC, - VPCAttachment: testAttachment, - } - ipRes := &ipamResult{ipv6Subnet: subnet, ipv6Gateway: gateway, routes: []*net.IPNet{route}} - - result := buildTapResult(conf, ipRes, "H0abc123", "aa:bb:cc:dd:ee:ff", 1500) - - // Result should be structurally valid for kraftlet (host netns) workloads. - if result.CNIVersion != testCNIVersion { - t.Errorf("CNIVersion = %q, want %q", result.CNIVersion, testCNIVersion) - } - if len(result.Interfaces) != 1 { - t.Fatalf("Interfaces count = %d, want 1", len(result.Interfaces)) - } - // Host tap interface must not reference a sandbox (kraftlet has no netns). - if result.Interfaces[0].Sandbox != "" { - t.Errorf("Interfaces[0].Sandbox = %q, want empty (host netns, no sandbox)", result.Interfaces[0].Sandbox) - } - if len(result.IPs) != 1 { - t.Fatalf("IPs count = %d, want 1", len(result.IPs)) + if result.IPs[0].Interface == nil || *result.IPs[0].Interface != 1 { + t.Errorf("IPs[0].Interface = %v, want 1 (guest)", result.IPs[0].Interface) } if len(result.Routes) != 1 { - t.Fatalf("Routes count = %d, want 1", len(result.Routes)) + t.Errorf("Routes count = %d, want 1", len(result.Routes)) } } @@ -1248,14 +290,10 @@ func TestCmdDelIdempotent(t *testing.T) { // TestCmdDelIdempotentMissingResources returns nil even when the config is // valid but all resources are missing (k8s client creation fails in tests). func TestCmdDelIdempotentMissingResources(t *testing.T) { - // Save and restore the original enableLocalIPAM state. - original := enableLocalIPAM - defer func() { enableLocalIPAM = original }() - conf := fmt.Sprintf( `{"cniVersion":"1.0.0","name":"test",`+ `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","interface_type":"veth"}`, + `"vpcattachment":"%s"}`, testVPC, testAttachment, ) args := &skel.CmdArgs{ @@ -1292,7 +330,7 @@ func TestCmdDelFlushesGuestNetnsConfig(t *testing.T) { conf := fmt.Sprintf( `{"cniVersion":"1.0.0","name":"test",`+ `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","interface_type":"veth"}`, + `"vpcattachment":"%s"}`, testVPC, testAttachment, ) args := &skel.CmdArgs{ @@ -1340,27 +378,6 @@ func TestCmdCheckInvalidConfig(t *testing.T) { } } -func TestCmdCheckInvalidInterfaceType(t *testing.T) { - conf := fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","interface_type":"bogus"}`, - testVPC, testAttachment, - ) - args := &skel.CmdArgs{ - ContainerID: testContainerID, - StdinData: []byte(conf), - } - - err := cmdCheck(args) - if err == nil { - t.Fatalf("expected error for invalid interface_type, got nil") - } - if !strings.Contains(err.Error(), `invalid interface_type "bogus"`) { - t.Fatalf("error %q does not contain expected message", err.Error()) - } -} - func TestCmdCheckValidConfigMissingResources(t *testing.T) { conf := fmt.Sprintf( `{"cniVersion":"1.0.0","name":"test",`+ @@ -1384,27 +401,6 @@ func TestCmdCheckValidConfigMissingResources(t *testing.T) { } } -func TestCmdCheckTapModeValidConfigMissingResources(t *testing.T) { - conf := fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","interface_type":"tap"}`, - testVPC, testAttachment, - ) - args := &skel.CmdArgs{ - ContainerID: testContainerID, - StdinData: []byte(conf), - } - - err := cmdCheck(args) - if err == nil { - t.Fatalf("expected CHECK failure for missing resources, got nil") - } - if !strings.Contains(err.Error(), "CHECK failed") { - t.Fatalf("error %q does not contain 'CHECK failed'", err.Error()) - } -} - func TestCmdCheckMissingVPC(t *testing.T) { conf := `{"cniVersion":"1.0.0","name":"test","type":"galactic-cni"}` args := &skel.CmdArgs{ @@ -1488,8 +484,7 @@ func TestResourceTrackerCleanupZeroValue(t *testing.T) { // cleanup with a zero-value tracker must not panic — it's called in a // defer and the caller may have failed before setting any fields. tracker := &resourceTracker{} - ctx := context.Background() - tracker.cleanup(ctx) // should not panic + tracker.cleanup() // should not panic } func TestResourceTrackerCleanupPartialState(t *testing.T) { @@ -1498,19 +493,14 @@ func TestResourceTrackerCleanupPartialState(t *testing.T) { tracker := &resourceTracker{ vpc: testVPC, vpcAttachment: testAttachment, - ifaceType: interfaceTypeVeth, - namespace: "default", } - ctx := context.Background() - tracker.cleanup(ctx) // should not panic; vrf.Delete will fail but is logged + tracker.cleanup() // should not panic; vrf.Delete will fail but is logged } func TestResourceTrackerFieldsSet(t *testing.T) { tracker := &resourceTracker{ vpc: testVPC, vpcAttachment: testAttachment, - ifaceType: interfaceTypeTap, - namespace: "test-ns", } if tracker.vpc != testVPC { @@ -1519,18 +509,9 @@ func TestResourceTrackerFieldsSet(t *testing.T) { if tracker.vpcAttachment != testAttachment { t.Errorf("vpcAttachment = %q, want %q", tracker.vpcAttachment, testAttachment) } - if tracker.ifaceType != interfaceTypeTap { - t.Errorf("ifaceType = %q, want %q", tracker.ifaceType, interfaceTypeTap) - } - if tracker.namespace != "test-ns" { - t.Errorf("namespace = %q, want %q", tracker.namespace, "test-ns") - } if tracker.vrfCreated { t.Error("vrfCreated should be false by default") } - if tracker.advCreated { - t.Error("advCreated should be false by default") - } } // ---- cmdStatus --------------------------------------------------------- @@ -1545,29 +526,13 @@ func TestCmdStatusInvalidConfig(t *testing.T) { assertCNIError(t, err, 7, "invalid CNI config") } -func TestCmdStatusInvalidInterfaceType(t *testing.T) { - conf := fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test",`+ - `"type":"galactic-cni","vpc":"%s",`+ - `"vpcattachment":"%s","interface_type":"bogus"}`, - testVPC, testAttachment, - ) - args := &skel.CmdArgs{ - ContainerID: testContainerID, - StdinData: []byte(conf), - } - - err := cmdStatus(args) - assertCNIError(t, err, 7, `invalid interface_type "bogus"`) -} - func TestCmdStatusValidConfigMissingResources(t *testing.T) { // STATUS should succeed with valid config even when VRF/interface // resources don't exist — STATUS answers "is the plugin ready to ADD?" // not "does a prior ADD's state persist?". - original := probeAPIServer - probeAPIServer = func() error { return nil } - defer func() { probeAPIServer = original }() + original := cnimaster.ProbeAPIServer + cnimaster.ProbeAPIServer = func() error { return nil } + defer func() { cnimaster.ProbeAPIServer = original }() conf := fmt.Sprintf( `{"cniVersion":"1.0.0","name":"test",`+ @@ -1589,9 +554,9 @@ func TestCmdStatusValidConfigMissingResources(t *testing.T) { func TestCmdStatusMissingVPC(t *testing.T) { // STATUS does not validate attachment-specific fields — it only checks // that the config is parseable and the API server is reachable. - original := probeAPIServer - probeAPIServer = func() error { return nil } - defer func() { probeAPIServer = original }() + original := cnimaster.ProbeAPIServer + cnimaster.ProbeAPIServer = func() error { return nil } + defer func() { cnimaster.ProbeAPIServer = original }() conf := fmt.Sprintf( `{"cniVersion":"1.0.0","name":"test",`+ @@ -1612,9 +577,9 @@ func TestCmdStatusMissingVPC(t *testing.T) { func TestCmdStatusMissingVPCAttachment(t *testing.T) { // STATUS does not validate attachment-specific fields — it only checks // that the config is parseable and the API server is reachable. - original := probeAPIServer - probeAPIServer = func() error { return nil } - defer func() { probeAPIServer = original }() + original := cnimaster.ProbeAPIServer + cnimaster.ProbeAPIServer = func() error { return nil } + defer func() { cnimaster.ProbeAPIServer = original }() conf := fmt.Sprintf( `{"cniVersion":"1.0.0","name":"test",`+ @@ -1634,9 +599,9 @@ func TestCmdStatusMissingVPCAttachment(t *testing.T) { func TestCmdStatusAPIProbeFailure(t *testing.T) { // STATUS should return CNI error code 50 when the API server probe fails. - original := probeAPIServer - probeAPIServer = func() error { return errors.New("connection refused") } - defer func() { probeAPIServer = original }() + original := cnimaster.ProbeAPIServer + cnimaster.ProbeAPIServer = func() error { return errors.New("connection refused") } + defer func() { cnimaster.ProbeAPIServer = original }() conf := fmt.Sprintf( `{"cniVersion":"1.0.0","name":"test",`+ @@ -1653,205 +618,12 @@ func TestCmdStatusAPIProbeFailure(t *testing.T) { assertCNIError(t, err, 50, "API server health check failed") } -// ---- isTransientError ---------------------------------------------------- - -func TestIsTransientError(t *testing.T) { - tests := []struct { - name string - err error - wantTrans bool - }{ - { - name: "nil error is not transient", - err: nil, - wantTrans: false, - }, - { - name: "context deadline exceeded is transient", - err: context.DeadlineExceeded, - wantTrans: true, - }, - { - name: "context canceled is transient", - err: context.Canceled, - wantTrans: true, - }, - { - name: "wrapped context deadline exceeded is transient", - err: fmt.Errorf("k8s: %w", context.DeadlineExceeded), - wantTrans: true, - }, - { - name: "wrapped context canceled is transient", - err: fmt.Errorf("k8s: %w", context.Canceled), - wantTrans: true, - }, - { - name: "generic error is not transient", - err: errors.New("some error"), - wantTrans: false, - }, - { - name: "validation error is not transient", - err: apierrors.NewBadRequest("bad request"), - wantTrans: false, - }, - { - name: "not found error is not transient", - err: apierrors.NewNotFound( - schema.GroupResource{Group: "network.datumapis.com", Resource: "bgpadvertisements"}, "test"), - wantTrans: false, - }, - { - name: "503 service unavailable is transient", - err: apierrors.NewServiceUnavailable("service unavailable"), - // apierrors.IsServiceUnavailable catches 503. - wantTrans: true, - }, - { - name: "429 too many requests is transient", - err: apierrors.NewTooManyRequests("too many requests", 0), - // apierrors.IsTooManyRequests catches 429. - wantTrans: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := isTransientError(tt.err) - if got != tt.wantTrans { - t.Errorf("isTransientError(%v) = %v, want %v", tt.err, got, tt.wantTrans) - } - }) - } -} - -// ---- retryK8sOps --------------------------------------------------------- - -func TestRetryK8sOpsSucceedsImmediately(t *testing.T) { - calls := 0 - err := retryK8sOps(100*time.Millisecond, func(ctx context.Context) error { - calls++ - return nil - }) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if calls != 1 { - t.Errorf("expected 1 call, got %d", calls) - } -} - -func TestRetryK8sOpsRetriesOnTransientError(t *testing.T) { - calls := 0 - err := retryK8sOps(2*time.Second, func(ctx context.Context) error { - calls++ - if calls < 3 { - return context.DeadlineExceeded - } - return nil - }) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if calls != 3 { - t.Errorf("expected 3 calls (initial + 2 retries), got %d", calls) - } -} - -func TestRetryK8sOpsFailsAfterMaxRetries(t *testing.T) { - calls := 0 - err := retryK8sOps(2*time.Second, func(ctx context.Context) error { - calls++ - return context.DeadlineExceeded - }) - if err == nil { - t.Fatal("expected error, got nil") - } - if calls != maxRetries+1 { - t.Errorf("expected %d calls (initial + maxRetries), got %d", maxRetries+1, calls) - } -} - -func TestRetryK8sOpsNoRetryOnNonTransientError(t *testing.T) { - calls := 0 - permanentErr := errors.New("validation failed") - err := retryK8sOps(2*time.Second, func(ctx context.Context) error { - calls++ - return permanentErr - }) - if !errors.Is(err, permanentErr) { - t.Fatalf("expected %v, got %v", permanentErr, err) - } - if calls != 1 { - t.Errorf("expected 1 call (no retry), got %d", calls) - } -} - -func TestRetryK8sOpsExhaustsDeadline(t *testing.T) { - // When the timeout is very short, retries still happen but the fn - // completes instantly — so we exhaust maxRetries and get the last - // transient error back (not a context timeout, since fn is fast). - calls := 0 - err := retryK8sOps(1*time.Millisecond, func(ctx context.Context) error { - calls++ - return apierrors.NewServiceUnavailable("unavailable") - }) - if err == nil { - t.Fatal("expected error, got nil") - } - // Should have made maxRetries+1 attempts (initial + 2 retries). - if calls != maxRetries+1 { - t.Errorf("expected %d calls, got %d", maxRetries+1, calls) - } - // Final error is the last transient error returned by fn. - if !strings.Contains(err.Error(), "unavailable") { - t.Errorf("expected 'unavailable' in error, got %v", err) - } -} - -// ---- probeAPIServer ------------------------------------------------------ - -func TestProbeAPIServerErrNotInCluster(t *testing.T) { - // When probeAPIServerFn returns ErrNotInCluster, probeAPIServer should - // return nil (not running in-cluster; skip API check). - original := probeAPIServer - probeAPIServer = func() error { return nil } - defer func() { probeAPIServer = original }() - - if err := probeAPIServer(); err != nil { - t.Fatalf("expected nil for ErrNotInCluster, got %v", err) - } -} - -func TestProbeAPIServerMalformedKubeconfig(t *testing.T) { - // When probeAPIServerFn returns a non-ErrNotInCluster error (e.g. a - // malformed kubeconfig file), probeAPIServer should surface it wrapped. - original := probeAPIServer - probeAPIServer = func() error { - return errors.New("load kubeconfig: invalid kubeconfig: permission denied") - } - defer func() { probeAPIServer = original }() - - err := probeAPIServer() - if err == nil { - t.Fatal("expected error for malformed kubeconfig, got nil") - } - if !strings.Contains(err.Error(), "load kubeconfig") { - t.Fatalf("error %q does not contain 'load kubeconfig'", err.Error()) - } - if !strings.Contains(err.Error(), "permission denied") { - t.Fatalf("error %q does not contain original error", err.Error()) - } -} - // ---- cmdAdd prevResult validation ---------------------------------------- func TestCmdAddPrevResultValid(t *testing.T) { t.Setenv("GALACTIC_CNI_NODE_NAME", "") t.Setenv("NODE_NAME", "") // prevResult that is a valid CNI result. cmdAdd should pass prevResult - // validation and fail later due to missing node name. conf := fmt.Sprintf( `{"cniVersion":"1.0.0","name":"test",`+ @@ -1870,229 +642,3 @@ func TestCmdAddPrevResultValid(t *testing.T) { // not code 6 for prevResult. assertCNIError(t, err, 4, "node name is required") } - -// ---- loadHostConf ----------------------------------------------------------- - -func TestLoadHostConf(t *testing.T) { - tmpDir := t.TempDir() - conflistPath := filepath.Join(tmpDir, "10-galactic.conflist") - - // 1. Missing file tolerated, defaults to galactic-system namespace. - conf, err := loadHostConf(conflistPath) - if err != nil { - t.Fatalf("unexpected error for missing conflist: %v", err) - } - if conf.Namespace != config.DefaultNamespace { - t.Errorf("Namespace = %q, want %q", conf.Namespace, config.DefaultNamespace) - } - - // 2. Conflist parses but lacks galactic-cni entry. - badContent := `{"cniVersion":"1.0.0","name":"test","plugins":[{"type":"some-other-plugin"}]}` - if err := os.WriteFile(conflistPath, []byte(badContent), 0644); err != nil { - t.Fatalf("os.WriteFile: %v", err) - } - _, err = loadHostConf(conflistPath) - if err == nil { - t.Fatal("expected error for missing plugin type, got nil") - } - - // 3. Conflist parses correctly. - goodContent := `{ - "cniVersion": "1.0.0", - "name": "galactic", - "plugins": [ - { - "type": "galactic-cni", - "node_name": "test-worker", - "kubeconfig": "/etc/custom-kubeconfig", - "namespace": "custom-namespace", - "log_file": "/var/log/custom.log", - "log_level": "debug" - } - ] - }` - if err := os.WriteFile(conflistPath, []byte(goodContent), 0644); err != nil { - t.Fatalf("os.WriteFile: %v", err) - } - conf, err = loadHostConf(conflistPath) - if err != nil { - t.Fatalf("unexpected error for good conflist: %v", err) - } - if conf.NodeName != "test-worker" { - t.Errorf("NodeName = %q, want %q", conf.NodeName, "test-worker") - } - if conf.Kubeconfig != "/etc/custom-kubeconfig" { - t.Errorf("Kubeconfig = %q, want %q", conf.Kubeconfig, "/etc/custom-kubeconfig") - } - if conf.Namespace != "custom-namespace" { - t.Errorf("Namespace = %q, want %q", conf.Namespace, "custom-namespace") - } - if conf.LogFile != "/var/log/custom.log" { - t.Errorf("LogFile = %q, want %q", conf.LogFile, "/var/log/custom.log") - } - if conf.LogLevel != config.LogLevelDebug { - t.Errorf("LogLevel = %q, want %q", conf.LogLevel, config.LogLevelDebug) - } -} - -// ---- enableLocalIPAM required check ----------------------------------------- - -func TestEnableLocalIPAMRequired(t *testing.T) { - // Set local IPAM enabled - t.Setenv("GALACTIC_CNI_ENABLE_LOCAL_IPAM", "true") - t.Setenv("GALACTIC_CNI_NODE_NAME", "test-node") - - // Missing IPAM block should cause a hard error. - inputNoIPAM := fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test","type":"galactic-cni","vpc":"%s","vpcattachment":"%s"}`, - testVPC, testAttachment, - ) - _, err := parseConf([]byte(inputNoIPAM)) - if err == nil { - t.Fatal("expected error for missing ipam block when local IPAM is enabled, got nil") - } - if !strings.Contains(err.Error(), "no 'ipam' block is present") { - t.Fatalf("expected error containing 'no 'ipam' block', got: %v", err) - } - - // Present IPAM block should succeed. - inputWithIPAM := fmt.Sprintf( - `{"cniVersion":"1.0.0","name":"test","type":"galactic-cni","vpc":"%s","vpcattachment":"%s","ipam":{"type":"pool"}}`, - testVPC, testAttachment, - ) - conf, err := parseConf([]byte(inputWithIPAM)) - if err != nil { - t.Fatalf("unexpected error with present ipam block: %v", err) - } - if conf.IPAM == nil { - t.Fatal("expected IPAM block to be non-nil") - } -} - -// ---- annotation key length ------------------------------------------------- - -// TestAnnotationKeyNameLength verifies that every annotation key builder stays -// within Kubernetes' 63-byte limit on the "name" part of an annotation key -// (the segment after the last "/"), using a realistic 64-character container -// ID (containerd/Docker use full SHA256 hex digests). This guards against a -// real production incident: annotationContainerIDLen was sized for the old -// "allocated-subnet." prefix (17 bytes) and wasn't updated when the prefix -// grew by 5 bytes to "allocated-subnet-ipv6."/"-ipv4." — every BGPAdvertisement -// apply failed with "name part must be no more than 63 bytes" until fixed. -func TestAnnotationKeyNameLength(t *testing.T) { - const maxAnnotationNameLen = 63 - // A realistic full-length container ID (64 hex chars, as containerd/Docker use). - fullContainerID := strings.Repeat("a", 64) - - tests := []struct { - name string - key string - }{ - {"subnetAnnotationKeyIPv6", subnetAnnotationKeyIPv6(fullContainerID)}, - {"subnetAnnotationKeyIPv4", subnetAnnotationKeyIPv4(fullContainerID)}, - {"netnsAnnotationKey", netnsAnnotationKey(fullContainerID)}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - slash := strings.LastIndex(tt.key, "/") - namePart := tt.key - if slash != -1 { - namePart = tt.key[slash+1:] - } - if len(namePart) > maxAnnotationNameLen { - t.Errorf("%s(%d-char containerID) name part %q is %d bytes, want <= %d", - tt.name, len(fullContainerID), namePart, len(namePart), maxAnnotationNameLen) - } - }) - } -} - -// ---- logging setup ---------------------------------------------------------- - -func TestLoggingSetup(t *testing.T) { - tmpDir := t.TempDir() - logPath := filepath.Join(tmpDir, "sub", "test.log") - - // Setup logging, which should create the directory and open/write to the file. - setupLogging(logPath, config.DefaultLogLevel) - slog.Info("test log message") - - // Read the log file to verify the message was logged. - data, err := os.ReadFile(logPath) - if err != nil { - t.Fatalf("read log file: %v", err) - } - if !strings.Contains(string(data), "test log message") { - t.Fatalf("log content does not contain message: %s", string(data)) - } -} - -func TestParseLogLevel(t *testing.T) { - tests := []struct { - name string - in string - want slog.Level - wantErr bool - }{ - {"empty defaults to info", "", slog.LevelInfo, false}, - {"debug", config.LogLevelDebug, slog.LevelDebug, false}, - {"info", config.DefaultLogLevel, slog.LevelInfo, false}, - {"warn", config.LogLevelWarn, slog.LevelWarn, false}, - {"warning alias", config.LogLevelWarning, slog.LevelWarn, false}, - {"error", config.LogLevelError, slog.LevelError, false}, - {"case insensitive", "DEBUG", slog.LevelDebug, false}, - {"surrounding whitespace", " warn ", slog.LevelWarn, false}, - {"unknown falls back to info with error", "verbose", slog.LevelInfo, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := parseLogLevel(tt.in) - if (err != nil) != tt.wantErr { - t.Fatalf("parseLogLevel(%q) error = %v, wantErr %v", tt.in, err, tt.wantErr) - } - if got != tt.want { - t.Errorf("parseLogLevel(%q) = %v, want %v", tt.in, got, tt.want) - } - }) - } -} - -func TestLoggingSetupRespectsLevel(t *testing.T) { - tmpDir := t.TempDir() - logPath := filepath.Join(tmpDir, "test.log") - - setupLogging(logPath, config.LogLevelWarn) - slog.Info("should be suppressed at warn level") - slog.Warn("should appear at warn level") - - data, err := os.ReadFile(logPath) - if err != nil { - t.Fatalf("read log file: %v", err) - } - content := string(data) - if strings.Contains(content, "should be suppressed at warn level") { - t.Errorf("expected info message to be filtered out at warn level, got: %s", content) - } - if !strings.Contains(content, "should appear at warn level") { - t.Errorf("expected warn message to be present, got: %s", content) - } -} - -func TestLoggingSetupInvalidLevelFallsBackToInfo(t *testing.T) { - tmpDir := t.TempDir() - logPath := filepath.Join(tmpDir, "test.log") - - // An invalid level must not fail the CNI operation; it should fall back - // to DefaultLogLevel (info) rather than panic or drop all logging. - setupLogging(logPath, "verbose") - slog.Info("should appear at fallback info level") - - data, err := os.ReadFile(logPath) - if err != nil { - t.Fatalf("read log file: %v", err) - } - if !strings.Contains(string(data), "should appear at fallback info level") { - t.Errorf("expected info message to be present after fallback, got: %s", string(data)) - } -} diff --git a/internal/cni/config.go b/internal/cni/config.go index 3ff10a04..c6e9f7cf 100644 --- a/internal/cni/config.go +++ b/internal/cni/config.go @@ -5,24 +5,7 @@ package cni import ( - "context" - "encoding/json" - "errors" - "fmt" - "log/slog" - "net" - "os" - "path/filepath" - "strings" - - "github.com/containernetworking/cni/pkg/types" - type100 "github.com/containernetworking/cni/pkg/types/100" - "github.com/vishvananda/netlink" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - + "go.datum.net/galactic/internal/cnimaster" "go.datum.net/galactic/internal/config" ) @@ -39,505 +22,11 @@ func InitCNIConfig() { cniConfig = config.NewCNIConfig() } -const sanitizeForErrorBinary = "" - -// errInvalidCNIConfig is the message for CNI config parse errors (code 7). -const errInvalidCNIConfig = "invalid CNI config" - -// errVPCRequired and errVPCAttachmentRequired are messages for missing -// identifier fields (code 7). -const ( - errVPCRequired = "vpc is required and must be a non-empty base62 string" - errVPCAttachmentRequired = "vpcattachment is required and must be a non-empty base62 string" -) - -const ( - // maxIPv6SubnetPrefixLen is the maximum (longest) prefix length allowed - // for ipv6_subnet. It matches ipam.PoolAllocator's constraint that the - // pool prefix must be no longer than the per-allocation subnet length; - // dual-stack tenant addressing allocates /96 endpoints from this subnet, - // so the subnet itself must be a /96 or shorter. - maxIPv6SubnetPrefixLen = 96 - - // maxIPv4SubnetPrefixLen is the maximum (longest) prefix length allowed - // for ipv4_subnet: a full IPv4 host route. - maxIPv4SubnetPrefixLen = 32 -) - -// addressFamilyIPv6 and addressFamilyIPv4 are the only valid entries for -// the address_families config field. -const ( - addressFamilyIPv6 = "ipv6" - addressFamilyIPv4 = "ipv4" -) - -// isValidBase62 reports whether s contains only valid base62 characters -// ([0-9a-zA-Z]) and is non-empty. VPC and VPCAttachment identifiers are -// base62-encoded and used throughout the ADD path (interface naming, -// BGP CRD population). Rejecting them early in parseConf prevents cryptic -// errors deep in the stack after partial kernel state has been created. -func isValidBase62(s string) bool { - if len(s) == 0 { - return false - } - for _, c := range s { - if (c < '0' || c > '9') && (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') { - return false - } - } - return true -} - -// conflistEnvelope matches standard CNI conflist JSON structure. -type conflistEnvelope struct { - CNIVersion string `json:"cniVersion"` - Name string `json:"name"` - Plugins []json.RawMessage `json:"plugins"` -} - -// loadHostConf loads node-local settings from the CNI conflist. -// If the file is missing, it returns a zero-value HostConf (tolerating local test runs) -// but still defaulting Namespace to config.DefaultNamespace. -func loadHostConf(filePath string) (*HostConf, error) { - if filePath == "" { - filePath = config.DefaultConfFile - } - data, err := os.ReadFile(filePath) - if err != nil { - if os.IsNotExist(err) { - // Tolerated, return defaulted config. - return &HostConf{ - Namespace: config.DefaultNamespace, - }, nil - } - return nil, fmt.Errorf("read conflist file %q: %w", filePath, err) - } - - var env conflistEnvelope - if err := json.Unmarshal(data, &env); err != nil { - return nil, fmt.Errorf("parse conflist envelope: %w", err) - } - - for _, raw := range env.Plugins { - var meta struct { - Type string `json:"type"` - } - if err := json.Unmarshal(raw, &meta); err != nil { - continue - } - if meta.Type == "galactic-cni" { - var conf HostConf - if err := json.Unmarshal(raw, &conf); err != nil { - return nil, fmt.Errorf("parse host CNI config: %w", err) - } - if conf.Namespace == "" { - conf.Namespace = config.DefaultNamespace - } - return &conf, nil - } - } - - return nil, fmt.Errorf("conflist at %q does not contain a plugin with type \"galactic-cni\"", filePath) -} - -// detectNodeNameFromAPI queries the Kubernetes API and matches the node's -// InternalIP addresses against local interface addresses. Returns the first -// matching node name, or empty string with no error if detection fails -// (allowing callers to fall through to other resolution methods). -func detectNodeNameFromAPI() (string, error) { - restCfg, err := ctrl.GetConfig() - if err != nil { - return "", fmt.Errorf("get kubeconfig: %w", err) - } - - k8sClient, err := client.New(restCfg, client.Options{ - Scheme: buildDetectScheme(), - }) - if err != nil { - return "", fmt.Errorf("create k8s client: %w", err) - } - - var nodeList corev1.NodeList - if err := k8sClient.List(context.Background(), &nodeList, &client.ListOptions{ - Limit: 1000, - }); err != nil { - return "", fmt.Errorf("list nodes: %w", err) - } - - // Collect all local interface addresses - addrs, err := netlink.AddrList(nil, netlink.FAMILY_ALL) - if err != nil { - return "", fmt.Errorf("list local addresses: %w", err) - } - - localIPs := make(map[string]bool, len(addrs)) - for _, addr := range addrs { - localIPs[addr.IP.String()] = true - } - - // Match against node InternalIPs - for _, node := range nodeList.Items { - for _, addr := range node.Status.Addresses { - if addr.Type == corev1.NodeInternalIP && localIPs[addr.Address] { - slog.Info("Auto-detected node name from Kubernetes API", - "nodeName", node.Name, "matchedIP", addr.Address) - return node.Name, nil - } - } - } - - return "", errors.New("no local interface address matched any node InternalIP") -} - -// buildDetectScheme returns a minimal scheme containing only corev1 types -// needed for node name detection. -func buildDetectScheme() *runtime.Scheme { - scheme := runtime.NewScheme() - _ = corev1.AddToScheme(scheme) - return scheme -} - -// parseLogLevel maps a config-supplied level name to a slog.Level. Matching is -// case-insensitive. An empty string resolves to config.DefaultLogLevel. -// Unrecognized values return an error alongside the info-level fallback, so -// callers can warn without failing the CNI operation over a typo'd setting. -func parseLogLevel(s string) (slog.Level, error) { - switch strings.ToLower(strings.TrimSpace(s)) { - case "": - return parseLogLevel(config.DefaultLogLevel) - case config.LogLevelDebug: - return slog.LevelDebug, nil - case config.DefaultLogLevel: - return slog.LevelInfo, nil - case config.LogLevelWarn, config.LogLevelWarning: - return slog.LevelWarn, nil - case config.LogLevelError: - return slog.LevelError, nil - default: - return slog.LevelInfo, fmt.Errorf("unknown log level %q (want %s, %s, %s, or %s)", - s, config.LogLevelDebug, config.DefaultLogLevel, config.LogLevelWarn, config.LogLevelError) - } -} - -// setupLogging configures the slog default logger to write to the specified -// path at the specified verbosity. If opening the file fails, it logs a -// warning to os.Stderr and falls back. An unrecognized logLevel also logs a -// warning and falls back to config.DefaultLogLevel rather than failing the -// operation. -func setupLogging(logPath, logLevel string) { - if logPath == "" { - logPath = config.DefaultLogFile - } - level, err := parseLogLevel(logLevel) - if err != nil { - slog.Warn("Invalid log level, falling back to default", - "value", logLevel, "default", config.DefaultLogLevel, "err", err) - } - // Ensure parent directory exists. - if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { - slog.Warn("Failed to create log directory", "path", filepath.Dir(logPath), "err", err) - return - } - file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - slog.Warn("Failed to open log file, falling back to Stderr", "path", logPath, "err", err) - return - } - // Use JSON handler for structured logging to file. - handler := slog.NewJSONHandler(file, &slog.HandlerOptions{Level: level}) - slog.SetDefault(slog.New(handler)) -} - -// statusConf holds the minimal CNI config fields needed for STATUS validation. - -// STATUS only checks that the config is parseable and the API server is reachable; -// it does not validate attachment-specific fields (VPC, VPCAttachment) because -// STATUS must succeed before any ADD has ever run. -type statusConf struct { - CNIVersion string `json:"cniVersion"` - Type string `json:"type"` - InterfaceType string `json:"interface_type"` -} - -// parseStatusConf validates that the CNI config is parseable and contains the -// required top-level fields (cniVersion, type). Unlike parseConf, it does not -// validate VPC or VPCAttachment because STATUS must succeed on a freshly -// started node before any ADD has run. However, interface_type is validated -// if present because it is a structural config field, not an attachment -// identifier. -func parseStatusConf(data []byte) error { - var sc statusConf - if err := json.Unmarshal(data, &sc); err != nil { - return &types.Error{Code: 7, Msg: errInvalidCNIConfig, Details: err.Error()} - } - if sc.CNIVersion == "" { - return &types.Error{Code: 7, Msg: "cniVersion is required"} - } - if sc.Type == "" { - return &types.Error{Code: 7, Msg: "type is required"} - } - // Validate interface_type if present. - if sc.InterfaceType != "" { - switch sc.InterfaceType { - case interfaceTypeVeth, interfaceTypeTap: - default: - return &types.Error{Code: 7, Msg: fmt.Sprintf( - "invalid interface_type %q: must be %q or %q", - sc.InterfaceType, interfaceTypeVeth, interfaceTypeTap, - )} - } - } - return nil -} - -// validatePrevResult checks that the prevResult (from a preceding plugin in -// the CNI chain) is a valid, parseable CNI result. Returns an error if the -// result is non-nil but cannot be parsed as a versioned CNI result, ensuring -// galactic-cni fails fast rather than silently operating on garbage state. -func validatePrevResult(res types.Result) error { - if res == nil { - return nil - } - // Marshal to JSON and re-parse to verify the result is structurally valid. - // This catches malformed results that survived CNI framework unmarshaling. - jsonBytes, err := json.Marshal(res) - if err != nil { - return fmt.Errorf("marshal prevResult: %w", err) - } - if _, err := type100.NewResult(jsonBytes); err != nil { - return fmt.Errorf("parse prevResult: %w", err) - } - return nil -} - -// validatePrevResultAdd performs content-level validation of prevResult during -// the ADD operation. It ensures the preceding plugin produced a result with at -// least one interface or IP assignment, which is the minimum expected structure -// for any meaningful CNI chain. Returns nil when prevResult is nil (no -// preceding plugin) or structurally valid with expected content. -func validatePrevResultAdd(res types.Result) error { - if res == nil { - return nil - } - jsonBytes, err := json.Marshal(res) - if err != nil { - return fmt.Errorf("marshal prevResult: %w", err) - } - result, err := type100.NewResult(jsonBytes) - if err != nil { - return fmt.Errorf("parse prevResult: %w", err) - } - versioned, err := type100.GetResult(result) - if err != nil { - return fmt.Errorf("get prevResult version: %w", err) - } - // A valid prevResult must declare at least one interface or IP assignment. - if len(versioned.Interfaces) == 0 && len(versioned.IPs) == 0 { - return errors.New("prevResult declares no interfaces or IP assignments") - } - return nil -} - -// parseConf unmarshals the CNI configuration from stdin data and validates -// the interface type and base62-encoded identifier fields. It resolves the -// host configuration and sets up process environment variables and logging. +// parseConf unmarshals the CNI configuration from stdin data, validates the +// base62-encoded identifier fields, and resolves logging. The actual logic +// is shared with galactic-tap-cni — see internal/cnimaster.ParseConf — since +// none of it is veth-specific; this is a thin wrapper binding it to this +// binary's own cniConfig/ConfFile. func parseConf(data []byte) (*PluginConf, error) { - conf := &PluginConf{} - if err := json.Unmarshal(data, &conf); err != nil { - return nil, &types.Error{Code: 7, Msg: errInvalidCNIConfig, Details: err.Error()} - } - if !isValidBase62(conf.VPC) { - if len(conf.VPC) == 0 { - return nil, &types.Error{Code: 7, Msg: errVPCRequired} - } - return nil, &types.Error{ - Code: 7, - Msg: fmt.Sprintf("invalid base62 value for field 'vpc': %q", sanitizeForError(conf.VPC)), - } - } - if !isValidBase62(conf.VPCAttachment) { - if len(conf.VPCAttachment) == 0 { - return nil, &types.Error{Code: 7, Msg: errVPCAttachmentRequired} - } - return nil, &types.Error{ - Code: 7, - Msg: fmt.Sprintf("invalid base62 value for field 'vpcattachment': %q", sanitizeForError(conf.VPCAttachment)), - } - } - - // Load host CNI config - hostConf, err := loadHostConf(ConfFile) - if err != nil { - return nil, fmt.Errorf("load host CNI config: %w", err) - } - - // Resolve config: env var > conflist > default. - cniConfig.Resolve(&config.ConflistValues{ - NodeName: hostConf.NodeName, - Kubeconfig: hostConf.Kubeconfig, - Namespace: hostConf.Namespace, - LogFile: hostConf.LogFile, - LogLevel: hostConf.LogLevel, - }) - - // NodeName fallback: auto-detect from the Kubernetes API by matching local - // interface addresses against node InternalIPs. This handles cases where - // the conflist file is missing (e.g. hostPath mount issues in container- - // based environments like Kind). - if cniConfig.NodeName == "" { - detected, detectErr := detectNodeNameFromAPI() - if detectErr != nil { - slog.Warn("Node name auto-detection failed", "err", detectErr) - } - cniConfig.NodeName = detected - } - if cniConfig.NodeName == "" { - return nil, &types.Error{Code: 4, Msg: "node name is required (or set GALACTIC_CNI_NODE_NAME)"} - } - _ = os.Setenv("NODE_NAME", cniConfig.NodeName) - - // Propagate Kubeconfig - _ = os.Setenv("KUBECONFIG", cniConfig.Kubeconfig) - - // Resolve and propagate Namespace fallback - namespace := conf.Namespace - if namespace == "" { - namespace = cniConfig.Namespace - } - conf.Namespace = namespace - - // Setup Logging - setupLogging(cniConfig.LogFile, cniConfig.LogLevel) - slog.Debug("CNI config received", "stdin", string(data)) - - // Resolve local IPAM flag - enableLocalIPAM = config.CNIGetEnableLocalIPAM() - - // Enforce required IPAM block if local IPAM is enabled - if enableLocalIPAM && conf.IPAM == nil { - return nil, &types.Error{Code: 7, Msg: "local IPAM is enabled, but no 'ipam' block is present in the configuration"} - } - - // Validate dual-stack addressing fields (ipv6_subnet, ipv4_subnet, - // address_families). Both subnet fields stay optional at the parseConf - // level: whether one is actually required depends on which IPAM path a - // given ADD takes (static, local-IPAM fallback, or pool), which is - // resolved in allocateIPAM — see wantsIPAM/allocateIPAM in ipam_ops.go. - // When present, both are validated for CIDR shape so misconfigurations - // are caught early regardless of which path runs. - if conf.IPv6Subnet != "" { - ip, mask, err := net.ParseCIDR(conf.IPv6Subnet) - if err != nil { - return nil, &types.Error{Code: 7, Msg: fmt.Sprintf( - "invalid CIDR value for field 'ipv6_subnet': %q", sanitizeForError(conf.IPv6Subnet)), - } - } - if ip.To4() != nil { - return nil, &types.Error{Code: 7, Msg: fmt.Sprintf( - "ipv6_subnet must be an IPv6 CIDR, got IPv4: %q", sanitizeForError(conf.IPv6Subnet)), - } - } - if prefixLen, _ := mask.Mask.Size(); prefixLen > maxIPv6SubnetPrefixLen { - return nil, &types.Error{Code: 7, Msg: fmt.Sprintf( - "ipv6_subnet prefix length %d exceeds maximum of %d: %q", - prefixLen, maxIPv6SubnetPrefixLen, sanitizeForError(conf.IPv6Subnet)), - } - } - } - if conf.IPv4Subnet != "" { - ip, mask, err := net.ParseCIDR(conf.IPv4Subnet) - if err != nil { - return nil, &types.Error{Code: 7, Msg: fmt.Sprintf( - "invalid CIDR value for field 'ipv4_subnet': %q", sanitizeForError(conf.IPv4Subnet)), - } - } - if ip.To4() == nil { - return nil, &types.Error{Code: 7, Msg: fmt.Sprintf( - "ipv4_subnet must be an IPv4 CIDR, got IPv6: %q", sanitizeForError(conf.IPv4Subnet)), - } - } - if prefixLen, _ := mask.Mask.Size(); prefixLen > maxIPv4SubnetPrefixLen { - return nil, &types.Error{Code: 7, Msg: fmt.Sprintf( - "ipv4_subnet prefix length %d exceeds maximum of %d: %q", - prefixLen, maxIPv4SubnetPrefixLen, sanitizeForError(conf.IPv4Subnet)), - } - } - } - if len(conf.AddressFamilies) == 0 { - conf.AddressFamilies = []string{addressFamilyIPv6} - } else { - for _, af := range conf.AddressFamilies { - switch af { - case addressFamilyIPv6, addressFamilyIPv4: - default: - return nil, &types.Error{Code: 7, Msg: fmt.Sprintf( - "invalid address_families entry %q: must be %q or %q", - sanitizeForError(af), addressFamilyIPv6, addressFamilyIPv4), - } - } - } - } - - if conf.PrevResult != nil { - if err := validatePrevResult(conf.PrevResult); err != nil { - return nil, &types.Error{Code: 6, Msg: fmt.Sprintf("invalid prevResult: %v", err)} - } - } - if conf.InterfaceType == "" { - conf.InterfaceType = interfaceTypeVeth - } - switch conf.InterfaceType { - case interfaceTypeVeth, interfaceTypeTap: - default: - return nil, &types.Error{Code: 7, Msg: fmt.Sprintf( - "invalid interface_type %q: must be %q or %q", - conf.InterfaceType, interfaceTypeVeth, interfaceTypeTap, - )} - - } - return conf, nil -} - -// sanitizeForError returns s unchanged if it contains only printable ASCII -// characters; otherwise returns "" to avoid corrupting log output. -func sanitizeForError(s string) string { - for _, c := range s { - if c < 0x20 || c > 0x7e { - return sanitizeForErrorBinary - } - } - return s -} - -// subnetAnnotationKeyIPv6 returns the annotation key for storing the -// allocated IPv6 subnet for the given container ID. Kubernetes limits the -// name part of an annotation key to 63 bytes; "allocated-subnet-ipv6." is 22 -// bytes, leaving 41 bytes for the container ID prefix. -func subnetAnnotationKeyIPv6(containerID string) string { - id := containerID - if len(id) > annotationContainerIDLen { - id = id[:annotationContainerIDLen] - } - return fmt.Sprintf("%s.%s", annotationAllocatedSubnetIPv6, id) -} - -// subnetAnnotationKeyIPv4 returns the annotation key for storing the -// allocated IPv4 address for the given container ID. Mirrors -// subnetAnnotationKeyIPv6. -func subnetAnnotationKeyIPv4(containerID string) string { - id := containerID - if len(id) > annotationContainerIDLen { - id = id[:annotationContainerIDLen] - } - return fmt.Sprintf("%s.%s", annotationAllocatedSubnetIPv4, id) -} - -// netnsAnnotationKey returns the annotation key for storing the network -// namespace path used by the given container ID. Mirrors subnetAnnotationKeyIPv6. -func netnsAnnotationKey(containerID string) string { - id := containerID - if len(id) > annotationContainerIDLen { - id = id[:annotationContainerIDLen] - } - return fmt.Sprintf("%s.%s", annotationNetNS, id) + return cnimaster.ParseConf(data, cniConfig, ConfFile) } diff --git a/internal/cni/doc.go b/internal/cni/doc.go index 8c9c896c..0813e35a 100644 --- a/internal/cni/doc.go +++ b/internal/cni/doc.go @@ -2,21 +2,30 @@ // // SPDX-License-Identifier: AGPL-3.0-or-later -// Package cni implements the Galactic CNI plugin for wiring containers into -// SRv6-backed VPC networks. +// Package cni implements galactic-cni, the veth master plugin for wiring +// container workloads into SRv6-backed VPC networks. Tap-based workloads +// (Kata, Firecracker, kraftlet/Unikraft) are galactic-tap-cni's own master +// plugin (internal/cnitap) — interface kind is which binary is invoked now, +// not a config field either binary branches on. // -// On ADD the plugin creates a VRF, a veth or tap interface, installs -// termination routes in the VRF table, allocates a pod subnet via IPAM, -// and publishes BGPAdvertisement/BGPVRFInstance CRDs for route distribution. -// On DEL it performs best-effort cleanup in reverse order. CHECK and STATUS -// validate that managed kernel resources are intact. +// On ADD the plugin creates a VRF, a veth pair, and patches the pod's NAD +// with the host interface name. On DEL it performs best-effort cleanup in +// reverse order. CHECK and STATUS validate that managed kernel resources are +// intact. IPAM allocation, termination-route installation, and +// BGPAdvertisement/BGPVRFInstance publish are no longer this package's +// concern — they're galactic-ipam's, galactic-route's, and galactic-bgp's +// own, chained after this plugin per the conflist (see +// internal/cniipam, internal/cniroute, internal/cnibgp). // // Subpackages isolate kernel primitives: // -// - ipam: IPv6 subnet allocation from a CIDR pool or static address -// - route: VRF route add/delete for termination gateways // - veth: veth pair creation for container workloads -// - tap: TAP device creation for VM workloads (Kata, Firecracker) +// +// internal/cni/ipam, internal/cni/route, and internal/cni/tap are the same +// kind of kernel-primitive package, but are no longer used by this package +// itself — they're used exclusively by internal/cniipam, internal/cniroute, +// and internal/cnitap respectively, now that IPAM, termination routes, and +// tap are each their own chain-invoked binary. // // Usage: // diff --git a/internal/cni/hostgw/hostgw.go b/internal/cni/hostgw/hostgw.go new file mode 100644 index 00000000..9d036e20 --- /dev/null +++ b/internal/cni/hostgw/hostgw.go @@ -0,0 +1,212 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package hostgw configures the host-side gateway address and VRF-table +// pod-subnet route for a VPC attachment's allocated IPAM addresses. +// +// This is kernel-interface work (netlink address/route/neighbor +// manipulation on the interface a master plugin — galactic-cni, +// galactic-tap-cni — itself created), not BGP/SRv6/eBPF publish, so it +// lives here rather than in internal/cnibgp: once galactic-bgp became its +// own chain-invoked plugin (a separate process, invoked after the master +// has already printed its own result), it no longer has any interface to +// configure — "zero kernel-interface dependency" is the whole reason that +// split was worth doing in the first place. Both master plugins call this +// directly, before building their own CNI result; galactic-bgp reads +// whatever addresses ended up in prevResult and never touches the kernel +// interface at all. +package hostgw + +import ( + "errors" + "fmt" + "net" + "syscall" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + + "go.datum.net/galactic/internal/cniipam" + "go.datum.net/galactic/internal/plumbing/intf" + "go.datum.net/galactic/internal/plumbing/vrf" +) + +// ConfigureHostGateway assigns each configured family's gateway address as a +// host address (/128 for IPv6, /32 for IPv4 on veth) on the host-side +// interface (veth or tap) and installs an explicit pod-subnet route for that +// family into the VRF table. IPv4 is skipped entirely when the attachment is +// IPv6-only. +// +// Using a full-length host address (not the pod subnet mask) prevents the +// kernel from auto-creating a subnet-router anycast entry in the VRF local +// table. When the pod address equals the subnet network address the anycast +// absorbs seg6local-decapped inner packets before they reach the guest +// interface. The explicit subnet route replaces the one the kernel would +// have created from the wider mask. +// +// For tap interfaces, the IPv4 gateway is instead assigned as a /25 so the +// address reported on the interface reflects a real subnet (VM guests expect +// this). That reintroduces the wider-mask hazard described above, so the +// address is added with IFA_F_NOPREFIXROUTE: the kernel skips auto-creating +// the connected /25 route entirely, leaving the explicit pod-subnet route +// below as the only thing that governs delivery to this VM's address. +// +// guestHWAddr is the guest-side veth's MAC address, used to prime a +// permanent neighbor table entry for the pod's own address (see +// installGatewayNeighbor). It is nil for tap attachments, which have no +// separate guest-side link in this netns to resolve a MAC from. +func ConfigureHostGateway(vpc, vpcAttachment string, res *cniipam.IPAMResult, guestHWAddr net.HardwareAddr) error { + if res == nil { + return nil + } + hostName := intf.GenerateInterfaceNameHost(vpc, vpcAttachment) + hostLink, err := netlink.LinkByName(hostName) + if err != nil { + return fmt.Errorf("get host interface %q: %w", hostName, err) + } + tableID, err := vrf.TableID(vpc, vpcAttachment) + if err != nil { + return fmt.Errorf("get VRF table ID for pod subnet route: %w", err) + } + + if res.IPv6Gateway != nil { + gwNet := &net.IPNet{IP: res.IPv6Gateway, Mask: net.CIDRMask(128, 128)} + if err := installGatewayRoute(hostLink, gwNet, res.IPv6Subnet, netlink.FAMILY_V6, int(tableID), 0); err != nil { + return err + } + if guestHWAddr != nil { + if err := installGatewayNeighbor(hostLink, res.IPv6Subnet.IP, netlink.FAMILY_V6, guestHWAddr); err != nil { + return err + } + } + } + if res.IPv4Gateway != nil { + ipv4Mask, addrFlags := ipv4GatewayAddrParams(hostLink) + gwNet := &net.IPNet{IP: res.IPv4Gateway, Mask: ipv4Mask} + ipv4Subnet := &net.IPNet{IP: res.IPv4Address, Mask: net.CIDRMask(32, 32)} + if err := installGatewayRoute(hostLink, gwNet, ipv4Subnet, netlink.FAMILY_V4, int(tableID), addrFlags); err != nil { + return err + } + if guestHWAddr != nil { + if err := installGatewayNeighbor(hostLink, res.IPv4Address, netlink.FAMILY_V4, guestHWAddr); err != nil { + return err + } + } + } + return nil +} + +// installGatewayNeighbor installs a permanent neighbor table entry mapping +// podIP to guestHWAddr on hostLink. +// +// The eBPF uSID ingress datapath (internal/plumbing/ebpf/prog/usid.c) +// decapsulates SRv6 traffic and calls bpf_fib_lookup() to resolve the +// egress path for the inner packet, then redirects it straight to the +// resolved neighbor — entirely in-kernel, never touching the normal +// forwarding stack. bpf_fib_lookup() does not itself trigger ARP/NDP +// resolution the way ordinary kernel packet forwarding does, so without a +// pre-existing neighbor table entry it fails with BPF_FIB_LKUP_RET_NO_NEIGH +// and the datapath drops the packet. A permanent entry (installed once, at +// CNI ADD, using the guest veth's own known MAC) means this resolution +// never depends on dynamic ARP/NDP at all. +func installGatewayNeighbor(hostLink netlink.Link, podIP net.IP, family int, guestHWAddr net.HardwareAddr) error { + neigh := &netlink.Neigh{ + LinkIndex: hostLink.Attrs().Index, + Family: family, + State: netlink.NUD_PERMANENT, + IP: podIP, + HardwareAddr: guestHWAddr, + } + if err := netlink.NeighSet(neigh); err != nil { + return fmt.Errorf("add permanent neighbor %s -> %s on host interface %q: %w", + podIP, guestHWAddr, hostLink.Attrs().Name, err) + } + return nil +} + +// ipv4GatewayAddrParams returns the IPv4 gateway mask and netlink address +// flags to use for hostLink. Tap interfaces get a /25 (so the address +// reported on the interface reflects a real subnet) with +// IFA_F_NOPREFIXROUTE, which stops the kernel from auto-creating a connected +// route for the wider mask. Veth interfaces keep the plain /32 host address +// with no flags. +func ipv4GatewayAddrParams(hostLink netlink.Link) (net.IPMask, int) { + if _, isTap := hostLink.(*netlink.Tuntap); isTap { + return net.CIDRMask(25, 32), unix.IFA_F_NOPREFIXROUTE + } + return net.CIDRMask(32, 32), 0 +} + +// installGatewayRoute assigns gwNet as a host address on hostLink and +// installs an explicit route to subnet into the given VRF table, for one +// address family. Idempotent: existing matching routes/addresses are left +// alone, and conflicting ones return an error rather than being overwritten. +func installGatewayRoute(hostLink netlink.Link, gwNet, subnet *net.IPNet, family, tableID, addrFlags int) error { + hostName := hostLink.Attrs().Name + if err := netlink.AddrAdd(hostLink, &netlink.Addr{IPNet: gwNet, Flags: addrFlags}); err != nil { + if !errors.Is(err, syscall.EEXIST) { + return fmt.Errorf("add gateway address %s to host interface %q: %w", gwNet, hostName, err) + } + } + + desiredRoute := &netlink.Route{ + Dst: subnet, + LinkIndex: hostLink.Attrs().Index, + Table: tableID, + } + + existingRoutes, err := netlink.RouteListFiltered( + family, + &netlink.Route{Table: tableID}, + netlink.RT_FILTER_TABLE, + ) + if err != nil { + return fmt.Errorf("list routes in VRF table: %w", err) + } + for _, r := range existingRoutes { + if r.Dst == nil { + continue + } + if r.Dst.String() != desiredRoute.Dst.String() { + continue + } + if routeConflicts(&r, desiredRoute) { + return fmt.Errorf( + "existing route %v to %s conflicts with desired route %v", + r, desiredRoute.Dst, desiredRoute, + ) + } + return nil + } + + if err := netlink.RouteAdd(desiredRoute); err != nil { + if errors.Is(err, syscall.EEXIST) { + return nil + } + return fmt.Errorf("add pod subnet route to VRF table: %w", err) + } + return nil +} + +// routeConflicts reports whether an existing route conflicts with the desired +// pod-subnet route. A conflict occurs when the destination matches but the +// gateway or link index differs. +func routeConflicts(existing, desired *netlink.Route) bool { + if existing.Dst == nil || desired.Dst == nil { + return false + } + if existing.Dst.String() != desired.Dst.String() { + return false + } + if (existing.Gw != nil) != (desired.Gw != nil) { + return true + } + if existing.Gw != nil && !existing.Gw.Equal(desired.Gw) { + return true + } + if existing.LinkIndex != 0 && desired.LinkIndex != 0 && existing.LinkIndex != desired.LinkIndex { + return true + } + return false +} diff --git a/internal/cni/hostgw/hostgw_test.go b/internal/cni/hostgw/hostgw_test.go new file mode 100644 index 00000000..f251994a --- /dev/null +++ b/internal/cni/hostgw/hostgw_test.go @@ -0,0 +1,115 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package hostgw + +import ( + "net" + "testing" + + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +func mustParseCIDR(t *testing.T, cidr string) *net.IPNet { + t.Helper() + _, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + t.Fatalf("parse CIDR %q: %v", cidr, err) + } + return ipnet +} + +func TestIPv4GatewayAddrParams(t *testing.T) { + tests := []struct { + name string + hostLink netlink.Link + wantMask net.IPMask + wantFlags int + }{ + { + name: "tap gets /25 with NOPREFIXROUTE", + hostLink: &netlink.Tuntap{LinkAttrs: netlink.LinkAttrs{Name: "tap0"}}, + wantMask: net.CIDRMask(25, 32), + wantFlags: unix.IFA_F_NOPREFIXROUTE, + }, + { + name: "veth gets /32 with no flags", + hostLink: &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: "veth0"}}, + wantMask: net.CIDRMask(32, 32), + wantFlags: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotMask, gotFlags := ipv4GatewayAddrParams(tt.hostLink) + if gotMask.String() != tt.wantMask.String() { + t.Errorf("mask = %v, want %v", gotMask, tt.wantMask) + } + if gotFlags != tt.wantFlags { + t.Errorf("flags = %v, want %v", gotFlags, tt.wantFlags) + } + }) + } +} + +func TestRouteConflicts(t *testing.T) { + dst := mustParseCIDR(t, "fd00:10:ff01::1234/80") + gw1 := net.ParseIP("fd00:10:ff01::1") + gw2 := net.ParseIP("fd00:10:ff01::2") + otherDst := mustParseCIDR(t, "fd00:10:ff02::1234/80") + + tests := []struct { + name string + existing *netlink.Route + desired *netlink.Route + want bool + }{ + {"nil existing destination — no conflict", &netlink.Route{Dst: nil}, &netlink.Route{Dst: dst}, false}, + {"nil desired destination — no conflict", &netlink.Route{Dst: dst}, &netlink.Route{Dst: nil}, false}, + {"different destinations — no conflict", &netlink.Route{Dst: otherDst}, &netlink.Route{Dst: dst}, false}, + { + "same destination, no gateway on either — no conflict", + &netlink.Route{Dst: dst, LinkIndex: 5}, &netlink.Route{Dst: dst, LinkIndex: 5}, false, + }, + { + "same destination, same gateway — no conflict", + &netlink.Route{Dst: dst, Gw: gw1, LinkIndex: 5}, &netlink.Route{Dst: dst, Gw: gw1, LinkIndex: 5}, false, + }, + { + "same destination, different gateway — conflict", + &netlink.Route{Dst: dst, Gw: gw1}, &netlink.Route{Dst: dst, Gw: gw2}, true, + }, + { + "existing has gateway, desired does not — conflict", + &netlink.Route{Dst: dst, Gw: gw1}, &netlink.Route{Dst: dst}, true, + }, + { + "desired has gateway, existing does not — conflict", + &netlink.Route{Dst: dst}, &netlink.Route{Dst: dst, Gw: gw1}, true, + }, + { + "same destination, same gateway, different link index — conflict", + &netlink.Route{Dst: dst, Gw: gw1, LinkIndex: 5}, &netlink.Route{Dst: dst, Gw: gw1, LinkIndex: 7}, true, + }, + { + "same destination, gateway set, link index zero on existing — no conflict", + &netlink.Route{Dst: dst, Gw: gw1, LinkIndex: 0}, &netlink.Route{Dst: dst, Gw: gw1, LinkIndex: 5}, false, + }, + { + "same destination, no gateway, different link index — conflict", + &netlink.Route{Dst: dst, LinkIndex: 5}, &netlink.Route{Dst: dst, LinkIndex: 7}, true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := routeConflicts(tt.existing, tt.desired) + if got != tt.want { + t.Errorf("routeConflicts() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/cni/ipam/dualstack.go b/internal/cni/ipam/dualstack.go index 9a0c687b..bf9edfad 100644 --- a/internal/cni/ipam/dualstack.go +++ b/internal/cni/ipam/dualstack.go @@ -31,16 +31,17 @@ type DualStackResult struct { // the resulting allocator only allocates IPv4 addresses (IPv6 fields in // DualStackResult are left nil). If ipv4Pool is empty, the resulting // allocator only allocates IPv6 addresses (IPv4 fields in DualStackResult -// are left nil) and ipv4LockDir is ignored. ipv4LockDir is passed straight -// through to NewIPv4PoolAllocator (see DefaultIPv4LockDir for the production -// path). +// are left nil). lockDir is passed straight through to both +// NewPoolAllocator and NewIPv4PoolAllocator (see DefaultLockDir for the +// production path) — one shared root serves both families, since each +// pool's own CIDR namespaces its state into a distinct subdirectory. func NewDualStackAllocator( - ipv6Pool, ipv6Gateway, ipv4Pool, ipv4Gateway, ipv4LockDir string, + ipv6Pool, ipv6Gateway, ipv4Pool, ipv4Gateway, lockDir string, ) (*DualStackAllocator, error) { a := &DualStackAllocator{} if ipv6Pool != "" { - ipv6, err := NewPoolAllocator(ipv6Pool, ipv6Gateway, DefaultSubnetLen) + ipv6, err := NewPoolAllocator(ipv6Pool, ipv6Gateway, DefaultSubnetLen, lockDir) if err != nil { return nil, err } @@ -48,7 +49,7 @@ func NewDualStackAllocator( } if ipv4Pool != "" { - ipv4, err := NewIPv4PoolAllocator(ipv4Pool, ipv4Gateway, ipv4LockDir) + ipv4, err := NewIPv4PoolAllocator(ipv4Pool, ipv4Gateway, lockDir) if err != nil { return nil, err } diff --git a/internal/cni/ipam/dualstack_test.go b/internal/cni/ipam/dualstack_test.go index bc72e67a..ab231993 100644 --- a/internal/cni/ipam/dualstack_test.go +++ b/internal/cni/ipam/dualstack_test.go @@ -4,7 +4,10 @@ package ipam -import "testing" +import ( + "fmt" + "testing" +) func TestNewDualStackAllocator(t *testing.T) { tests := []struct { @@ -167,8 +170,12 @@ func TestDualStackAllocatorAllocate(t *testing.T) { t.Fatalf("unexpected error: %v", err) } + // Each iteration uses a distinct container ID -- Allocate is + // idempotent per containerID, so reusing one ID would only ever + // consume a single IPv4 address instead of exhausting the pool. for i := range 4 { - if _, err := a.Allocate("container"); err != nil { + containerID := fmt.Sprintf("container-%d", i) + if _, err := a.Allocate(containerID); err != nil { t.Fatalf("unexpected error on allocation %d: %v", i, err) } } diff --git a/internal/cni/ipam/ipam.go b/internal/cni/ipam/ipam.go index b4a90eab..f78b99d0 100644 --- a/internal/cni/ipam/ipam.go +++ b/internal/cni/ipam/ipam.go @@ -2,17 +2,25 @@ // // SPDX-License-Identifier: AGPL-3.0-or-later -// Package ipam provides IPv6 subnet allocation for the Galactic CNI. -// Each allocation returns a subnet (default /96) from a larger CIDR pool -// (e.g. a /64 region subnet). Allocations are kept ephemeral in memory; -// separate CNI plugin processes (each invocation is a separate process) rely -// on the BGPAdvertisement CRD annotation to look up the allocated subnet -// during teardown. +// Package ipam provides IPv6 subnet allocation for the Galactic CNI. Each +// allocation returns a subnet (default /96) from a larger CIDR pool (e.g. a +// /64 region subnet). +// +// Allocations persist as an on-disk marker file per allocated subnet, keyed +// by the pool CIDR (mirroring IPv4PoolAllocator's own scheme) — required +// because each CNI ADD/DEL is a separate OS process: an in-memory-only +// record (as this package used before) is discarded the moment the ADD +// process that created it exits, leaving DEL with nothing to look up. package ipam import ( + "errors" "fmt" "net" + "os" + "path/filepath" + "strconv" + "strings" "sync" ) @@ -23,23 +31,41 @@ const ( // DefaultSubnetLen is the default prefix length returned per allocation. // A /96 gives 2^32 addresses per pod subnet. DefaultSubnetLen = 96 + + // poolLockFileName is the flock target within each pool's state + // directory; every other entry in that directory is an allocation + // marker file named after the subnet it reserves. + poolLockFileName = "lock" ) -// PoolAllocator allocates IPv6 subnets from a CIDR pool, tracking -// allocations by subnet CIDR string in memory. All bindings are ephemeral. +// DefaultLockDir is the well-known parent directory for the node-local +// on-disk lock and allocation state both PoolAllocator (IPv6) and +// IPv4PoolAllocator use to stay correct across separate CNI plugin +// invocations. Both families share one root — each pool's own CIDR +// namespaces its state into a distinct subdirectory (sanitizePoolDirName), +// so an IPv6 pool and an IPv4 pool never collide here. +const DefaultLockDir = "/var/lib/cni/galactic-ipam" + +// PoolAllocator allocates IPv6 subnets from a CIDR pool, persisting each +// allocation as a marker file under a lock directory, guarded by a +// cross-process flock — see IPv4PoolAllocator's own doc comment for why +// this is required rather than optional. type PoolAllocator struct { - pool *net.IPNet // the master pool (e.g. a /64 region subnet) - subnetLen int // prefix length per allocation (e.g. 96) - gateway net.IP // gateway IP address - poolIP net.IP // immutable copy of pool.IP for boundary checks - reserved string // subnet CIDR string containing the gateway; never allocated - allocations sync.Map // allocated subnet CIDR string -> struct{}{} - mu sync.Mutex // serializes Allocate calls + pool *net.IPNet // the master pool (e.g. a /64 region subnet) + subnetLen int // prefix length per allocation (e.g. 96) + gateway net.IP // gateway IP address + poolIP net.IP // immutable copy of pool.IP for boundary checks + reserved string // subnet CIDR string containing the gateway; never allocated + mu sync.Mutex // serializes Allocate/Deallocate within this process + stateDir string // directory holding the lock file and one allocation marker file per allocated subnet + state lockedState } -// NewPoolAllocator creates a new pool allocator from an IPv6 CIDR pool, -// an optional gateway address, and a subnet prefix length. The pool must be -// an IPv6 prefix with a length of subnetLen or fewer bits (e.g. a /64 region +// NewPoolAllocator creates a new pool allocator from an IPv6 CIDR pool, an +// optional gateway address, a subnet prefix length, and a parent directory +// for this pool's on-disk lock and allocation state (see DefaultLockDir for +// the production path; lockDir must not be empty). The pool must be an +// IPv6 prefix with a length of subnetLen or fewer bits (e.g. a /64 region // subnet when subnetLen is the default /96, though any pool length <= // subnetLen is accepted). If gateway is empty, the first address in the pool // (host bits = 1) is used as the gateway. If subnetLen is 0, DefaultSubnetLen @@ -48,7 +74,7 @@ type PoolAllocator struct { // self-assign the gateway's own address to one of its secondary/pod // addresses, colliding with the address every other endpoint in the pool // routes its default route through. -func NewPoolAllocator(poolCIDR, gateway string, subnetLen int) (*PoolAllocator, error) { +func NewPoolAllocator(poolCIDR, gateway string, subnetLen int, lockDir string) (*PoolAllocator, error) { _, pool, err := net.ParseCIDR(poolCIDR) if err != nil { return nil, fmt.Errorf("parse pool CIDR %q: %w", poolCIDR, err) @@ -66,11 +92,14 @@ func NewPoolAllocator(poolCIDR, gateway string, subnetLen int) (*PoolAllocator, return nil, fmt.Errorf("pool prefix length %d is longer than subnet length %d", mask, subnetLen) } + if lockDir == "" { + return nil, errors.New("lockDir must not be empty") + } + pa := &PoolAllocator{ - pool: pool, - subnetLen: subnetLen, - poolIP: make(net.IP, ipv6Bits/8), - allocations: sync.Map{}, + pool: pool, + subnetLen: subnetLen, + poolIP: make(net.IP, ipv6Bits/8), } copy(pa.poolIP, pool.IP) @@ -97,65 +126,164 @@ func NewPoolAllocator(poolCIDR, gateway string, subnetLen int) (*PoolAllocator, } pa.reserved = reservedSubnet.String() + stateDir := filepath.Join(lockDir, sanitizePoolDirName(pool.String())) + if err := os.MkdirAll(stateDir, 0o700); err != nil { + return nil, fmt.Errorf("create pool state dir %q: %w", stateDir, err) + } + pa.stateDir = stateDir + pa.state = lockedState{stateDir: stateDir, lockFileName: poolLockFileName} + return pa, nil } // Allocate assigns the next available IPv6 subnet from the pool for the // given container ID, skipping the subnet that contains the pool's gateway -// address. Returns the allocated subnet CIDR or an error if the pool is -// exhausted. Thread-safe. -func (a *PoolAllocator) Allocate(_ string) (*net.IPNet, error) { +// address and any subnet another allocation already holds (per the on-disk +// marker files, so this is correct across separate CNI plugin invocations +// on the same pool). If containerID already holds an allocation in this +// pool, that same subnet is returned rather than a fresh one being handed +// out — the CNI spec permits a runtime to retry ADD for the same container +// after a transient failure, and without this check each retry would leak +// the marker file from the previous attempt (findContainerMarker only ever +// returns the first match, so only one of the leaked markers would ever be +// recoverable via DEL). Returns the allocated subnet CIDR or an error if the +// pool is exhausted. Thread-safe. +func (a *PoolAllocator) Allocate(containerID string) (*net.IPNet, error) { a.mu.Lock() defer a.mu.Unlock() - // Collect currently allocated subnets for fast lookup. - used := make(map[string]struct{}) - a.allocations.Range(func(key, _ any) bool { - used[key.(string)] = struct{}{} - return true - }) - - // Iterate subnet boundaries within the pool. - subnetStart := make(net.IP, ipv6Bits/8) - copy(subnetStart, a.poolIP) - - for ; a.pool.Contains(subnetStart); subnetStart = incSubnet(subnetStart, a.subnetLen) { - // Build the subnet CIDR for this boundary. - subnet := &net.IPNet{ - IP: make(net.IP, ipv6Bits/8), - Mask: net.CIDRMask(a.subnetLen, ipv6Bits), + var result *net.IPNet + err := a.state.withLock(func() error { + if name, ok := a.state.findContainerMarkerLocked(containerID); ok { + existing, err := parseAllocatedSubnet(desanitizeMarkerName(name)) + if err != nil { + return fmt.Errorf("parse existing allocation marker %q: %w", name, err) + } + result = existing + return nil } - copy(subnet.IP, subnetStart) - subnetStr := subnet.String() - // Skip the subnet reserved for the gateway. - if subnetStr == a.reserved { - continue + used, err := a.usedSubnets() + if err != nil { + return err } - // Skip already allocated. - if _, ok := used[subnetStr]; ok { - continue + // Iterate subnet boundaries within the pool. + subnetStart := make(net.IP, ipv6Bits/8) + copy(subnetStart, a.poolIP) + + for ; a.pool.Contains(subnetStart); subnetStart = incSubnet(subnetStart, a.subnetLen) { + subnet := &net.IPNet{ + IP: make(net.IP, ipv6Bits/8), + Mask: net.CIDRMask(a.subnetLen, ipv6Bits), + } + copy(subnet.IP, subnetStart) + subnetStr := subnet.String() + + if subnetStr == a.reserved { + continue + } + if _, ok := used[subnetStr]; ok { + continue + } + + markerPath := filepath.Join(a.stateDir, sanitizePoolDirName(subnetStr)) + if err := os.WriteFile(markerPath, []byte(containerID), 0o600); err != nil { + return fmt.Errorf("write allocation marker %q: %w", markerPath, err) + } + result = subnet + return nil } - // Allocate. - a.allocations.Store(subnetStr, struct{}{}) - return subnet, nil + return fmt.Errorf("pool %s exhausted (subnet /%d)", a.pool.String(), a.subnetLen) + }) + if err != nil { + return nil, err } + return result, nil +} - return nil, fmt.Errorf("pool %s exhausted (subnet /%d)", a.pool.String(), a.subnetLen) +// usedSubnets reads the pool's state directory and returns the set of +// subnet CIDR strings currently marked allocated. Callers must hold both mu +// and the pool's flock. +func (a *PoolAllocator) usedSubnets() (map[string]struct{}, error) { + entries, err := a.state.entries() + if err != nil { + return nil, err + } + used := make(map[string]struct{}, len(entries)) + for _, e := range entries { + used[desanitizeMarkerName(e.Name())] = struct{}{} + } + return used, nil } // Deallocate removes the allocation for the given subnet CIDR string. -// Silently ignores unknown subnets. +// Silently ignores unknown subnets. Serialized the same way as Allocate. +// Callers that only know the containerID (not the allocated value) should +// use DeallocateContainer instead. func (a *PoolAllocator) Deallocate(subnetCIDR string) { - a.allocations.Delete(subnetCIDR) + a.mu.Lock() + defer a.mu.Unlock() + + _ = a.state.withLock(func() error { + return os.Remove(filepath.Join(a.stateDir, sanitizePoolDirName(subnetCIDR))) + }) +} + +// LookupContainer reports the subnet CIDR, if any, allocated to +// containerID, without removing it — used by CHECK to confirm an +// allocation is still in place. Returns ("", false) if none is found. +func (a *PoolAllocator) LookupContainer(containerID string) (string, bool) { + a.mu.Lock() + defer a.mu.Unlock() + + var name string + var ok bool + _ = a.state.withLock(func() error { + name, ok = a.state.findContainerMarkerLocked(containerID) + return nil + }) + if !ok { + return "", false + } + return desanitizeMarkerName(name), true +} + +// DeallocateContainer removes the allocation, if any, held by containerID, +// without the caller needing to already know the allocated subnet — the +// on-disk marker file records which containerID holds each subnet, so this +// is a direct scan of this pool's own state, no external lookup (e.g. a +// CRD read) required. The scan and the removal happen under a single flock +// acquisition (via lockedState.withLock) so a concurrent process sharing +// this pool can never interleave between them. Returns the deallocated +// subnet CIDR and true if one was found; ("", false) otherwise. +func (a *PoolAllocator) DeallocateContainer(containerID string) (string, bool) { + a.mu.Lock() + defer a.mu.Unlock() + + var subnet string + var ok bool + _ = a.state.withLock(func() error { + name, found := a.state.findContainerMarkerLocked(containerID) + if !found { + return nil + } + subnet = desanitizeMarkerName(name) + ok = true + return os.Remove(filepath.Join(a.stateDir, name)) + }) + if !ok { + return "", false + } + return subnet, true } -// IsAllocated reports whether the given subnet CIDR string is actively allocated. +// IsAllocated reports whether the given subnet CIDR string is actively +// allocated, by checking for its on-disk marker file. func (a *PoolAllocator) IsAllocated(subnetCIDR string) bool { - _, ok := a.allocations.Load(subnetCIDR) - return ok + _, err := os.Stat(filepath.Join(a.stateDir, sanitizePoolDirName(subnetCIDR))) + return err == nil } // Gateway returns the gateway IP for the pool. @@ -209,3 +337,40 @@ func incSubnet(ip net.IP, subnetLen int) net.IP { } return ip } + +// desanitizeMarkerName reverses sanitizePoolDirName's "/" -> "-" replacement +// for a subnet CIDR marker filename. A CIDR string carries exactly one "/", +// so replacing the first "-" back is unambiguous (subnet strings otherwise +// contain only hex digits and ":"). +func desanitizeMarkerName(name string) string { + before, after, found := strings.Cut(name, "-") + if !found { + return name + } + return before + "/" + after +} + +// parseAllocatedSubnet parses a subnet CIDR string previously produced by +// Allocate (via (*net.IPNet).String()) back into a *net.IPNet, preserving +// its IP exactly as allocated. net.ParseCIDR is deliberately not used here: +// it returns the *masked* network address, which would zero out incSubnet's +// per-subnet counter byte — the byte incSubnet advances sits inside what a +// strict /subnetLen mask treats as host bits (see incSubnet's own doc +// comment), so re-masking a stored subnet string would silently collapse +// every allocated subnet in the pool back down to the same reserved-subnet +// address. +func parseAllocatedSubnet(cidr string) (*net.IPNet, error) { + ipStr, prefixStr, found := strings.Cut(cidr, "/") + if !found { + return nil, fmt.Errorf("missing '/' in subnet CIDR %q", cidr) + } + ip := net.ParseIP(ipStr) + if ip == nil { + return nil, fmt.Errorf("invalid IP in subnet CIDR %q", cidr) + } + prefixLen, err := strconv.Atoi(prefixStr) + if err != nil { + return nil, fmt.Errorf("invalid prefix length in subnet CIDR %q: %w", cidr, err) + } + return &net.IPNet{IP: ip.To16(), Mask: net.CIDRMask(prefixLen, ipv6Bits)}, nil +} diff --git a/internal/cni/ipam/ipam_test.go b/internal/cni/ipam/ipam_test.go index 3c23bfeb..2a588ab9 100644 --- a/internal/cni/ipam/ipam_test.go +++ b/internal/cni/ipam/ipam_test.go @@ -93,7 +93,7 @@ func TestNewPoolAllocator(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - pa, err := NewPoolAllocator(tt.poolCIDR, tt.gateway, tt.subnetLen) + pa, err := NewPoolAllocator(tt.poolCIDR, tt.gateway, tt.subnetLen, t.TempDir()) if tt.wantErr { if err == nil { t.Fatal("expected error, got nil") @@ -115,7 +115,7 @@ func TestNewPoolAllocator(t *testing.T) { } func TestPoolAllocatorAllocate(t *testing.T) { - pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen) + pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen, t.TempDir()) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -160,8 +160,40 @@ func TestPoolAllocatorAllocate(t *testing.T) { } } +func TestPoolAllocatorAllocateIdempotentPerContainer(t *testing.T) { + pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen, t.TempDir()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + first, err := pa.Allocate("container-1") + if err != nil { + t.Fatalf("unexpected error on first allocation: %v", err) + } + + // A CNI runtime may retry ADD for the same container after a transient + // failure; a retry must get back the same subnet, not a fresh one (and + // must not leak the first attempt's marker file). + second, err := pa.Allocate("container-1") + if err != nil { + t.Fatalf("unexpected error on retry allocation: %v", err) + } + if second.String() != first.String() { + t.Errorf("retry Allocate() = %q, want %q (same as first allocation)", second.String(), first.String()) + } + + // A different container must still get a distinct subnet. + other, err := pa.Allocate("container-2") + if err != nil { + t.Fatalf("unexpected error on other container's allocation: %v", err) + } + if other.String() == first.String() { + t.Errorf("other container's Allocate() = %q, want distinct from %q", other.String(), first.String()) + } +} + func TestPoolAllocatorSkipsAllocatedSubnets(t *testing.T) { - pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen) + pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen, t.TempDir()) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -187,7 +219,7 @@ func TestPoolAllocatorSkipsAllocatedSubnets(t *testing.T) { } func TestPoolAllocatorReservesGatewaySubnet(t *testing.T) { - pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen) + pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen, t.TempDir()) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -216,7 +248,7 @@ func TestPoolAllocatorReservesGatewaySubnet(t *testing.T) { } func TestPoolAllocatorDeallocate(t *testing.T) { - pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen) + pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen, t.TempDir()) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -241,7 +273,7 @@ func TestPoolAllocatorDeallocate(t *testing.T) { } func TestPoolAllocatorDeallocateUnknown(t *testing.T) { - pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen) + pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen, t.TempDir()) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -250,8 +282,62 @@ func TestPoolAllocatorDeallocateUnknown(t *testing.T) { pa.Deallocate("fd00:dead::/80") } +func TestPoolAllocatorRejectsEmptyLockDir(t *testing.T) { + if _, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen, ""); err == nil { + t.Fatal("expected error for empty lockDir, got nil") + } +} + +// TestPoolAllocatorPersistsAcrossInstances is the regression test for the +// bug this package's on-disk persistence fixes: each CNI ADD/DEL is a +// separate OS process, so a fresh *PoolAllocator constructed by DEL must +// still see the allocation ADD's own (now-exited) *PoolAllocator made. +func TestPoolAllocatorPersistsAcrossInstances(t *testing.T) { + lockDir := t.TempDir() + + addPA, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen, lockDir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + subnet, err := addPA.Allocate("container-a") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // A brand new instance, as DEL's own process would construct, must see + // the allocation the (conceptually already-exited) ADD process made. + delPA, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen, lockDir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !delPA.IsAllocated(subnet.String()) { + t.Fatalf("IsAllocated(%q) = false on a fresh instance, want true (persisted)", subnet) + } + + gotSubnet, ok := delPA.DeallocateContainer("container-a") + if !ok { + t.Fatal("DeallocateContainer(\"container-a\") = false, want true") + } + if gotSubnet != subnet.String() { + t.Errorf("DeallocateContainer returned %q, want %q", gotSubnet, subnet.String()) + } + if delPA.IsAllocated(subnet.String()) { + t.Error("IsAllocated after DeallocateContainer = true, want false") + } +} + +func TestPoolAllocatorDeallocateContainerUnknown(t *testing.T) { + pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen, t.TempDir()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, ok := pa.DeallocateContainer("no-such-container"); ok { + t.Error("DeallocateContainer for unknown container = true, want false") + } +} + func TestPoolAllocatorIsAllocated(t *testing.T) { - pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen) + pa, err := NewPoolAllocator(testPoolCIDR, testPoolGw, testSubnetLen, t.TempDir()) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/internal/cni/ipam/ipv4.go b/internal/cni/ipam/ipv4.go index 9d8d1093..c439ae56 100644 --- a/internal/cni/ipam/ipv4.go +++ b/internal/cni/ipam/ipv4.go @@ -18,12 +18,6 @@ const ( // ipv4Bits is the number of bits in an IPv4 address. ipv4Bits = 32 - // DefaultIPv4LockDir is the well-known parent directory for the - // node-local on-disk lock and allocation state IPv4PoolAllocator uses to - // stay correct across separate CNI plugin invocations (each ADD/DEL is - // its own OS process) sharing the same site-wide IPv4 pool. - DefaultIPv4LockDir = "/var/lib/cni/galactic-ipv4" - // ipv4LockFileName is the flock target within each pool's state // directory; every other entry in that directory is an allocation // marker file named after the address it reserves. @@ -49,13 +43,14 @@ type IPv4PoolAllocator struct { gateway net.IP // gateway IP address mu sync.Mutex // serializes Allocate/Deallocate within this process stateDir string // directory holding the lock file and one allocation marker file per allocated address + state lockedState } // NewIPv4PoolAllocator creates a new IPv4 pool allocator from a CIDR pool and // an optional gateway address. The pool must be an IPv4 prefix. If gateway is // empty, the network address plus 1 is used as the gateway. lockDir is the // parent directory for this pool's on-disk lock and allocation state (see -// DefaultIPv4LockDir for the production path); it must not be empty, and a +// DefaultLockDir for the production path); it must not be empty, and a // pool-scoped subdirectory under it is created if it doesn't already exist. func NewIPv4PoolAllocator(poolCIDR, gateway, lockDir string) (*IPv4PoolAllocator, error) { _, pool, err := net.ParseCIDR(poolCIDR) @@ -98,61 +93,68 @@ func NewIPv4PoolAllocator(poolCIDR, gateway, lockDir string) (*IPv4PoolAllocator return nil, fmt.Errorf("create pool state dir %q: %w", stateDir, err) } a.stateDir = stateDir + a.state = lockedState{stateDir: stateDir, lockFileName: ipv4LockFileName} return a, nil } // Allocate assigns the next available IPv4 /32 address from the pool for the // given container ID, skipping reserved addresses (the network address, the -// gateway, the second-to-last address, and the last address of the pool). -// Returns an error if the pool is exhausted. The read-modify-write against -// the on-disk allocation state is serialized both within this process (via -// mu) and across processes sharing the same pool (via a flock on the pool's -// lock file), so concurrent ADDs from different VPCs on the same node never -// return the same address. +// gateway, the second-to-last address, and the last address of the pool). If +// containerID already holds an allocation in this pool, that same address is +// returned rather than a fresh one being handed out — see +// PoolAllocator.Allocate's doc comment (IPv6) for why this idempotency check +// matters for CNI ADD retries. Returns an error if the pool is exhausted. +// The read-modify-write against the on-disk allocation state is serialized +// both within this process (via mu) and across processes sharing the same +// pool (via a flock on the pool's lock file), so concurrent ADDs from +// different VPCs on the same node never return the same address. func (a *IPv4PoolAllocator) Allocate(containerID string) (net.IP, error) { a.mu.Lock() defer a.mu.Unlock() - lock, err := newFileLock(filepath.Join(a.stateDir, ipv4LockFileName)) - if err != nil { - return nil, fmt.Errorf("open lock for pool %s: %w", a.pool.String(), err) - } - defer func() { _ = lock.close() }() + var result net.IP + err := a.state.withLock(func() error { + if addrStr, ok := a.state.findContainerMarkerLocked(containerID); ok { + result = net.ParseIP(addrStr).To4() + return nil + } - if err := lock.lock(); err != nil { - return nil, fmt.Errorf("lock pool %s: %w", a.pool.String(), err) - } + used, err := a.usedAddresses() + if err != nil { + return err + } - used, err := a.usedAddresses() - if err != nil { - return nil, err - } + reserved := a.reservedAddresses() - reserved := a.reservedAddresses() + ones, bits := a.pool.Mask.Size() + total := uint64(1) << uint(bits-ones) - ones, bits := a.pool.Mask.Size() - total := uint64(1) << uint(bits-ones) + for i := range total { + addr := offsetIP4(a.pool.IP, i) + addrStr := addr.String() - for i := range total { - addr := offsetIP4(a.pool.IP, i) - addrStr := addr.String() + if _, ok := reserved[addrStr]; ok { + continue + } + if _, ok := used[addrStr]; ok { + continue + } - if _, ok := reserved[addrStr]; ok { - continue - } - if _, ok := used[addrStr]; ok { - continue + markerPath := filepath.Join(a.stateDir, addrStr) + if err := os.WriteFile(markerPath, []byte(containerID), 0o600); err != nil { + return fmt.Errorf("write allocation marker %q: %w", markerPath, err) + } + result = addr + return nil } - markerPath := filepath.Join(a.stateDir, addrStr) - if err := os.WriteFile(markerPath, []byte(containerID), 0o600); err != nil { - return nil, fmt.Errorf("write allocation marker %q: %w", markerPath, err) - } - return addr, nil + return fmt.Errorf("pool %s exhausted", a.pool.String()) + }) + if err != nil { + return nil, err } - - return nil, fmt.Errorf("pool %s exhausted", a.pool.String()) + return result, nil } // Deallocate removes the allocation for the given address string. Silently @@ -161,17 +163,52 @@ func (a *IPv4PoolAllocator) Deallocate(addr string) { a.mu.Lock() defer a.mu.Unlock() - lock, err := newFileLock(filepath.Join(a.stateDir, ipv4LockFileName)) - if err != nil { - return - } - defer func() { _ = lock.close() }() + _ = a.state.withLock(func() error { + return os.Remove(filepath.Join(a.stateDir, addr)) + }) +} - if err := lock.lock(); err != nil { - return - } +// LookupContainer reports the address, if any, allocated to containerID, +// without removing it — used by CHECK to confirm an allocation is still in +// place. Returns ("", false) if none is found. +func (a *IPv4PoolAllocator) LookupContainer(containerID string) (string, bool) { + a.mu.Lock() + defer a.mu.Unlock() - _ = os.Remove(filepath.Join(a.stateDir, addr)) + var addr string + var ok bool + _ = a.state.withLock(func() error { + addr, ok = a.state.findContainerMarkerLocked(containerID) + return nil + }) + return addr, ok +} + +// DeallocateContainer removes the allocation, if any, held by containerID, +// without the caller needing to already know the allocated address — +// mirrors PoolAllocator.DeallocateContainer (IPv6); see its doc comment for +// why the scan and the removal must happen under a single flock acquisition. +// Returns the deallocated address and true if one was found; ("", false) +// otherwise. +func (a *IPv4PoolAllocator) DeallocateContainer(containerID string) (string, bool) { + a.mu.Lock() + defer a.mu.Unlock() + + var addr string + var ok bool + _ = a.state.withLock(func() error { + found, wasFound := a.state.findContainerMarkerLocked(containerID) + if !wasFound { + return nil + } + addr = found + ok = true + return os.Remove(filepath.Join(a.stateDir, found)) + }) + if !ok { + return "", false + } + return addr, true } // IsAllocated reports whether the given address string is actively @@ -185,16 +222,13 @@ func (a *IPv4PoolAllocator) IsAllocated(addr string) bool { // addresses currently marked allocated by any process sharing this pool. // Callers must hold both mu and the pool's flock. func (a *IPv4PoolAllocator) usedAddresses() (map[string]struct{}, error) { - entries, err := os.ReadDir(a.stateDir) + entries, err := a.state.entries() if err != nil { - return nil, fmt.Errorf("read pool state dir %q: %w", a.stateDir, err) + return nil, err } used := make(map[string]struct{}, len(entries)) for _, e := range entries { - if e.Name() == ipv4LockFileName { - continue - } used[e.Name()] = struct{}{} } return used, nil diff --git a/internal/cni/ipam/ipv4_test.go b/internal/cni/ipam/ipv4_test.go index 5585f2be..4d357402 100644 --- a/internal/cni/ipam/ipv4_test.go +++ b/internal/cni/ipam/ipv4_test.go @@ -4,7 +4,10 @@ package ipam -import "testing" +import ( + "fmt" + "testing" +) const ( // testIPv4PoolCIDR is a /29 (8 addresses) so tests can exercise @@ -117,6 +120,34 @@ func TestIPv4PoolAllocatorAllocate(t *testing.T) { } } +func TestIPv4PoolAllocatorAllocateIdempotentPerContainer(t *testing.T) { + a, err := NewIPv4PoolAllocator(testIPv4PoolCIDR, testIPv4Gw, t.TempDir()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + first, err := a.Allocate("container-1") + if err != nil { + t.Fatalf("unexpected error on first allocation: %v", err) + } + + second, err := a.Allocate("container-1") + if err != nil { + t.Fatalf("unexpected error on retry allocation: %v", err) + } + if !second.Equal(first) { + t.Errorf("retry Allocate() = %q, want %q (same as first allocation)", second, first) + } + + other, err := a.Allocate("container-2") + if err != nil { + t.Fatalf("unexpected error on other container's allocation: %v", err) + } + if other.Equal(first) { + t.Errorf("other container's Allocate() = %q, want distinct from %q", other, first) + } +} + func TestIPv4PoolAllocatorSkipsReservedAddresses(t *testing.T) { a, err := NewIPv4PoolAllocator(testIPv4PoolCIDR, testIPv4Gw, t.TempDir()) if err != nil { @@ -150,9 +181,13 @@ func TestIPv4PoolAllocatorExhaustion(t *testing.T) { } // The /29 has exactly 4 usable addresses (.2-.5); the 5th allocation - // must fail with an exhaustion error. + // must fail with an exhaustion error. Each iteration uses a distinct + // container ID -- Allocate is idempotent per containerID (a CNI ADD + // retry must get back the same address, not a fresh one), so reusing + // one ID here would only ever consume a single address. for i := range 4 { - if _, err := a.Allocate("container"); err != nil { + containerID := fmt.Sprintf("container-%d", i) + if _, err := a.Allocate(containerID); err != nil { t.Fatalf("unexpected error on allocation %d: %v", i, err) } } @@ -184,6 +219,46 @@ func TestIPv4PoolAllocatorDeallocate(t *testing.T) { } } +func TestIPv4PoolAllocatorDeallocateContainer(t *testing.T) { + lockDir := t.TempDir() + + addA, err := NewIPv4PoolAllocator(testIPv4PoolCIDR, testIPv4Gw, lockDir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + addr, err := addA.Allocate("container-a") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // A fresh instance, as DEL's own process would construct, must see the + // allocation the (conceptually already-exited) ADD process made. + delA, err := NewIPv4PoolAllocator(testIPv4PoolCIDR, testIPv4Gw, lockDir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + gotAddr, ok := delA.DeallocateContainer("container-a") + if !ok { + t.Fatal("DeallocateContainer(\"container-a\") = false, want true") + } + if gotAddr != addr.String() { + t.Errorf("DeallocateContainer returned %q, want %q", gotAddr, addr.String()) + } + if delA.IsAllocated(addr.String()) { + t.Error("IsAllocated after DeallocateContainer = true, want false") + } +} + +func TestIPv4PoolAllocatorDeallocateContainerUnknown(t *testing.T) { + a, err := NewIPv4PoolAllocator(testIPv4PoolCIDR, testIPv4Gw, t.TempDir()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, ok := a.DeallocateContainer("no-such-container"); ok { + t.Error("DeallocateContainer for unknown container = true, want false") + } +} + func TestIPv4PoolAllocatorDeallocateUnknown(t *testing.T) { a, err := NewIPv4PoolAllocator(testIPv4PoolCIDR, testIPv4Gw, t.TempDir()) if err != nil { diff --git a/internal/cni/ipam/lockedstate.go b/internal/cni/ipam/lockedstate.go new file mode 100644 index 00000000..a4e21b74 --- /dev/null +++ b/internal/cni/ipam/lockedstate.go @@ -0,0 +1,79 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package ipam + +import ( + "fmt" + "os" + "path/filepath" +) + +// lockedState wraps the on-disk state directory and cross-process flock +// pattern shared by PoolAllocator (IPv6) and IPv4PoolAllocator: a directory +// holding a "lock" file plus one allocation marker file per allocated +// subnet/address, guarded by a flock so separate CNI plugin invocations +// (each ADD/DEL/CHECK is its own OS process) never race against each other +// over the same pool. +type lockedState struct { + stateDir string + lockFileName string +} + +// withLock acquires the pool's cross-process flock, runs fn, and releases +// the lock unconditionally before returning. Every read-modify-write against +// stateDir must go through this — in particular, a scan (to find which +// marker file, if any, belongs to a given containerID) and any removal that +// follows it must happen inside the *same* withLock call. Splitting them +// across two separate lock/unlock cycles, as an earlier version of +// findContainerMarker plus its callers did, leaves a window between the scan +// and the removal where a concurrent process sharing this pool can allocate +// or deallocate the very entry being acted on. +func (s lockedState) withLock(fn func() error) error { + lock, err := newFileLock(filepath.Join(s.stateDir, s.lockFileName)) + if err != nil { + return fmt.Errorf("open lock for %q: %w", s.stateDir, err) + } + defer func() { _ = lock.close() }() + + if err := lock.lock(); err != nil { + return fmt.Errorf("lock %q: %w", s.stateDir, err) + } + return fn() +} + +// entries reads stateDir and returns every marker filename except the lock +// file itself. Callers must already hold the flock (call from inside +// withLock). +func (s lockedState) entries() ([]os.DirEntry, error) { + all, err := os.ReadDir(s.stateDir) + if err != nil { + return nil, fmt.Errorf("read pool state dir %q: %w", s.stateDir, err) + } + markers := make([]os.DirEntry, 0, len(all)) + for _, e := range all { + if e.Name() == s.lockFileName { + continue + } + markers = append(markers, e) + } + return markers, nil +} + +// findContainerMarkerLocked scans stateDir for the marker file whose +// content matches containerID, returning its filename. Callers must already +// hold the flock (call from inside withLock). +func (s lockedState) findContainerMarkerLocked(containerID string) (string, bool) { + entries, err := s.entries() + if err != nil { + return "", false + } + for _, e := range entries { + content, err := os.ReadFile(filepath.Join(s.stateDir, e.Name())) + if err == nil && string(content) == containerID { + return e.Name(), true + } + } + return "", false +} diff --git a/internal/cni/ipam_ops.go b/internal/cni/ipam_ops.go deleted file mode 100644 index ac76e05d..00000000 --- a/internal/cni/ipam_ops.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright 2025 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package cni - -import ( - "context" - "errors" - "fmt" - "log/slog" - "net" - - "github.com/containernetworking/cni/pkg/skel" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - "go.datum.net/galactic/internal/cni/ipam" - bgpv1alpha1 "go.datum.net/network/api/v1alpha1" -) - -// ipv4LockDir is the IPv4PoolAllocator lock/state directory used by -// allocatePoolIPAM/deallocateIPAM. Overridable in tests (mirrors the -// ConfFile pattern in config.go) so unit tests never touch the real -// production path. -var ipv4LockDir = ipam.DefaultIPv4LockDir - -// wantsIPAM reports whether the given config should trigger IPAM allocation -// at all. Four independent signals opt in: an explicit "static" IPAM type, a -// configured IPv6Subnet or IPv4Subnet (the NAD-driven pool-IPAM path, either -// family alone or both), or the --enable-local-ipam dev fallback. A config -// with none of these (e.g. a tap workload that manages its own addressing) -// allocates nothing, matching today's behavior of skipping IPAM entirely -// rather than erroring. -func wantsIPAM(pluginConf *PluginConf) bool { - if pluginConf.IPAM != nil && pluginConf.IPAM.Type == ipamTypeStatic { - return true - } - return pluginConf.IPv6Subnet != "" || pluginConf.IPv4Subnet != "" || enableLocalIPAM -} - -// allocateIPAM allocates addresses for the given container. This is -// interface-agnostic — it does not touch any kernel state or network -// namespaces. Returns (nil, nil) when wantsIPAM reports no allocation is -// requested. When enableLocalIPAM is true and IPv6Subnet is unset, falls back -// to a built-in default IPv6 pool CIDR. -func allocateIPAM(args *skel.CmdArgs, pluginConf *PluginConf) (*ipamResult, error) { - if !wantsIPAM(pluginConf) { - return nil, nil - } - - if pluginConf.IPAM != nil && pluginConf.IPAM.Type == ipamTypeStatic { - return allocateStaticIPAM(args, pluginConf.IPAM) - } - - return allocatePoolIPAM(args, pluginConf) -} - -// allocateStaticIPAM validates and returns the pre-assigned static IPv6 -// address from the "static" IPAM block. No IPv4 address is ever allocated -// for static IPAM — it is a single fixed address, not a dual-stack pool. -func allocateStaticIPAM(args *skel.CmdArgs, ipamConf *IPAM) (*ipamResult, error) { - alloc := ipam.NewStaticAllocator() - allocIP, err := alloc.Allocate(args.ContainerID, ipamConf.StaticIP) - if err != nil { - return nil, fmt.Errorf("allocate static IP: %w", err) - } - subnet := &net.IPNet{ - IP: allocIP, - Mask: net.CIDRMask(64, 128), - } - slog.Debug("IPAM: allocated static", "containerID", args.ContainerID, "subnet", subnet) - return &ipamResult{ipv6Subnet: subnet}, nil -} - -// allocatePoolIPAM allocates a dual-stack, IPv6-only, or IPv4-only pool-based -// endpoint address for the given container, via ipam.DualStackAllocator. -// IPv6Subnet and IPv4Subnet each independently supply a pool CIDR for their -// family; at least one must be set (falling back to localIPAMDefaultPool for -// IPv6 when enableLocalIPAM and both are unset). -func allocatePoolIPAM(args *skel.CmdArgs, pluginConf *PluginConf) (*ipamResult, error) { - ipv6Pool := pluginConf.IPv6Subnet - if ipv6Pool == "" && pluginConf.IPv4Subnet == "" { - if !enableLocalIPAM { - return nil, errors.New("ipv6_subnet or ipv4_subnet is required (or enable local IPAM)") - } - ipv6Pool = localIPAMDefaultPool - } - - alloc, err := ipam.NewDualStackAllocator(ipv6Pool, "", pluginConf.IPv4Subnet, "", ipv4LockDir) - if err != nil { - return nil, fmt.Errorf("create dual-stack allocator: %w", err) - } - - res, err := alloc.Allocate(args.ContainerID) - if err != nil { - return nil, fmt.Errorf("allocate dual-stack addresses: %w", err) - } - - var routes []*net.IPNet - if res.IPv6Subnet != nil { - routes = append(routes, &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)}) - } - if res.IPv4Address != nil { - routes = append(routes, &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}) - } - - slog.Debug("IPAM: allocated", "containerID", args.ContainerID, - "ipv6Subnet", res.IPv6Subnet, "ipv6Gateway", res.IPv6Gateway, - "ipv4Address", res.IPv4Address, "ipv4Gateway", res.IPv4Gateway) - - return &ipamResult{ - ipv6Subnet: res.IPv6Subnet, - ipv6Gateway: res.IPv6Gateway, - ipv4Address: res.IPv4Address, - ipv4Gateway: res.IPv4Gateway, - routes: routes, - }, nil -} - -// configureIPAM allocates addresses and configures the guest interface inside -// the container network namespace with both families (when dual-stack). This -// is veth-only; for tap mode, use allocateIPAM directly (the VM manages its -// own guest interface). -func configureIPAM(args *skel.CmdArgs, pluginConf *PluginConf, guestName string) (*ipamResult, error) { - ipamResult, err := allocateIPAM(args, pluginConf) - if err != nil { - return nil, err - } - if ipamResult == nil { - return nil, nil - } - - var ipv4Net *net.IPNet - if ipamResult.ipv4Address != nil { - ipv4Net = &net.IPNet{IP: ipamResult.ipv4Address, Mask: net.CIDRMask(32, 32)} - } - if err := configureInterfaceInNetns( - args.Netns, guestName, - ipamResult.ipv6Subnet, ipamResult.ipv6Gateway, - ipv4Net, ipamResult.ipv4Gateway, - ); err != nil { - return nil, err - } - - return ipamResult, nil -} - -// deallocateIPAM releases the IPAM allocation for the given container. -// Reads the allocated IPv6 subnet and (if present) IPv4 address from the -// BGPAdvertisement CRD annotations, then deallocates each independently and -// non-fatally: a missing annotation for one family (e.g. a pre-existing -// v6-only pod, or a partial ADD failure that never reached IPv4 allocation) -// must not prevent cleanup of the other. -func deallocateIPAM(args *skel.CmdArgs, pluginConf *PluginConf, k8s client.Client) { - if pluginConf.IPAM != nil && pluginConf.IPAM.Type == ipamTypeStatic { - // Static allocations don't need deallocation. - return - } - - ipv6Subnet, ipv4Addr := getAllocatedSubnetsFromCRD(args.ContainerID, pluginConf, k8s) - if ipv6Subnet == "" && ipv4Addr == "" { - // No allocation found — either allocation was never completed, - // or the advertisement was already deleted. Nothing to clean up. - slog.Debug("IPAM: no allocation found to deallocate", "containerID", args.ContainerID) - return - } - - if ipv6Subnet != "" { - ipv6Pool := pluginConf.IPv6Subnet - if ipv6Pool == "" && enableLocalIPAM { - ipv6Pool = localIPAMDefaultPool - } - pa, err := ipam.NewPoolAllocator(ipv6Pool, "", 0) - if err != nil { - slog.Warn("IPAM: failed to build IPv6 pool allocator for deallocation, skipping", "err", err, - "containerID", args.ContainerID, "subnet", ipv6Subnet) - } else { - pa.Deallocate(ipv6Subnet) - slog.Debug("IPAM: deallocated IPv6", "containerID", args.ContainerID, "subnet", ipv6Subnet) - } - } - - if ipv4Addr != "" { - if pluginConf.IPv4Subnet == "" { - slog.Warn("IPAM: found allocated IPv4 address but no ipv4_subnet in config, skipping deallocation", - "containerID", args.ContainerID, "address", ipv4Addr) - } else if pa, err := ipam.NewIPv4PoolAllocator(pluginConf.IPv4Subnet, "", ipv4LockDir); err != nil { - slog.Warn("IPAM: failed to build IPv4 pool allocator for deallocation, skipping", "err", err, - "containerID", args.ContainerID, "address", ipv4Addr) - } else { - pa.Deallocate(ipv4Addr) - slog.Debug("IPAM: deallocated IPv4", "containerID", args.ContainerID, "address", ipv4Addr) - } - } -} - -// getAllocatedSubnetsFromCRD reads the allocated IPv6 subnet and (if present) -// IPv4 address for the given container from the BGPAdvertisement CRD -// annotations. Either return value is empty when not found. -func getAllocatedSubnetsFromCRD( - containerID string, pluginConf *PluginConf, k8s client.Client, -) (ipv6Subnet, ipv4Addr string) { - namespace := pluginConf.Namespace - - ctx, cancel := context.WithTimeout(context.Background(), cniTimeout) - defer cancel() - - adv := &bgpv1alpha1.BGPAdvertisement{ - ObjectMeta: metav1.ObjectMeta{ - Name: bgpAdvertisementName(pluginConf.VPC, pluginConf.VPCAttachment), - Namespace: namespace, - }, - } - if err := k8s.Get(ctx, client.ObjectKeyFromObject(adv), adv); err != nil { - return "", "" - } - - return adv.Annotations[subnetAnnotationKeyIPv6(containerID)], adv.Annotations[subnetAnnotationKeyIPv4(containerID)] -} diff --git a/internal/cni/ipam_ops_test.go b/internal/cni/ipam_ops_test.go deleted file mode 100644 index fc31bcda..00000000 --- a/internal/cni/ipam_ops_test.go +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright 2025 Datum Cloud, Inc. -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package cni - -import ( - "net" - "testing" - - "github.com/containernetworking/cni/pkg/skel" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "go.datum.net/galactic/internal/cni/ipam" - "go.datum.net/galactic/internal/config" - bgpv1alpha1 "go.datum.net/network/api/v1alpha1" -) - -// ---- wantsIPAM ------------------------------------------------------------ - -func TestWantsIPAM(t *testing.T) { - original := enableLocalIPAM - defer func() { enableLocalIPAM = original }() - - tests := []struct { - name string - pluginConf *PluginConf - enableLocalIPA bool - want bool - }{ - { - name: "no ipam block, no ipv6_subnet, local IPAM disabled", - pluginConf: &PluginConf{}, - want: false, - }, - { - name: "static ipam type opts in regardless of other fields", - pluginConf: &PluginConf{IPAM: &IPAM{Type: ipamTypeStatic}}, - want: true, - }, - { - name: "ipv6_subnet set opts in", - pluginConf: &PluginConf{IPv6Subnet: localIPAMDefaultPool}, - want: true, - }, - { - name: "ipv4_subnet set opts in", - pluginConf: &PluginConf{IPv4Subnet: testIPv4Subnet}, - want: true, - }, - { - name: "local IPAM enabled opts in even without ipv6_subnet", - pluginConf: &PluginConf{}, - enableLocalIPA: true, - want: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - enableLocalIPAM = tt.enableLocalIPA - if got := wantsIPAM(tt.pluginConf); got != tt.want { - t.Errorf("wantsIPAM(%+v) = %v, want %v", tt.pluginConf, got, tt.want) - } - }) - } -} - -// ---- allocateIPAM ---------------------------------------------------------- - -func TestAllocateIPAMNoAllocation(t *testing.T) { - original := enableLocalIPAM - defer func() { enableLocalIPAM = original }() - enableLocalIPAM = false - - args := &skel.CmdArgs{ContainerID: testContainerID} - res, err := allocateIPAM(args, &PluginConf{}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res != nil { - t.Errorf("allocateIPAM() = %+v, want nil (wantsIPAM should have been false)", res) - } -} - -func TestAllocateIPAMStatic(t *testing.T) { - args := &skel.CmdArgs{ContainerID: testContainerID} - pluginConf := &PluginConf{ - IPAM: &IPAM{Type: ipamTypeStatic, StaticIP: "fd00:10:ff01::1234"}, - } - - res, err := allocateIPAM(args, pluginConf) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res == nil { - t.Fatal("allocateIPAM() = nil, want a result") - } - if res.ipv6Subnet == nil || !res.ipv6Subnet.IP.Equal(net.ParseIP("fd00:10:ff01::1234")) { - t.Errorf("ipv6Subnet = %v, want fd00:10:ff01::1234", res.ipv6Subnet) - } - if res.ipv4Address != nil { - t.Errorf("ipv4Address = %v, want nil for static IPAM", res.ipv4Address) - } -} - -func TestAllocateIPAMPoolIPv6Only(t *testing.T) { - args := &skel.CmdArgs{ContainerID: testContainerID} - pluginConf := &PluginConf{IPv6Subnet: localIPAMDefaultPool} - - res, err := allocateIPAM(args, pluginConf) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res == nil { - t.Fatal("allocateIPAM() = nil, want a result") - } - if res.ipv6Subnet == nil { - t.Fatal("ipv6Subnet = nil, want an allocated /96") - } - if ones, bits := res.ipv6Subnet.Mask.Size(); ones != 96 || bits != 128 { - t.Errorf("ipv6Subnet mask = /%d, want /96", ones) - } - if res.ipv6Gateway == nil { - t.Error("ipv6Gateway = nil, want the pool's default gateway (::1 of the /64)") - } - if res.ipv4Address != nil { - t.Errorf("ipv4Address = %v, want nil (no ipv4_subnet configured)", res.ipv4Address) - } - if len(res.routes) != 1 { - t.Errorf("routes = %v, want exactly one default IPv6 route", res.routes) - } -} - -func TestAllocateIPAMPoolIPv4Only(t *testing.T) { - origLockDir := ipv4LockDir - ipv4LockDir = t.TempDir() - defer func() { ipv4LockDir = origLockDir }() - - args := &skel.CmdArgs{ContainerID: testContainerID} - pluginConf := &PluginConf{IPv4Subnet: testIPv4Subnet} - - res, err := allocateIPAM(args, pluginConf) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res == nil { - t.Fatal("allocateIPAM() = nil, want a result") - } - if res.ipv6Subnet != nil { - t.Errorf("ipv6Subnet = %v, want nil (no ipv6_subnet configured)", res.ipv6Subnet) - } - if res.ipv4Address == nil { - t.Fatal("ipv4Address = nil, want an allocated /32") - } - if res.ipv4Gateway == nil { - t.Error("ipv4Gateway = nil, want the pool's default gateway") - } - if len(res.routes) != 1 { - t.Errorf("routes = %v, want exactly one default IPv4 route", res.routes) - } -} - -func TestAllocateIPAMPoolDualStack(t *testing.T) { - origLockDir := ipv4LockDir - ipv4LockDir = t.TempDir() - defer func() { ipv4LockDir = origLockDir }() - - args := &skel.CmdArgs{ContainerID: testContainerID} - pluginConf := &PluginConf{ - IPv6Subnet: localIPAMDefaultPool, - IPv4Subnet: testIPv4Subnet, - } - - res, err := allocateIPAM(args, pluginConf) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res == nil { - t.Fatal("allocateIPAM() = nil, want a result") - } - if res.ipv6Subnet == nil { - t.Error("ipv6Subnet = nil, want an allocated /96") - } - if res.ipv4Address == nil { - t.Fatal("ipv4Address = nil, want an allocated /32") - } - if res.ipv4Gateway == nil { - t.Error("ipv4Gateway = nil, want the pool's default gateway") - } - if len(res.routes) != 2 { - t.Errorf("routes = %v, want one default route per family", res.routes) - } -} - -func TestAllocateIPAMPoolMissingBothSubnetsErrors(t *testing.T) { - original := enableLocalIPAM - defer func() { enableLocalIPAM = original }() - enableLocalIPAM = false - - // wantsIPAM only opts in here via the static/ipv6_subnet/ipv4_subnet/ - // local-IPAM signals, so force the pool path directly to exercise the - // "no source for any pool CIDR" error without relying on wantsIPAM's - // gating. - args := &skel.CmdArgs{ContainerID: testContainerID} - _, err := allocatePoolIPAM(args, &PluginConf{}) - if err == nil { - t.Fatal("expected error when both ipv6_subnet and ipv4_subnet are unset and local IPAM is disabled, got nil") - } -} - -// ---- deallocateIPAM --------------------------------------------------------- - -func TestDeallocateIPAMStaticNoop(t *testing.T) { - // Static IPAM never wrote a CRD annotation for cmdDel to look up, and - // deallocateIPAM must return immediately without attempting a k8s lookup. - pluginConf := &PluginConf{ - VPC: testVPC, VPCAttachment: testAttachment, - IPAM: &IPAM{Type: ipamTypeStatic}, - } - args := &skel.CmdArgs{ContainerID: testContainerID} - // A nil client would panic if deallocateIPAM tried to use it; passing nil - // here asserts the static-type early return happens first. - deallocateIPAM(args, pluginConf, nil) -} - -func TestDeallocateIPAMDualStack(t *testing.T) { - origLockDir := ipv4LockDir - ipv4LockDir = t.TempDir() - defer func() { ipv4LockDir = origLockDir }() - - pluginConf := &PluginConf{ - VPC: testVPC, VPCAttachment: testAttachment, - IPv6Subnet: localIPAMDefaultPool, - IPv4Subnet: testIPv4Subnet, - } - args := &skel.CmdArgs{ContainerID: testContainerID} - - // Allocate an IPv4 address the same way ADD would, so there's a real - // marker file for Deallocate to remove. - alloc, err := ipam.NewDualStackAllocator(pluginConf.IPv6Subnet, "", pluginConf.IPv4Subnet, "", ipv4LockDir) - if err != nil { - t.Fatalf("NewDualStackAllocator: %v", err) - } - dsRes, err := alloc.Allocate(args.ContainerID) - if err != nil { - t.Fatalf("Allocate: %v", err) - } - - ipv4Pool, err := ipam.NewIPv4PoolAllocator(pluginConf.IPv4Subnet, "", ipv4LockDir) - if err != nil { - t.Fatalf("NewIPv4PoolAllocator: %v", err) - } - if !ipv4Pool.IsAllocated(dsRes.IPv4Address.String()) { - t.Fatalf("setup: IPv4 address %s not marked allocated", dsRes.IPv4Address) - } - - adv := &bgpv1alpha1.BGPAdvertisement{ - ObjectMeta: metav1.ObjectMeta{ - Name: bgpAdvertisementName(pluginConf.VPC, pluginConf.VPCAttachment), - Namespace: config.DefaultNamespace, - Annotations: map[string]string{ - subnetAnnotationKeyIPv6(args.ContainerID): dsRes.IPv6Subnet.String(), - subnetAnnotationKeyIPv4(args.ContainerID): dsRes.IPv4Address.String(), - }, - }, - } - pluginConf.Namespace = config.DefaultNamespace - k8s := fakeClient(adv) - - deallocateIPAM(args, pluginConf, k8s) - - if ipv4Pool.IsAllocated(dsRes.IPv4Address.String()) { - t.Errorf("IPv4 address %s still marked allocated after deallocateIPAM", dsRes.IPv4Address) - } -} - -func TestDeallocateIPAMIPv4Only(t *testing.T) { - origLockDir := ipv4LockDir - ipv4LockDir = t.TempDir() - defer func() { ipv4LockDir = origLockDir }() - - pluginConf := &PluginConf{ - VPC: testVPC, VPCAttachment: testAttachment, - Namespace: config.DefaultNamespace, - IPv4Subnet: testIPv4Subnet, - } - args := &skel.CmdArgs{ContainerID: testContainerID} - - ipv4Pool, err := ipam.NewIPv4PoolAllocator(pluginConf.IPv4Subnet, "", ipv4LockDir) - if err != nil { - t.Fatalf("NewIPv4PoolAllocator: %v", err) - } - ipv4Addr, err := ipv4Pool.Allocate(args.ContainerID) - if err != nil { - t.Fatalf("Allocate: %v", err) - } - if !ipv4Pool.IsAllocated(ipv4Addr.String()) { - t.Fatalf("setup: IPv4 address %s not marked allocated", ipv4Addr) - } - - adv := &bgpv1alpha1.BGPAdvertisement{ - ObjectMeta: metav1.ObjectMeta{ - Name: bgpAdvertisementName(pluginConf.VPC, pluginConf.VPCAttachment), - Namespace: config.DefaultNamespace, - Annotations: map[string]string{ - // No IPv6 annotation — this is an IPv4-only allocation. - subnetAnnotationKeyIPv4(args.ContainerID): ipv4Addr.String(), - }, - }, - } - k8s := fakeClient(adv) - - // Must not panic despite no ipv6_subnet in config, and must deallocate - // the IPv4 address. - deallocateIPAM(args, pluginConf, k8s) - - if ipv4Pool.IsAllocated(ipv4Addr.String()) { - t.Errorf("IPv4 address %s still marked allocated after deallocateIPAM", ipv4Addr) - } -} - -func TestDeallocateIPAMPartialAllocationNonFatal(t *testing.T) { - // A v6-only pod (no IPv4 annotation, e.g. pre-existing or a partial ADD - // failure) must still have its IPv6 side cleaned up without erroring, - // and must not attempt to touch a nonexistent IPv4 pool. - pluginConf := &PluginConf{ - VPC: testVPC, VPCAttachment: testAttachment, - Namespace: config.DefaultNamespace, - IPv6Subnet: localIPAMDefaultPool, - // IPv4Subnet intentionally unset. - } - args := &skel.CmdArgs{ContainerID: testContainerID} - - adv := &bgpv1alpha1.BGPAdvertisement{ - ObjectMeta: metav1.ObjectMeta{ - Name: bgpAdvertisementName(pluginConf.VPC, pluginConf.VPCAttachment), - Namespace: config.DefaultNamespace, - Annotations: map[string]string{ - subnetAnnotationKeyIPv6(args.ContainerID): "fd00:10:ff01::1234/96", - }, - }, - } - k8s := fakeClient(adv) - - // Must not panic despite no ipv4_subnet in config. - deallocateIPAM(args, pluginConf, k8s) -} - -func TestDeallocateIPAMNoAllocationFound(t *testing.T) { - pluginConf := &PluginConf{ - VPC: testVPC, VPCAttachment: testAttachment, - Namespace: config.DefaultNamespace, - } - args := &skel.CmdArgs{ContainerID: testContainerID} - // No BGPAdvertisement exists at all. - k8s := fakeClient() - - // Must return cleanly with nothing to deallocate. - deallocateIPAM(args, pluginConf, k8s) -} diff --git a/internal/cni/ops_add.go b/internal/cni/ops_add.go index f1b6aa1f..5d9282f9 100644 --- a/internal/cni/ops_add.go +++ b/internal/cni/ops_add.go @@ -6,31 +6,28 @@ package cni import ( "context" - "errors" "fmt" "log/slog" - "net" "os" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" "github.com/vishvananda/netlink" - "go.datum.net/galactic/internal/cni/route" - "go.datum.net/galactic/internal/cni/tap" "go.datum.net/galactic/internal/cni/veth" + "go.datum.net/galactic/internal/cnimaster" + "go.datum.net/galactic/internal/nadpatch" "go.datum.net/galactic/internal/plumbing/intf" "go.datum.net/galactic/internal/plumbing/vrf" ) // cmdAdd uses a named return (err) so that the deferred selective rollback -// below always observes the real failure: several branches check errors via -// "if err := f(); err != nil" inside nested if/switch blocks, which declares -// a block-scoped err that would otherwise shadow this function's err and -// leave the deferred rollback thinking the call succeeded. A plain -// "return expr" always assigns expr to a named result, regardless of that -// local shadowing, so naming the return here is what makes rollback fire on -// every failure path instead of just the ones using top-level "x, err := f()". +// below always observes the real failure — see the doc comment on the +// original version of this function for why a named return matters here; +// still true with fewer branches. galactic-cni's own ADD ends by printing +// its own result and returning: BGP/SRv6/eBPF publish is galactic-bgp's +// job, invoked next by the CNI runtime per conflist order, not by this +// process. func cmdAdd(args *skel.CmdArgs) (err error) { pluginConf, err := parseConf(args.StdinData) if err != nil { @@ -42,7 +39,7 @@ func cmdAdd(args *skel.CmdArgs) (err error) { // assignment. A nil or structurally broken prevResult indicates a mis- // configured chain that galactic-cni should not silently ignore. if pluginConf.PrevResult != nil { - if err := validatePrevResultAdd(pluginConf.PrevResult); err != nil { + if err := cnimaster.ValidatePrevResultAdd(pluginConf.PrevResult); err != nil { return &types.Error{Code: 6, Msg: fmt.Sprintf("prevResult validation in ADD: %v", err)} } } @@ -57,26 +54,30 @@ func cmdAdd(args *skel.CmdArgs) (err error) { slog.Info("ADD: starting", "containerID", args.ContainerID, "netns", args.Netns, "ifName", args.IfName, "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment, - "interfaceType", pluginConf.InterfaceType, "namespace", namespace, "nodeName", nodeName) + "namespace", namespace, "nodeName", nodeName) // Track resources for selective rollback on failure. tracker := &resourceTracker{ vpc: pluginConf.VPC, vpcAttachment: pluginConf.VPCAttachment, - ifaceType: pluginConf.InterfaceType, - namespace: namespace, + } + // Record IPAM delegation intent up front, before configureIPAM (called + // from buildVethResult below) ever runs — see resourceTracker's + // ipamDelegated doc comment for why rollback needs this set + // unconditionally on "ipam" block presence, not just after a + // successful ExecAdd. + if pluginConf.IPAM != nil { + tracker.ipamDelegated = true + tracker.ipamType = pluginConf.IPAM.Type + tracker.ipamStdin = args.StdinData } // Selective rollback: clean up only resources that were created. - // We need a context for k8s operations in rollback; the k8s client - // will be populated by publishBGPState before it's needed. - rollbackCtx, rollbackCancel := context.WithTimeout(context.Background(), cniTimeout) defer func() { if err != nil { slog.Error("ADD: failed, rolling back created resources", "err", err, "containerID", args.ContainerID, "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) - tracker.cleanup(rollbackCtx) - rollbackCancel() + tracker.cleanup() } }() @@ -86,16 +87,8 @@ func cmdAdd(args *skel.CmdArgs) (err error) { tracker.vrfCreated = true slog.Debug("ADD: VRF ready", "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) - // Create the appropriate interface type (veth or tap). - switch pluginConf.InterfaceType { - case interfaceTypeVeth: - if err := veth.Add(pluginConf.VPC, pluginConf.VPCAttachment, pluginConf.MTU); err != nil { - return fmt.Errorf("add veth: %w", err) - } - case interfaceTypeTap: - if err := tap.Add(pluginConf.VPC, pluginConf.VPCAttachment, pluginConf.MTU); err != nil { - return fmt.Errorf("add tap: %w", err) - } + if err := veth.Add(pluginConf.VPC, pluginConf.VPCAttachment, pluginConf.MTU); err != nil { + return fmt.Errorf("add veth: %w", err) } hostName := intf.GenerateInterfaceNameHost(pluginConf.VPC, pluginConf.VPCAttachment) @@ -110,84 +103,20 @@ func cmdAdd(args *skel.CmdArgs) (err error) { // Annotate the NAD with the host interface name. The NAD must already // exist (created by the external VPC operator); a missing or otherwise // unpatchable NAD is a hard failure. - k8sClient, err := newK8sClient() + k8sClient, err := cnimaster.NewK8sClient() if err != nil { return fmt.Errorf("create k8s client: %w", err) } - tracker.k8s = k8sClient - podNamespace := parsePodNamespace(args.Args) - if err := annotateNAD(rollbackCtx, k8sClient, pluginConf.Name, podNamespace, hostName); err != nil { + podNamespace := nadpatch.ParsePodNamespace(args.Args) + nadCtx, nadCancel := context.WithTimeout(context.Background(), cniTimeout) + defer nadCancel() + if err := nadpatch.AnnotateNAD(nadCtx, k8sClient, pluginConf.Name, podNamespace, hostName); err != nil { return fmt.Errorf("annotate NAD: %w", err) } - dev := hostName - for _, termination := range pluginConf.Terminations { - if err := route.Add(pluginConf.VPC, pluginConf.VPCAttachment, termination.Network, termination.Via, dev); err != nil { - return fmt.Errorf("add route %s: %w", termination.Network, err) - } - tracker.routesCreated++ - } - if tracker.routesCreated > 0 { - slog.Debug("ADD: termination routes installed", "count", tracker.routesCreated, "dev", dev) - } - - // Host-device delegation and IPAM are veth-only. - // In tap mode the guest VM manages its own networking. - var ipamResult *ipamResult - var guestHWAddr net.HardwareAddr - switch pluginConf.InterfaceType { - case interfaceTypeVeth: - guestName := intf.GenerateInterfaceNameGuest(pluginConf.VPC, pluginConf.VPCAttachment) - ipamResult, guestHWAddr, err = buildVethResult(args, pluginConf, hostName, guestName, hostMac, hostMTU) - if err != nil { - return err - } - if ipamResult != nil { - slog.Debug("ADD: IPAM allocated", "containerID", args.ContainerID, - "ipv6Subnet", ipamResult.ipv6Subnet, "ipv6Gateway", ipamResult.ipv6Gateway, - "ipv4Address", ipamResult.ipv4Address, "ipv4Gateway", ipamResult.ipv4Gateway) - } - case interfaceTypeTap: - // Allocate IPAM for the tap interface (same as veth). - // The VM manages its own guest interface; the CNI only configures the host side. - ipamResult, err = allocateIPAM(args, pluginConf) - if err != nil { - return fmt.Errorf("allocate IPAM: %w", err) - } - if ipamResult != nil { - slog.Debug("ADD: IPAM allocated", "containerID", args.ContainerID, - "ipv6Subnet", ipamResult.ipv6Subnet, "ipv6Gateway", ipamResult.ipv6Gateway, - "ipv4Address", ipamResult.ipv4Address, "ipv4Gateway", ipamResult.ipv4Gateway) - } - - // Configure the gateway address on the host tap and install the VRF route. - if err := configureHostGateway(pluginConf.VPC, pluginConf.VPCAttachment, ipamResult, nil); err != nil { - return err - } - if ipamResult != nil && ipamResult.ipv6Gateway != nil { - slog.Debug("ADD: host gateway configured", "name", hostName, "gateway", ipamResult.ipv6Gateway) - } - - // Print the CNI result with IP info. - result := buildTapResult(pluginConf, ipamResult, hostName, hostMac, hostMTU) - if err := types.PrintResult(result, pluginConf.CNIVersion); err != nil { - return fmt.Errorf("print CNI result: %w", err) - } - - // Decode VPC for BGP state publish. - vpcHex, err := intf.Base62ToHex(pluginConf.VPC) - if err != nil { - return fmt.Errorf("decode VPC: %w", err) - } - - // Publish BGP state (SRv6 ingress + BGP CRDs). - if tracker.k8s == nil { - return errors.New("k8s client not set in tracker") - } - slog.Debug("ADD: publishing BGP state", "containerID", args.ContainerID, "interfaceType", interfaceTypeTap) - return publishBGPStateK8s(args, pluginConf, nodeName, namespace, ipamResult, vpcHex, tracker.k8s, tracker) - } + // Termination routes are galactic-route's job now — chained next after + // this plugin, when the attachment has any (see internal/cniroute). - slog.Debug("ADD: publishing BGP state", "containerID", args.ContainerID, "interfaceType", pluginConf.InterfaceType) - return publishBGPState(args, pluginConf, nodeName, namespace, ipamResult, guestHWAddr, tracker) + guestName := intf.GenerateInterfaceNameGuest(pluginConf.VPC, pluginConf.VPCAttachment) + return buildVethResult(args, pluginConf, hostName, guestName, hostMac, hostMTU) } diff --git a/internal/cni/ops_check.go b/internal/cni/ops_check.go index 866b27be..2f443747 100644 --- a/internal/cni/ops_check.go +++ b/internal/cni/ops_check.go @@ -5,27 +5,20 @@ package cni import ( - "context" "encoding/json" "errors" "fmt" "log/slog" "net" - "net/http" - "os" - "time" "github.com/containernetworking/cni/pkg/skel" - "github.com/containernetworking/cni/pkg/types" type100 "github.com/containernetworking/cni/pkg/types/100" + "github.com/containernetworking/plugins/pkg/ipam" "github.com/containernetworking/plugins/pkg/ns" "github.com/vishvananda/netlink" - "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" - "go.datum.net/galactic/internal/config" + "go.datum.net/galactic/internal/cnimaster" "go.datum.net/galactic/internal/plumbing/intf" - "go.datum.net/galactic/internal/plumbing/vrf" ) // cmdCheck validates that the container's network state matches what was @@ -38,27 +31,24 @@ func cmdCheck(args *skel.CmdArgs) error { return err } slog.Info("CHECK: starting", "containerID", args.ContainerID, - "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment, "interfaceType", pluginConf.InterfaceType) + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) var errs []error // Check node-level state (VRF + host interface). - hostName, nodeErrs := checkNodeLevelState(pluginConf.VPC, pluginConf.VPCAttachment) + hostName, nodeErrs := cnimaster.CheckNodeLevelState(pluginConf.VPC, pluginConf.VPCAttachment) errs = append(errs, nodeErrs...) - // For veth mode, verify the guest interface is in the container netns. - if pluginConf.InterfaceType == interfaceTypeVeth { - guestName := intf.GenerateInterfaceNameGuest(pluginConf.VPC, pluginConf.VPCAttachment) - if err := checkGuestInterface(args.Netns, guestName); err != nil { - errs = append(errs, fmt.Errorf("guest interface %q: %w", guestName, err)) - } - - // Verify termination routes exist in the VRF table. - if err := checkTerminationRoutes(pluginConf.VPC, pluginConf.VPCAttachment, pluginConf.Terminations); err != nil { - errs = append(errs, fmt.Errorf("termination routes: %w", err)) - } + // Verify the guest interface is in the container netns. + guestName := intf.GenerateInterfaceNameGuest(pluginConf.VPC, pluginConf.VPCAttachment) + if err := checkGuestInterface(args.Netns, guestName); err != nil { + errs = append(errs, fmt.Errorf("guest interface %q: %w", guestName, err)) } + // Termination routes are galactic-route's own CHECK now (see + // internal/cniroute's checkTerminationRoutes) — this plugin's CHECK no + // longer verifies them. + // Validate kernel state against prevResult (CNI spec §4.3). if pluginConf.RawPrevResult != nil { if err := checkPrevResult(pluginConf.RawPrevResult, hostName, args.Netns); err != nil { @@ -66,6 +56,14 @@ func cmdCheck(args *skel.CmdArgs) error { } } + // Delegate CHECK to the IPAM plugin so a lost or corrupted allocation + // marker file is caught here too, not just at ADD/DEL time. + if pluginConf.IPAM != nil { + if err := ipam.ExecCheck(pluginConf.IPAM.Type, args.StdinData); err != nil { + errs = append(errs, fmt.Errorf("IPAM CHECK: %w", err)) + } + } + if len(errs) > 0 { err := fmt.Errorf("CHECK failed: %w", errors.Join(errs...)) slog.Error("CHECK: failed", "err", err, "containerID", args.ContainerID, @@ -77,107 +75,11 @@ func cmdCheck(args *skel.CmdArgs) error { return nil } -// cmdStatus implements the CNI spec STATUS operation. It is called by the -// runtime to determine whether the plugin is ready to service ADD requests. -// Unlike cmdCheck, no container is attached so there is no Netns to inspect. -// STATUS validates the plugin's own readiness: config is parseable and the -// API server is reachable for BGPAdvertisement CRD operations. Attachment- -// specific kernel resources (VRF, host interface) are NOT checked because -// STATUS must succeed before any ADD has ever run. +// cmdStatus implements the CNI spec STATUS operation — see +// internal/cnimaster.RunStatus for the full reasoning, shared verbatim with +// galactic-tap-cni. func cmdStatus(args *skel.CmdArgs) error { - // Validate config is parseable (minimal check — no VPC/VPCAttachment - // validation since STATUS must succeed before any ADD has run). - if err := parseStatusConf(args.StdinData); err != nil { - return err - } - - // Load host CNI config to resolve Kubeconfig and LogFile - hostConf, err := loadHostConf(ConfFile) - if err != nil { - return &types.Error{Code: 7, Msg: fmt.Sprintf("load host CNI config: %v", err)} - } - - // Resolve config: env var > conflist > default. - cniConfig.Resolve(&config.ConflistValues{ - Kubeconfig: hostConf.Kubeconfig, - Namespace: hostConf.Namespace, - LogFile: hostConf.LogFile, - LogLevel: hostConf.LogLevel, - }) - - // Propagate Kubeconfig - _ = os.Setenv("KUBECONFIG", cniConfig.Kubeconfig) - - // Setup Logging - setupLogging(cniConfig.LogFile, cniConfig.LogLevel) - slog.Debug("CNI config received", "stdin", string(args.StdinData)) - - // Config is parseable and API server is reachable. - slog.Info("STATUS: probing API server reachability") - if err := probeAPIServer(); err != nil { - slog.Error("STATUS: API server probe failed", "err", err) - return &types.Error{Code: 50, Msg: fmt.Sprintf("API server health check failed: %v", err)} - } - slog.Info("STATUS: ready") - return nil -} - -// probeAPIServer performs a lightweight GET against the in-cluster API server -// to verify reachability. Returns nil when the server responds (any HTTP -// status code) or when running outside a cluster with no kubeconfig. -// -// probeAPIServerFn is a variable so tests can override it. -var probeAPIServerFn = func() error { - kubeconfig, err := ctrl.GetConfig() - if err != nil { - if errors.Is(err, rest.ErrNotInCluster) { - // Not running in-cluster; skip API check. - return nil - } - return fmt.Errorf("load kubeconfig: %w", err) - } - kubeconfig.Timeout = 2 * time.Second - httpClient, err := rest.HTTPClientFor(kubeconfig) - if err != nil { - return fmt.Errorf("build http client: %w", err) - } - req, err := http.NewRequestWithContext( - context.Background(), - http.MethodGet, - kubeconfig.Host+"/healthz", - nil, - ) - if err != nil { - return fmt.Errorf("build healthz request: %w", err) - } - resp, err := httpClient.Do(req) - if err != nil { - return fmt.Errorf("healthz request failed: %w", err) - } - defer resp.Body.Close() //nolint:errcheck // best-effort probe - return nil -} - -var probeAPIServer = probeAPIServerFn - -// checkNodeLevelState verifies that node-level networking resources exist: -// the VRF interface and the host-side endpoint interface. Returns the host -// interface name (for callers that need it, e.g. cmdCheck's prevResult -// validation) and a slice of errors (nil when all checks pass) so callers -// can accumulate and report all failures at once. -func checkNodeLevelState(vpc, vpcAttachment string) (string, []error) { - var errs []error - - if err := vrf.Exists(vpc, vpcAttachment); err != nil { - errs = append(errs, fmt.Errorf("vrf %s-%s: %w", vpc, vpcAttachment, err)) - } - - hostName := intf.GenerateInterfaceNameHost(vpc, vpcAttachment) - if _, err := netlink.LinkByName(hostName); err != nil { - errs = append(errs, fmt.Errorf("host interface %q: %w", hostName, err)) - } - - return hostName, errs + return cnimaster.RunStatus(args.StdinData, cniConfig, ConfFile) } // checkGuestInterface verifies that the named interface exists inside the @@ -203,56 +105,6 @@ func checkGuestInterface(netnsPath, ifName string) error { }) } -// checkTerminationRoutes verifies that all termination routes exist in the -// VRF table for the given VPC/VPCAttachment pair. -func checkTerminationRoutes(vpc, vpcAttachment string, terminations []Termination) error { - tableID, err := vrf.TableID(vpc, vpcAttachment) - if err != nil { - return fmt.Errorf("get VRF table ID: %w", err) - } - - handle, err := netlink.NewHandle() - if err != nil { - return fmt.Errorf("create netlink handle: %w", err) - } - defer handle.Close() //nolint:errcheck // netlink cleanup on teardown - - routes, err := handle.RouteListFiltered( - netlink.FAMILY_V6, - &netlink.Route{Table: int(tableID)}, - netlink.RT_FILTER_TABLE, - ) - if err != nil { - return fmt.Errorf("list routes: %w", err) - } - - dev := intf.GenerateInterfaceNameHost(vpc, vpcAttachment) - for _, term := range terminations { - viaIP := net.ParseIP(term.Via) - if viaIP == nil { - return fmt.Errorf("invalid termination gateway %q", term.Via) - } - found := false - for _, r := range routes { - if r.Dst != nil && - r.Dst.String() == term.Network && - r.Gw != nil && - r.Gw.Equal(viaIP) && - r.LinkIndex > 0 { - // Verify the link name matches (defers to the veth/tap device). - if link, linkErr := handle.LinkByIndex(r.LinkIndex); linkErr == nil && link.Attrs().Name == dev { - found = true - break - } - } - } - if !found { - return fmt.Errorf("missing route %s via %s in VRF table %d", term.Network, term.Via, tableID) - } - } - return nil -} - // checkPrevResult validates that kernel state matches the interfaces and IPs // recorded in the prevResult returned by the most recent ADD. Per the CNI spec // §4.3, CHECK must verify that managed resources have not drifted. @@ -280,7 +132,7 @@ func checkPrevResult(rawPrevResult map[string]interface{}, _ string, netns strin // Host-side interface: validate MAC and MTU from the host namespace. if iface.Sandbox == "" { - if err := validateHostInterface(iface.Name, iface.Mac, iface.Mtu); err != nil { + if err := cnimaster.ValidateHostInterface(iface.Name, iface.Mac, iface.Mtu); err != nil { return fmt.Errorf("interface %q (host): %w", iface.Name, err) } continue @@ -314,22 +166,6 @@ func checkPrevResult(rawPrevResult map[string]interface{}, _ string, netns strin return nil } -// validateHostInterface checks that a host-side interface's MAC and MTU match -// the values recorded in prevResult. -func validateHostInterface(name, wantMac string, wantMtu int) error { - link, err := netlink.LinkByName(name) - if err != nil { - return fmt.Errorf("find link: %w", err) - } - if wantMac != "" && link.Attrs().HardwareAddr.String() != wantMac { - return fmt.Errorf("MAC mismatch: expected %q, got %q", wantMac, link.Attrs().HardwareAddr.String()) - } - if wantMtu > 0 && link.Attrs().MTU != wantMtu { - return fmt.Errorf("MTU mismatch: expected %d, got %d", wantMtu, link.Attrs().MTU) - } - return nil -} - // validateGuestInterface checks that a guest-side interface's MAC and MTU match // the values recorded in prevResult, reading from inside the container netns. func validateGuestInterface(name, wantMac string, wantMtu int, netns string) error { diff --git a/internal/cni/ops_del.go b/internal/cni/ops_del.go index 8539f9c2..054fafbf 100644 --- a/internal/cni/ops_del.go +++ b/internal/cni/ops_del.go @@ -10,6 +10,7 @@ import ( "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" type100 "github.com/containernetworking/cni/pkg/types/100" + "github.com/containernetworking/plugins/pkg/ipam" ) func cmdDel(args *skel.CmdArgs) error { @@ -30,12 +31,13 @@ func cmdDel(args *skel.CmdArgs) error { vpc, vpcAtt := pluginConf.VPC, pluginConf.VPCAttachment // Deallocate the pod's IPAM subnet. This is pod-specific and safe to - // release immediately. Applies to both veth and tap modes. - if wantsIPAM(pluginConf) { - if k8s, err := newK8sClient(); err == nil { - deallocateIPAM(args, pluginConf, k8s) - } else { - slog.Warn("DEL: failed to create k8s client, skipping IPAM deallocation", "err", err, + // release immediately. Delegating at all (or not) is entirely + // pluginConf.IPAM's own presence — no k8s client needed here at all + // now that galactic-ipam's own DEL looks its allocation up locally + // (see internal/cniipam's doc comment). + if pluginConf.IPAM != nil { + if err := ipam.ExecDel(pluginConf.IPAM.Type, args.StdinData); err != nil { + slog.Warn("DEL: IPAM delegation failed, allocation may not have been released", "err", err, "containerID", args.ContainerID) } } @@ -49,31 +51,25 @@ func cmdDel(args *skel.CmdArgs) error { // Multus secondary attachment) — the move is then a no-op, and the // leftover route survives indefinitely since there's no ephemeral // sandbox netns to reclaim it, wedging the next ADD with "file exists". - // Only applies to veth mode; tap mode has no guest-side netns config. - if pluginConf.InterfaceType == interfaceTypeVeth { - if err := flushGuestNetnsConfig(args.Netns, args.IfName); err != nil { - slog.Warn("DEL: failed to flush guest interface address/route, may still be in the netns", - "err", err, "containerID", args.ContainerID, "netns", args.Netns) - } + if err := flushGuestNetnsConfig(args.Netns, args.IfName); err != nil { + slog.Warn("DEL: failed to flush guest interface address/route, may still be in the netns", + "err", err, "containerID", args.ContainerID, "netns", args.Netns) } // Forward DEL to host-device delegated plugin (CNI spec §4). This moves // the guest veth end back out of the container netns and restores its - // original (host-side) name. Only applies to veth mode; tap mode has no - // host-device delegation. + // original (host-side) name. // // DEL must always return success per the CNI spec, so an error here // (e.g. the device was never moved into the netns because ADD failed // before reaching that step, or the netns is already gone) is logged // rather than propagated. - if pluginConf.InterfaceType == interfaceTypeVeth { - if err := hostDevice("DEL", args, pluginConf); err != nil { - slog.Warn("DEL: host-device DEL failed, guest interface may still be in the netns", - "err", err, "containerID", args.ContainerID, "netns", args.Netns) - } + if err := hostDevice("DEL", args, pluginConf); err != nil { + slog.Warn("DEL: host-device DEL failed, guest interface may still be in the netns", + "err", err, "containerID", args.ContainerID, "netns", args.Netns) } - // Shared resources (VRF, veth/tap, routes, SRv6 ingress, BGPAdvertisement, + // Shared resources (VRF, veth, routes, SRv6 ingress, BGPAdvertisement, // BGPVRFInstance) are keyed by (vpc, vpcAttachment) and may still be in use // by another pod. Deleting them here races with cmdAdd during pod restarts — // the old pod's DEL can destroy resources the new pod just created. diff --git a/internal/cni/resource.go b/internal/cni/resource.go index 5810ecf6..9f9c5aa1 100644 --- a/internal/cni/resource.go +++ b/internal/cni/resource.go @@ -5,150 +5,60 @@ package cni import ( - "context" "log/slog" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/containernetworking/plugins/pkg/ipam" - "go.datum.net/galactic/internal/cni/tap" "go.datum.net/galactic/internal/cni/veth" - "go.datum.net/galactic/internal/plumbing/ebpf/attach" - "go.datum.net/galactic/internal/plumbing/vrf" - bgpv1alpha1 "go.datum.net/network/api/v1alpha1" + "go.datum.net/galactic/internal/cnimaster" ) -var cniScheme = runtime.NewScheme() - -// enableLocalIPAM controls whether the plugin performs IP allocation when -// no explicit ipam block is present in the CNI config. Defaults to false. -var enableLocalIPAM bool - -// SetEnableLocalIPAM sets the local IPAM flag from the CLI. -func SetEnableLocalIPAM(v bool) { - enableLocalIPAM = v -} - -func init() { - utilruntime.Must(clientgoscheme.AddToScheme(cniScheme)) - utilruntime.Must(bgpv1alpha1.AddToScheme(cniScheme)) -} - -// resourceTracker tracks resources created during cmdAdd for selective rollback. +// resourceTracker tracks resources created during cmdAdd for selective +// rollback. galactic-cni is veth-only, and its ADD only ever creates the +// VRF, the veth pair, and (if delegated) an IPAM allocation — BGP/SRv6/eBPF +// publish is galactic-bgp's own, separately chain-invoked plugin now, with +// its own smaller tracker (internal/cnibgp); termination routes are +// galactic-route's own, with its own smaller tracker (internal/cniroute); +// so this one no longer needs to know anything about either. type resourceTracker struct { vpc, vpcAttachment string - ifaceType string vrfCreated bool - routesCreated int - vrfInstanceCreated bool - advCreated bool - k8s client.Client - namespace string - // ebpfRegistered, ebpfBlock, and ebpfArgument record the eBPF uSID - // datapath's vrf_table registration (registerEBPFDatapath, Milestone - // 7.1), if one actually happened (the BGPRouter may not be - // configured, in which case ebpfRegistered stays false and cleanup - // has nothing to unregister). Only vrf_table - // is rolled back here -- locator_table/function_table entries are - // keyed by Block, not by this specific (vpc, vpcAttachment), and - // typically shared across many attachments on the same node, so they - // are never this attachment's rollback's responsibility to remove - // (Milestone 7.2). - ebpfRegistered bool - ebpfBlock uint64 - ebpfArgument uint16 + // ipamDelegated, ipamType, and ipamStdin record enough to release the + // IPAM allocation during rollback. Set as soon as pluginConf.IPAM != nil + // is known (before configureIPAM/ipam.ExecAdd is even attempted) rather + // than only after a successful ExecAdd: ipam.ExecDel is idempotent per + // the CNI IPAM delegation protocol (galactic-ipam's own cmdDel no-ops + // when it finds no allocation for the containerID), so calling it + // unconditionally whenever an "ipam" block was configured is safe and + // covers every failure path, including ones where ExecAdd itself never + // ran. Without this, a failed ADD that got past IPAM permanently burns + // an address/subnet out of the pool — the on-disk marker file has no + // implicit teardown the way the old in-memory-only allocator did. + ipamDelegated bool + ipamType string + ipamStdin []byte } // cleanup rolls back all tracked resources in reverse creation order. // Errors are logged but never returned — the caller already has a failure. -func (rt *resourceTracker) cleanup(ctx context.Context) { - slog.Info("Selective rollback: cleaning up resources created during failed ADD", - "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) - - // 1. Delete BGPAdvertisement (withdraws prefixes) - if rt.advCreated && rt.k8s != nil { - adv := &bgpv1alpha1.BGPAdvertisement{ - ObjectMeta: metav1.ObjectMeta{ - Name: bgpAdvertisementName(rt.vpc, rt.vpcAttachment), - Namespace: rt.namespace, - }, - } - if err := rt.k8s.Delete(ctx, adv); client.IgnoreNotFound(err) != nil { - slog.Error("Rollback: failed to delete BGPAdvertisement", "err", err, - "name", adv.Name, "namespace", rt.namespace) - } else { - slog.Debug("Rollback: deleted BGPAdvertisement", "name", adv.Name, "namespace", rt.namespace) - } - } - - // 2. Delete BGPVRFInstance - if rt.vrfInstanceCreated && rt.k8s != nil { - vrfInst := &bgpv1alpha1.BGPVRFInstance{ - ObjectMeta: metav1.ObjectMeta{ - Name: bgpVRFInstanceName(rt.vpc, rt.vpcAttachment), - Namespace: rt.namespace, - }, - } - if err := rt.k8s.Delete(ctx, vrfInst); client.IgnoreNotFound(err) != nil { - slog.Error("Rollback: failed to delete BGPVRFInstance", "err", err, - "name", vrfInst.Name, "namespace", rt.namespace) +// Takes no context: unlike before this split, the only non-kernel call left +// here is ipam.ExecDel, which (like ExecAdd) shells out to the delegated +// plugin binary rather than making a k8s API call (BGP CRD/eBPF rollback is +// galactic-bgp's own tracker now). +func (rt *resourceTracker) cleanup() { + // Release the IPAM allocation first (if pluginConf carried an "ipam" + // block at all — see the ipamDelegated field doc comment for why this + // fires unconditionally on that alone, not just after a confirmed + // ExecAdd). Interface/VRF cleanup is shared with galactic-tap-cni's own + // tracker, so it lives in cnimaster.CleanupAttachment. + if rt.ipamDelegated { + if err := ipam.ExecDel(rt.ipamType, rt.ipamStdin); err != nil { + slog.Error("Rollback: failed to release IPAM allocation", "err", err, "ipamType", rt.ipamType) } else { - slog.Debug("Rollback: deleted BGPVRFInstance", "name", vrfInst.Name, "namespace", rt.namespace) + slog.Debug("Rollback: released IPAM allocation", "ipamType", rt.ipamType) } } - // 3. Unregister the eBPF uSID datapath's vrf_table entry (only if - // registerEBPFDatapath actually wrote one, Milestone 7.2). A pinned - // BPF map entry has no implicit teardown when the VRF/interfaces are - // deleted below, so it must be removed explicitly here and nowhere - // else in the normal cmdDel path is expected to (design plan §5.1). - if rt.ebpfRegistered { - // Recomputed fresh rather than cached at registration time: this is - // exactly the value unregisterEBPFDatapath needs to confirm the - // vrf_table slot still belongs to this attachment before deleting it - // (see its doc comment). The VRF interface itself isn't deleted - // until step 6 below, so it's still resolvable here. - if vrfTableID, err := vrf.TableID(rt.vpc, rt.vpcAttachment); err != nil { - slog.Error("Rollback: failed to resolve VRF table id, skipping eBPF vrf_table unregister", "err", err, - "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) - } else if err := unregisterEBPFDatapath(rt.ebpfBlock, rt.ebpfArgument, vrfTableID, attach.PinDir); err != nil { - slog.Error("Rollback: failed to unregister eBPF vrf_table entry", "err", err, - "block", rt.ebpfBlock, "argument", rt.ebpfArgument) - } else { - slog.Debug("Rollback: unregistered eBPF vrf_table entry", - "block", rt.ebpfBlock, "argument", rt.ebpfArgument) - } - } - - // 4. Delete host veth (veth mode only) - if rt.ifaceType == interfaceTypeVeth { - if err := veth.Delete(rt.vpc, rt.vpcAttachment); err != nil { - slog.Error("Rollback: failed to delete veth", "err", err, - "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) - } else { - slog.Debug("Rollback: deleted veth", "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) - } - } - - // 5. Delete tap (tap mode only) - if rt.ifaceType == interfaceTypeTap { - if err := tap.Delete(rt.vpc, rt.vpcAttachment); err != nil { - slog.Error("Rollback: failed to delete tap", "err", err, - "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) - } else { - slog.Debug("Rollback: deleted tap", "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) - } - } - - // 6. Delete VRF (flushes all routes, removes VRF interface) - if err := vrf.Delete(rt.vpc, rt.vpcAttachment); err != nil { - slog.Error("Rollback: failed to delete VRF", "err", err, - "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) - } else { - slog.Debug("Rollback: deleted VRF", "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) - } + cnimaster.CleanupAttachment(rt.vpc, rt.vpcAttachment, "veth", veth.Delete) } diff --git a/internal/cni/result.go b/internal/cni/result.go index 3c2ee3a1..0238cbe7 100644 --- a/internal/cni/result.go +++ b/internal/cni/result.go @@ -6,18 +6,23 @@ package cni import ( "fmt" + "log/slog" "net" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" type100 "github.com/containernetworking/cni/pkg/types/100" + "github.com/containernetworking/plugins/pkg/ipam" "github.com/vishvananda/netlink" + + "go.datum.net/galactic/internal/cni/hostgw" + "go.datum.net/galactic/internal/cniipam" ) // buildResult constructs the CNI result, including IPAM data if configured. func buildResult( pluginConf *PluginConf, - ipRes *ipamResult, + ipRes *cniipam.IPAMResult, hostName, guestName string, hostMac, guestMac string, hostMTU, guestMTU int, @@ -46,32 +51,28 @@ func buildResult( // appendIPConfigs adds one IPConfig per allocated address family in ipRes // (IPv6, and IPv4 when present) plus any default routes, all pointing at the -// given Interfaces index. ipv4Mask sets the prefix length reported for the -// IPv4 address — /32 for veth, /25 for tap (matching the host gateway mask -// installed by ipv4GatewayAddrParams, so downstream consumers such as -// kraftlet configure the guest with the same real subnet the host side -// advertises). No-op when ipRes is nil. -func appendIPConfigs(result *type100.Result, ipRes *ipamResult, ifaceIndex int, ipv4Mask net.IPMask) { +// given Interfaces index. No-op when ipRes is nil. +func appendIPConfigs(result *type100.Result, ipRes *cniipam.IPAMResult, ifaceIndex int, ipv4Mask net.IPMask) { if ipRes == nil { return } - if ipRes.ipv6Subnet != nil { + if ipRes.IPv6Subnet != nil { result.IPs = append(result.IPs, &type100.IPConfig{ - Address: *ipRes.ipv6Subnet, - Gateway: ipRes.ipv6Gateway, + Address: *ipRes.IPv6Subnet, + Gateway: ipRes.IPv6Gateway, Interface: type100.Int(ifaceIndex), }) } - if ipRes.ipv4Address != nil { + if ipRes.IPv4Address != nil { result.IPs = append(result.IPs, &type100.IPConfig{ - Address: net.IPNet{IP: ipRes.ipv4Address, Mask: ipv4Mask}, - Gateway: ipRes.ipv4Gateway, + Address: net.IPNet{IP: ipRes.IPv4Address, Mask: ipv4Mask}, + Gateway: ipRes.IPv4Gateway, Interface: type100.Int(ifaceIndex), }) } - if len(ipRes.routes) > 0 { - result.Routes = make([]*types.Route, 0, len(ipRes.routes)) - for _, dst := range ipRes.routes { + if len(ipRes.Routes) > 0 { + result.Routes = make([]*types.Route, 0, len(ipRes.Routes)) + for _, dst := range ipRes.Routes { result.Routes = append(result.Routes, &types.Route{ Dst: *dst, }) @@ -80,15 +81,18 @@ func appendIPConfigs(result *type100.Result, ipRes *ipamResult, ifaceIndex int, } // buildVethResult handles veth-specific result building: host-device -// delegation, IPAM, guest interface reading, and result printing. -// Returns the IPAM result for BGP advertisement, or nil if no IPAM. +// delegation, IPAM, host gateway configuration, guest interface reading, +// and result printing. galactic-bgp (chained next by the runtime) picks up +// everything it needs — the allocated addresses, and that this was a veth +// attachment — from the result this prints, not from a Go-level return +// value. func buildVethResult( args *skel.CmdArgs, pluginConf *PluginConf, hostName, guestName string, hostMac string, hostMTU int, -) (*ipamResult, net.HardwareAddr, error) { +) error { // Only call host-device ADD if the guest interface is still in the host // namespace. If a prior attempt already moved it to the container netns but // failed at a later step, we must not try to move it again. @@ -97,62 +101,86 @@ func buildVethResult( // previous run. The host-device plugin renames the moved interface // to args.IfName, so a prior run may have left that name behind. if err := cleanupContainerNetns(args.Netns, args.IfName); err != nil { - return nil, nil, fmt.Errorf("cleanup container netns: %w", err) + return fmt.Errorf("cleanup container netns: %w", err) } if err := hostDevice("ADD", args, pluginConf); err != nil { - return nil, nil, fmt.Errorf("host-device ADD: %w", err) + return fmt.Errorf("host-device ADD: %w", err) } } // Configure IP address on the guest interface inside the container netns. - var ipamResult *ipamResult - if wantsIPAM(pluginConf) { + // Delegating at all is this plugin's own call, decided solely by "ipam" + // block presence — no config field or env var elsewhere can override + // that (see internal/cniipam's doc comment for the explicit contract). + var ipamResult *cniipam.IPAMResult + if pluginConf.IPAM != nil { result, err := configureIPAM(args, pluginConf, args.IfName) if err != nil { - return nil, nil, fmt.Errorf("configure IPAM: %w", err) + return fmt.Errorf("configure IPAM: %w", err) } ipamResult = result } + if ipamResult != nil { + slog.Debug("ADD: IPAM allocated", "containerID", args.ContainerID, + "ipv6Subnet", ipamResult.IPv6Subnet, "ipv6Gateway", ipamResult.IPv6Gateway, + "ipv4Address", ipamResult.IPv4Address, "ipv4Gateway", ipamResult.IPv4Gateway) + } // Read guest veth attributes inside the container netns. guestMac, guestMTU, err := readGuestInterface(args.Netns, args.IfName) if err != nil { - return nil, nil, fmt.Errorf("read guest interface: %w", err) + return fmt.Errorf("read guest interface: %w", err) } guestHWAddr, err := net.ParseMAC(guestMac) if err != nil { - return nil, nil, fmt.Errorf("parse guest interface MAC %q: %w", guestMac, err) + return fmt.Errorf("parse guest interface MAC %q: %w", guestMac, err) + } + + // Configure the host-side gateway address and VRF route before printing + // the result — kernel-interface work this plugin owns (see + // internal/cni/hostgw's doc comment for why galactic-bgp no longer does + // this itself). + if err := hostgw.ConfigureHostGateway(pluginConf.VPC, pluginConf.VPCAttachment, ipamResult, guestHWAddr); err != nil { + return fmt.Errorf("configure host gateway: %w", err) } + result := buildResult(pluginConf, ipamResult, hostName, args.IfName, hostMac, guestMac, hostMTU, guestMTU, args.Netns) if err := types.PrintResult(result, pluginConf.CNIVersion); err != nil { - return nil, nil, fmt.Errorf("print CNI result: %w", err) + return fmt.Errorf("print CNI result: %w", err) } - return ipamResult, guestHWAddr, nil + return nil } -// buildTapResult constructs the CNI result for tap mode: a single host -// interface with optional IPAM data. The guest VM manages its own interface; -// the IP here describes the allocated subnet for BGP advertisement. The IPv4 -// address is reported with a /25 mask, matching the mask -// ipv4GatewayAddrParams installs on the host side of the tap (see bgp.go). -func buildTapResult( - pluginConf *PluginConf, - ipRes *ipamResult, - hostName, hostMac string, - hostMTU int, -) *type100.Result { - result := &type100.Result{ - CNIVersion: pluginConf.CNIVersion, - Interfaces: []*type100.Interface{ - { - Name: hostName, - Mac: hostMac, - Mtu: hostMTU, - Sandbox: "", - }, - }, +// configureIPAM delegates IPAM allocation to whatever binary pluginConf's +// own "ipam.type" names (per the CNI IPAM delegation protocol — see +// github.com/containernetworking/plugins/pkg/ipam.ExecAdd), then applies +// the returned addresses to the guest interface inside the container +// network namespace with both families (when dual-stack). args.StdinData +// is passed straight through as the delegate's own netconf: it already +// contains the "ipam" block (plus everything else in this plugin's own +// config, which the delegate simply ignores). +func configureIPAM(args *skel.CmdArgs, pluginConf *PluginConf, guestName string) (*cniipam.IPAMResult, error) { + cniResult, err := ipam.ExecAdd(pluginConf.IPAM.Type, args.StdinData) + if err != nil { + return nil, fmt.Errorf("delegate to %s ADD: %w", pluginConf.IPAM.Type, err) } - appendIPConfigs(result, ipRes, 0, net.CIDRMask(25, 32)) // index into Interfaces (host tap) - return result + ipamResult, err := cniipam.ResultToIPAMResult(cniResult) + if err != nil { + return nil, fmt.Errorf("convert IPAM result: %w", err) + } + + var ipv4Net *net.IPNet + if ipamResult.IPv4Address != nil { + ipv4Net = &net.IPNet{IP: ipamResult.IPv4Address, Mask: net.CIDRMask(32, 32)} + } + if err := configureInterfaceInNetns( + args.Netns, guestName, + ipamResult.IPv6Subnet, ipamResult.IPv6Gateway, + ipv4Net, ipamResult.IPv4Gateway, + ); err != nil { + return nil, err + } + + return ipamResult, nil } diff --git a/internal/cni/types.go b/internal/cni/types.go index 6398f6aa..46696b5a 100644 --- a/internal/cni/types.go +++ b/internal/cni/types.go @@ -5,77 +5,21 @@ package cni import ( - "net" - "github.com/containernetworking/cni/pkg/types" -) - -// Termination represents a network termination point with a destination -// CIDR and next-hop gateway address. -type Termination struct { - Network string `json:"network"` - Via string `json:"via,omitempty"` -} - -// IPAM holds IP address management configuration passed in the CNI config. -// Pool CIDR fields (formerly Pool/Gateway/SubnetLen) have been retired in -// favor of PluginConf.IPv6Subnet/IPv4Subnet — see allocateIPAM. -type IPAM struct { - Type string `json:"type"` // "pool" (default) or "static" - StaticIP string `json:"static_ip,omitempty"` // used when type="static" - Routes []Route `json:"routes,omitempty"` - Addresses []Address `json:"addresses,omitempty"` -} - -// Route describes a static route to install. -type Route struct { - Dst string `json:"dst"` - GW string `json:"gw,omitempty"` -} -// Address describes a static IP address assignment. -type Address struct { - Address string `json:"address"` -} + "go.datum.net/galactic/internal/cnimaster" + "go.datum.net/galactic/internal/hostconf" +) -// PluginConf is the CNI plugin configuration passed via stdin on each invocation. -// -// IPv6Subnet, IPv4Subnet, and AddressFamilies feed the dual-stack IPAM -// allocators (internal/cni/ipam, IPv4PoolAllocator/DualStackAllocator); as of -// this change allocateIPAM does not yet consume them, and format/requiredness -// validation in parseConf lands in a later phase. -type PluginConf struct { - types.PluginConf - VPC string `json:"vpc"` - VPCAttachment string `json:"vpcattachment"` - MTU int `json:"mtu,omitempty"` - InterfaceType string `json:"interface_type,omitempty"` // interfaceTypeVeth or interfaceTypeTap - Terminations []Termination `json:"terminations,omitempty"` - IPAM *IPAM `json:"ipam"` - Namespace string `json:"namespace,omitempty"` - IPv6Subnet string `json:"ipv6_subnet,omitempty"` // region IPv6 pool CIDR; endpoints alloc /96 - IPv4Subnet string `json:"ipv4_subnet,omitempty"` // optional site IPv4 pool CIDR; endpoints alloc /32 - AddressFamilies []string `json:"address_families,omitempty"` // families to allocate; default ["ipv6"] -} +// PluginConf is the CNI plugin configuration passed via stdin on each +// invocation of galactic-cni, the veth master plugin. It's the same shape +// galactic-tap-cni (internal/cnitap) uses — see internal/cnimaster's own +// doc comment — so both packages alias the one canonical definition rather +// than each declaring their own copy. +type PluginConf = cnimaster.PluginConf // HostConf holds node-local settings read from /etc/cni/net.d/10-galactic.conflist. -type HostConf struct { - NodeName string `json:"node_name"` - Kubeconfig string `json:"kubeconfig"` - Namespace string `json:"namespace"` - LogFile string `json:"log_file"` - LogLevel string `json:"log_level,omitempty"` -} - -// ipamResult holds the IPAM allocation details for building the CNI result. -// ipv4Address/ipv4Gateway are nil when the attachment is IPv6-only. -type ipamResult struct { - ipv6Subnet *net.IPNet - ipv6Gateway net.IP - ipv4Address net.IP - ipv4Gateway net.IP - routes []*net.IPNet -} +type HostConf = hostconf.HostConf // HostDevicePluginConf is the configuration for the host-device CNI plugin // delegation used to move the guest veth endpoint into the container netns. diff --git a/internal/cnibgp/bgp.go b/internal/cnibgp/bgp.go new file mode 100644 index 00000000..5669d1df --- /dev/null +++ b/internal/cnibgp/bgp.go @@ -0,0 +1,512 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package cnibgp implements galactic-bgp, the SRv6/BGP/eBPF publish plugin +// in the galactic CNI chain. It is chain-invoked (per CNI conflist order) +// after the master plugin (galactic-cni or galactic-tap-cni), not called +// as a library — it has zero kernel-interface dependency: every address it +// advertises comes from prevResult (see prevresult.go), never from a +// runtime call into the interface it doesn't own. Host-interface gateway +// configuration lives in internal/cni/hostgw instead, called directly by +// the master plugins, for exactly this reason. +package cnibgp + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/netip" + "sort" + "strconv" + "strings" + "time" + + "github.com/containernetworking/cni/pkg/skel" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "go.datum.net/galactic/internal/cniipam" + "go.datum.net/galactic/internal/crdnames" + "go.datum.net/galactic/internal/plumbing/ebpf/uformat" + "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" + "go.datum.net/galactic/internal/plumbing/vrf" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// maxRetries is the maximum number of retry attempts for transient k8s API +// errors during the BGP state publish phase. The total number of attempts +// is maxRetries+1 (initial + retries). +const maxRetries = 2 + +// ifaceTypeVeth and ifaceTypeTap are the two values publishConfig.ifaceType +// accepts, inferred from prevResult (see prevresult.go) rather than a +// config field. +const ( + ifaceTypeVeth = "veth" + ifaceTypeTap = "tap" +) + +// publishConfig carries the subset of this plugin's own config that +// publishing needs. +type publishConfig struct { + vpc, vpcAttachment string + // ifaceType selects the eBPF vrf_table egress_kind (veth vs tap) — see + // egressKindForInterfaceType. Inferred from prevResult, never a config + // field. + ifaceType string +} + +// publishResult records what publishBGPState actually created, so cmdAdd +// can fold it into its own rollback tracker. +type publishResult struct { + vrfInstanceCreated bool + advertisementCreated bool + // ebpfRegistered, ebpfBlock, and ebpfArgument record the eBPF uSID + // datapath's vrf_table registration, if one actually happened (the + // BGPRouter may not be configured, in which case ebpfRegistered stays + // false). See unregisterEBPFDatapath for rolling this back. + ebpfRegistered bool + ebpfBlock uint64 + ebpfArgument uint16 +} + +// isTransientError reports whether err is a transient failure that may +// resolve itself on retry (API server unavailable, timeout, network blip). +// Returns false for validation errors, not-found, and other permanent +// failures that should not be retried. +func isTransientError(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return true + } + unwrapped := errors.Unwrap(err) + if unwrapped != nil { + if errors.Is(unwrapped, context.DeadlineExceeded) || errors.Is(unwrapped, context.Canceled) { + return true + } + } + if apierrors.IsServiceUnavailable(err) || + apierrors.IsInternalError(err) || + apierrors.IsServerTimeout(err) || + apierrors.IsTooManyRequests(err) { + return true + } + if netErr, ok := unwrapped.(interface{ Temporary() bool }); ok && netErr.Temporary() { + return true + } + return false +} + +// retryK8sOps runs fn with up to maxRetries+1 attempts, retrying on +// transient k8s API errors with exponential backoff. The context passed to +// fn has a timeout derived from timeout. Non-transient errors are returned +// immediately without retry. +func retryK8sOps(timeout time.Duration, fn func(ctx context.Context) error) error { + var lastErr error + for attempt := 0; attempt <= maxRetries; attempt++ { + if attempt > 0 { + backoff := time.Duration(1< uformat.NodeIDMax { + return false, 0, fmt.Errorf("eBPF registration: nodeID %d out of range [%#x,%#x]", + bgp.nodeID, uint16(uformat.NodeIDMin), uint16(uformat.NodeIDMax)) + } + + egressKind, err := egressKindForInterfaceType(ifaceType) + if err != nil { + return false, 0, fmt.Errorf("determine eBPF egress kind: %w", err) + } + + prefix, err := netip.ParsePrefix(bgp.srv6Locator) + if err != nil { + return false, 0, fmt.Errorf("parse SRv6 locator %q for eBPF registration: %w", bgp.srv6Locator, err) + } + block, err = uformat.Block(prefix.Addr()) + if err != nil { + return false, 0, fmt.Errorf("derive eBPF uSID Block from locator %q: %w", bgp.srv6Locator, err) + } + + vrfTableID, err := vrf.TableID(vpc, vpcAttachment) + if err != nil { + return false, 0, fmt.Errorf("look up VRF table id for eBPF registration: %w", err) + } + + registry, closer, err := usidmap.OpenPinnedRegistry(pinDir) + if err != nil { + return false, 0, fmt.Errorf("open pinned eBPF uSID maps: %w", err) + } + defer func() { _ = closer.Close() }() + + if err := registry.Locator.Register(block, uint16(bgp.nodeID)); err != nil { + return false, 0, fmt.Errorf("register eBPF locator_table entry: %w", err) + } + if err := registry.Function.Register(block, uformat.FunctionEndDT46); err != nil { + return false, 0, fmt.Errorf("register eBPF function_table entry: %w", err) + } + + if err := registry.VRF.Register(block, argument, vrfTableID, egressKind); err != nil { + return false, 0, fmt.Errorf("register eBPF vrf_table entry: %w", err) + } + return true, block, nil +} + +// egressKindForInterfaceType maps a "veth"/"tap" interface type string to the +// vrf_table egress_kind value usid.c's step 9 uses to pick between +// bpf_redirect_peer (veth, crosses into the container's netns) and plain +// bpf_redirect (tap, which never leaves this netns). +func egressKindForInterfaceType(ifaceType string) (uint32, error) { + switch ifaceType { + case ifaceTypeVeth: + return usidmap.EgressKindVeth, nil + case ifaceTypeTap: + return usidmap.EgressKindTap, nil + default: + return 0, fmt.Errorf("unknown interface type %q", ifaceType) + } +} + +// unregisterEBPFDatapath removes the vrf_table entry registerEBPFDatapath +// wrote for this (block, argument) pair, from cmdAdd's failed-ADD rollback +// path. Idempotent: not an error if the entry is already gone. +// +// expectedVRFTableID must be this attachment's own VRF table id (recomputed +// by the caller via vrf.TableID). Only deletes the entry when it still +// resolves to expectedVRFTableID, and leaves it alone otherwise — see +// resourceTracker.cleanup's doc comment for the race this guards against. +func unregisterEBPFDatapath(block uint64, argument uint16, expectedVRFTableID uint32, pinDir string) error { + registry, closer, err := usidmap.OpenPinnedRegistry(pinDir) + if err != nil { + return fmt.Errorf("open pinned eBPF uSID maps: %w", err) + } + defer func() { _ = closer.Close() }() + + entry, ok, err := registry.VRF.Get(block, argument) + if err != nil { + return fmt.Errorf("read eBPF vrf_table entry before unregister: %w", err) + } + if !ok { + return nil + } + if entry.VRFTableID != expectedVRFTableID { + slog.Warn("Rollback: eBPF vrf_table entry no longer belongs to this attachment, leaving it in place", + "block", block, "argument", argument, + "expectedVRFTableID", expectedVRFTableID, "currentVRFTableID", entry.VRFTableID) + return nil + } + + if err := registry.VRF.Unregister(block, argument); err != nil { + return fmt.Errorf("unregister eBPF vrf_table entry: %w", err) + } + return nil +} diff --git a/internal/cni/bgp_ebpf_test.go b/internal/cnibgp/bgp_ebpf_test.go similarity index 54% rename from internal/cni/bgp_ebpf_test.go rename to internal/cnibgp/bgp_ebpf_test.go index 49e810ff..4d34838e 100644 --- a/internal/cni/bgp_ebpf_test.go +++ b/internal/cnibgp/bgp_ebpf_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0-or-later -package cni +package cnibgp import ( "fmt" @@ -15,6 +15,16 @@ import ( "go.datum.net/galactic/internal/plumbing/vrf" ) +// requireRoot skips the test unless running as root — pinned eBPF maps and +// real VRF/netlink state need CAP_NET_ADMIN/CAP_BPF and a real kernel. See +// internal/cni's own requireRoot for the project-wide pattern. +func requireRoot(t *testing.T) { + t.Helper() + if os.Geteuid() != 0 { + t.Skip("requires root (CAP_NET_ADMIN/CAP_BPF); run under scripts/ci.sh unittest-root") + } +} + // TestRegisterEBPFDatapath_NotConfiguredIsNoOp covers the short-circuit for // a node whose BGPRouter has no SRv6Locator/NodeID configured at all: SRv6 // is intentionally not set up for this attachment, so registerEBPFDatapath @@ -22,7 +32,7 @@ import ( func TestRegisterEBPFDatapath_NotConfiguredIsNoOp(t *testing.T) { cfg := bgpConfig{srv6Locator: "", nodeID: 0} registered, _, err := registerEBPFDatapath( - cfg, testVPC, testAttachment, interfaceTypeVeth, 42, "/sys/fs/bpf/galactic-does-not-exist") + cfg, testVPC, testAttachment, ifaceTypeVeth, 42, "/sys/fs/bpf/galactic-does-not-exist") if err != nil { t.Errorf("registerEBPFDatapath with unconfigured BGPRouter = %v, want nil (no-op)", err) } @@ -35,18 +45,11 @@ func TestRegisterEBPFDatapath_NotConfiguredIsNoOp(t *testing.T) { // bounds check on the raw nodeID *before* it narrows to uint16 for // registration: an out-of-[uformat.NodeIDMin,NodeIDMax] value (here, one // that wraps to an in-range-looking uint16 if narrowed unchecked -- 0x10001 -// wraps to 1, which alone would otherwise pass uformat.ValidateNodeID inside -// registry.Locator.Register) must be rejected here, before any pinned map -// is even opened -- proven the same way -// TestRegisterEBPFDatapath_NotConfiguredIsNoOp proves its own no-op, via a -// pinDir that doesn't exist: if this check ran after the narrowing (or not -// at all), the call would instead fail later with an "open pinned eBPF uSID -// maps" error against that bogus path, not the bounds-check error this test -// asserts. +// wraps to 1) must be rejected here, before any pinned map is even opened. func TestRegisterEBPFDatapath_RejectsOutOfRangeNodeID(t *testing.T) { cfg := bgpConfig{srv6Locator: "2001:db8:1::/48", nodeID: 0x10001} // wraps to uint16(1) if narrowed unchecked registered, _, err := registerEBPFDatapath( - cfg, testVPC, testAttachment, interfaceTypeVeth, 42, "/sys/fs/bpf/galactic-does-not-exist") + cfg, testVPC, testAttachment, ifaceTypeVeth, 42, "/sys/fs/bpf/galactic-does-not-exist") if err == nil { t.Fatal("registerEBPFDatapath with nodeID=0x10001 = nil error, want an out-of-range rejection") } @@ -58,11 +61,11 @@ func TestRegisterEBPFDatapath_RejectsOutOfRangeNodeID(t *testing.T) { } } -// TestRegisterEBPFDatapath_RegistersAllThreeTables is Milestone 7.1's exit -// criterion: a single registerEBPFDatapath call populates locator_table, -// function_table, and vrf_table consistently for the same (vpc, -// vpcAttachment), against real pinned eBPF maps under a throwaway pin -// directory -- not the production attach.PinDir. +// TestRegisterEBPFDatapath_RegistersAllThreeTables: a single +// registerEBPFDatapath call populates locator_table, function_table, and +// vrf_table consistently for the same (vpc, vpcAttachment), against real +// pinned eBPF maps under a throwaway pin directory — not the production +// attach.PinDir. func TestRegisterEBPFDatapath_RegistersAllThreeTables(t *testing.T) { requireRoot(t) @@ -88,7 +91,7 @@ func TestRegisterEBPFDatapath_RegistersAllThreeTables(t *testing.T) { t.Cleanup(func() { _ = loaderObjs.Close() }) cfg := bgpConfig{srv6Locator: locator, nodeID: nodeID} - registered, _, err := registerEBPFDatapath(cfg, vpc, vpcAttachment, interfaceTypeVeth, uint16(vrfID), pinDir) + registered, _, err := registerEBPFDatapath(cfg, vpc, vpcAttachment, ifaceTypeVeth, uint16(vrfID), pinDir) if err != nil { t.Fatalf("registerEBPFDatapath: %v", err) } @@ -119,8 +122,8 @@ func TestRegisterEBPFDatapath_RegistersAllThreeTables(t *testing.T) { entries[0].VRFTableID, vrfTableID) } if entries[0].EgressKind != usidmap.EgressKindVeth { - t.Errorf("vrf_table entry EgressKind = %d, want %d (EgressKindVeth, from InterfaceType %q)", - entries[0].EgressKind, usidmap.EgressKindVeth, interfaceTypeVeth) + t.Errorf("vrf_table entry EgressKind = %d, want %d (EgressKindVeth, from interfaceType %q)", + entries[0].EgressKind, usidmap.EgressKindVeth, ifaceTypeVeth) } locEntries, err := reg.Locator.List() @@ -140,24 +143,10 @@ func TestRegisterEBPFDatapath_RegistersAllThreeTables(t *testing.T) { } } -// TestResourceTrackerCleanup_UnregistersEBPFVRFEntry is Milestone 7.2's -// exit criterion: a failed ADD's rollback (resourceTracker.cleanup) cleans -// up both the kernel route (existing behavior, already covered by -// TestResourceTrackerCleanupPartialState) and the new eBPF vrf_table map -// entry, when one was actually registered. cleanup's own unregister step -// always targets the real, production attach.PinDir (it is not -// parameterized, unlike registerEBPFDatapath -- see resource.go), so this -// test loads/pins the real datapath there for the duration of the test, -// cleaning it up fully afterward; this mirrors the same "real global -// state" pattern this file's other resourceTracker tests already use for -// vrf.Delete/veth.Delete. -// -// cleanup's unregister step now recomputes this attachment's own VRF table -// id (vrf.TableID) and only deletes the vrf_table entry if it still -// resolves there, so a real VRF interface for (testVPC, testAttachment) -// must exist for the duration of this test -- unlike before this fix, -// where the seeded entry's VRFTableID was an arbitrary, unrelated value. -func TestResourceTrackerCleanup_UnregistersEBPFVRFEntry(t *testing.T) { +// TestUnregisterEBPFDatapath_RemovesOwnEntry covers UnregisterEBPFDatapath's +// normal path: an entry this attachment registered gets removed when its +// VRFTableID still matches. +func TestUnregisterEBPFDatapath_RemovesOwnEntry(t *testing.T) { requireRoot(t) if err := vrf.Add(testVPC, testAttachment); err != nil { @@ -169,16 +158,17 @@ func TestResourceTrackerCleanup_UnregistersEBPFVRFEntry(t *testing.T) { t.Fatalf("vrf.TableID: %v", err) } - loaderObjs, err := attach.Load(attach.PinDir) + pinDir := fmt.Sprintf("/sys/fs/bpf/galactic-bgp-test-%d", os.Getpid()) + t.Cleanup(func() { _ = os.RemoveAll(pinDir) }) + loaderObjs, err := attach.Load(pinDir) if err != nil { - t.Fatalf("attach.Load(attach.PinDir): %v", err) + t.Fatalf("attach.Load: %v", err) } t.Cleanup(func() { _ = loaderObjs.Close() }) - t.Cleanup(func() { _ = os.RemoveAll(attach.PinDir) }) - reg, closer, err := usidmap.OpenPinnedRegistry(attach.PinDir) + reg, closer, err := usidmap.OpenPinnedRegistry(pinDir) if err != nil { - t.Fatalf("OpenPinnedRegistry(attach.PinDir): %v", err) + t.Fatalf("OpenPinnedRegistry: %v", err) } defer func() { _ = closer.Close() }() @@ -188,62 +178,46 @@ func TestResourceTrackerCleanup_UnregistersEBPFVRFEntry(t *testing.T) { if err := reg.VRF.Register(testBlock, testArgument, vrfTableID, usidmap.EgressKindVeth); err != nil { t.Fatalf("seed vrf_table entry: %v", err) } - if _, ok, err := reg.VRF.Get(testBlock, testArgument); err != nil || !ok { - t.Fatalf("seeded entry not visible before cleanup: ok=%v err=%v", ok, err) - } - tracker := &resourceTracker{ - vpc: testVPC, - vpcAttachment: testAttachment, - namespace: "ebpf-cleanup-test", - ebpfRegistered: true, - ebpfBlock: testBlock, - ebpfArgument: testArgument, - } - tracker.cleanup(t.Context()) + if err := unregisterEBPFDatapath(testBlock, testArgument, vrfTableID, pinDir); err != nil { + t.Fatalf("UnregisterEBPFDatapath: %v", err) + } if _, ok, err := reg.VRF.Get(testBlock, testArgument); err != nil || ok { - t.Errorf("vrf_table entry after cleanup: ok=%v err=%v, want ok=false (unregistered)", ok, err) + t.Errorf("vrf_table entry after unregister: ok=%v err=%v, want ok=false", ok, err) } } -// TestResourceTrackerCleanup_LeavesEBPFVRFEntryOwnedByAnotherAttachment -// covers the race this fix closes: retryK8sOps can re-run -// publishBGPStateK8s's whole closure on a later attempt without -// re-registering the eBPF entry (registerEBPFDatapath only runs again if -// that attempt gets that far), so by the time a later attempt's -// checkArgumentCollision failure triggers this rollback, the (block, +// TestUnregisterEBPFDatapath_LeavesEntryOwnedByAnotherAttachment covers the +// race UnregisterEBPFDatapath guards against: retryK8sOps can re-run +// PublishBGPStateK8s's whole closure on a later attempt without +// re-registering the eBPF entry, so by the time a later attempt's +// checkArgumentCollision failure triggers a caller's rollback, the (block, // argument) slot this attachment originally wrote may have since been // overwritten by the very other attachment the collision was detected -// against -- unregistering unconditionally would delete a live -// attachment's forwarding entry instead of this rolled-back one's own. If -// the current entry's VRFTableID no longer matches this attachment's own -// (recomputed fresh, not read from the tracker), cleanup must leave it in -// place. -func TestResourceTrackerCleanup_LeavesEBPFVRFEntryOwnedByAnotherAttachment(t *testing.T) { +// against. Unregistering unconditionally would delete a live attachment's +// forwarding entry instead of this rolled-back one's own. +func TestUnregisterEBPFDatapath_LeavesEntryOwnedByAnotherAttachment(t *testing.T) { requireRoot(t) - if err := vrf.Add(testVPC, testAttachment); err != nil { - t.Fatalf("vrf.Add: %v", err) - } - t.Cleanup(func() { _ = vrf.Delete(testVPC, testAttachment) }) - - loaderObjs, err := attach.Load(attach.PinDir) + pinDir := fmt.Sprintf("/sys/fs/bpf/galactic-bgp-test-%d", os.Getpid()) + t.Cleanup(func() { _ = os.RemoveAll(pinDir) }) + loaderObjs, err := attach.Load(pinDir) if err != nil { - t.Fatalf("attach.Load(attach.PinDir): %v", err) + t.Fatalf("attach.Load: %v", err) } t.Cleanup(func() { _ = loaderObjs.Close() }) - t.Cleanup(func() { _ = os.RemoveAll(attach.PinDir) }) - reg, closer, err := usidmap.OpenPinnedRegistry(attach.PinDir) + reg, closer, err := usidmap.OpenPinnedRegistry(pinDir) if err != nil { - t.Fatalf("OpenPinnedRegistry(attach.PinDir): %v", err) + t.Fatalf("OpenPinnedRegistry: %v", err) } defer func() { _ = closer.Close() }() const testBlock uint64 = 0x0102030405 const testArgument uint16 = 0x042 const anotherAttachmentsVRFTableID uint32 = 0x9999 + const thisAttachmentsVRFTableID uint32 = 0x1111 // Simulate the colliding attachment having since overwritten this same // (block, argument) slot with its own, different VRF table id. @@ -251,23 +225,17 @@ func TestResourceTrackerCleanup_LeavesEBPFVRFEntryOwnedByAnotherAttachment(t *te t.Fatalf("seed vrf_table entry: %v", err) } - tracker := &resourceTracker{ - vpc: testVPC, - vpcAttachment: testAttachment, - namespace: "ebpf-cleanup-test", - ebpfRegistered: true, - ebpfBlock: testBlock, - ebpfArgument: testArgument, - } - tracker.cleanup(t.Context()) + if err := unregisterEBPFDatapath(testBlock, testArgument, thisAttachmentsVRFTableID, pinDir); err != nil { + t.Fatalf("UnregisterEBPFDatapath: %v", err) + } entry, ok, err := reg.VRF.Get(testBlock, testArgument) if err != nil || !ok { - t.Fatalf("vrf_table entry after cleanup: ok=%v err=%v, want ok=true (must survive, it's not this attachment's)", + t.Fatalf("vrf_table entry after unregister: ok=%v err=%v, want ok=true (must survive, it's not this attachment's)", ok, err) } if entry.VRFTableID != anotherAttachmentsVRFTableID { - t.Errorf("vrf_table entry VRFTableID after cleanup = %#x, want unchanged %#x", + t.Errorf("vrf_table entry VRFTableID after unregister = %#x, want unchanged %#x", entry.VRFTableID, anotherAttachmentsVRFTableID) } } diff --git a/internal/cni/bgp_test.go b/internal/cnibgp/bgp_test.go similarity index 57% rename from internal/cni/bgp_test.go rename to internal/cnibgp/bgp_test.go index 91cd70e3..aa95068f 100644 --- a/internal/cni/bgp_test.go +++ b/internal/cnibgp/bgp_test.go @@ -2,156 +2,85 @@ // // SPDX-License-Identifier: AGPL-3.0-or-later -package cni +package cnibgp import ( "context" + "errors" "fmt" "net" "reflect" "strings" "testing" + "time" - "github.com/vishvananda/netlink" - "golang.org/x/sys/unix" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "go.datum.net/galactic/internal/cniipam" + "go.datum.net/galactic/internal/crdnames" "go.datum.net/galactic/internal/plumbing/ebpf/uformat" "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" bgpv1alpha1 "go.datum.net/network/api/v1alpha1" ) -// ---- ipv4GatewayAddrParams ------------------------------------------------ +const ( + testVPC = "abc" + testAttachment = "def" + testRouterName = "overlay-router" + testRD65000_1 = "65000:1" + testVPCHex1234 = "0000000004d2" // decimal 1234 + testNetns = "/proc/1/ns/net" +) -func TestIPv4GatewayAddrParams(t *testing.T) { - tests := []struct { - name string - hostLink netlink.Link - wantMask net.IPMask - wantFlags int - }{ - { - name: "tap gets /25 with NOPREFIXROUTE", - hostLink: &netlink.Tuntap{LinkAttrs: netlink.LinkAttrs{Name: "tap0"}}, - wantMask: net.CIDRMask(25, 32), - wantFlags: unix.IFA_F_NOPREFIXROUTE, - }, - { - name: "veth gets /32 with no flags", - hostLink: &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: "veth0"}}, - wantMask: net.CIDRMask(32, 32), - wantFlags: 0, - }, - } +var testScheme = func() *runtime.Scheme { + s := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(s)) + utilruntime.Must(bgpv1alpha1.AddToScheme(s)) + return s +}() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotMask, gotFlags := ipv4GatewayAddrParams(tt.hostLink) - if gotMask.String() != tt.wantMask.String() { - t.Errorf("mask = %v, want %v", gotMask, tt.wantMask) - } - if gotFlags != tt.wantFlags { - t.Errorf("flags = %v, want %v", gotFlags, tt.wantFlags) - } - }) - } +func fakeClient(objs ...client.Object) client.Client { + return fake.NewClientBuilder().WithScheme(testScheme).WithObjects(objs...).Build() } -// ---- routeConflicts ------------------------------------------------------ - -func TestRouteConflicts(t *testing.T) { - dst := mustParseCIDR(t, "fd00:10:ff01::1234/80") - gw1 := net.ParseIP("fd00:10:ff01::1") - gw2 := net.ParseIP("fd00:10:ff01::2") - otherDst := mustParseCIDR(t, "fd00:10:ff02::1234/80") +func mustParseCIDR(t *testing.T, cidr string) *net.IPNet { + t.Helper() + _, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + t.Fatalf("parse CIDR %q: %v", cidr, err) + } + return ipnet +} - tests := []struct { - name string - existing *netlink.Route - desired *netlink.Route - want bool - }{ - { - name: "nil existing destination — no conflict", - existing: &netlink.Route{Dst: nil}, - desired: &netlink.Route{Dst: dst}, - want: false, - }, - { - name: "nil desired destination — no conflict", - existing: &netlink.Route{Dst: dst}, - desired: &netlink.Route{Dst: nil}, - want: false, - }, - { - name: "different destinations — no conflict", - existing: &netlink.Route{Dst: otherDst}, - desired: &netlink.Route{Dst: dst}, - want: false, - }, - { - name: "same destination, no gateway on either — no conflict", - existing: &netlink.Route{Dst: dst, LinkIndex: 5}, - desired: &netlink.Route{Dst: dst, LinkIndex: 5}, - want: false, - }, - { - name: "same destination, same gateway — no conflict", - existing: &netlink.Route{Dst: dst, Gw: gw1, LinkIndex: 5}, - desired: &netlink.Route{Dst: dst, Gw: gw1, LinkIndex: 5}, - want: false, +// routerForNode builds a BGPRouter with spec.targetRef.name set to nodeName. +func routerForNode(name, nodeName, namespace string, asn int64) *bgpv1alpha1.BGPRouter { + return &bgpv1alpha1.BGPRouter{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, }, - { - name: "same destination, different gateway — conflict", - existing: &netlink.Route{Dst: dst, Gw: gw1}, - desired: &netlink.Route{Dst: dst, Gw: gw2}, - want: true, - }, - { - name: "existing has gateway, desired does not — conflict", - existing: &netlink.Route{Dst: dst, Gw: gw1}, - desired: &netlink.Route{Dst: dst}, - want: true, - }, - { - name: "desired has gateway, existing does not — conflict", - existing: &netlink.Route{Dst: dst}, - desired: &netlink.Route{Dst: dst, Gw: gw1}, - want: true, + Spec: bgpv1alpha1.BGPRouterSpec{ + TargetRef: bgpv1alpha1.TargetRef{ + Kind: "Node", + Name: nodeName, + }, + LocalASN: asn, + RouterID: "10.0.0.1", + Roles: []bgpv1alpha1.RouterRole{bgpv1alpha1.RouterRoleTenant}, + AddressFamilies: []bgpv1alpha1.AddressFamily{ + {AFI: bgpv1alpha1.AFIL2VPN, SAFI: bgpv1alpha1.SAFIEVPN}, + }, }, - { - name: "same destination, same gateway, different link index — conflict", - existing: &netlink.Route{Dst: dst, Gw: gw1, LinkIndex: 5}, - desired: &netlink.Route{Dst: dst, Gw: gw1, LinkIndex: 7}, - want: true, - }, - { - name: "same destination, gateway set, link index zero on existing — no conflict", - existing: &netlink.Route{Dst: dst, Gw: gw1, LinkIndex: 0}, - desired: &netlink.Route{Dst: dst, Gw: gw1, LinkIndex: 5}, - want: false, - }, - { - name: "same destination, no gateway, different link index — conflict", - existing: &netlink.Route{Dst: dst, LinkIndex: 5}, - desired: &netlink.Route{Dst: dst, LinkIndex: 7}, - want: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := routeConflicts(tt.existing, tt.desired) - if got != tt.want { - t.Errorf("routeConflicts() = %v, want %v", got, tt.want) - } - }) } } -// ---- allocateArgument ------------------------------------------------------ - // vrfInstanceForRouter builds a BGPVRFInstance targeting routerName with the // given VRFID (the allocated Argument), for allocateArgument's test fixtures. func vrfInstanceForRouter(name, namespace, routerName string, vrfID int32) *bgpv1alpha1.BGPVRFInstance { @@ -164,6 +93,8 @@ func vrfInstanceForRouter(name, namespace, routerName string, vrfID int32) *bgpv } } +// ---- allocateArgument ------------------------------------------------------ + func TestAllocateArgument(t *testing.T) { const ( namespace = "default" @@ -196,9 +127,6 @@ func TestAllocateArgument(t *testing.T) { t.Run("skips values used by this router and ignores other routers", func(t *testing.T) { used1 := vrfInstanceForRouter("other-att-1", namespace, routerName, 1) used2 := vrfInstanceForRouter("other-att-2", namespace, routerName, 2) - // Same VRFID (1) under a different router -- must not count toward - // this router's used set, since Argument allocation is per node - // (i.e. per BGPRouter), not platform-wide. differentRouter := vrfInstanceForRouter("different-router-att", namespace, "other-router", 1) k8s := fakeClient(used1, used2, differentRouter) got, err := allocateArgument(context.Background(), k8s, namespace, routerName, "new-att") @@ -241,12 +169,6 @@ func TestAllocateArgument(t *testing.T) { // ---- checkArgumentCollision ------------------------------------------------- -// TestCheckArgumentCollision guards against a regression of the fix where a -// lexicographic-name tie-break let exactly one of two colliding instances -// "win" without ever proving the other side's check would run after this -// one's create -- concurrent create+check interleaving could let both sides -// pass. Detection must not depend on name ordering: it must fire regardless -// of whether the other instance's name sorts before or after this one's. func TestCheckArgumentCollision(t *testing.T) { const ( namespace = "default" @@ -304,9 +226,9 @@ func TestEgressKindForInterfaceType(t *testing.T) { want uint32 wantErr bool }{ - {name: "veth maps to EgressKindVeth", iface: interfaceTypeVeth, want: usidmap.EgressKindVeth}, - {name: "empty defaults to EgressKindVeth", iface: "", want: usidmap.EgressKindVeth}, - {name: "tap maps to EgressKindTap", iface: interfaceTypeTap, want: usidmap.EgressKindTap}, + {name: "veth maps to EgressKindVeth", iface: ifaceTypeVeth, want: usidmap.EgressKindVeth}, + {name: "tap maps to EgressKindTap", iface: ifaceTypeTap, want: usidmap.EgressKindTap}, + {name: "empty type errors", iface: "", wantErr: true}, {name: "unknown type errors", iface: "bogus", wantErr: true}, } @@ -337,9 +259,6 @@ func TestBuildVRFInstanceSpec(t *testing.T) { if spec.RouterRef == nil || spec.RouterRef.Name != testRouterName { t.Errorf("RouterRef = %+v, want Name %q", spec.RouterRef, testRouterName) } - if spec.RouterSelector != nil { - t.Errorf("RouterSelector = %+v, want nil", spec.RouterSelector) - } if spec.VRFID != 1234 { t.Errorf("VRFID = %d, want 1234", spec.VRFID) } @@ -364,7 +283,7 @@ func TestIPAMAdvertisementPrefixesNil(t *testing.T) { } func TestIPAMAdvertisementPrefixesIPv4Only(t *testing.T) { - res := &ipamResult{ipv4Address: net.ParseIP("10.128.0.5")} + res := &cniipam.IPAMResult{IPv4Address: net.ParseIP("10.128.0.5")} prefixes, ipv6Subnet, ipv4Addr := ipamAdvertisementPrefixes(res) @@ -381,7 +300,7 @@ func TestIPAMAdvertisementPrefixesIPv4Only(t *testing.T) { func TestIPAMAdvertisementPrefixesDualStack(t *testing.T) { ipv6Subnet := mustParseCIDR(t, "fd00:10:ff01::1234/96") - res := &ipamResult{ipv6Subnet: ipv6Subnet, ipv4Address: net.ParseIP("10.128.0.5")} + res := &cniipam.IPAMResult{IPv6Subnet: ipv6Subnet, IPv4Address: net.ParseIP("10.128.0.5")} prefixes, gotIPv6Subnet, gotIPv4Addr := ipamAdvertisementPrefixes(res) @@ -410,9 +329,9 @@ func TestAllAdvertisedPrefixesEmpty(t *testing.T) { func TestAllAdvertisedPrefixesSingleContainer(t *testing.T) { const v6, v4 = "fd00:20:ff01::1234/96", "172.20.1.5" annotations := map[string]string{ - netnsAnnotationKey("cid-a"): testNetns, - subnetAnnotationKeyIPv6("cid-a"): v6, - subnetAnnotationKeyIPv4("cid-a"): v4, + crdnames.NetNSKey("cid-a"): testNetns, + crdnames.SubnetKeyIPv6("cid-a"): v6, + crdnames.SubnetKeyIPv4("cid-a"): v4, } got := allAdvertisedPrefixes(annotations) @@ -431,11 +350,11 @@ func TestAllAdvertisedPrefixesSingleContainer(t *testing.T) { func TestAllAdvertisedPrefixesMultipleContainers(t *testing.T) { const aV4, bV6, bV4 = "172.20.1.5", "fd00:20:ff01::1234/96", "172.21.1.2" annotations := map[string]string{ - netnsAnnotationKey("cid-a"): testNetns, - subnetAnnotationKeyIPv4("cid-a"): aV4, - netnsAnnotationKey("cid-b"): testNetns, - subnetAnnotationKeyIPv6("cid-b"): bV6, - subnetAnnotationKeyIPv4(("cid-b")): bV4, + crdnames.NetNSKey("cid-a"): testNetns, + crdnames.SubnetKeyIPv4("cid-a"): aV4, + crdnames.NetNSKey("cid-b"): testNetns, + crdnames.SubnetKeyIPv6("cid-b"): bV6, + crdnames.SubnetKeyIPv4("cid-b"): bV4, } got := allAdvertisedPrefixes(annotations) @@ -449,9 +368,9 @@ func TestAllAdvertisedPrefixesMultipleContainers(t *testing.T) { func TestAllAdvertisedPrefixesIgnoresOtherAnnotations(t *testing.T) { const v4 = "172.20.1.5" annotations := map[string]string{ - netnsAnnotationKey("cid-a"): testNetns, - subnetAnnotationKeyIPv4("cid-a"): v4, - "some.other/annotation": "should be ignored", + crdnames.NetNSKey("cid-a"): testNetns, + crdnames.SubnetKeyIPv4("cid-a"): v4, + "some.other/annotation": "should be ignored", } got := allAdvertisedPrefixes(annotations) @@ -503,3 +422,255 @@ func TestBuildAdvertisementSpecDualStack(t *testing.T) { t.Errorf("Prefixes[1] = %q, want %q", spec.Prefixes[1], ipv4Prefix) } } + +// ---- routeTarget --------------------------------------------------------- + +func TestRouteTarget(t *testing.T) { + tests := []struct { + name string + asNumber int64 + vpcHex string + want string + wantErr bool + }{ + {name: "VPC value fits in 32 bits", asNumber: 65000, vpcHex: testVPCHex1234, want: "65000:1234"}, + {name: "upper bits beyond 32 stripped", asNumber: 65000, vpcHex: "000100000001", want: testRD65000_1}, + {name: "low 32 bits all set", asNumber: 65000, vpcHex: "0000ffffffff", want: "65000:4294967295"}, + {name: "different ASN", asNumber: 4200000000, vpcHex: testVPCHex1234, want: "4200000000:1234"}, + {name: "invalid hex string", vpcHex: "zzzzzz", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := routeTarget(tt.asNumber, tt.vpcHex) + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("routeTarget(%d, %q) = %q, want %q", tt.asNumber, tt.vpcHex, got, tt.want) + } + }) + } +} + +// ---- lookupBGPRouter ----------------------------------------------------- + +func TestLookupBGPRouter(t *testing.T) { + ctx := context.Background() + const ( + nodeName = "node1" + namespace = "default" + ) + + matchingRouter := routerForNode(testRouterName, nodeName, namespace, 65000) + + tests := []struct { + name string + objects []client.Object + wantErr string + check func(t *testing.T, cfg bgpConfig) + }{ + {name: "no router for node", objects: nil, wantErr: "no BGPRouter found"}, + { + name: "single matching router returns correct config", + objects: []client.Object{matchingRouter}, + check: func(t *testing.T, cfg bgpConfig) { + t.Helper() + if cfg.asNumber != 65000 { + t.Errorf("asNumber = %d, want 65000", cfg.asNumber) + } + if cfg.routerName != testRouterName { + t.Errorf("routerName = %q, want %q", cfg.routerName, testRouterName) + } + }, + }, + { + name: "router with SRv6Locator and NodeID configured", + objects: []client.Object{ + func() *bgpv1alpha1.BGPRouter { + r := routerForNode("srv6-router", nodeName, namespace, 65000) + r.Spec.SRv6Locator = "fd00:10::/48" + r.Spec.NodeID = 7 + return r + }(), + }, + check: func(t *testing.T, cfg bgpConfig) { + t.Helper() + if cfg.srv6Locator != "fd00:10::/48" { + t.Errorf("srv6Locator = %q, want %q", cfg.srv6Locator, "fd00:10::/48") + } + if cfg.nodeID != 7 { + t.Errorf("nodeID = %d, want 7", cfg.nodeID) + } + }, + }, + { + name: "router in different namespace is ignored", + objects: []client.Object{routerForNode("other-ns-router", nodeName, "other-ns", 65001)}, + wantErr: "no BGPRouter found", + }, + { + name: "non-matching node router is ignored", + objects: []client.Object{ + routerForNode("other-node-router", "node2", namespace, 65001), + matchingRouter, + }, + check: func(t *testing.T, cfg bgpConfig) { + t.Helper() + if cfg.routerName != testRouterName { + t.Errorf("routerName = %q, want %q", cfg.routerName, testRouterName) + } + }, + }, + { + name: "ambiguous: two routers target same node", + objects: []client.Object{ + routerForNode("router-a", nodeName, namespace, 65000), + routerForNode("router-b", nodeName, namespace, 65001), + }, + wantErr: "ambiguous", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8s := fakeClient(tt.objects...) + + cfg, err := lookupBGPRouter(ctx, k8s, nodeName, namespace) + if tt.wantErr != "" { + if err == nil { + t.Fatalf("expected error containing %q, got nil", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("error %q does not contain %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tt.check != nil { + tt.check(t, cfg) + } + }) + } +} + +// ---- isTransientError ---------------------------------------------------- + +func TestIsTransientError(t *testing.T) { + tests := []struct { + name string + err error + wantTrans bool + }{ + {"nil error is not transient", nil, false}, + {"context deadline exceeded is transient", context.DeadlineExceeded, true}, + {"context canceled is transient", context.Canceled, true}, + {"wrapped context deadline exceeded is transient", fmt.Errorf("k8s: %w", context.DeadlineExceeded), true}, + {"wrapped context canceled is transient", fmt.Errorf("k8s: %w", context.Canceled), true}, + {"generic error is not transient", errors.New("some error"), false}, + {"validation error is not transient", apierrors.NewBadRequest("bad request"), false}, + { + "not found error is not transient", + apierrors.NewNotFound(schema.GroupResource{Group: "network.datumapis.com", Resource: "bgpadvertisements"}, "test"), + false, + }, + {"503 service unavailable is transient", apierrors.NewServiceUnavailable("service unavailable"), true}, + {"429 too many requests is transient", apierrors.NewTooManyRequests("too many requests", 0), true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isTransientError(tt.err) + if got != tt.wantTrans { + t.Errorf("isTransientError(%v) = %v, want %v", tt.err, got, tt.wantTrans) + } + }) + } +} + +// ---- retryK8sOps --------------------------------------------------------- + +func TestRetryK8sOpsSucceedsImmediately(t *testing.T) { + calls := 0 + err := retryK8sOps(100*time.Millisecond, func(ctx context.Context) error { + calls++ + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if calls != 1 { + t.Errorf("expected 1 call, got %d", calls) + } +} + +func TestRetryK8sOpsRetriesOnTransientError(t *testing.T) { + calls := 0 + err := retryK8sOps(2*time.Second, func(ctx context.Context) error { + calls++ + if calls < 3 { + return context.DeadlineExceeded + } + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if calls != 3 { + t.Errorf("expected 3 calls (initial + 2 retries), got %d", calls) + } +} + +func TestRetryK8sOpsFailsAfterMaxRetries(t *testing.T) { + calls := 0 + err := retryK8sOps(2*time.Second, func(ctx context.Context) error { + calls++ + return context.DeadlineExceeded + }) + if err == nil { + t.Fatal("expected error, got nil") + } + if calls != maxRetries+1 { + t.Errorf("expected %d calls (initial + maxRetries), got %d", maxRetries+1, calls) + } +} + +func TestRetryK8sOpsNoRetryOnNonTransientError(t *testing.T) { + calls := 0 + permanentErr := errors.New("validation failed") + err := retryK8sOps(2*time.Second, func(ctx context.Context) error { + calls++ + return permanentErr + }) + if !errors.Is(err, permanentErr) { + t.Fatalf("expected %v, got %v", permanentErr, err) + } + if calls != 1 { + t.Errorf("expected 1 call (no retry), got %d", calls) + } +} + +func TestRetryK8sOpsExhaustsDeadline(t *testing.T) { + calls := 0 + err := retryK8sOps(1*time.Millisecond, func(ctx context.Context) error { + calls++ + return apierrors.NewServiceUnavailable("unavailable") + }) + if err == nil { + t.Fatal("expected error, got nil") + } + if calls != maxRetries+1 { + t.Errorf("expected %d calls, got %d", maxRetries+1, calls) + } + if !strings.Contains(err.Error(), "unavailable") { + t.Errorf("expected 'unavailable' in error, got %v", err) + } +} diff --git a/internal/cnibgp/cnibgp.go b/internal/cnibgp/cnibgp.go new file mode 100644 index 00000000..20384b4c --- /dev/null +++ b/internal/cnibgp/cnibgp.go @@ -0,0 +1,42 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnibgp + +import ( + "time" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/version" + + "go.datum.net/galactic/internal/metadata" + "go.datum.net/galactic/internal/plumbing/ebpf/attach" +) + +const cniTimeout = 10 * time.Second + +// ebpfPinDir is the bpffs directory this package's own eBPF registration +// (registerEBPFDatapath, bgp.go), rollback (resourceTracker.cleanup, +// resource.go), and CHECK (checkEBPFEntry, ops_check.go) all read/write +// pinned uSID maps under. A package var defaulting to attach.PinDir, rather +// than every call site reading that constant directly, so tests can point +// it at a throwaway pin directory instead of the real production one — +// attach.PinDir being a const otherwise gives production callers no seam +// for that (see resource_test.go's resourceTracker.cleanup tests). +var ebpfPinDir = attach.PinDir + +// RunPlugin starts galactic-bgp, handling the CNI ADD, DEL, CHECK, and +// STATUS operations for the BGP/SRv6/eBPF publish stage of the chain. +func RunPlugin() { + skel.PluginMainFuncs( + skel.CNIFuncs{ + Add: cmdAdd, + Check: cmdCheck, + Del: cmdDel, + Status: cmdStatus, + }, + version.All, + "CNI galactic-bgp plugin "+metadata.Version, + ) +} diff --git a/internal/cnibgp/config.go b/internal/cnibgp/config.go new file mode 100644 index 00000000..d18a8355 --- /dev/null +++ b/internal/cnibgp/config.go @@ -0,0 +1,252 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnibgp + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + + "github.com/containernetworking/cni/pkg/types" + type100 "github.com/containernetworking/cni/pkg/types/100" + + "go.datum.net/galactic/internal/config" + "go.datum.net/galactic/internal/hostconf" +) + +var ConfFile = config.DefaultConfFile + +// cniConfig is the shared config resolver for env var resolution. +// Initialized by InitCNIConfig() (called from cmd/galactic-bgp/main.go). +// +// Reuses internal/config.CNIConfig (the GALACTIC_CNI_* env var names) as-is +// rather than defining a GALACTIC_BGP_* set: node_name/kubeconfig/namespace +// are shared node-level settings, not domain-specific behavior the way +// galactic-ipam's own enable-local-ipam flag is — every binary in the chain +// resolves them from the same static conflist file (see +// go.datum.net/galactic/internal/hostconf's doc comment). +var cniConfig *config.CNIConfig + +// InitCNIConfig initializes the shared config resolver for CNI env var +// resolution. Callers should invoke this once at process startup before any +// config lookups. +func InitCNIConfig() { + cniConfig = config.NewCNIConfig() +} + +const errInvalidCNIConfig = "invalid CNI config" + +const ( + errVPCRequired = "vpc is required and must be a non-empty base62 string" + errVPCAttachmentRequired = "vpcattachment is required and must be a non-empty base62 string" +) + +const sanitizeForErrorBinary = "" + +// isValidBase62 reports whether s contains only valid base62 characters +// ([0-9a-zA-Z]) and is non-empty. +func isValidBase62(s string) bool { + if len(s) == 0 { + return false + } + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') { + return false + } + } + return true +} + +// loadHostConf loads node-local settings from the static per-node conflist. +// If the file is missing, it returns a zero-value HostConf (tolerating local +// test runs) but still defaulting Namespace to config.DefaultNamespace. +func loadHostConf(filePath string) (*HostConf, error) { + if filePath == "" { + filePath = config.DefaultConfFile + } + conf, err := hostconf.Load(filePath, hostconf.PluginType) + if err != nil { + if os.IsNotExist(unwrapPathError(err)) { + return &HostConf{Namespace: config.DefaultNamespace}, nil + } + return nil, err + } + if conf.Namespace == "" { + conf.Namespace = config.DefaultNamespace + } + return conf, nil +} + +// unwrapPathError returns the innermost error wrapped by err, so +// os.IsNotExist (which does not itself traverse %w wrapping) can still +// recognize a missing conflist file wrapped by hostconf.Load. +func unwrapPathError(err error) error { + for { + unwrapped := errors.Unwrap(err) + if unwrapped == nil { + return err + } + err = unwrapped + } +} + +// parseLogLevel maps a config-supplied level name to a slog.Level. +func parseLogLevel(s string) (slog.Level, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "": + return parseLogLevel(config.DefaultLogLevel) + case config.LogLevelDebug: + return slog.LevelDebug, nil + case config.DefaultLogLevel: + return slog.LevelInfo, nil + case config.LogLevelWarn, config.LogLevelWarning: + return slog.LevelWarn, nil + case config.LogLevelError: + return slog.LevelError, nil + default: + return slog.LevelInfo, fmt.Errorf("unknown log level %q (want %s, %s, %s, or %s)", + s, config.LogLevelDebug, config.DefaultLogLevel, config.LogLevelWarn, config.LogLevelError) + } +} + +// setupLogging configures the slog default logger to write to the specified +// path at the specified verbosity. +func setupLogging(logPath, logLevel string) { + if logPath == "" { + logPath = config.DefaultLogFile + } + level, err := parseLogLevel(logLevel) + if err != nil { + slog.Warn("Invalid log level, falling back to default", + "value", logLevel, "default", config.DefaultLogLevel, "err", err) + } + if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { + slog.Warn("Failed to create log directory", "path", filepath.Dir(logPath), "err", err) + return + } + file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + slog.Warn("Failed to open log file, falling back to Stderr", "path", logPath, "err", err) + return + } + handler := slog.NewJSONHandler(file, &slog.HandlerOptions{Level: level}) + slog.SetDefault(slog.New(handler)) +} + +// statusConf holds the minimal CNI config fields needed for STATUS validation. +type statusConf struct { + CNIVersion string `json:"cniVersion"` + Type string `json:"type"` +} + +func parseStatusConf(data []byte) error { + var sc statusConf + if err := json.Unmarshal(data, &sc); err != nil { + return &types.Error{Code: 7, Msg: errInvalidCNIConfig, Details: err.Error()} + } + if sc.CNIVersion == "" { + return &types.Error{Code: 7, Msg: "cniVersion is required"} + } + if sc.Type == "" { + return &types.Error{Code: 7, Msg: "type is required"} + } + return nil +} + +// parseConf unmarshals the CNI configuration from stdin data (the same +// document the master plugin received), validates the base62-encoded +// identifier fields, and resolves node-level settings and logging. +func parseConf(data []byte) (*PluginConf, error) { + conf := &PluginConf{} + if err := json.Unmarshal(data, &conf); err != nil { + return nil, &types.Error{Code: 7, Msg: errInvalidCNIConfig, Details: err.Error()} + } + if !isValidBase62(conf.VPC) { + if len(conf.VPC) == 0 { + return nil, &types.Error{Code: 7, Msg: errVPCRequired} + } + return nil, &types.Error{ + Code: 7, + Msg: fmt.Sprintf("invalid base62 value for field 'vpc': %q", sanitizeForError(conf.VPC)), + } + } + if !isValidBase62(conf.VPCAttachment) { + if len(conf.VPCAttachment) == 0 { + return nil, &types.Error{Code: 7, Msg: errVPCAttachmentRequired} + } + return nil, &types.Error{ + Code: 7, + Msg: fmt.Sprintf("invalid base62 value for field 'vpcattachment': %q", sanitizeForError(conf.VPCAttachment)), + } + } + + hostConf, err := loadHostConf(ConfFile) + if err != nil { + return nil, fmt.Errorf("load host CNI config: %w", err) + } + + cniConfig.Resolve(&config.ConflistValues{ + NodeName: hostConf.NodeName, + Kubeconfig: hostConf.Kubeconfig, + Namespace: hostConf.Namespace, + LogFile: hostConf.LogFile, + LogLevel: hostConf.LogLevel, + }) + + if cniConfig.NodeName == "" { + detected, detectErr := hostconf.DetectNodeNameFromAPI() + if detectErr != nil { + slog.Warn("Node name auto-detection failed", "err", detectErr) + } + cniConfig.NodeName = detected + } + if cniConfig.NodeName == "" { + return nil, &types.Error{Code: 4, Msg: "node name is required (or set GALACTIC_CNI_NODE_NAME)"} + } + _ = os.Setenv("KUBECONFIG", cniConfig.Kubeconfig) + + namespace := conf.Namespace + if namespace == "" { + namespace = cniConfig.Namespace + } + conf.Namespace = namespace + + setupLogging(cniConfig.LogFile, cniConfig.LogLevel) + slog.Debug("CNI config received", "stdin", string(data)) + + if conf.PrevResult != nil { + if err := validatePrevResult(conf.PrevResult); err != nil { + return nil, &types.Error{Code: 6, Msg: fmt.Sprintf("invalid prevResult: %v", err)} + } + } + return conf, nil +} + +func validatePrevResult(res types.Result) error { + if res == nil { + return nil + } + jsonBytes, err := json.Marshal(res) + if err != nil { + return fmt.Errorf("marshal prevResult: %w", err) + } + if _, err := type100.NewResult(jsonBytes); err != nil { + return fmt.Errorf("parse prevResult: %w", err) + } + return nil +} + +func sanitizeForError(s string) string { + for _, c := range s { + if c < 0x20 || c > 0x7e { + return sanitizeForErrorBinary + } + } + return s +} diff --git a/internal/cnibgp/ops_add.go b/internal/cnibgp/ops_add.go new file mode 100644 index 00000000..17fe9fc5 --- /dev/null +++ b/internal/cnibgp/ops_add.go @@ -0,0 +1,92 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnibgp + +import ( + "context" + "fmt" + "log/slog" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + + "go.datum.net/galactic/internal/plumbing/intf" +) + +// cmdAdd is galactic-bgp's own ADD: the last (per the design note's sample +// conflists) plugin in the chain, publishing SRv6/BGP/eBPF state for +// whatever the master plugin already created. It never touches a kernel +// interface — everything it needs (which interface kind, which addresses) +// comes from prevResult (see prevresult.go). +func cmdAdd(args *skel.CmdArgs) (err error) { + pluginConf, err := parseConf(args.StdinData) + if err != nil { + return err + } + + ifaceType, ipamResult, prevResult, err := inferFromPrevResult(pluginConf.RawPrevResult) + if err != nil { + return &types.Error{Code: 6, Msg: fmt.Sprintf("infer from prevResult: %v", err)} + } + + nodeName := cniConfig.NodeName + namespace := pluginConf.Namespace + + slog.Info("ADD: starting", "containerID", args.ContainerID, + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment, + "ifaceType", ifaceType, "namespace", namespace, "nodeName", nodeName) + + tracker := &resourceTracker{ + vpc: pluginConf.VPC, + vpcAttachment: pluginConf.VPCAttachment, + namespace: namespace, + } + + defer func() { + if err != nil { + slog.Error("ADD: failed, rolling back created resources", "err", err, + "containerID", args.ContainerID, "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + // rollbackCtx is created here, fresh, rather than up front and + // shared with the work that just failed: publishBGPState's own + // retryK8sOps (bgp.go) can burn up to ~30s across its retries, + // each on its own fresh cniTimeout context. A context created + // before that call and reused here could already be expired by + // the time cleanup runs, so Delete would fail with "context + // deadline exceeded" instead of NotFound — client.IgnoreNotFound + // doesn't catch that, and the just-created CRDs would leak. + // Giving rollback its own full cniTimeout budget, on the failure + // path only, also means a successful ADD never allocates (and + // must remember to cancel) a context it doesn't use. + rollbackCtx, rollbackCancel := context.WithTimeout(context.Background(), cniTimeout) + tracker.cleanup(rollbackCtx) + rollbackCancel() + } + }() + + k8sClient, err := newK8sClient() + if err != nil { + return fmt.Errorf("create k8s client: %w", err) + } + tracker.k8s = k8sClient + + vpcHex, err := intf.Base62ToHex(pluginConf.VPC) + if err != nil { + return fmt.Errorf("decode VPC: %w", err) + } + + cfg := publishConfig{vpc: pluginConf.VPC, vpcAttachment: pluginConf.VPCAttachment, ifaceType: ifaceType} + result, err := publishBGPState(args, cfg, nodeName, namespace, ipamResult, vpcHex, k8sClient) + tracker.publishResult = result + if err != nil { + return err + } + + // Pass prevResult through unchanged: this plugin adds no new interfaces + // or IPs of its own, so its own CNI result is exactly what it received. + // Per the design note's sample conflists, galactic-bgp is the last + // plugin in the chain, so this becomes the runtime's authoritative + // result for the ADD. + return types.PrintResult(prevResult, pluginConf.CNIVersion) +} diff --git a/internal/cnibgp/ops_check.go b/internal/cnibgp/ops_check.go new file mode 100644 index 00000000..89a3e925 --- /dev/null +++ b/internal/cnibgp/ops_check.go @@ -0,0 +1,231 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnibgp + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "net/netip" + "os" + "time" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "go.datum.net/galactic/internal/config" + "go.datum.net/galactic/internal/crdnames" + "go.datum.net/galactic/internal/plumbing/ebpf/uformat" + "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" + "go.datum.net/galactic/internal/plumbing/vrf" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +// cmdCheck verifies that the BGP state cmdAdd published is still in place: +// the BGPVRFInstance and BGPAdvertisement CRDs exist, and — when this +// node's BGPRouter has SRv6 configured — the eBPF vrf_table entry for this +// attachment is still registered. None of this is a move from +// internal/cni's own CHECK; it's genuinely new, since nothing before this +// split ever verified CRD/eBPF state independently of kernel interface +// state. +func cmdCheck(args *skel.CmdArgs) error { + pluginConf, err := parseConf(args.StdinData) + if err != nil { + return err + } + slog.Info("CHECK: starting", "containerID", args.ContainerID, + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + + k8s, err := newK8sClient() + if err != nil { + return fmt.Errorf("create k8s client: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), cniTimeout) + defer cancel() + + var errs []error + + vrfInst := &bgpv1alpha1.BGPVRFInstance{} + vrfName := crdnames.BGPVRFInstanceName(pluginConf.VPC, pluginConf.VPCAttachment) + vrfErr := k8s.Get(ctx, client.ObjectKey{Name: vrfName, Namespace: pluginConf.Namespace}, vrfInst) + if vrfErr != nil { + errs = append(errs, fmt.Errorf("BGPVRFInstance %s: %w", vrfName, vrfErr)) + } + + adv := &bgpv1alpha1.BGPAdvertisement{} + advName := crdnames.BGPAdvertisementName(pluginConf.VPC, pluginConf.VPCAttachment) + if err := k8s.Get(ctx, client.ObjectKey{Name: advName, Namespace: pluginConf.Namespace}, adv); err != nil { + errs = append(errs, fmt.Errorf("BGPAdvertisement %s: %w", advName, err)) + } + + // The eBPF vrf_table entry is only checkable once the BGPVRFInstance + // lookup succeeded (it carries the Argument value the entry is keyed + // on) and this node's router actually has SRv6 configured — matches + // registerEBPFDatapath's own no-op case. + if vrfErr == nil { + if err := checkEBPFEntry(ctx, k8s, pluginConf, uint16(vrfInst.Spec.VRFID)); err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + err := fmt.Errorf("CHECK failed: %w", errors.Join(errs...)) + slog.Error("CHECK: failed", "err", err, "containerID", args.ContainerID, + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + return err + } + slog.Info("CHECK: passed", "containerID", args.ContainerID, + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + return nil +} + +// checkEBPFEntry verifies every eBPF table registerEBPFDatapath wrote for +// this attachment still exists and is intact: the locator_table entry (this +// router's own node), the function_table entry (SRv6 End.DT46 behavior), +// and the vrf_table entry (still resolving to this attachment's own VRF +// table id) — plus the same nodeID range check ADD treats as a hard error. +// Checking vrf_table alone would miss a corrupted/missing locator or +// function entry, or a nodeID that drifted out of range after ADD, while +// still reporting the attachment healthy. Returns nil (not an error) when +// this node's router has no SRv6Locator/nodeID configured — SRv6 was +// intentionally never set up for this attachment, matching +// registerEBPFDatapath's own no-op case. +func checkEBPFEntry(ctx context.Context, k8s client.Client, pluginConf *PluginConf, argument uint16) error { + bgp, err := lookupBGPRouter(ctx, k8s, cniConfig.NodeName, pluginConf.Namespace) + if err != nil { + return fmt.Errorf("look up BGPRouter: %w", err) + } + if bgp.srv6Locator == "" || bgp.nodeID == 0 { + return nil + } + + if bgp.nodeID < uformat.NodeIDMin || bgp.nodeID > uformat.NodeIDMax { + return fmt.Errorf("eBPF check: nodeID %d out of range [%#x,%#x]", + bgp.nodeID, uint16(uformat.NodeIDMin), uint16(uformat.NodeIDMax)) + } + + prefix, err := netip.ParsePrefix(bgp.srv6Locator) + if err != nil { + return fmt.Errorf("parse SRv6 locator %q: %w", bgp.srv6Locator, err) + } + block, err := uformat.Block(prefix.Addr()) + if err != nil { + return fmt.Errorf("derive eBPF uSID Block from locator %q: %w", bgp.srv6Locator, err) + } + + vrfTableID, err := vrf.TableID(pluginConf.VPC, pluginConf.VPCAttachment) + if err != nil { + return fmt.Errorf("get VRF table ID: %w", err) + } + + registry, closer, err := usidmap.OpenPinnedRegistry(ebpfPinDir) + if err != nil { + return fmt.Errorf("open pinned eBPF uSID maps: %w", err) + } + defer func() { _ = closer.Close() }() + + var errs []error + + if _, ok, err := registry.Locator.Get(block, uint16(bgp.nodeID)); err != nil { + errs = append(errs, fmt.Errorf("read eBPF locator_table entry: %w", err)) + } else if !ok { + errs = append(errs, fmt.Errorf( + "eBPF locator_table entry for block %#x node-id %#x not found", block, uint16(bgp.nodeID))) + } + + if funcEntry, ok, err := registry.Function.Get(block, uformat.FunctionEndDT46); err != nil { + errs = append(errs, fmt.Errorf("read eBPF function_table entry: %w", err)) + } else if !ok { + errs = append(errs, fmt.Errorf( + "eBPF function_table entry for block %#x function %#x not found", block, uformat.FunctionEndDT46)) + } else if funcEntry.Behavior != usidmap.BehaviorEndDT46 { + errs = append(errs, fmt.Errorf( + "eBPF function_table entry Behavior = %#x, want %#x", funcEntry.Behavior, usidmap.BehaviorEndDT46)) + } + + entry, ok, err := registry.VRF.Get(block, argument) + if err != nil { + errs = append(errs, fmt.Errorf("read eBPF vrf_table entry: %w", err)) + } else if !ok { + errs = append(errs, fmt.Errorf("eBPF vrf_table entry for block %#x argument %#x not found", block, argument)) + } else if entry.VRFTableID != vrfTableID { + errs = append(errs, fmt.Errorf("eBPF vrf_table entry VRFTableID = %#x, want %#x", entry.VRFTableID, vrfTableID)) + } + + return errors.Join(errs...) +} + +// cmdStatus implements the CNI spec STATUS operation — galactic-bgp talks +// to the API server (BGP CRD reads/writes), so this probes it the same way +// internal/cni's own cmdStatus does. +func cmdStatus(args *skel.CmdArgs) error { + if err := parseStatusConf(args.StdinData); err != nil { + return err + } + + hostConf, err := loadHostConf(ConfFile) + if err != nil { + return &types.Error{Code: 7, Msg: fmt.Sprintf("load host CNI config: %v", err)} + } + + cniConfig.Resolve(&config.ConflistValues{ + Kubeconfig: hostConf.Kubeconfig, + Namespace: hostConf.Namespace, + LogFile: hostConf.LogFile, + LogLevel: hostConf.LogLevel, + }) + + _ = os.Setenv("KUBECONFIG", cniConfig.Kubeconfig) + + setupLogging(cniConfig.LogFile, cniConfig.LogLevel) + slog.Debug("CNI config received", "stdin", string(args.StdinData)) + + slog.Info("STATUS: probing API server reachability") + if err := probeAPIServer(); err != nil { + slog.Error("STATUS: API server probe failed", "err", err) + return &types.Error{Code: 50, Msg: fmt.Sprintf("API server health check failed: %v", err)} + } + slog.Info("STATUS: ready") + return nil +} + +// probeAPIServerFn is a variable so tests can override it. +var probeAPIServerFn = func() error { + kubeconfig, err := ctrl.GetConfig() + if err != nil { + if errors.Is(err, rest.ErrNotInCluster) { + return nil + } + return fmt.Errorf("load kubeconfig: %w", err) + } + kubeconfig.Timeout = 2 * time.Second + httpClient, err := rest.HTTPClientFor(kubeconfig) + if err != nil { + return fmt.Errorf("build http client: %w", err) + } + req, err := http.NewRequestWithContext( + context.Background(), + http.MethodGet, + kubeconfig.Host+"/healthz", + nil, + ) + if err != nil { + return fmt.Errorf("build healthz request: %w", err) + } + resp, err := httpClient.Do(req) + if err != nil { + return fmt.Errorf("healthz request failed: %w", err) + } + defer resp.Body.Close() //nolint:errcheck // best-effort probe + return nil +} + +var probeAPIServer = probeAPIServerFn diff --git a/internal/cnibgp/ops_del.go b/internal/cnibgp/ops_del.go new file mode 100644 index 00000000..5bd8abba --- /dev/null +++ b/internal/cnibgp/ops_del.go @@ -0,0 +1,42 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnibgp + +import ( + "log/slog" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + type100 "github.com/containernetworking/cni/pkg/types/100" +) + +// cmdDel is a no-op, same as every other binary in the chain: the +// BGPVRFInstance/BGPAdvertisement CRDs and eBPF vrf_table entry this +// plugin's own ADD created are keyed by (vpc, vpcAttachment) and may still +// be in use by another pod/VM sharing the same attachment. Deleting them +// here would race with a concurrent ADD during restarts. Cleanup is left +// entirely to galactic-router's GC controller — see internal/cni's own +// cmdDel for the full reasoning, identical here. +func cmdDel(args *skel.CmdArgs) error { + // DEL is idempotent per the CNI spec: always return success, even if + // parsing the config fails — logging vpc/vpcAttachment (when parseable) + // is the only reason to parse at all here, since there's no cleanup to + // gate on it. + pluginConf, parseErr := parseConf(args.StdinData) + if parseErr != nil { + slog.Error("DEL: failed to parse CNI config, skipping cleanup logging", "err", parseErr, + "containerID", args.ContainerID) + result := &type100.Result{} + _ = types.PrintResult(result, "1.0.0") + return nil + } + + slog.Info("DEL: skipping shared resource cleanup (handled by GC)", "containerID", args.ContainerID, + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + + result := &type100.Result{} + _ = types.PrintResult(result, pluginConf.CNIVersion) + return nil +} diff --git a/internal/cnibgp/ops_del_test.go b/internal/cnibgp/ops_del_test.go new file mode 100644 index 00000000..f12c363d --- /dev/null +++ b/internal/cnibgp/ops_del_test.go @@ -0,0 +1,58 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnibgp + +import ( + "fmt" + "testing" + + "github.com/containernetworking/cni/pkg/skel" + + "go.datum.net/galactic/internal/config" +) + +// TestCmdDelIdempotent returns nil even when the CNI config is invalid, per +// the CNI spec's DEL idempotency requirement. +func TestCmdDelIdempotent(t *testing.T) { + args := &skel.CmdArgs{ + ContainerID: "test-container", + StdinData: []byte("not valid json"), + } + + if err := cmdDel(args); err != nil { + t.Fatalf("cmdDel with invalid config returned error = %v, want nil", err) + } +} + +// TestCmdDelParsesConfigForLogging exercises the valid-config path: DEL has +// nothing to clean up itself (GC handles it), but it should still be able to +// parse the conflist it's given — vpc/vpcAttachment come from that parse, +// and CNIVersion for the printed result comes from pluginConf.CNIVersion, +// not a hardcoded "1.0.0", so a conflist authored with a different version +// still gets a matching result back on DEL the same way ADD/CHECK do. +func TestCmdDelParsesConfigForLogging(t *testing.T) { + // parseConf resolves cniConfig's node-level env vars, which is normally + // done once at process startup by InitCNIConfig() (cmd/galactic-bgp's + // main); tests that go through parseConf need the same setup. Setting + // GALACTIC_CNI_NODE_NAME explicitly also skips parseConf's fallback to + // hostconf.DetectNodeNameFromAPI, which would otherwise try (and hang + // retrying) to reach a real API server that doesn't exist in this test. + t.Setenv(config.EnvCNINodeName, "test-node") + cniConfig = config.NewCNIConfig() + t.Cleanup(func() { cniConfig = nil }) + + conf := fmt.Sprintf( + `{"cniVersion":"1.1.0","name":"test","type":"galactic-bgp","vpc":"%s","vpcattachment":"%s"}`, + testVPC, testAttachment, + ) + args := &skel.CmdArgs{ + ContainerID: "test-container", + StdinData: []byte(conf), + } + + if err := cmdDel(args); err != nil { + t.Fatalf("cmdDel with valid config returned error = %v, want nil", err) + } +} diff --git a/internal/cnibgp/prevresult.go b/internal/cnibgp/prevresult.go new file mode 100644 index 00000000..02756e36 --- /dev/null +++ b/internal/cnibgp/prevresult.go @@ -0,0 +1,106 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnibgp + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/containernetworking/cni/pkg/types" + type100 "github.com/containernetworking/cni/pkg/types/100" + + "go.datum.net/galactic/internal/cniipam" +) + +// inferFromPrevResult reconstructs the ipamResult and infers whether the +// preceding master plugin created a veth or tap interface, both from +// rawPrevResult — the raw JSON form of the CNI chain's accumulated result +// so far (PluginConf.RawPrevResult; the typed PluginConf.PrevResult field +// is never populated by plain JSON unmarshal, per its "json:\"-\"" tag — +// ops_check.go elsewhere in this repo already reads prevResult the same, +// correct way). +// +// type100.NewResult only accepts a Result whose own CNIVersion field is +// exactly "1.0.0" or "1.1.0" (github.com/containernetworking/cni's +// pkg/types/100 supportedVersions) — and the master plugin's printed Result +// carries the conflist's own cniVersion value verbatim (buildResult/ +// buildTapResult set CNIVersion: pluginConf.CNIVersion), not a value this +// plugin controls. A conflist authored with an older cniVersion (e.g. +// "0.4.0") therefore fails ADD here for every attachment in the chain. This +// is a real, hard requirement on every conflist in this repo's CNI chain — +// see docs/cni/configuration.md's cniVersion paragraph — not just an +// implementation detail of this function. +// +// galactic-bgp has no kernel-interface access or IPAM knowledge of its own +// — this is the only way it learns either. Interface-kind inference: a +// veth master's own result declares one interface with a non-empty Sandbox +// (the guest end, moved into the container netns — see internal/cni/ +// result.go's buildResult); a tap master's declares zero (the tap device +// stays on the host — internal/cnitap/result.go's buildTapResult). This +// counts Sandbox-carrying interfaces rather than len(Interfaces) itself, +// on purpose: #306 chains galactic-route into this same prevResult next, +// and total interface count is not this plugin's to own — if anything +// appended later adds a host-side interface entry of its own, a raw count +// would silently reclassify a tap master as veth (or vice versa) instead +// of failing loudly, because both 1 and 2 are valid switch cases. Whether +// an interface was moved into the container's netns is the actual property +// that distinguishes veth from tap, not how many entries happen to be in +// the slice — see the design note's "Split the veth/tap master into two +// binaries" section for why no interface_type field exists anywhere in the +// chain to make this explicit instead. +func inferFromPrevResult( + rawPrevResult map[string]interface{}, +) (ifaceType string, ipamResult *cniipam.IPAMResult, parsed types.Result, err error) { + if rawPrevResult == nil { + return "", nil, nil, errors.New("no prevResult: galactic-bgp must be chained after a master plugin") + } + + jsonBytes, err := json.Marshal(rawPrevResult) + if err != nil { + return "", nil, nil, fmt.Errorf("marshal prevResult: %w", err) + } + parsed, err = type100.NewResult(jsonBytes) + if err != nil { + return "", nil, nil, fmt.Errorf("parse prevResult: %w", err) + } + versioned, err := type100.GetResult(parsed) + if err != nil { + return "", nil, nil, fmt.Errorf("get prevResult: %w", err) + } + + if len(versioned.Interfaces) == 0 { + return "", nil, nil, errors.New("prevResult declares no interfaces") + } + + var sandboxed int + for _, iface := range versioned.Interfaces { + if iface.Sandbox != "" { + sandboxed++ + } + } + switch sandboxed { + case 0: + ifaceType = ifaceTypeTap + case 1: + ifaceType = ifaceTypeVeth + default: + return "", nil, nil, fmt.Errorf( + "prevResult declares %d interfaces with a non-empty Sandbox, want 0 (tap master) or 1 (veth master)", + sandboxed) + } + + ipamResult, err = cniipam.ResultToIPAMResult(parsed) + if err != nil { + return "", nil, nil, fmt.Errorf("convert prevResult IPs: %w", err) + } + if ipamResult.IPv6Subnet == nil && ipamResult.IPv4Address == nil { + // No IPAM allocation at all (e.g. a tap workload managing its own + // addressing) — nil, not a zero-value result, matches what every + // other caller in this chain already treats as "no IPAM." + ipamResult = nil + } + return ifaceType, ipamResult, parsed, nil +} diff --git a/internal/cnibgp/prevresult_test.go b/internal/cnibgp/prevresult_test.go new file mode 100644 index 00000000..fa8a7702 --- /dev/null +++ b/internal/cnibgp/prevresult_test.go @@ -0,0 +1,185 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnibgp + +import ( + "encoding/json" + "net" + "testing" + + type100 "github.com/containernetworking/cni/pkg/types/100" +) + +const ( + testCNIVersion100 = "1.0.0" + testTapMAC = "aa:bb:cc:dd:ee:ff" + testVethHostMAC = "aa:bb:cc:dd:ee:f0" + testVethGuestMAC = "aa:bb:cc:dd:ee:f1" + testTapIfaceName = "tap0" + testVethHostName = "veth0" + testVethGuestName = "eth0" +) + +// rawPrevResultFrom round-trips result through JSON, the same way the CNI +// runtime hands prevResult to a chained plugin's stdin (skel.CmdArgs.StdinData +// -> PluginConf.RawPrevResult), so tests exercise inferFromPrevResult exactly +// as it receives real input. +func rawPrevResultFrom(t *testing.T, result *type100.Result) map[string]interface{} { + t.Helper() + b, err := json.Marshal(result) + if err != nil { + t.Fatalf("marshal result: %v", err) + } + var raw map[string]interface{} + if err := json.Unmarshal(b, &raw); err != nil { + t.Fatalf("unmarshal result: %v", err) + } + return raw +} + +func TestInferFromPrevResult_TapMaster(t *testing.T) { + // A tap master's own result declares one host-only interface — no + // Sandbox — see internal/cnitap/result.go's buildTapResult. + raw := rawPrevResultFrom(t, &type100.Result{ + CNIVersion: testCNIVersion100, + Interfaces: []*type100.Interface{ + {Name: testTapIfaceName, Mac: testTapMAC, Sandbox: ""}, + }, + }) + + ifaceType, ipamResult, _, err := inferFromPrevResult(raw) + if err != nil { + t.Fatalf("inferFromPrevResult() error = %v", err) + } + if ifaceType != ifaceTypeTap { + t.Errorf("ifaceType = %q, want %q", ifaceType, ifaceTypeTap) + } + if ipamResult != nil { + t.Errorf("ipamResult = %+v, want nil (no IPAM in this result)", ipamResult) + } +} + +func TestInferFromPrevResult_VethMaster(t *testing.T) { + // A veth master's own result declares two interfaces: host (no Sandbox) + // and guest (Sandbox = the container netns it was moved into) — see + // internal/cni/result.go's buildResult. + raw := rawPrevResultFrom(t, &type100.Result{ + CNIVersion: testCNIVersion100, + Interfaces: []*type100.Interface{ + {Name: testVethHostName, Mac: testVethHostMAC, Sandbox: ""}, + {Name: testVethGuestName, Mac: testVethGuestMAC, Sandbox: testNetns}, + }, + }) + + ifaceType, _, _, err := inferFromPrevResult(raw) + if err != nil { + t.Fatalf("inferFromPrevResult() error = %v", err) + } + if ifaceType != ifaceTypeVeth { + t.Errorf("ifaceType = %q, want %q", ifaceType, ifaceTypeVeth) + } +} + +func TestInferFromPrevResult_ExtraHostInterfaceStillClassifiesCorrectly(t *testing.T) { + // A hypothetical extra host-side interface appended by a later + // chain-invoked plugin (e.g. #306's galactic-route) must not flip a tap + // master's classification to veth just because len(Interfaces) grew + // from 1 to 2 — only Sandbox-carrying interfaces count. + raw := rawPrevResultFrom(t, &type100.Result{ + CNIVersion: testCNIVersion100, + Interfaces: []*type100.Interface{ + {Name: testTapIfaceName, Mac: testTapMAC, Sandbox: ""}, + {Name: "route0", Mac: "aa:bb:cc:dd:ee:fe", Sandbox: ""}, + }, + }) + + ifaceType, _, _, err := inferFromPrevResult(raw) + if err != nil { + t.Fatalf("inferFromPrevResult() error = %v", err) + } + if ifaceType != ifaceTypeTap { + t.Errorf("ifaceType = %q, want %q", ifaceType, ifaceTypeTap) + } +} + +func TestInferFromPrevResult_MultipleSandboxedInterfacesIsHardError(t *testing.T) { + // More than one Sandbox-carrying interface is genuinely ambiguous — no + // known master plugin produces this — and must fail loudly rather than + // guess. + raw := rawPrevResultFrom(t, &type100.Result{ + CNIVersion: testCNIVersion100, + Interfaces: []*type100.Interface{ + {Name: testVethHostName, Mac: testVethHostMAC, Sandbox: ""}, + {Name: testVethGuestName, Mac: testVethGuestMAC, Sandbox: testNetns}, + {Name: "eth1", Mac: "aa:bb:cc:dd:ee:f2", Sandbox: testNetns}, + }, + }) + + if _, _, _, err := inferFromPrevResult(raw); err == nil { + t.Fatal("inferFromPrevResult() error = nil, want error for ambiguous interface shape") + } +} + +func TestInferFromPrevResult_NoInterfacesIsError(t *testing.T) { + raw := rawPrevResultFrom(t, &type100.Result{CNIVersion: testCNIVersion100}) + + if _, _, _, err := inferFromPrevResult(raw); err == nil { + t.Fatal("inferFromPrevResult() error = nil, want error for empty Interfaces") + } +} + +func TestInferFromPrevResult_NilRawPrevResult(t *testing.T) { + if _, _, _, err := inferFromPrevResult(nil); err == nil { + t.Fatal("inferFromPrevResult() error = nil, want error for nil rawPrevResult") + } +} + +// TestInferFromPrevResult_RejectsOlderCNIVersion documents (and locks in) +// the hard cniVersion requirement described in this function's doc comment +// and docs/cni/configuration.md: type100.NewResult only accepts a Result +// whose own CNIVersion is exactly "1.0.0" or "1.1.0", and the master +// plugin's printed Result carries the conflist's cniVersion straight +// through, so an older value must fail loudly here rather than proceed with +// stale/wrong assumptions about the Result's shape. +func TestInferFromPrevResult_RejectsOlderCNIVersion(t *testing.T) { + raw := rawPrevResultFrom(t, &type100.Result{ + CNIVersion: "0.4.0", + Interfaces: []*type100.Interface{ + {Name: testTapIfaceName, Mac: testTapMAC, Sandbox: ""}, + }, + }) + + if _, _, _, err := inferFromPrevResult(raw); err == nil { + t.Fatal("inferFromPrevResult() error = nil, want error for cniVersion \"0.4.0\"") + } +} + +func TestInferFromPrevResult_CarriesIPAMResult(t *testing.T) { + ipv6 := net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(64, 128)} + raw := rawPrevResultFrom(t, &type100.Result{ + CNIVersion: testCNIVersion100, + Interfaces: []*type100.Interface{ + {Name: testVethHostName, Mac: testVethHostMAC, Sandbox: ""}, + {Name: testVethGuestName, Mac: testVethGuestMAC, Sandbox: testNetns}, + }, + IPs: []*type100.IPConfig{ + {Address: ipv6, Interface: type100.Int(1)}, + }, + }) + + ifaceType, ipamResult, _, err := inferFromPrevResult(raw) + if err != nil { + t.Fatalf("inferFromPrevResult() error = %v", err) + } + if ifaceType != ifaceTypeVeth { + t.Errorf("ifaceType = %q, want %q", ifaceType, ifaceTypeVeth) + } + if ipamResult == nil || ipamResult.IPv6Subnet == nil { + t.Fatalf("ipamResult = %+v, want a non-nil IPv6Subnet", ipamResult) + } + if ipamResult.IPv6Subnet.String() != "2001:db8::1/64" { + t.Errorf("ipamResult.IPv6Subnet = %s, want 2001:db8::1/64", ipamResult.IPv6Subnet.String()) + } +} diff --git a/internal/cnibgp/resource.go b/internal/cnibgp/resource.go new file mode 100644 index 00000000..de314d07 --- /dev/null +++ b/internal/cnibgp/resource.go @@ -0,0 +1,113 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnibgp + +import ( + "context" + "fmt" + "log/slog" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "go.datum.net/galactic/internal/crdnames" + "go.datum.net/galactic/internal/plumbing/vrf" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +var cniScheme = runtime.NewScheme() + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(cniScheme)) + utilruntime.Must(bgpv1alpha1.AddToScheme(cniScheme)) +} + +// newK8sClient creates a new Kubernetes client using the in-cluster config, +// scoped to cniScheme (BGP CRDs). +func newK8sClient() (client.Client, error) { + restCfg, err := ctrl.GetConfig() + if err != nil { + return nil, fmt.Errorf("get kubeconfig: %w", err) + } + c, err := client.New(restCfg, client.Options{Scheme: cniScheme}) + if err != nil { + return nil, fmt.Errorf("create k8s client: %w", err) + } + return c, nil +} + +// resourceTracker tracks resources created during cmdAdd for selective +// rollback. galactic-bgp's own ADD only ever creates BGP CRDs and an eBPF +// vrf_table entry — the kernel-interface/VRF cleanup that used to live +// alongside these in one process-wide tracker is now each master plugin's +// own, smaller tracker (internal/cni, internal/cnitap), scoped to exactly +// what its own ADD creates. +// +// publishResult is embedded, rather than its five fields being copied over +// field-by-field, so a future field added to one struct can't silently stop +// being tracked in the other with no compiler error to catch it — cmdAdd +// assigns the whole publishResult from publishBGPState in one shot +// (tracker.publishResult = result). +type resourceTracker struct { + vpc, vpcAttachment string + namespace string + k8s client.Client + + publishResult +} + +// cleanup rolls back all tracked resources. Errors are logged but never +// returned — the caller already has a failure. +func (rt *resourceTracker) cleanup(ctx context.Context) { + slog.Info("Selective rollback: cleaning up resources created during failed ADD", + "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) + + if rt.advertisementCreated && rt.k8s != nil { + adv := &bgpv1alpha1.BGPAdvertisement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crdnames.BGPAdvertisementName(rt.vpc, rt.vpcAttachment), + Namespace: rt.namespace, + }, + } + if err := rt.k8s.Delete(ctx, adv); client.IgnoreNotFound(err) != nil { + slog.Error("Rollback: failed to delete BGPAdvertisement", "err", err, + "name", adv.Name, "namespace", rt.namespace) + } else { + slog.Debug("Rollback: deleted BGPAdvertisement", "name", adv.Name, "namespace", rt.namespace) + } + } + + if rt.vrfInstanceCreated && rt.k8s != nil { + vrfInst := &bgpv1alpha1.BGPVRFInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: crdnames.BGPVRFInstanceName(rt.vpc, rt.vpcAttachment), + Namespace: rt.namespace, + }, + } + if err := rt.k8s.Delete(ctx, vrfInst); client.IgnoreNotFound(err) != nil { + slog.Error("Rollback: failed to delete BGPVRFInstance", "err", err, + "name", vrfInst.Name, "namespace", rt.namespace) + } else { + slog.Debug("Rollback: deleted BGPVRFInstance", "name", vrfInst.Name, "namespace", rt.namespace) + } + } + + if rt.ebpfRegistered { + if vrfTableID, err := vrf.TableID(rt.vpc, rt.vpcAttachment); err != nil { + slog.Error("Rollback: failed to resolve VRF table id, skipping eBPF vrf_table unregister", "err", err, + "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) + } else if err := unregisterEBPFDatapath(rt.ebpfBlock, rt.ebpfArgument, vrfTableID, ebpfPinDir); err != nil { + slog.Error("Rollback: failed to unregister eBPF vrf_table entry", "err", err, + "block", rt.ebpfBlock, "argument", rt.ebpfArgument) + } else { + slog.Debug("Rollback: unregistered eBPF vrf_table entry", + "block", rt.ebpfBlock, "argument", rt.ebpfArgument) + } + } +} diff --git a/internal/cnibgp/resource_test.go b/internal/cnibgp/resource_test.go new file mode 100644 index 00000000..190f792e --- /dev/null +++ b/internal/cnibgp/resource_test.go @@ -0,0 +1,279 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnibgp + +import ( + "context" + "fmt" + "os" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "go.datum.net/galactic/internal/crdnames" + "go.datum.net/galactic/internal/plumbing/ebpf/attach" + "go.datum.net/galactic/internal/plumbing/ebpf/usidmap" + "go.datum.net/galactic/internal/plumbing/vrf" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +const testDefaultNamespace = "default" + +// ---- resourceTracker.cleanup: zero-value / partial state ------------------- + +// TestResourceTrackerCleanup_ZeroValue verifies cleanup with a zero-value +// tracker doesn't panic — it's called from cmdAdd's defer, and the caller +// may have failed before setting any fields (e.g. newK8sClient itself +// failed, so tracker.k8s is nil). +func TestResourceTrackerCleanup_ZeroValue(t *testing.T) { + tracker := &resourceTracker{} + tracker.cleanup(context.Background()) // must not panic +} + +// TestResourceTrackerCleanup_NilK8sClientSkipsCRDDeletes verifies cleanup +// doesn't attempt a Delete through a nil client even when the created-flags +// say there's something to roll back — this shouldn't happen in practice +// (tracker.k8s is set right after newK8sClient succeeds, before anything +// else can set vrfInstanceCreated/advertisementCreated), but cleanup's own +// nil check is what actually prevents the panic if it ever does. +func TestResourceTrackerCleanup_NilK8sClientSkipsCRDDeletes(t *testing.T) { + tracker := &resourceTracker{ + vpc: testVPC, + vpcAttachment: testAttachment, + namespace: testDefaultNamespace, + publishResult: publishResult{ + vrfInstanceCreated: true, + advertisementCreated: true, + }, + } + tracker.cleanup(context.Background()) // must not panic +} + +// ---- resourceTracker.cleanup: CRD rollback --------------------------------- + +// TestResourceTrackerCleanup_DeletesOnlyWhatWasCreated exercises cleanup's +// actual wiring end to end: given a tracker whose publishResult says only +// the BGPVRFInstance was created (advertisementCreated stays false, as it +// would if publishBGPState failed between the two), cleanup must delete the +// BGPVRFInstance and leave any BGPAdvertisement alone. +func TestResourceTrackerCleanup_DeletesOnlyWhatWasCreated(t *testing.T) { + namespace := testDefaultNamespace + vrfName := crdnames.BGPVRFInstanceName(testVPC, testAttachment) + advName := crdnames.BGPAdvertisementName(testVPC, testAttachment) + + existingVRFInst := &bgpv1alpha1.BGPVRFInstance{ + ObjectMeta: metav1.ObjectMeta{Name: vrfName, Namespace: namespace}, + } + // A BGPAdvertisement that was NOT created by this ADD (e.g. left over + // from another container sharing the same attachment) — cleanup must + // not touch it, since advertisementCreated is false. + untouchedAdv := &bgpv1alpha1.BGPAdvertisement{ + ObjectMeta: metav1.ObjectMeta{Name: advName, Namespace: namespace}, + } + + k8s := fakeClient(existingVRFInst, untouchedAdv) + tracker := &resourceTracker{ + vpc: testVPC, + vpcAttachment: testAttachment, + namespace: namespace, + k8s: k8s, + publishResult: publishResult{vrfInstanceCreated: true}, + } + + tracker.cleanup(context.Background()) + + if err := k8s.Get(context.Background(), client.ObjectKey{Name: vrfName, Namespace: namespace}, + &bgpv1alpha1.BGPVRFInstance{}); err == nil { + t.Error("BGPVRFInstance still exists after cleanup, want deleted") + } + if err := k8s.Get(context.Background(), client.ObjectKey{Name: advName, Namespace: namespace}, + &bgpv1alpha1.BGPAdvertisement{}); err != nil { + t.Errorf("BGPAdvertisement Get after cleanup = %v, want it left untouched (advertisementCreated was false)", err) + } +} + +// TestResourceTrackerCleanup_DeletesBothCRDsWhenBothCreated covers the +// common failure path: both CRDs were created, then something later in +// publishBGPState (or types.PrintResult) failed, so both must roll back. +func TestResourceTrackerCleanup_DeletesBothCRDsWhenBothCreated(t *testing.T) { + namespace := testDefaultNamespace + vrfName := crdnames.BGPVRFInstanceName(testVPC, testAttachment) + advName := crdnames.BGPAdvertisementName(testVPC, testAttachment) + + k8s := fakeClient( + &bgpv1alpha1.BGPVRFInstance{ObjectMeta: metav1.ObjectMeta{Name: vrfName, Namespace: namespace}}, + &bgpv1alpha1.BGPAdvertisement{ObjectMeta: metav1.ObjectMeta{Name: advName, Namespace: namespace}}, + ) + tracker := &resourceTracker{ + vpc: testVPC, + vpcAttachment: testAttachment, + namespace: namespace, + k8s: k8s, + publishResult: publishResult{vrfInstanceCreated: true, advertisementCreated: true}, + } + + tracker.cleanup(context.Background()) + + if err := k8s.Get(context.Background(), client.ObjectKey{Name: vrfName, Namespace: namespace}, + &bgpv1alpha1.BGPVRFInstance{}); err == nil { + t.Error("BGPVRFInstance still exists after cleanup, want deleted") + } + if err := k8s.Get(context.Background(), client.ObjectKey{Name: advName, Namespace: namespace}, + &bgpv1alpha1.BGPAdvertisement{}); err == nil { + t.Error("BGPAdvertisement still exists after cleanup, want deleted") + } +} + +// TestResourceTrackerCleanup_MissingCRDsAreNotError covers cleanup's use of +// client.IgnoreNotFound: a CRD already gone (e.g. a retry after a partial +// prior rollback) must not surface as an error — cleanup never returns one. +func TestResourceTrackerCleanup_MissingCRDsAreNotError(t *testing.T) { + tracker := &resourceTracker{ + vpc: testVPC, + vpcAttachment: testAttachment, + namespace: testDefaultNamespace, + k8s: fakeClient(), + publishResult: publishResult{vrfInstanceCreated: true, advertisementCreated: true}, + } + + tracker.cleanup(context.Background()) // must not panic; nothing to delete +} + +// ---- resourceTracker.cleanup: eBPF rollback -------------------------------- + +// TestResourceTrackerCleanup_UnregistersOwnEBPFEntry covers the eBPF branch +// of cleanup's own wiring (not unregisterEBPFDatapath in isolation, which +// bgp_ebpf_test.go already covers): a tracker with ebpfRegistered=true must +// resolve this attachment's own VRF table id and remove exactly the +// vrf_table entry it registered. +func TestResourceTrackerCleanup_UnregistersOwnEBPFEntry(t *testing.T) { + requireRoot(t) + + if err := vrf.Add(testVPC, testAttachment); err != nil { + t.Fatalf("vrf.Add: %v", err) + } + t.Cleanup(func() { _ = vrf.Delete(testVPC, testAttachment) }) + vrfTableID, err := vrf.TableID(testVPC, testAttachment) + if err != nil { + t.Fatalf("vrf.TableID: %v", err) + } + + pinDir := fmt.Sprintf("/sys/fs/bpf/galactic-bgp-test-%d", os.Getpid()) + t.Cleanup(func() { _ = os.RemoveAll(pinDir) }) + // cleanup (resource.go) reads the pin directory from the package-level + // ebpfPinDir var, not attach.PinDir directly — point it at this test's + // own throwaway directory instead of the real production one. + origPinDir := ebpfPinDir + ebpfPinDir = pinDir + t.Cleanup(func() { ebpfPinDir = origPinDir }) + + loaderObjs, err := attach.Load(pinDir) + if err != nil { + t.Fatalf("attach.Load: %v", err) + } + t.Cleanup(func() { _ = loaderObjs.Close() }) + + reg, closer, err := usidmap.OpenPinnedRegistry(pinDir) + if err != nil { + t.Fatalf("OpenPinnedRegistry: %v", err) + } + defer func() { _ = closer.Close() }() + + const testBlock uint64 = 0x0102030405 + const testArgument uint16 = 0x042 + if err := reg.VRF.Register(testBlock, testArgument, vrfTableID, usidmap.EgressKindVeth); err != nil { + t.Fatalf("seed vrf_table entry: %v", err) + } + + tracker := &resourceTracker{ + vpc: testVPC, + vpcAttachment: testAttachment, + publishResult: publishResult{ + ebpfRegistered: true, + ebpfBlock: testBlock, + ebpfArgument: testArgument, + }, + } + + tracker.cleanup(context.Background()) + + if _, ok, err := reg.VRF.Get(testBlock, testArgument); err != nil || ok { + t.Errorf("vrf_table entry after cleanup: ok=%v err=%v, want ok=false", ok, err) + } +} + +// TestResourceTrackerCleanup_LeavesEBPFEntryOwnedByAnotherAttachment is the +// rollback-collision race exercised at resourceTracker.cleanup's own level +// (bgp_ebpf_test.go's TestUnregisterEBPFDatapath_LeavesEntryOwnedByAnotherAttachment +// covers the same guard one layer lower, calling unregisterEBPFDatapath +// directly). retryK8sOps can re-run publishBGPState's whole closure without +// re-registering the eBPF entry, so a later attempt's checkArgumentCollision +// failure can trigger cleanup after the (block, argument) slot this +// attachment originally wrote has already been overwritten by the very +// other attachment the collision was detected against. cleanup must resolve +// its own vrf.TableID and leave the slot alone when it no longer matches, +// rather than deleting a live attachment's forwarding entry. +func TestResourceTrackerCleanup_LeavesEBPFEntryOwnedByAnotherAttachment(t *testing.T) { + requireRoot(t) + + if err := vrf.Add(testVPC, testAttachment); err != nil { + t.Fatalf("vrf.Add: %v", err) + } + t.Cleanup(func() { _ = vrf.Delete(testVPC, testAttachment) }) + + pinDir := fmt.Sprintf("/sys/fs/bpf/galactic-bgp-test-%d", os.Getpid()) + t.Cleanup(func() { _ = os.RemoveAll(pinDir) }) + // cleanup (resource.go) reads the pin directory from the package-level + // ebpfPinDir var, not attach.PinDir directly — point it at this test's + // own throwaway directory instead of the real production one. + origPinDir := ebpfPinDir + ebpfPinDir = pinDir + t.Cleanup(func() { ebpfPinDir = origPinDir }) + + loaderObjs, err := attach.Load(pinDir) + if err != nil { + t.Fatalf("attach.Load: %v", err) + } + t.Cleanup(func() { _ = loaderObjs.Close() }) + + reg, closer, err := usidmap.OpenPinnedRegistry(pinDir) + if err != nil { + t.Fatalf("OpenPinnedRegistry: %v", err) + } + defer func() { _ = closer.Close() }() + + const testBlock uint64 = 0x0102030405 + const testArgument uint16 = 0x042 + const anotherAttachmentsVRFTableID uint32 = 0x9999 + + // Simulate the colliding (winning) attachment having since overwritten + // this same (block, argument) slot with its own, different VRF table id. + if err := reg.VRF.Register(testBlock, testArgument, anotherAttachmentsVRFTableID, usidmap.EgressKindVeth); err != nil { + t.Fatalf("seed vrf_table entry: %v", err) + } + + tracker := &resourceTracker{ + vpc: testVPC, + vpcAttachment: testAttachment, + publishResult: publishResult{ + ebpfRegistered: true, + ebpfBlock: testBlock, + ebpfArgument: testArgument, + }, + } + + tracker.cleanup(context.Background()) + + entry, ok, err := reg.VRF.Get(testBlock, testArgument) + if err != nil || !ok { + t.Fatalf("vrf_table entry after cleanup: ok=%v err=%v, want ok=true (must survive, it's not this attachment's)", + ok, err) + } + if entry.VRFTableID != anotherAttachmentsVRFTableID { + t.Errorf("vrf_table entry VRFTableID after cleanup = %#x, want unchanged %#x", + entry.VRFTableID, anotherAttachmentsVRFTableID) + } +} diff --git a/internal/cnibgp/types.go b/internal/cnibgp/types.go new file mode 100644 index 00000000..f2c34fa4 --- /dev/null +++ b/internal/cnibgp/types.go @@ -0,0 +1,27 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnibgp + +import ( + "github.com/containernetworking/cni/pkg/types" + + "go.datum.net/galactic/internal/hostconf" +) + +// PluginConf is the CNI plugin configuration passed via stdin on each +// invocation of galactic-bgp — the same document the master plugin itself +// received, since the CNI runtime passes each chain entry its own stanza +// plus prevResult; galactic-bgp only reads vpc/vpcattachment/namespace out +// of it (mtu, terminations, ipam are the master's/galactic-ipam's own +// concerns). +type PluginConf struct { + types.PluginConf + VPC string `json:"vpc"` + VPCAttachment string `json:"vpcattachment"` + Namespace string `json:"namespace,omitempty"` +} + +// HostConf holds node-local settings read from /etc/cni/net.d/10-galactic.conflist. +type HostConf = hostconf.HostConf diff --git a/internal/cniipam/allocate.go b/internal/cniipam/allocate.go new file mode 100644 index 00000000..0f4d68e2 --- /dev/null +++ b/internal/cniipam/allocate.go @@ -0,0 +1,177 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniipam + +import ( + "errors" + "fmt" + "log/slog" + "net" + + "github.com/containernetworking/cni/pkg/skel" + + "go.datum.net/galactic/internal/cni/ipam" +) + +// localIPAMDefaultPool is the IPv6 CIDR pool used when local IPAM is enabled +// but neither static_ip nor ipv6_subnet/ipv4_subnet is set in the ipam +// block. Allocations from it use ipam.DefaultSubnetLen (/96). +const localIPAMDefaultPool = "fd00:10:ff01::/64" + +// lockDir is the on-disk allocation-state root both PoolAllocator and +// IPv4PoolAllocator persist to. Overridable in tests so unit tests never +// touch the real production path. +var lockDir = ipam.DefaultLockDir + +// allocate allocates addresses for the given container according to conf's +// mode: presence of StaticIP selects the static path; otherwise the pool +// path (either family alone, or both). +func allocate(args *skel.CmdArgs, conf *IPAM) (*IPAMResult, error) { + if conf.StaticIP != "" { + return allocateStatic(args, conf) + } + return allocatePool(args, conf) +} + +// allocateStatic validates and returns the pre-assigned static IPv6 address +// from static_ip. No IPv4 address is ever allocated for static IPAM — it is +// a single fixed address, not a dual-stack pool. +func allocateStatic(args *skel.CmdArgs, conf *IPAM) (*IPAMResult, error) { + alloc := ipam.NewStaticAllocator() + allocIP, err := alloc.Allocate(args.ContainerID, conf.StaticIP) + if err != nil { + return nil, fmt.Errorf("allocate static IP: %w", err) + } + subnet := &net.IPNet{ + IP: allocIP, + Mask: net.CIDRMask(64, 128), + } + slog.Debug("IPAM: allocated static", "containerID", args.ContainerID, "subnet", subnet) + return &IPAMResult{IPv6Subnet: subnet}, nil +} + +// allocatePool allocates a dual-stack, IPv6-only, or IPv4-only pool-based +// endpoint address for the given container, via ipam.DualStackAllocator. +// IPv6Subnet and IPv4Subnet each independently supply a pool CIDR for their +// family; at least one must be set (falling back to localIPAMDefaultPool +// for IPv6 when GALACTIC_IPAM_ENABLE_LOCAL_IPAM is set and both are unset — +// see parseConf, which fills that default in before this ever runs). +func allocatePool(args *skel.CmdArgs, conf *IPAM) (*IPAMResult, error) { + if conf.IPv6Subnet == "" && conf.IPv4Subnet == "" { + return nil, errors.New("ipam.ipv6_subnet or ipam.ipv4_subnet is required (or enable GALACTIC_IPAM_ENABLE_LOCAL_IPAM)") + } + + alloc, err := ipam.NewDualStackAllocator(conf.IPv6Subnet, "", conf.IPv4Subnet, "", lockDir) + if err != nil { + return nil, fmt.Errorf("create dual-stack allocator: %w", err) + } + + res, err := alloc.Allocate(args.ContainerID) + if err != nil { + return nil, fmt.Errorf("allocate dual-stack addresses: %w", err) + } + + var routes []*net.IPNet + if res.IPv6Subnet != nil { + routes = append(routes, &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)}) + } + if res.IPv4Address != nil { + routes = append(routes, &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}) + } + + slog.Debug("IPAM: allocated", "containerID", args.ContainerID, + "ipv6Subnet", res.IPv6Subnet, "ipv6Gateway", res.IPv6Gateway, + "ipv4Address", res.IPv4Address, "ipv4Gateway", res.IPv4Gateway) + + return &IPAMResult{ + IPv6Subnet: res.IPv6Subnet, + IPv6Gateway: res.IPv6Gateway, + IPv4Address: res.IPv4Address, + IPv4Gateway: res.IPv4Gateway, + Routes: routes, + }, nil +} + +// effectiveIPv6Subnet returns conf.IPv6Subnet if either family's subnet was +// ever explicitly set. Otherwise — neither ipv6_subnet nor ipv4_subnet is +// set — the only pool an allocation could possibly have come from is +// parseConf's default-filler pool, so that's returned directly instead of +// re-deriving it from GALACTIC_IPAM_ENABLE_LOCAL_IPAM. deallocate/ +// checkAllocation must not depend on that env var still agreeing at DEL/ +// CHECK time with whatever it resolved to at ADD time: if it flips in +// between, re-checking it here would see an empty subnet and silently skip +// cleanup/verification, leaking the allocation instead of releasing it. +func effectiveIPv6Subnet(conf *IPAM) string { + if conf.IPv6Subnet != "" || conf.IPv4Subnet != "" { + return conf.IPv6Subnet + } + return localIPAMDefaultPool +} + +// deallocate releases whatever allocation containerID holds against conf's +// pools — entirely local: each family's own on-disk marker file is looked +// up directly by containerID (internal/cni/ipam's DeallocateContainer), no +// external state (a CRD read, a Kubernetes client) required. A missing +// allocation for one family (e.g. a v6-only pod, or a partial ADD failure +// that never reached IPv4 allocation) does not prevent cleanup of the +// other — each call is independent and silently no-ops if nothing is +// found. +func deallocate(containerID string, conf *IPAM) { + if conf.StaticIP != "" { + // Static allocations don't need deallocation. + return + } + + if ipv6Subnet := effectiveIPv6Subnet(conf); ipv6Subnet != "" { + pa, err := ipam.NewPoolAllocator(ipv6Subnet, "", 0, lockDir) + if err != nil { + slog.Warn("IPAM: failed to build IPv6 pool allocator for deallocation, skipping", "err", err, + "containerID", containerID) + } else if subnet, ok := pa.DeallocateContainer(containerID); ok { + slog.Debug("IPAM: deallocated IPv6", "containerID", containerID, "subnet", subnet) + } + } + + if conf.IPv4Subnet != "" { + pa, err := ipam.NewIPv4PoolAllocator(conf.IPv4Subnet, "", lockDir) + if err != nil { + slog.Warn("IPAM: failed to build IPv4 pool allocator for deallocation, skipping", "err", err, + "containerID", containerID) + } else if addr, ok := pa.DeallocateContainer(containerID); ok { + slog.Debug("IPAM: deallocated IPv4", "containerID", containerID, "address", addr) + } + } +} + +// checkAllocation verifies that containerID still holds an allocation +// against every family conf configures — used by CHECK. A static +// allocation has nothing persisted to check (it's validated once, at ADD, +// and never stored), so it always passes. Returns one error per +// missing/unreachable family; nil means every configured family checked +// out. +func checkAllocation(containerID string, conf *IPAM) []error { + if conf.StaticIP != "" { + return nil + } + + var errs []error + if ipv6Subnet := effectiveIPv6Subnet(conf); ipv6Subnet != "" { + pa, err := ipam.NewPoolAllocator(ipv6Subnet, "", 0, lockDir) + if err != nil { + errs = append(errs, fmt.Errorf("open IPv6 pool: %w", err)) + } else if _, ok := pa.LookupContainer(containerID); !ok { + errs = append(errs, errors.New("no IPv6 allocation found for container")) + } + } + if conf.IPv4Subnet != "" { + pa, err := ipam.NewIPv4PoolAllocator(conf.IPv4Subnet, "", lockDir) + if err != nil { + errs = append(errs, fmt.Errorf("open IPv4 pool: %w", err)) + } else if _, ok := pa.LookupContainer(containerID); !ok { + errs = append(errs, errors.New("no IPv4 allocation found for container")) + } + } + return errs +} diff --git a/internal/cniipam/allocate_test.go b/internal/cniipam/allocate_test.go new file mode 100644 index 00000000..298fe3aa --- /dev/null +++ b/internal/cniipam/allocate_test.go @@ -0,0 +1,129 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniipam + +import ( + "net" + "testing" + + "github.com/containernetworking/cni/pkg/skel" +) + +const ( + testContainerID = "test-container" + testIPv6PoolDefault = "fd00:10:ff01::/64" + testIPv4Subnet = "10.128.0.0/20" +) + +func withTempLockDir(t *testing.T) { + t.Helper() + original := lockDir + lockDir = t.TempDir() + t.Cleanup(func() { lockDir = original }) +} + +func TestAllocateStatic(t *testing.T) { + args := &skel.CmdArgs{ContainerID: testContainerID} + conf := &IPAM{Type: testIPAMType, StaticIP: "fd00:10:ff01::1234"} + + res, err := allocate(args, conf) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res.IPv6Subnet == nil || !res.IPv6Subnet.IP.Equal(net.ParseIP("fd00:10:ff01::1234")) { + t.Errorf("IPv6Subnet = %v, want fd00:10:ff01::1234", res.IPv6Subnet) + } + if res.IPv4Address != nil { + t.Errorf("IPv4Address = %v, want nil for static IPAM", res.IPv4Address) + } +} + +func TestAllocatePoolDualStack(t *testing.T) { + withTempLockDir(t) + args := &skel.CmdArgs{ContainerID: testContainerID} + conf := &IPAM{Type: testIPAMType, IPv6Subnet: testIPv6PoolDefault, IPv4Subnet: testIPv4Subnet} + + res, err := allocate(args, conf) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res.IPv6Subnet == nil { + t.Error("IPv6Subnet = nil, want an allocated /96") + } + if res.IPv4Address == nil { + t.Fatal("IPv4Address = nil, want an allocated /32") + } + if len(res.Routes) != 2 { + t.Errorf("Routes = %v, want one default route per family", res.Routes) + } +} + +func TestAllocatePoolMissingBothSubnetsErrors(t *testing.T) { + withTempLockDir(t) + args := &skel.CmdArgs{ContainerID: testContainerID} + if _, err := allocate(args, &IPAM{Type: testIPAMType}); err == nil { + t.Fatal("expected error when neither subnet is set, got nil") + } +} + +func TestDeallocateStaticNoop(t *testing.T) { + // Static allocations persist nothing; deallocate must be a pure no-op + // (this asserts it doesn't panic touching pool state that was never + // created). + deallocate(testContainerID, &IPAM{Type: testIPAMType, StaticIP: "fd00::1"}) +} + +func TestAllocateDeallocateRoundTripIPv6(t *testing.T) { + withTempLockDir(t) + args := &skel.CmdArgs{ContainerID: testContainerID} + conf := &IPAM{Type: testIPAMType, IPv6Subnet: testIPv6PoolDefault} + + res, err := allocate(args, conf) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // A fresh "DEL process" must be able to look the allocation up and + // deallocate it, without ever being told the allocated subnet. + deallocate(testContainerID, conf) + + if errs := checkAllocation(testContainerID, conf); len(errs) == 0 { + t.Errorf("checkAllocation after deallocate = no errors, want a not-found error (subnet %v)", res.IPv6Subnet) + } +} + +func TestAllocateDeallocateRoundTripIPv4(t *testing.T) { + withTempLockDir(t) + args := &skel.CmdArgs{ContainerID: testContainerID} + conf := &IPAM{Type: testIPAMType, IPv4Subnet: testIPv4Subnet} + + if _, err := allocate(args, conf); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if errs := checkAllocation(testContainerID, conf); len(errs) != 0 { + t.Fatalf("checkAllocation before deallocate = %v, want no errors", errs) + } + + deallocate(testContainerID, conf) + + if errs := checkAllocation(testContainerID, conf); len(errs) == 0 { + t.Error("checkAllocation after deallocate = no errors, want a not-found error") + } +} + +func TestCheckAllocationStaticAlwaysPasses(t *testing.T) { + if errs := checkAllocation(testContainerID, &IPAM{Type: testIPAMType, StaticIP: "fd00::1"}); errs != nil { + t.Errorf("checkAllocation for static IPAM = %v, want nil (nothing persisted to check)", errs) + } +} + +func TestCheckAllocationUnknownContainer(t *testing.T) { + withTempLockDir(t) + conf := &IPAM{Type: testIPAMType, IPv6Subnet: testIPv6PoolDefault, IPv4Subnet: testIPv4Subnet} + errs := checkAllocation("no-such-container", conf) + if len(errs) != 2 { + t.Fatalf("checkAllocation for unknown container = %v, want 2 errors (one per family)", errs) + } +} diff --git a/internal/cniipam/cniipam.go b/internal/cniipam/cniipam.go new file mode 100644 index 00000000..654e9992 --- /dev/null +++ b/internal/cniipam/cniipam.go @@ -0,0 +1,27 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniipam + +import ( + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/version" + + "go.datum.net/galactic/internal/metadata" +) + +// RunPlugin starts galactic-ipam, handling the CNI IPAM delegation +// protocol's ADD, DEL, CHECK, and STATUS operations. +func RunPlugin() { + skel.PluginMainFuncs( + skel.CNIFuncs{ + Add: cmdAdd, + Check: cmdCheck, + Del: cmdDel, + Status: cmdStatus, + }, + version.All, + "CNI galactic-ipam plugin "+metadata.Version, + ) +} diff --git a/internal/cniipam/config.go b/internal/cniipam/config.go new file mode 100644 index 00000000..4c78165e --- /dev/null +++ b/internal/cniipam/config.go @@ -0,0 +1,156 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniipam + +import ( + "encoding/json" + "fmt" + "net" + + "github.com/containernetworking/cni/pkg/types" + + "go.datum.net/galactic/internal/config" +) + +const errInvalidCNIConfig = "invalid CNI config" + +const ( + // maxIPv6SubnetPrefixLen mirrors internal/cni's own constraint: pool + // prefix must be no longer than the per-allocation subnet length, and + // dual-stack tenant addressing allocates /96 endpoints. + maxIPv6SubnetPrefixLen = 96 + maxIPv4SubnetPrefixLen = 32 +) + +const ( + addressFamilyIPv6 = "ipv6" + addressFamilyIPv4 = "ipv4" +) + +// sanitizeForErrorBinary is substituted for a config value that fails +// sanitizeForError's printable-ASCII check. +const sanitizeForErrorBinary = "" + +// parseConf unmarshals the full CNI config document (the same one the +// master plugin itself received) and validates/normalizes the "ipam" +// block. Returns a *types.Error (CNI error code 7) for anything a real IPAM +// invocation should never see, since a master plugin only ever delegates +// here when its own "ipam" block is present. +func parseConf(data []byte) (*pluginConf, error) { + conf := &pluginConf{} + if err := json.Unmarshal(data, conf); err != nil { + return nil, &types.Error{Code: 7, Msg: errInvalidCNIConfig, Details: err.Error()} + } + if conf.IPAM == nil { + return nil, &types.Error{Code: 7, Msg: "ipam block is required"} + } + + if conf.IPAM.IPv6Subnet != "" { + if err := validateIPv6Subnet(conf.IPAM.IPv6Subnet); err != nil { + return nil, err + } + } + if conf.IPAM.IPv4Subnet != "" { + if err := validateIPv4Subnet(conf.IPAM.IPv4Subnet); err != nil { + return nil, err + } + } + + // Default-filler: only when the ipam block is present but specifies + // neither a static address nor a pool CIDR for either family. Cannot + // manufacture an ipam block out of thin air — that decision already + // happened in the master plugin, before this process was even execed. + if conf.IPAM.StaticIP == "" && conf.IPAM.IPv6Subnet == "" && conf.IPAM.IPv4Subnet == "" { + if config.IPAMGetEnableLocalIPAM() { + conf.IPAM.IPv6Subnet = localIPAMDefaultPool + } + } + + if len(conf.IPAM.AddressFamilies) == 0 { + conf.IPAM.AddressFamilies = []string{addressFamilyIPv6} + } else { + for _, af := range conf.IPAM.AddressFamilies { + switch af { + case addressFamilyIPv6, addressFamilyIPv4: + default: + return nil, &types.Error{Code: 7, Msg: fmt.Sprintf( + "invalid ipam.address_families entry %q: must be %q or %q", + sanitizeForError(af), addressFamilyIPv6, addressFamilyIPv4), + } + } + } + } + + return conf, nil +} + +func validateIPv6Subnet(subnet string) error { + ip, mask, err := net.ParseCIDR(subnet) + if err != nil { + return &types.Error{Code: 7, Msg: fmt.Sprintf( + "invalid CIDR value for field 'ipam.ipv6_subnet': %q", sanitizeForError(subnet)), + } + } + if ip.To4() != nil { + return &types.Error{Code: 7, Msg: fmt.Sprintf( + "ipam.ipv6_subnet must be an IPv6 CIDR, got IPv4: %q", sanitizeForError(subnet)), + } + } + if prefixLen, _ := mask.Mask.Size(); prefixLen > maxIPv6SubnetPrefixLen { + return &types.Error{Code: 7, Msg: fmt.Sprintf( + "ipam.ipv6_subnet prefix length %d exceeds maximum of %d: %q", + prefixLen, maxIPv6SubnetPrefixLen, sanitizeForError(subnet)), + } + } + return nil +} + +func validateIPv4Subnet(subnet string) error { + ip, mask, err := net.ParseCIDR(subnet) + if err != nil { + return &types.Error{Code: 7, Msg: fmt.Sprintf( + "invalid CIDR value for field 'ipam.ipv4_subnet': %q", sanitizeForError(subnet)), + } + } + if ip.To4() == nil { + return &types.Error{Code: 7, Msg: fmt.Sprintf( + "ipam.ipv4_subnet must be an IPv4 CIDR, got IPv6: %q", sanitizeForError(subnet)), + } + } + if prefixLen, _ := mask.Mask.Size(); prefixLen > maxIPv4SubnetPrefixLen { + return &types.Error{Code: 7, Msg: fmt.Sprintf( + "ipam.ipv4_subnet prefix length %d exceeds maximum of %d: %q", + prefixLen, maxIPv4SubnetPrefixLen, sanitizeForError(subnet)), + } + } + return nil +} + +// sanitizeForError returns s unchanged if it contains only printable ASCII +// characters; otherwise returns "" to avoid corrupting log output. +func sanitizeForError(s string) string { + for _, c := range s { + if c < 0x20 || c > 0x7e { + return sanitizeForErrorBinary + } + } + return s +} + +// parseStatusConf validates that STATUS's config is minimally parseable. +// galactic-ipam has no attachment-specific or API-server state to check — +// STATUS just confirms the binary can parse a well-formed CNI config. +func parseStatusConf(data []byte) error { + var sc struct { + CNIVersion string `json:"cniVersion"` + } + if err := json.Unmarshal(data, &sc); err != nil { + return &types.Error{Code: 7, Msg: errInvalidCNIConfig, Details: err.Error()} + } + if sc.CNIVersion == "" { + return &types.Error{Code: 7, Msg: "cniVersion is required"} + } + return nil +} diff --git a/internal/cniipam/config_test.go b/internal/cniipam/config_test.go new file mode 100644 index 00000000..7206bc93 --- /dev/null +++ b/internal/cniipam/config_test.go @@ -0,0 +1,149 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniipam + +import ( + "fmt" + "strings" + "testing" +) + +const ( + testCNIVersion = "1.0.0" + testIPAMType = "galactic-ipam" +) + +// confJSON builds a minimal CNI config document carrying the given "ipam" +// block body (already-JSON-encoded, e.g. `"type":"galactic-ipam"`), keeping +// every test case below short enough to stay under the project's line +// length limit. +func confJSON(ipamBody string) string { + return fmt.Sprintf(`{"cniVersion":"%s","name":"test","ipam":{%s}}`, testCNIVersion, ipamBody) +} + +func TestParseConf(t *testing.T) { + tests := []struct { + name string + input string + wantErr string + wantType string + }{ + { + name: "MissingIPAMBlockRejected", + input: fmt.Sprintf(`{"cniVersion":"%s","name":"test","type":"%s"}`, testCNIVersion, testIPAMType), + wantErr: "ipam block is required", + }, + { + name: "StaticIPAccepted", + input: confJSON(fmt.Sprintf(`"type":%q,"static_ip":"fd00::1234"`, testIPAMType)), + wantType: testIPAMType, + }, + { + name: "InvalidIPv6SubnetCIDRRejected", + input: confJSON(fmt.Sprintf(`"type":%q,"ipv6_subnet":"not-a-cidr"`, testIPAMType)), + wantErr: "invalid CIDR value for field 'ipam.ipv6_subnet'", + }, + { + name: "IPv4GivenWhereIPv6SubnetExpectedRejected", + input: confJSON(fmt.Sprintf(`"type":%q,"ipv6_subnet":"10.0.0.0/24"`, testIPAMType)), + wantErr: "ipam.ipv6_subnet must be an IPv6 CIDR, got IPv4", + }, + { + name: "IPv6SubnetPrefixLengthOver96Rejected", + input: confJSON(fmt.Sprintf(`"type":%q,"ipv6_subnet":"fd00:10:ff01::/112"`, testIPAMType)), + wantErr: "ipam.ipv6_subnet prefix length 112 exceeds maximum of 96", + }, + { + name: "InvalidIPv4SubnetCIDRRejected", + input: confJSON(fmt.Sprintf(`"type":%q,"ipv4_subnet":"not-a-cidr"`, testIPAMType)), + wantErr: "invalid CIDR value for field 'ipam.ipv4_subnet'", + }, + { + name: "IPv6GivenWhereIPv4SubnetExpectedRejected", + input: confJSON(fmt.Sprintf(`"type":%q,"ipv4_subnet":"2001:db8::/64"`, testIPAMType)), + wantErr: "ipam.ipv4_subnet must be an IPv4 CIDR, got IPv6", + }, + { + name: "IPv4SubnetPrefixLengthOver32Rejected", + input: confJSON(fmt.Sprintf(`"type":%q,"ipv4_subnet":"::ffff:10.0.0.0/40"`, testIPAMType)), + wantErr: "ipam.ipv4_subnet prefix length 40 exceeds maximum of 32", + }, + { + name: "InvalidAddressFamiliesEntryRejected", + input: confJSON(fmt.Sprintf( + `"type":%q,"ipv6_subnet":"fd00::/64","address_families":["ipv6","bogus"]`, testIPAMType)), + wantErr: `invalid ipam.address_families entry "bogus"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conf, err := parseConf([]byte(tt.input)) + if tt.wantErr != "" { + if err == nil { + t.Fatalf("expected error containing %q, got nil", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("error %q does not contain %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if conf.IPAM.Type != tt.wantType { + t.Errorf("IPAM.Type = %q, want %q", conf.IPAM.Type, tt.wantType) + } + }) + } +} + +func TestParseConfDefaultFillerOnlyWhenUnderspecified(t *testing.T) { + t.Setenv("GALACTIC_IPAM_ENABLE_LOCAL_IPAM", "true") + + // ipam present but specifies neither static_ip nor a subnet: filled in. + conf, err := parseConf([]byte(confJSON(fmt.Sprintf(`"type":%q`, testIPAMType)))) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if conf.IPAM.IPv6Subnet != localIPAMDefaultPool { + t.Errorf("IPv6Subnet = %q, want default-filled %q", conf.IPAM.IPv6Subnet, localIPAMDefaultPool) + } + + // ipam present and already specifies a subnet: default-filler must not + // override it. + conf, err = parseConf([]byte(confJSON(fmt.Sprintf(`"type":%q,"ipv4_subnet":"10.0.0.0/24"`, testIPAMType)))) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if conf.IPAM.IPv6Subnet != "" { + t.Errorf("IPv6Subnet = %q, want empty (ipv4_subnet already specified)", conf.IPAM.IPv6Subnet) + } +} + +func TestParseConfDefaultFillerRequiresEnvVar(t *testing.T) { + t.Setenv("GALACTIC_IPAM_ENABLE_LOCAL_IPAM", "false") + + conf, err := parseConf([]byte(confJSON(fmt.Sprintf(`"type":%q`, testIPAMType)))) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if conf.IPAM.IPv6Subnet != "" || conf.IPAM.IPv4Subnet != "" { + t.Errorf("IPv6Subnet/IPv4Subnet = %q/%q, want both empty (default-filler disabled)", + conf.IPAM.IPv6Subnet, conf.IPAM.IPv4Subnet) + } +} + +func TestParseStatusConf(t *testing.T) { + if err := parseStatusConf([]byte(`{"cniVersion":"1.0.0"}`)); err != nil { + t.Errorf("unexpected error: %v", err) + } + if err := parseStatusConf([]byte(`not json`)); err == nil { + t.Error("expected error for invalid JSON, got nil") + } + if err := parseStatusConf([]byte(`{}`)); err == nil { + t.Error("expected error for missing cniVersion, got nil") + } +} diff --git a/internal/cniipam/ops.go b/internal/cniipam/ops.go new file mode 100644 index 00000000..3c906579 --- /dev/null +++ b/internal/cniipam/ops.go @@ -0,0 +1,85 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniipam + +import ( + "errors" + "fmt" + "log/slog" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" +) + +// cmdAdd implements the CNI IPAM delegation ADD path. Being invoked at all +// is the master plugin's own signal that its "ipam" block was present — +// this function always allocates, it never re-checks whether it should. +func cmdAdd(args *skel.CmdArgs) error { + conf, err := parseConf(args.StdinData) + if err != nil { + return err + } + + slog.Info("ADD: starting", "containerID", args.ContainerID, "type", conf.IPAM.Type) + + result, err := allocate(args, conf.IPAM) + if err != nil { + return fmt.Errorf("allocate: %w", err) + } + + cniResult := BuildCNIResult(conf.CNIVersion, result) + if err := types.PrintResult(cniResult, conf.CNIVersion); err != nil { + return fmt.Errorf("print CNI result: %w", err) + } + slog.Info("ADD: allocated", "containerID", args.ContainerID, + "ipv6Subnet", result.IPv6Subnet, "ipv4Address", result.IPv4Address) + return nil +} + +// cmdDel implements the CNI IPAM delegation DEL path. Per the CNI spec, +// DEL is idempotent: a config parse failure or a missing allocation is not +// an error, since there may be nothing left to clean up. +func cmdDel(args *skel.CmdArgs) error { + slog.Info("DEL: starting", "containerID", args.ContainerID) + + conf, err := parseConf(args.StdinData) + if err != nil { + slog.Warn("DEL: failed to parse CNI config, skipping deallocation", "err", err, + "containerID", args.ContainerID) + return nil + } + + deallocate(args.ContainerID, conf.IPAM) + return nil +} + +// cmdCheck implements the CNI IPAM delegation CHECK path: confirm the +// containerID's allocation, if any, is still present in each family conf +// configures. +func cmdCheck(args *skel.CmdArgs) error { + conf, err := parseConf(args.StdinData) + if err != nil { + return err + } + + if errs := checkAllocation(args.ContainerID, conf.IPAM); len(errs) > 0 { + err := fmt.Errorf("CHECK failed: %w", errors.Join(errs...)) + slog.Error("CHECK: failed", "err", err, "containerID", args.ContainerID) + return err + } + slog.Info("CHECK: passed", "containerID", args.ContainerID) + return nil +} + +// cmdStatus implements the CNI spec STATUS operation. galactic-ipam has no +// API server or attachment-specific state to probe — it either parses a +// well-formed config or it doesn't. +func cmdStatus(args *skel.CmdArgs) error { + if err := parseStatusConf(args.StdinData); err != nil { + return err + } + slog.Info("STATUS: ready") + return nil +} diff --git a/internal/cniipam/ops_test.go b/internal/cniipam/ops_test.go new file mode 100644 index 00000000..4c80cdd9 --- /dev/null +++ b/internal/cniipam/ops_test.go @@ -0,0 +1,81 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniipam + +import ( + "fmt" + "strings" + "testing" + + "github.com/containernetworking/cni/pkg/skel" +) + +func TestCmdAddCmdDelCmdCheckRoundTrip(t *testing.T) { + withTempLockDir(t) + + conf := confJSON(fmt.Sprintf(`"type":%q,"ipv6_subnet":%q`, testIPAMType, testIPv6PoolDefault)) + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte(conf)} + + if err := cmdAdd(args); err != nil { + t.Fatalf("cmdAdd: unexpected error: %v", err) + } + if err := cmdCheck(args); err != nil { + t.Fatalf("cmdCheck after cmdAdd: unexpected error: %v", err) + } + if err := cmdDel(args); err != nil { + t.Fatalf("cmdDel: unexpected error: %v", err) + } + if err := cmdCheck(args); err == nil { + t.Fatal("cmdCheck after cmdDel: expected error (allocation released), got nil") + } +} + +func TestCmdAddMissingIPAMBlock(t *testing.T) { + args := &skel.CmdArgs{ + ContainerID: testContainerID, + StdinData: []byte(fmt.Sprintf(`{"cniVersion":"%s","name":"test","type":%q}`, testCNIVersion, testIPAMType)), + } + err := cmdAdd(args) + if err == nil || !strings.Contains(err.Error(), "ipam block is required") { + t.Fatalf("expected 'ipam block is required' error, got: %v", err) + } +} + +func TestCmdDelIdempotentOnUnparseableConfig(t *testing.T) { + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte("not valid json")} + if err := cmdDel(args); err != nil { + t.Fatalf("cmdDel with invalid config returned error = %v, want nil (idempotent)", err) + } +} + +func TestCmdDelStaticIsNoop(t *testing.T) { + conf := confJSON(fmt.Sprintf(`"type":%q,"static_ip":"fd00::1234"`, testIPAMType)) + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte(conf)} + if err := cmdDel(args); err != nil { + t.Fatalf("cmdDel for static IPAM returned error = %v, want nil", err) + } +} + +func TestCmdStatusValid(t *testing.T) { + args := &skel.CmdArgs{StdinData: []byte(fmt.Sprintf(`{"cniVersion":"%s"}`, testCNIVersion))} + if err := cmdStatus(args); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestCmdStatusInvalidConfig(t *testing.T) { + args := &skel.CmdArgs{StdinData: []byte("not valid json")} + if err := cmdStatus(args); err == nil { + t.Fatal("expected error for invalid config, got nil") + } +} + +func TestCmdCheckStaticAlwaysPasses(t *testing.T) { + conf := confJSON(fmt.Sprintf(`"type":%q,"static_ip":"fd00::1234"`, testIPAMType)) + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte(conf)} + if err := cmdCheck(args); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/internal/cniipam/result.go b/internal/cniipam/result.go new file mode 100644 index 00000000..9991d77c --- /dev/null +++ b/internal/cniipam/result.go @@ -0,0 +1,81 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniipam + +import ( + "encoding/json" + "fmt" + "net" + + "github.com/containernetworking/cni/pkg/types" + type100 "github.com/containernetworking/cni/pkg/types/100" +) + +// BuildCNIResult constructs the type100.Result IPAM delegation returns — +// ips/routes only, no interfaces. The master plugin owns interface +// creation entirely; keeping interfaces out of this result is exactly what +// delegation exists to enforce (see the package doc comment). +func BuildCNIResult(cniVersion string, res *IPAMResult) *type100.Result { + result := &type100.Result{CNIVersion: cniVersion} + if res == nil { + return result + } + if res.IPv6Subnet != nil { + result.IPs = append(result.IPs, &type100.IPConfig{ + Address: *res.IPv6Subnet, + Gateway: res.IPv6Gateway, + }) + } + if res.IPv4Address != nil { + result.IPs = append(result.IPs, &type100.IPConfig{ + Address: net.IPNet{IP: res.IPv4Address, Mask: net.CIDRMask(32, 32)}, + Gateway: res.IPv4Gateway, + }) + } + for _, r := range res.Routes { + result.Routes = append(result.Routes, &types.Route{Dst: *r}) + } + return result +} + +// ResultToIPAMResult converts a CNI result — as returned by +// github.com/containernetworking/cni/pkg/ipam.ExecAdd back to the master +// plugin that just delegated an ADD — into the local shape callers apply +// directly (configureInterfaceInNetns for veth, or read straight into a +// tap/BGP-advertisement result). Marshals and re-parses via type100 rather +// than a direct type assertion, since the concrete type returned by +// ExecAdd depends on CNI version negotiation (mirrors the same pattern +// internal/cni's own prevResult validation already uses). +func ResultToIPAMResult(res types.Result) (*IPAMResult, error) { + jsonBytes, err := json.Marshal(res) + if err != nil { + return nil, fmt.Errorf("marshal IPAM result: %w", err) + } + parsed, err := type100.NewResult(jsonBytes) + if err != nil { + return nil, fmt.Errorf("parse IPAM result: %w", err) + } + versioned, err := type100.GetResult(parsed) + if err != nil { + return nil, fmt.Errorf("get IPAM result: %w", err) + } + + r := &IPAMResult{} + for _, ipConf := range versioned.IPs { + if ipConf.Address.IP.To4() != nil { + r.IPv4Address = ipConf.Address.IP + r.IPv4Gateway = ipConf.Gateway + continue + } + subnet := ipConf.Address + r.IPv6Subnet = &subnet + r.IPv6Gateway = ipConf.Gateway + } + for _, rt := range versioned.Routes { + dst := rt.Dst + r.Routes = append(r.Routes, &dst) + } + return r, nil +} diff --git a/internal/cniipam/result_test.go b/internal/cniipam/result_test.go new file mode 100644 index 00000000..2d3aee1c --- /dev/null +++ b/internal/cniipam/result_test.go @@ -0,0 +1,111 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniipam + +import ( + "net" + "testing" + + type100 "github.com/containernetworking/cni/pkg/types/100" +) + +func mustParseCIDR(t *testing.T, cidr string) *net.IPNet { + t.Helper() + _, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + t.Fatalf("parse CIDR %q: %v", cidr, err) + } + return ipnet +} + +func TestBuildCNIResultNil(t *testing.T) { + result := BuildCNIResult(testCNIVersion, nil) + if result.CNIVersion != testCNIVersion { + t.Errorf("CNIVersion = %q, want %q", result.CNIVersion, testCNIVersion) + } + if len(result.IPs) != 0 { + t.Errorf("IPs = %v, want empty", result.IPs) + } +} + +func TestBuildCNIResultDualStack(t *testing.T) { + subnet := mustParseCIDR(t, "fd00:10:ff01::1234/96") + gw6 := net.ParseIP("fd00:10:ff01::1") + addr4 := net.ParseIP("10.128.0.5") + gw4 := net.ParseIP("10.128.0.1") + route6 := mustParseCIDR(t, "::/0") + + result := BuildCNIResult(testCNIVersion, &IPAMResult{ + IPv6Subnet: subnet, IPv6Gateway: gw6, + IPv4Address: addr4, IPv4Gateway: gw4, + Routes: []*net.IPNet{route6}, + }) + + if len(result.IPs) != 2 { + t.Fatalf("IPs count = %d, want 2", len(result.IPs)) + } + if result.IPs[0].Address.String() != subnet.String() { + t.Errorf("IPs[0].Address = %v, want %v", result.IPs[0].Address, subnet) + } + if !result.IPs[0].Gateway.Equal(gw6) { + t.Errorf("IPs[0].Gateway = %v, want %v", result.IPs[0].Gateway, gw6) + } + wantMask := net.CIDRMask(32, 32).String() + if result.IPs[1].Address.IP.String() != addr4.String() || result.IPs[1].Address.Mask.String() != wantMask { + t.Errorf("IPs[1].Address = %v, want %s/32", result.IPs[1].Address, addr4) + } + if len(result.Routes) != 1 { + t.Errorf("Routes count = %d, want 1", len(result.Routes)) + } + // No interfaces should ever be set — that's the master plugin's job. + if len(result.Interfaces) != 0 { + t.Errorf("Interfaces = %v, want empty (IPAM delegation never returns interfaces)", result.Interfaces) + } +} + +func TestResultToIPAMResultRoundTrip(t *testing.T) { + subnet := mustParseCIDR(t, "fd00:10:ff01::1234/96") + gw6 := net.ParseIP("fd00:10:ff01::1") + addr4 := net.ParseIP("10.128.0.5") + gw4 := net.ParseIP("10.128.0.1") + route6 := mustParseCIDR(t, "::/0") + + built := BuildCNIResult(testCNIVersion, &IPAMResult{ + IPv6Subnet: subnet, IPv6Gateway: gw6, + IPv4Address: addr4, IPv4Gateway: gw4, + Routes: []*net.IPNet{route6}, + }) + + roundTripped, err := ResultToIPAMResult(built) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if roundTripped.IPv6Subnet == nil || roundTripped.IPv6Subnet.String() != subnet.String() { + t.Errorf("IPv6Subnet = %v, want %v", roundTripped.IPv6Subnet, subnet) + } + if !roundTripped.IPv6Gateway.Equal(gw6) { + t.Errorf("IPv6Gateway = %v, want %v", roundTripped.IPv6Gateway, gw6) + } + if roundTripped.IPv4Address == nil || !roundTripped.IPv4Address.Equal(addr4) { + t.Errorf("IPv4Address = %v, want %v", roundTripped.IPv4Address, addr4) + } + if !roundTripped.IPv4Gateway.Equal(gw4) { + t.Errorf("IPv4Gateway = %v, want %v", roundTripped.IPv4Gateway, gw4) + } + if len(roundTripped.Routes) != 1 { + t.Fatalf("Routes count = %d, want 1", len(roundTripped.Routes)) + } +} + +func TestResultToIPAMResultEmpty(t *testing.T) { + empty := &type100.Result{CNIVersion: testCNIVersion} + res, err := ResultToIPAMResult(empty) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res.IPv6Subnet != nil || res.IPv4Address != nil { + t.Errorf("res = %+v, want all-nil fields", res) + } +} diff --git a/internal/cniipam/types.go b/internal/cniipam/types.go new file mode 100644 index 00000000..3fb8e686 --- /dev/null +++ b/internal/cniipam/types.go @@ -0,0 +1,84 @@ +// Copyright 2025 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package cniipam implements galactic-ipam, the delegated CNI IPAM plugin +// in the galactic CNI chain (see github.com/containernetworking/cni/pkg/ipam +// for the delegation protocol both master plugins, galactic-cni and +// galactic-tap-cni, invoke this through via ExecAdd/ExecDel/ExecCheck). +// +// Explicit contract: a master plugin delegates here if and only if its own +// "ipam" block is present at all — no environment variable or sibling +// config field can trigger or suppress that decision (that's the master's +// own call, made before this package is ever invoked). Once delegated to, +// mode selection is entirely this package's own: presence of +// ipam.static_ip selects the static single-address path; otherwise +// ipam.ipv6_subnet/ipv4_subnet (either family alone, or both) select the +// pool path. GALACTIC_IPAM_ENABLE_LOCAL_IPAM only fills in a default IPv6 +// pool CIDR when the ipam block is present but specifies neither +// static_ip nor a subnet — it can no longer manufacture an ipam block out +// of thin air the way its GALACTIC_CNI_ENABLE_LOCAL_IPAM predecessor did. +// +// Allocation state persists in on-disk marker files (internal/cni/ipam), +// keyed by containerID, so this package never needs a Kubernetes client +// at all: DEL looks its own allocation up locally instead of reading it +// back from a BGPAdvertisement CRD annotation galactic-bgp wrote. +package cniipam + +import ( + "net" + + "github.com/containernetworking/cni/pkg/types" +) + +// IPAM is the JSON shape of a CNI config's "ipam" block, as galactic-ipam +// itself parses it (the master plugins each embed the same shape as +// *IPAM in their own PluginConf, since the full netconf — including this +// block — is what gets passed through to the delegate unmodified). +type IPAM struct { + // Type names the delegated binary (e.g. "galactic-ipam") — a CNI IPAM + // delegation implementation detail (github.com/containernetworking/cni/ + // pkg/ipam.ExecAdd/ExecDel read this to know which binary to exec), not + // a mode selector. Mode is decided from which of the fields below are + // present instead — see the package doc comment. + Type string `json:"type"` + StaticIP string `json:"static_ip,omitempty"` + IPv6Subnet string `json:"ipv6_subnet,omitempty"` + IPv4Subnet string `json:"ipv4_subnet,omitempty"` + AddressFamilies []string `json:"address_families,omitempty"` + Routes []Route `json:"routes,omitempty"` + Addresses []Address `json:"addresses,omitempty"` +} + +// Route describes a static route to install. +type Route struct { + Dst string `json:"dst"` + GW string `json:"gw,omitempty"` +} + +// Address describes a static IP address assignment. +type Address struct { + Address string `json:"address"` +} + +// IPAMResult holds the allocation details a master plugin uses to build +// its own CNI result and, for veth, to configure the guest interface. +// IPv4Address/IPv4Gateway are nil when the attachment is IPv6-only. +type IPAMResult struct { + IPv6Subnet *net.IPNet + IPv6Gateway net.IP + IPv4Address net.IP + IPv4Gateway net.IP + Routes []*net.IPNet +} + +// pluginConf is the full CNI config document galactic-ipam receives as +// args.StdinData — the same document the master plugin itself parsed, +// passed through unmodified per the IPAM delegation protocol. Only the +// "ipam" key (plus cniVersion, for result versioning) is ever read; the +// master-plugin-specific fields (vpc, vpcattachment, terminations, ...) +// are present in the JSON but simply ignored here. +type pluginConf struct { + types.PluginConf + IPAM *IPAM `json:"ipam"` +} diff --git a/internal/cnimaster/check.go b/internal/cnimaster/check.go new file mode 100644 index 00000000..4d40f81e --- /dev/null +++ b/internal/cnimaster/check.go @@ -0,0 +1,147 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnimaster + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "time" + + "github.com/containernetworking/cni/pkg/types" + "github.com/vishvananda/netlink" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + + "go.datum.net/galactic/internal/config" + "go.datum.net/galactic/internal/plumbing/intf" + "go.datum.net/galactic/internal/plumbing/vrf" +) + +// RunStatus implements the CNI spec STATUS operation shared by both master +// plugins: config is parseable and the API server is reachable for +// BGPAdvertisement CRD operations. Attachment-specific kernel resources +// (VRF, host interface) are NOT checked because STATUS must succeed before +// any ADD has ever run. +// +// cniConfig and confFile are the caller's own package state — see +// ParseConf's doc comment for why these aren't shared globals. +func RunStatus(stdinData []byte, cniConfig *config.CNIConfig, confFile string) error { + // Validate config is parseable (minimal check — no VPC/VPCAttachment + // validation since STATUS must succeed before any ADD has run). + if err := ParseStatusConf(stdinData); err != nil { + return err + } + + // Load host CNI config to resolve Kubeconfig and LogFile + hostConf, err := LoadHostConf(confFile) + if err != nil { + return &types.Error{Code: 7, Msg: fmt.Sprintf("load host CNI config: %v", err)} + } + + // Resolve config: env var > conflist > default. + cniConfig.Resolve(&config.ConflistValues{ + Kubeconfig: hostConf.Kubeconfig, + Namespace: hostConf.Namespace, + LogFile: hostConf.LogFile, + LogLevel: hostConf.LogLevel, + }) + + // Propagate Kubeconfig + _ = os.Setenv("KUBECONFIG", cniConfig.Kubeconfig) + + // Setup Logging + SetupLogging(cniConfig.LogFile, cniConfig.LogLevel) + slog.Debug("CNI config received", "stdin", string(stdinData)) + + // Config is parseable and API server is reachable. + slog.Info("STATUS: probing API server reachability") + if err := ProbeAPIServer(); err != nil { + slog.Error("STATUS: API server probe failed", "err", err) + return &types.Error{Code: 50, Msg: fmt.Sprintf("API server health check failed: %v", err)} + } + slog.Info("STATUS: ready") + return nil +} + +// ProbeAPIServerFn performs a lightweight GET against the in-cluster API +// server to verify reachability. Returns nil when the server responds (any +// HTTP status code) or when running outside a cluster with no kubeconfig. +// +// ProbeAPIServer is a variable so tests can override it. +var ProbeAPIServerFn = func() error { + kubeconfig, err := ctrl.GetConfig() + if err != nil { + if errors.Is(err, rest.ErrNotInCluster) { + // Not running in-cluster; skip API check. + return nil + } + return fmt.Errorf("load kubeconfig: %w", err) + } + kubeconfig.Timeout = 2 * time.Second + httpClient, err := rest.HTTPClientFor(kubeconfig) + if err != nil { + return fmt.Errorf("build http client: %w", err) + } + req, err := http.NewRequestWithContext( + context.Background(), + http.MethodGet, + kubeconfig.Host+"/healthz", + nil, + ) + if err != nil { + return fmt.Errorf("build healthz request: %w", err) + } + resp, err := httpClient.Do(req) + if err != nil { + return fmt.Errorf("healthz request failed: %w", err) + } + defer resp.Body.Close() //nolint:errcheck // best-effort probe + return nil +} + +// ProbeAPIServer is the seam cmdStatus/RunStatus call through; tests +// override it (and restore ProbeAPIServerFn afterward) to simulate API +// server reachability failures without a real cluster. +var ProbeAPIServer = ProbeAPIServerFn + +// CheckNodeLevelState verifies that node-level networking resources exist: +// the VRF interface and the host-side endpoint interface. Returns the host +// interface name (for callers that need it, e.g. cmdCheck's prevResult +// validation) and a slice of errors (nil when all checks pass) so callers +// can accumulate and report all failures at once. +func CheckNodeLevelState(vpc, vpcAttachment string) (string, []error) { + var errs []error + + if err := vrf.Exists(vpc, vpcAttachment); err != nil { + errs = append(errs, fmt.Errorf("vrf %s-%s: %w", vpc, vpcAttachment, err)) + } + + hostName := intf.GenerateInterfaceNameHost(vpc, vpcAttachment) + if _, err := netlink.LinkByName(hostName); err != nil { + errs = append(errs, fmt.Errorf("host interface %q: %w", hostName, err)) + } + + return hostName, errs +} + +// ValidateHostInterface checks that a host-side interface's MAC and MTU +// match the values recorded in prevResult. +func ValidateHostInterface(name, wantMac string, wantMtu int) error { + link, err := netlink.LinkByName(name) + if err != nil { + return fmt.Errorf("find link: %w", err) + } + if wantMac != "" && link.Attrs().HardwareAddr.String() != wantMac { + return fmt.Errorf("MAC mismatch: expected %q, got %q", wantMac, link.Attrs().HardwareAddr.String()) + } + if wantMtu > 0 && link.Attrs().MTU != wantMtu { + return fmt.Errorf("MTU mismatch: expected %d, got %d", wantMtu, link.Attrs().MTU) + } + return nil +} diff --git a/internal/cnimaster/cnimaster_test.go b/internal/cnimaster/cnimaster_test.go new file mode 100644 index 00000000..b985dc0b --- /dev/null +++ b/internal/cnimaster/cnimaster_test.go @@ -0,0 +1,607 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnimaster + +import ( + "errors" + "fmt" + "log/slog" + "net" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/containernetworking/cni/pkg/types" + type100 "github.com/containernetworking/cni/pkg/types/100" + + "go.datum.net/galactic/internal/config" +) + +func TestMain(m *testing.M) { + _ = os.Setenv("GALACTIC_CNI_NODE_NAME", "test-node") + os.Exit(m.Run()) +} + +const ( + testVPC = "abc" + testAttachment = "def" + testInvalidBase62 = "abc-def" // shared invalid base62 string for tests + testNetns = "/proc/1/ns/net" + testMac = "aa:bb:cc:dd:ee:ff" + testIfName = "eth0" + testCNIVersion = "1.0.0" + + // testPrevResult is a valid CNI v1.0.0 result used in prevResult tests. + testPrevResult = `{"cniVersion":"1.0.0",` + + `"interfaces":[{"name":"` + testIfName + `","mac":"` + testMac + `",` + + `"sandbox":"/proc/1/ns/net"}],` + + `"ips":[{"version":"6","address":"fd00:1::1/64"}]}` +) + +// assertCNIError verifies that err is a *types.Error with the expected Code +// and that its Msg contains wantMsg (substring match). Pass wantMsg == "" to +// skip the message check. +func assertCNIError(t *testing.T, err error, wantCode uint, wantMsg string) { + t.Helper() + var cniErr *types.Error + if !errors.As(err, &cniErr) { + t.Fatalf("expected *types.Error, got %T: %v", err, err) + } + if cniErr.Code != wantCode { + t.Fatalf("expected code %d, got %d (Msg: %q)", wantCode, cniErr.Code, cniErr.Msg) + } + if wantMsg != "" && !strings.Contains(cniErr.Msg, wantMsg) { + t.Fatalf("expected Msg to contain %q, got %q", wantMsg, cniErr.Msg) + } +} + +func mustParseCIDR(t *testing.T, cidr string) *net.IPNet { + t.Helper() + ip, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + t.Fatalf("net.ParseCIDR(%q): %v", cidr, err) + } + ipNet.IP = ip + return ipNet +} + +// ---- ParseConf ------------------------------------------------------------- + +func TestParseConf(t *testing.T) { + cniConfig := config.NewCNIConfig() + + tests := []struct { + name string + input string + wantVPC string + wantErr string + wantCode uint // CNI error code; 0 means "don't check" + }{ + { + name: "valid config", + input: fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test",`+ + `"type":"galactic-cni","vpc":"%s",`+ + `"vpcattachment":"%s"}`, + testVPC, testAttachment, + ), + wantVPC: testVPC, + }, + { + name: "invalid JSON", + input: "not json", + wantErr: "invalid CNI config", + wantCode: 7, + }, + { + name: "empty input", + input: "", + wantErr: "invalid CNI config", + wantCode: 7, + }, + { + name: "missing vpc", + input: fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test",`+ + `"type":"galactic-cni","vpcattachment":"%s"}`, + testAttachment, + ), + wantErr: "vpc is required and must be a non-empty base62 string", + wantCode: 7, + }, + { + name: "empty vpc", + input: fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test",`+ + `"type":"galactic-cni","vpc":"",`+ + `"vpcattachment":"%s"}`, + testAttachment, + ), + wantErr: "vpc is required and must be a non-empty base62 string", + wantCode: 7, + }, + { + name: "vpc with invalid char hyphen", + input: fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test",`+ + `"type":"galactic-cni","vpc":"%s",`+ + `"vpcattachment":"%s"}`, + testInvalidBase62, testAttachment, + ), + wantErr: fmt.Sprintf("invalid base62 value for field 'vpc': %q", testInvalidBase62), + wantCode: 7, + }, + { + name: "vpc with invalid char underscore", + input: fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test",`+ + `"type":"galactic-cni","vpc":"abc_def",`+ + `"vpcattachment":"%s"}`, + testAttachment, + ), + wantErr: `invalid base62 value for field 'vpc': "abc_def"`, + wantCode: 7, + }, + { + name: "missing vpcattachment", + input: fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test",`+ + `"type":"galactic-cni","vpc":"%s"}`, + testVPC, + ), + wantErr: "vpcattachment is required and must be a non-empty base62 string", + wantCode: 7, + }, + { + name: "empty vpcattachment", + input: fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test",`+ + `"type":"galactic-cni","vpc":"%s",`+ + `"vpcattachment":""}`, + testVPC, + ), + wantErr: "vpcattachment is required and must be a non-empty base62 string", + wantCode: 7, + }, + { + name: "vpcattachment with invalid char space", + input: fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test",`+ + `"type":"galactic-cni","vpc":"%s",`+ + `"vpcattachment":"def ghi"}`, + testVPC, + ), + wantErr: `invalid base62 value for field 'vpcattachment': "def ghi"`, + wantCode: 7, + }, + { + name: "valid vpc and vpcattachment with mixed case base62", + input: `{"cniVersion":"1.0.0","name":"test",` + + `"type":"galactic-cni","vpc":"Abc123XYZ",` + + `"vpcattachment":"DeF456"}`, + wantVPC: "Abc123XYZ", + }, + { + name: "prevResult valid JSON result is accepted", + input: fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test",`+ + `"type":"galactic-cni","vpc":"%s",`+ + `"vpcattachment":"%s",`+ + `"prevResult":%s}`, + testVPC, testAttachment, testPrevResult, + ), + wantVPC: testVPC, + }, + { + name: "ipam block present is accepted, delegated per its own contract", + input: fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test",`+ + `"type":"galactic-cni","vpc":"%s",`+ + `"vpcattachment":"%s","ipam":{"type":"galactic-ipam","ipv6_subnet":"fd00:10:ff01::/48"}}`, + testVPC, testAttachment, + ), + wantVPC: testVPC, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conf, err := ParseConf([]byte(tt.input), cniConfig, config.DefaultConfFile) + if tt.wantErr != "" { + if err == nil { + t.Fatalf("expected error containing %q, got nil", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("error %q does not contain %q", err, tt.wantErr) + } + if tt.wantCode > 0 { + assertCNIError(t, err, tt.wantCode, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if conf.VPC != tt.wantVPC { + t.Errorf("VPC = %q, want %q", conf.VPC, tt.wantVPC) + } + }) + } +} + +// TestIPAMBlockPresenceIsTheOnlyTrigger is the regression test for the +// explicit contract internal/cniipam's doc comment describes: whether a +// master plugin delegates to IPAM at all is decided solely by whether +// "ipam" is present in its own config — no environment variable can +// manufacture (or suppress) that block. +func TestIPAMBlockPresenceIsTheOnlyTrigger(t *testing.T) { + cniConfig := config.NewCNIConfig() + + // Missing ipam block: no error, no delegation signal — conf.IPAM stays nil. + inputNoIPAM := fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test","type":"galactic-cni","vpc":"%s","vpcattachment":"%s"}`, + testVPC, testAttachment, + ) + conf, err := ParseConf([]byte(inputNoIPAM), cniConfig, config.DefaultConfFile) + if err != nil { + t.Fatalf("unexpected error for missing ipam block: %v", err) + } + if conf.IPAM != nil { + t.Fatalf("IPAM = %+v, want nil (absent block must never be manufactured)", conf.IPAM) + } + + // Present ipam block: conf.IPAM is populated, ready for delegation. + inputWithIPAM := fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test","type":"galactic-cni","vpc":"%s","vpcattachment":"%s",`+ + `"ipam":{"type":"galactic-ipam"}}`, + testVPC, testAttachment, + ) + conf, err = ParseConf([]byte(inputWithIPAM), cniConfig, config.DefaultConfFile) + if err != nil { + t.Fatalf("unexpected error with present ipam block: %v", err) + } + if conf.IPAM == nil { + t.Fatal("expected IPAM block to be non-nil") + } + if conf.IPAM.Type != "galactic-ipam" { + t.Errorf("IPAM.Type = %q, want %q", conf.IPAM.Type, "galactic-ipam") + } +} + +// ---- IsValidBase62 --------------------------------------------------------- + +func TestIsValidBase62(t *testing.T) { + tests := []struct { + name string + input string + want bool + }{ + {"empty", "", false}, + {"digits only", "1234567890", true}, + {"lowercase only", "abcdefghij", true}, + {"uppercase only", "ABCDEFGHIJ", true}, + {"mixed case", "aBcDeFgHiJ", true}, + {"mixed digits and letters", "abc123XYZ", true}, + {"hyphen", testInvalidBase62, false}, + {"underscore", "abc_def", false}, + {"space", "abc def", false}, + {"dot", "abc.def", false}, + {"slash", "abc/def", false}, + {"plus", "abc+def", false}, + {"equals", "abc=def", false}, + {"single digit", "0", true}, + {"single lowercase", "a", true}, + {"single uppercase", "Z", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := IsValidBase62(tt.input) + if got != tt.want { + t.Errorf("IsValidBase62(%q) = %v, want %v", tt.input, got, tt.want) + } + }) + } +} + +func TestSanitizeForError(t *testing.T) { + printable := "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!@#$%^&*()" + tests := []struct { + name string + input string + want string + }{ + {"normal string", testInvalidBase62, testInvalidBase62}, + {"empty", "", ""}, + {"newline", "abc\ndef", sanitizeForErrorBinary}, + {"null byte", "abc\x00def", sanitizeForErrorBinary}, + {"del char", "abc\x7fdef", sanitizeForErrorBinary}, + {"printable range", printable, printable}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := SanitizeForError(tt.input) + if got != tt.want { + t.Errorf("SanitizeForError(%q) = %q, want %q", tt.input, got, tt.want) + } + }) + } +} + +// ---- ValidatePrevResult / ValidatePrevResultAdd ---------------------------- + +func TestValidatePrevResult(t *testing.T) { + validResult := &type100.Result{ + CNIVersion: testCNIVersion, + Interfaces: []*type100.Interface{ + {Name: testIfName, Mac: testMac, Sandbox: testNetns}, + }, + IPs: []*type100.IPConfig{ + {Address: *mustParseCIDR(t, "fd00:1::1/64")}, + }, + } + + tests := []struct { + name string + input types.Result + wantErr bool + }{ + {"nil result allowed", nil, false}, + {"valid CNI result", validResult, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidatePrevResult(tt.input) + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} + +func TestValidatePrevResultAdd(t *testing.T) { + validWithInterface := &type100.Result{ + CNIVersion: testCNIVersion, + Interfaces: []*type100.Interface{ + {Name: testIfName, Mac: testMac, Sandbox: testNetns}, + }, + IPs: []*type100.IPConfig{ + {Address: *mustParseCIDR(t, "fd00:1::1/64")}, + }, + } + validWithIPsOnly := &type100.Result{ + CNIVersion: testCNIVersion, + IPs: []*type100.IPConfig{ + {Address: *mustParseCIDR(t, "fd00:1::1/64")}, + }, + } + emptyResult := &type100.Result{ + CNIVersion: testCNIVersion, + // No interfaces, no IPs — should fail content validation. + } + + tests := []struct { + name string + input types.Result + wantErr bool + }{ + {"nil result allowed", nil, false}, + {"valid result with interface", validWithInterface, false}, + {"valid result with IPs only", validWithIPsOnly, false}, + {"empty result (no interfaces or IPs)", emptyResult, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidatePrevResultAdd(tt.input) + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} + +// ---- ProbeAPIServer --------------------------------------------------------- + +func TestProbeAPIServerErrNotInCluster(t *testing.T) { + // When ProbeAPIServerFn returns ErrNotInCluster, ProbeAPIServer should + // return nil (not running in-cluster; skip API check). + original := ProbeAPIServer + ProbeAPIServer = func() error { return nil } + defer func() { ProbeAPIServer = original }() + + if err := ProbeAPIServer(); err != nil { + t.Fatalf("expected nil for ErrNotInCluster, got %v", err) + } +} + +func TestProbeAPIServerMalformedKubeconfig(t *testing.T) { + // When ProbeAPIServerFn returns a non-ErrNotInCluster error (e.g. a + // malformed kubeconfig file), ProbeAPIServer should surface it wrapped. + original := ProbeAPIServer + ProbeAPIServer = func() error { + return errors.New("load kubeconfig: invalid kubeconfig: permission denied") + } + defer func() { ProbeAPIServer = original }() + + err := ProbeAPIServer() + if err == nil { + t.Fatal("expected error for malformed kubeconfig, got nil") + } + if !strings.Contains(err.Error(), "load kubeconfig") { + t.Fatalf("error %q does not contain 'load kubeconfig'", err.Error()) + } + if !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("error %q does not contain original error", err.Error()) + } +} + +// ---- LoadHostConf ----------------------------------------------------------- + +func TestLoadHostConf(t *testing.T) { + tmpDir := t.TempDir() + conflistPath := filepath.Join(tmpDir, "10-galactic.conflist") + + // 1. Missing file tolerated, defaults to galactic-system namespace. + conf, err := LoadHostConf(conflistPath) + if err != nil { + t.Fatalf("unexpected error for missing conflist: %v", err) + } + if conf.Namespace != config.DefaultNamespace { + t.Errorf("Namespace = %q, want %q", conf.Namespace, config.DefaultNamespace) + } + + // 2. Conflist parses but lacks galactic-cni entry. + badContent := `{"cniVersion":"1.0.0","name":"test","plugins":[{"type":"some-other-plugin"}]}` + if err := os.WriteFile(conflistPath, []byte(badContent), 0644); err != nil { + t.Fatalf("os.WriteFile: %v", err) + } + _, err = LoadHostConf(conflistPath) + if err == nil { + t.Fatal("expected error for missing plugin type, got nil") + } + + // 3. Conflist parses correctly. + goodContent := `{ + "cniVersion": "1.0.0", + "name": "galactic", + "plugins": [ + { + "type": "galactic-cni", + "node_name": "test-worker", + "kubeconfig": "/etc/custom-kubeconfig", + "namespace": "custom-namespace", + "log_file": "/var/log/custom.log", + "log_level": "debug" + } + ] + }` + if err := os.WriteFile(conflistPath, []byte(goodContent), 0644); err != nil { + t.Fatalf("os.WriteFile: %v", err) + } + conf, err = LoadHostConf(conflistPath) + if err != nil { + t.Fatalf("unexpected error for good conflist: %v", err) + } + if conf.NodeName != "test-worker" { + t.Errorf("NodeName = %q, want %q", conf.NodeName, "test-worker") + } + if conf.Kubeconfig != "/etc/custom-kubeconfig" { + t.Errorf("Kubeconfig = %q, want %q", conf.Kubeconfig, "/etc/custom-kubeconfig") + } + if conf.Namespace != "custom-namespace" { + t.Errorf("Namespace = %q, want %q", conf.Namespace, "custom-namespace") + } + if conf.LogFile != "/var/log/custom.log" { + t.Errorf("LogFile = %q, want %q", conf.LogFile, "/var/log/custom.log") + } + if conf.LogLevel != config.LogLevelDebug { + t.Errorf("LogLevel = %q, want %q", conf.LogLevel, config.LogLevelDebug) + } +} + +// ---- logging setup ---------------------------------------------------------- + +func TestLoggingSetup(t *testing.T) { + tmpDir := t.TempDir() + logPath := filepath.Join(tmpDir, "sub", "test.log") + + // Setup logging, which should create the directory and open/write to the file. + SetupLogging(logPath, config.DefaultLogLevel) + slog.Info("test log message") + + // Read the log file to verify the message was logged. + data, err := os.ReadFile(logPath) + if err != nil { + t.Fatalf("read log file: %v", err) + } + if !strings.Contains(string(data), "test log message") { + t.Fatalf("log content does not contain message: %s", string(data)) + } +} + +func TestParseLogLevel(t *testing.T) { + tests := []struct { + name string + in string + want slog.Level + wantErr bool + }{ + {"empty defaults to info", "", slog.LevelInfo, false}, + {"debug", config.LogLevelDebug, slog.LevelDebug, false}, + {"info", config.DefaultLogLevel, slog.LevelInfo, false}, + {"warn", config.LogLevelWarn, slog.LevelWarn, false}, + {"warning alias", config.LogLevelWarning, slog.LevelWarn, false}, + {"error", config.LogLevelError, slog.LevelError, false}, + {"case insensitive", "DEBUG", slog.LevelDebug, false}, + {"surrounding whitespace", " warn ", slog.LevelWarn, false}, + {"unknown falls back to info with error", "verbose", slog.LevelInfo, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseLogLevel(tt.in) + if (err != nil) != tt.wantErr { + t.Fatalf("ParseLogLevel(%q) error = %v, wantErr %v", tt.in, err, tt.wantErr) + } + if got != tt.want { + t.Errorf("ParseLogLevel(%q) = %v, want %v", tt.in, got, tt.want) + } + }) + } +} + +func TestLoggingSetupRespectsLevel(t *testing.T) { + tmpDir := t.TempDir() + logPath := filepath.Join(tmpDir, "test.log") + + SetupLogging(logPath, config.LogLevelWarn) + slog.Info("should be suppressed at warn level") + slog.Warn("should appear at warn level") + + data, err := os.ReadFile(logPath) + if err != nil { + t.Fatalf("read log file: %v", err) + } + content := string(data) + if strings.Contains(content, "should be suppressed at warn level") { + t.Errorf("expected info message to be filtered out at warn level, got: %s", content) + } + if !strings.Contains(content, "should appear at warn level") { + t.Errorf("expected warn message to be present, got: %s", content) + } +} + +func TestLoggingSetupInvalidLevelFallsBackToInfo(t *testing.T) { + tmpDir := t.TempDir() + logPath := filepath.Join(tmpDir, "test.log") + + // An invalid level must not fail the CNI operation; it should fall back + // to DefaultLogLevel (info) rather than panic or drop all logging. + SetupLogging(logPath, "verbose") + slog.Info("should appear at fallback info level") + + data, err := os.ReadFile(logPath) + if err != nil { + t.Fatalf("read log file: %v", err) + } + if !strings.Contains(string(data), "should appear at fallback info level") { + t.Errorf("expected info message to be present after fallback, got: %s", string(data)) + } +} diff --git a/internal/cnimaster/config.go b/internal/cnimaster/config.go new file mode 100644 index 00000000..207edea4 --- /dev/null +++ b/internal/cnimaster/config.go @@ -0,0 +1,341 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnimaster + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + + "github.com/containernetworking/cni/pkg/types" + type100 "github.com/containernetworking/cni/pkg/types/100" + + "go.datum.net/galactic/internal/cniipam" + "go.datum.net/galactic/internal/config" + "go.datum.net/galactic/internal/hostconf" +) + +// PluginConf is the CNI plugin configuration passed via stdin on each +// invocation of either master plugin (galactic-cni or galactic-tap-cni). +// +// IPAM addressing fields (ipv6_subnet, ipv4_subnet, address_families, +// static_ip) live entirely inside the "ipam" block — see +// go.datum.net/galactic/internal/cniipam's doc comment for the explicit +// delegation contract: this struct only decides *whether* to delegate +// (IPAM != nil), never anything about how allocation itself works. +// Termination routes are galactic-route's own concern (see +// internal/cniroute) — neither master plugin's own JSON stanza carries a +// "terminations" field of its own to read. +type PluginConf struct { + types.PluginConf + VPC string `json:"vpc"` + VPCAttachment string `json:"vpcattachment"` + MTU int `json:"mtu,omitempty"` + IPAM *cniipam.IPAM `json:"ipam"` + Namespace string `json:"namespace,omitempty"` +} + +const sanitizeForErrorBinary = "" + +// errInvalidCNIConfig is the message for CNI config parse errors (code 7). +const errInvalidCNIConfig = "invalid CNI config" + +// errVPCRequired and errVPCAttachmentRequired are messages for missing +// identifier fields (code 7). +const ( + errVPCRequired = "vpc is required and must be a non-empty base62 string" + errVPCAttachmentRequired = "vpcattachment is required and must be a non-empty base62 string" +) + +// IsValidBase62 reports whether s contains only valid base62 characters +// ([0-9a-zA-Z]) and is non-empty. VPC and VPCAttachment identifiers are +// base62-encoded and used throughout the ADD path (interface naming, +// BGP CRD population). Rejecting them early in ParseConf prevents cryptic +// errors deep in the stack after partial kernel state has been created. +func IsValidBase62(s string) bool { + if len(s) == 0 { + return false + } + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') { + return false + } + } + return true +} + +// LoadHostConf loads node-local settings from the static per-node conflist. +// If the file is missing, it returns a zero-value HostConf (tolerating local +// test runs) but still defaulting Namespace to config.DefaultNamespace. +func LoadHostConf(filePath string) (*hostconf.HostConf, error) { + if filePath == "" { + filePath = config.DefaultConfFile + } + conf, err := hostconf.Load(filePath, hostconf.PluginType) + if err != nil { + if os.IsNotExist(UnwrapPathError(err)) { + return &hostconf.HostConf{Namespace: config.DefaultNamespace}, nil + } + return nil, err + } + if conf.Namespace == "" { + conf.Namespace = config.DefaultNamespace + } + return conf, nil +} + +// UnwrapPathError returns the innermost *os.PathError-shaped error wrapped +// by err, if any, so os.IsNotExist (which does not itself traverse %w +// wrapping) can still recognize a missing conflist file wrapped by +// hostconf.Load's fmt.Errorf("read conflist file %q: %w", ...). +func UnwrapPathError(err error) error { + for { + unwrapped := errors.Unwrap(err) + if unwrapped == nil { + return err + } + err = unwrapped + } +} + +// ParseLogLevel maps a config-supplied level name to a slog.Level. Matching +// is case-insensitive. An empty string resolves to config.DefaultLogLevel. +// Unrecognized values return an error alongside the info-level fallback, so +// callers can warn without failing the CNI operation over a typo'd setting. +func ParseLogLevel(s string) (slog.Level, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "": + return ParseLogLevel(config.DefaultLogLevel) + case config.LogLevelDebug: + return slog.LevelDebug, nil + case config.DefaultLogLevel: + return slog.LevelInfo, nil + case config.LogLevelWarn, config.LogLevelWarning: + return slog.LevelWarn, nil + case config.LogLevelError: + return slog.LevelError, nil + default: + return slog.LevelInfo, fmt.Errorf("unknown log level %q (want %s, %s, %s, or %s)", + s, config.LogLevelDebug, config.DefaultLogLevel, config.LogLevelWarn, config.LogLevelError) + } +} + +// SetupLogging configures the slog default logger to write to the specified +// path at the specified verbosity. If opening the file fails, it logs a +// warning to os.Stderr and falls back. An unrecognized logLevel also logs a +// warning and falls back to config.DefaultLogLevel rather than failing the +// operation. +func SetupLogging(logPath, logLevel string) { + if logPath == "" { + logPath = config.DefaultLogFile + } + level, err := ParseLogLevel(logLevel) + if err != nil { + slog.Warn("Invalid log level, falling back to default", + "value", logLevel, "default", config.DefaultLogLevel, "err", err) + } + // Ensure parent directory exists. + if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { + slog.Warn("Failed to create log directory", "path", filepath.Dir(logPath), "err", err) + return + } + file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + slog.Warn("Failed to open log file, falling back to Stderr", "path", logPath, "err", err) + return + } + // Use JSON handler for structured logging to file. + handler := slog.NewJSONHandler(file, &slog.HandlerOptions{Level: level}) + slog.SetDefault(slog.New(handler)) +} + +// statusConf holds the minimal CNI config fields needed for STATUS validation. +// +// STATUS only checks that the config is parseable and the API server is +// reachable; it does not validate attachment-specific fields (VPC, +// VPCAttachment) because STATUS must succeed before any ADD has ever run. +type statusConf struct { + CNIVersion string `json:"cniVersion"` + Type string `json:"type"` +} + +// ParseStatusConf validates that the CNI config is parseable and contains +// the required top-level fields (cniVersion, type). Unlike ParseConf, it +// does not validate VPC or VPCAttachment because STATUS must succeed on a +// freshly started node before any ADD has run. +func ParseStatusConf(data []byte) error { + var sc statusConf + if err := json.Unmarshal(data, &sc); err != nil { + return &types.Error{Code: 7, Msg: errInvalidCNIConfig, Details: err.Error()} + } + if sc.CNIVersion == "" { + return &types.Error{Code: 7, Msg: "cniVersion is required"} + } + if sc.Type == "" { + return &types.Error{Code: 7, Msg: "type is required"} + } + return nil +} + +// ValidatePrevResult checks that the prevResult (from a preceding plugin in +// the CNI chain) is a valid, parseable CNI result. Returns an error if the +// result is non-nil but cannot be parsed as a versioned CNI result, ensuring +// the master plugin fails fast rather than silently operating on garbage +// state. +func ValidatePrevResult(res types.Result) error { + if res == nil { + return nil + } + // Marshal to JSON and re-parse to verify the result is structurally valid. + // This catches malformed results that survived CNI framework unmarshaling. + jsonBytes, err := json.Marshal(res) + if err != nil { + return fmt.Errorf("marshal prevResult: %w", err) + } + if _, err := type100.NewResult(jsonBytes); err != nil { + return fmt.Errorf("parse prevResult: %w", err) + } + return nil +} + +// ValidatePrevResultAdd performs content-level validation of prevResult +// during the ADD operation. It ensures the preceding plugin produced a +// result with at least one interface or IP assignment, which is the minimum +// expected structure for any meaningful CNI chain. Returns nil when +// prevResult is nil (no preceding plugin) or structurally valid with +// expected content. +func ValidatePrevResultAdd(res types.Result) error { + if res == nil { + return nil + } + jsonBytes, err := json.Marshal(res) + if err != nil { + return fmt.Errorf("marshal prevResult: %w", err) + } + result, err := type100.NewResult(jsonBytes) + if err != nil { + return fmt.Errorf("parse prevResult: %w", err) + } + versioned, err := type100.GetResult(result) + if err != nil { + return fmt.Errorf("get prevResult version: %w", err) + } + // A valid prevResult must declare at least one interface or IP assignment. + if len(versioned.Interfaces) == 0 && len(versioned.IPs) == 0 { + return errors.New("prevResult declares no interfaces or IP assignments") + } + return nil +} + +// ParseConf unmarshals the CNI configuration from stdin data and validates +// the base62-encoded identifier fields. It resolves the host configuration +// and sets up process environment variables and logging. +// +// cniConfig is the caller's own *config.CNIConfig singleton (each master +// plugin binary owns one, initialized once via its own InitCNIConfig() at +// process startup) and confFile is the caller's own ConfFile package var — +// both are caller state, not shared, since each binary resolves its own +// GALACTIC_CNI_* environment independently. +func ParseConf(data []byte, cniConfig *config.CNIConfig, confFile string) (*PluginConf, error) { + conf := &PluginConf{} + if err := json.Unmarshal(data, &conf); err != nil { + return nil, &types.Error{Code: 7, Msg: errInvalidCNIConfig, Details: err.Error()} + } + if !IsValidBase62(conf.VPC) { + if len(conf.VPC) == 0 { + return nil, &types.Error{Code: 7, Msg: errVPCRequired} + } + return nil, &types.Error{ + Code: 7, + Msg: fmt.Sprintf("invalid base62 value for field 'vpc': %q", SanitizeForError(conf.VPC)), + } + } + if !IsValidBase62(conf.VPCAttachment) { + if len(conf.VPCAttachment) == 0 { + return nil, &types.Error{Code: 7, Msg: errVPCAttachmentRequired} + } + return nil, &types.Error{ + Code: 7, + Msg: fmt.Sprintf("invalid base62 value for field 'vpcattachment': %q", SanitizeForError(conf.VPCAttachment)), + } + } + + // Load host CNI config + hostConf, err := LoadHostConf(confFile) + if err != nil { + return nil, fmt.Errorf("load host CNI config: %w", err) + } + + // Resolve config: env var > conflist > default. + cniConfig.Resolve(&config.ConflistValues{ + NodeName: hostConf.NodeName, + Kubeconfig: hostConf.Kubeconfig, + Namespace: hostConf.Namespace, + LogFile: hostConf.LogFile, + LogLevel: hostConf.LogLevel, + }) + + // NodeName fallback: auto-detect from the Kubernetes API by matching local + // interface addresses against node InternalIPs. This handles cases where + // the conflist file is missing (e.g. hostPath mount issues in container- + // based environments like Kind). + if cniConfig.NodeName == "" { + detected, detectErr := hostconf.DetectNodeNameFromAPI() + if detectErr != nil { + slog.Warn("Node name auto-detection failed", "err", detectErr) + } + cniConfig.NodeName = detected + } + if cniConfig.NodeName == "" { + return nil, &types.Error{Code: 4, Msg: "node name is required (or set GALACTIC_CNI_NODE_NAME)"} + } + _ = os.Setenv("NODE_NAME", cniConfig.NodeName) + + // Propagate Kubeconfig + _ = os.Setenv("KUBECONFIG", cniConfig.Kubeconfig) + + // Resolve and propagate Namespace fallback + namespace := conf.Namespace + if namespace == "" { + namespace = cniConfig.Namespace + } + conf.Namespace = namespace + + // Setup Logging + SetupLogging(cniConfig.LogFile, cniConfig.LogLevel) + slog.Debug("CNI config received", "stdin", string(data)) + + // Whether IPAM runs at all is decided entirely by whether "ipam" is + // present — no environment variable or sibling field can trigger or + // suppress that. Addressing fields (ipv6_subnet, ipv4_subnet, + // address_families, static_ip) and their own default-filling/CIDR + // validation live inside internal/cniipam, since they're only ever + // read by whichever binary "ipam.type" names — the master plugin passes + // its own StdinData straight through unmodified when it delegates, so + // validating them here too would just be redundant work on the same + // bytes. + + if conf.PrevResult != nil { + if err := ValidatePrevResult(conf.PrevResult); err != nil { + return nil, &types.Error{Code: 6, Msg: fmt.Sprintf("invalid prevResult: %v", err)} + } + } + return conf, nil +} + +// SanitizeForError returns s unchanged if it contains only printable ASCII +// characters; otherwise returns "" to avoid corrupting log output. +func SanitizeForError(s string) string { + for _, c := range s { + if c < 0x20 || c > 0x7e { + return sanitizeForErrorBinary + } + } + return s +} diff --git a/internal/cnimaster/doc.go b/internal/cnimaster/doc.go new file mode 100644 index 00000000..18d6f1e1 --- /dev/null +++ b/internal/cnimaster/doc.go @@ -0,0 +1,20 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package cnimaster holds logic shared by galactic-cni (internal/cni, the +// veth master plugin) and galactic-tap-cni (internal/cnitap, the tap master +// plugin). Both own the same node-level lifecycle — parse the CNI config, +// resolve node/API settings, create a VRF, patch the pod's NAD, and answer +// CHECK/STATUS — differing only in which kernel interface primitive they +// call (veth vs tap) and, for CHECK, whether there's a guest-side netns to +// inspect (tap never enters one). That interface-specific sliver stays in +// each package; everything else lives here so a fix only has to happen +// once. +// +// PluginConf is the shared CNI config shape; internal/cni and internal/cnitap +// each declare their own `type PluginConf = cnimaster.PluginConf` alias +// (mirroring the existing HostConf alias pattern from internal/hostconf) so +// call sites in either package keep referring to their own package's +// PluginConf. +package cnimaster diff --git a/internal/cnimaster/resource.go b/internal/cnimaster/resource.go new file mode 100644 index 00000000..eb861b17 --- /dev/null +++ b/internal/cnimaster/resource.go @@ -0,0 +1,76 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnimaster + +import ( + "fmt" + "log/slog" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "go.datum.net/galactic/internal/plumbing/vrf" + bgpv1alpha1 "go.datum.net/network/api/v1alpha1" +) + +var scheme = runtime.NewScheme() + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + // bgpv1alpha1 is registered even though neither master plugin reads or + // writes BGP CRDs itself (that's galactic-bgp's own, chain-invoked + // concern now) — kept here only because nothing else needs its own + // scheme, and NAD annotation's unstructured.Unstructured Patch call + // doesn't require any scheme registration at all. Harmless to leave; + // trim if this scheme ever needs to shrink for another reason. + utilruntime.Must(bgpv1alpha1.AddToScheme(scheme)) +} + +// NewK8sClient creates a new Kubernetes client using the in-cluster config. +// The only k8s call either master plugin makes directly is the NAD +// annotation patch. +func NewK8sClient() (client.Client, error) { + restCfg, err := ctrl.GetConfig() + if err != nil { + return nil, fmt.Errorf("get kubeconfig: %w", err) + } + c, err := client.New(restCfg, client.Options{Scheme: scheme}) + if err != nil { + return nil, fmt.Errorf("create k8s client: %w", err) + } + return c, nil +} + +// CleanupAttachment rolls back a failed ADD's host-side interface and VRF, +// in that order, for selective rollback. Errors are logged but never +// returned — the caller already has a failure to report. +// +// ifaceKind names the interface kind for log messages ("veth", "tap"); del +// removes the attachment's own host/guest interface pair. Both the +// interface and the VRF (internal/plumbing/vrf, keyed by (vpc, +// vpcAttachment) here) are this attachment's own, so unconditional deletion +// of both is safe — neither call tears down state a sibling attachment +// still depends on. +func CleanupAttachment(vpc, vpcAttachment, ifaceKind string, del func(vpc, vpcAttachment string) error) { + slog.Info("Selective rollback: cleaning up resources created during failed ADD", + "vpc", vpc, "vpcAttachment", vpcAttachment) + + if err := del(vpc, vpcAttachment); err != nil { + slog.Error("Rollback: failed to delete "+ifaceKind, "err", err, + "vpc", vpc, "vpcAttachment", vpcAttachment) + } else { + slog.Debug("Rollback: deleted "+ifaceKind, "vpc", vpc, "vpcAttachment", vpcAttachment) + } + + if err := vrf.Delete(vpc, vpcAttachment); err != nil { + slog.Error("Rollback: failed to delete VRF", "err", err, + "vpc", vpc, "vpcAttachment", vpcAttachment) + } else { + slog.Debug("Rollback: deleted VRF", "vpc", vpc, "vpcAttachment", vpcAttachment) + } +} diff --git a/internal/cniroute/cniroute.go b/internal/cniroute/cniroute.go new file mode 100644 index 00000000..7784fa4c --- /dev/null +++ b/internal/cniroute/cniroute.go @@ -0,0 +1,27 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniroute + +import ( + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/version" + + "go.datum.net/galactic/internal/metadata" +) + +// RunPlugin starts galactic-route, handling the CNI ADD, DEL, CHECK, and +// STATUS operations for the termination-route stage of the chain. +func RunPlugin() { + skel.PluginMainFuncs( + skel.CNIFuncs{ + Add: cmdAdd, + Check: cmdCheck, + Del: cmdDel, + Status: cmdStatus, + }, + version.All, + "CNI galactic-route plugin "+metadata.Version, + ) +} diff --git a/internal/cniroute/cniroute_test.go b/internal/cniroute/cniroute_test.go new file mode 100644 index 00000000..4cfe5d25 --- /dev/null +++ b/internal/cniroute/cniroute_test.go @@ -0,0 +1,369 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniroute + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "testing" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" +) + +func TestMain(m *testing.M) { + InitCNIConfig() + os.Exit(m.Run()) +} + +const ( + testVPC = "abc" + testAttachment = "def" + testContainerID = "test-container" + testInvalidBase62 = "abc-def" + testMac = "aa:bb:cc:dd:ee:ff" + testIfName = "eth0" + + // testPrevResult is a valid CNI v1.0.0 result used in prevResult tests. + testPrevResult = `{"cniVersion":"1.0.0",` + + `"interfaces":[{"name":"` + testIfName + `","mac":"` + testMac + `",` + + `"sandbox":"/proc/1/ns/net"}],` + + `"ips":[{"version":"6","address":"fd00:1::1/64"}]}` +) + +// confJSON builds a minimal galactic-route CNI config document for tests. +func confJSON(vpc, vpcAttachment, prevResult string) string { + prevResultField := "" + if prevResult != "" { + prevResultField = `,"prevResult":` + prevResult + } + return fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test","type":"galactic-route",`+ + `"vpc":%q,"vpcattachment":%q%s}`, + vpc, vpcAttachment, prevResultField, + ) +} + +// assertCNIError verifies that err is a *types.Error with the expected Code +// and that its Msg contains wantMsg (substring match). Pass wantMsg == "" to +// skip the message check. +func assertCNIError(t *testing.T, err error, wantCode uint, wantMsg string) { + t.Helper() + var cniErr *types.Error + if !errors.As(err, &cniErr) { + t.Fatalf("expected *types.Error, got %T: %v", err, err) + } + if cniErr.Code != wantCode { + t.Fatalf("expected code %d, got %d (Msg: %q)", wantCode, cniErr.Code, cniErr.Msg) + } + if wantMsg != "" && !strings.Contains(cniErr.Msg, wantMsg) { + t.Fatalf("expected Msg to contain %q, got %q", wantMsg, cniErr.Msg) + } +} + +// ---- parseConf ------------------------------------------------------------- + +func TestParseConfInvalidJSON(t *testing.T) { + _, err := parseConf([]byte("not valid json")) + assertCNIError(t, err, 7, errInvalidCNIConfig) +} + +func TestParseConfMissingVPC(t *testing.T) { + _, err := parseConf([]byte(confJSON("", testAttachment, ""))) + assertCNIError(t, err, 7, errVPCRequired) +} + +func TestParseConfInvalidVPC(t *testing.T) { + _, err := parseConf([]byte(confJSON(testInvalidBase62, testAttachment, ""))) + assertCNIError(t, err, 7, "invalid base62 value for field 'vpc'") +} + +func TestParseConfMissingVPCAttachment(t *testing.T) { + _, err := parseConf([]byte(confJSON(testVPC, "", ""))) + assertCNIError(t, err, 7, errVPCAttachmentRequired) +} + +func TestParseConfInvalidVPCAttachment(t *testing.T) { + _, err := parseConf([]byte(confJSON(testVPC, testInvalidBase62, ""))) + assertCNIError(t, err, 7, "invalid base62 value for field 'vpcattachment'") +} + +func TestParseConfValid(t *testing.T) { + conf, err := parseConf([]byte(confJSON(testVPC, testAttachment, ""))) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if conf.VPC != testVPC || conf.VPCAttachment != testAttachment { + t.Errorf("got vpc=%q vpcAttachment=%q, want %q/%q", conf.VPC, conf.VPCAttachment, testVPC, testAttachment) + } +} + +func TestParseConfWithTerminations(t *testing.T) { + conf := fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test","type":"galactic-route",`+ + `"vpc":%q,"vpcattachment":%q,`+ + `"terminations":[{"network":"fd00:2::/64","via":"fd00:1::1"}]}`, + testVPC, testAttachment, + ) + parsed, err := parseConf([]byte(conf)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(parsed.Terminations) != 1 { + t.Fatalf("got %d terminations, want 1", len(parsed.Terminations)) + } + if parsed.Terminations[0].Network != "fd00:2::/64" || parsed.Terminations[0].Via != "fd00:1::1" { + t.Errorf("got termination %+v, want network=fd00:2::/64 via=fd00:1::1", parsed.Terminations[0]) + } +} + +func TestParseConfPrevResultNeverValidatedAtParseTime(t *testing.T) { + // types.PluginConf.PrevResult has json tag "-" and is never populated by + // plain json.Unmarshal (a pre-existing quirk of that library — see + // internal/cnibgp/prevresult.go's own doc comment). parseConf's + // validatePrevResult check therefore never fires here regardless of + // what "prevResult" contains; cmdAdd's own parsePrevResult (reading + // RawPrevResult instead) is what actually validates prevResult content. + conf, err := parseConf([]byte(confJSON(testVPC, testAttachment, `{"cniVersion":"garbage"}`))) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if conf.PrevResult != nil { + t.Error("PrevResult should remain nil after plain json.Unmarshal (see RawPrevResult instead)") + } + if conf.RawPrevResult == nil { + t.Error("RawPrevResult should be populated") + } +} + +// ---- isValidBase62 --------------------------------------------------------- + +func TestIsValidBase62(t *testing.T) { + tests := []struct { + name string + in string + want bool + }{ + {"empty", "", false}, + {"valid alnum", "aB3", true}, + {"hyphen", "abc-def", false}, + {"unicode", "abc€", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isValidBase62(tt.in); got != tt.want { + t.Errorf("isValidBase62(%q) = %v, want %v", tt.in, got, tt.want) + } + }) + } +} + +// ---- sanitizeForError ------------------------------------------------------- + +func TestSanitizeForError(t *testing.T) { + if got := sanitizeForError("printable-value"); got != "printable-value" { + t.Errorf("got %q, want unchanged", got) + } + if got := sanitizeForError("bad\x00value"); got != sanitizeForErrorBinary { + t.Errorf("got %q, want %q", got, sanitizeForErrorBinary) + } +} + +// ---- parseStatusConf -------------------------------------------------------- + +func TestParseStatusConfInvalidJSON(t *testing.T) { + err := parseStatusConf([]byte("not valid json")) + assertCNIError(t, err, 7, errInvalidCNIConfig) +} + +func TestParseStatusConfMissingCNIVersion(t *testing.T) { + err := parseStatusConf([]byte(`{"type":"galactic-route"}`)) + assertCNIError(t, err, 7, "cniVersion is required") +} + +func TestParseStatusConfMissingType(t *testing.T) { + err := parseStatusConf([]byte(`{"cniVersion":"1.0.0"}`)) + assertCNIError(t, err, 7, "type is required") +} + +func TestParseStatusConfValid(t *testing.T) { + if err := parseStatusConf([]byte(`{"cniVersion":"1.0.0","type":"galactic-route"}`)); err != nil { + t.Errorf("unexpected error: %v", err) + } +} + +// ---- cmdAdd ----------------------------------------------------------------- + +func TestCmdAddInvalidConfig(t *testing.T) { + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte("not valid json")} + err := cmdAdd(args) + assertCNIError(t, err, 7, errInvalidCNIConfig) +} + +func TestCmdAddNoPrevResult(t *testing.T) { + args := &skel.CmdArgs{ + ContainerID: testContainerID, + StdinData: []byte(confJSON(testVPC, testAttachment, "")), + } + err := cmdAdd(args) + assertCNIError(t, err, 6, "must be chained after a master plugin") +} + +func TestCmdAddInvalidPrevResult(t *testing.T) { + // A prevResult that unmarshals but isn't a parseable versioned CNI + // result. Caught by cmdAdd's own parsePrevResult, reading RawPrevResult + // — parseConf's validatePrevResult(conf.PrevResult) never sees this at + // all, since that field is never populated (see + // TestParseConfPrevResultNeverValidatedAtParseTime). + args := &skel.CmdArgs{ + ContainerID: testContainerID, + StdinData: []byte(confJSON(testVPC, testAttachment, `{"cniVersion":"garbage"}`)), + } + err := cmdAdd(args) + assertCNIError(t, err, 6, "parse prevResult") +} + +func TestCmdAddNoTerminationsPassesThroughPrevResult(t *testing.T) { + // With no terminations to install, cmdAdd never touches route.Add at + // all — it should succeed and simply echo prevResult back. + args := &skel.CmdArgs{ + ContainerID: testContainerID, + StdinData: []byte(confJSON(testVPC, testAttachment, testPrevResult)), + } + if err := cmdAdd(args); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +// ---- parsePrevResult -------------------------------------------------------- + +func TestParsePrevResultNil(t *testing.T) { + _, err := parsePrevResult(nil) + if err == nil { + t.Fatal("expected error for nil prevResult") + } +} + +func TestParsePrevResultValid(t *testing.T) { + var raw map[string]interface{} + if err := json.Unmarshal([]byte(testPrevResult), &raw); err != nil { + t.Fatalf("unmarshal test fixture: %v", err) + } + result, err := parsePrevResult(raw) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result == nil { + t.Fatal("expected non-nil result") + } +} + +// ---- cmdDel ----------------------------------------------------------------- + +func TestCmdDelIdempotent(t *testing.T) { + args := &skel.CmdArgs{ + ContainerID: testContainerID, + StdinData: []byte(confJSON(testVPC, testAttachment, "")), + } + if err := cmdDel(args); err != nil { + t.Errorf("cmdDel should never return an error, got: %v", err) + } +} + +func TestCmdDelIdempotentInvalidConfig(t *testing.T) { + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte("not valid json")} + if err := cmdDel(args); err != nil { + t.Errorf("cmdDel should never return an error even for unparseable config, got: %v", err) + } +} + +// ---- cmdCheck --------------------------------------------------------------- + +func TestCmdCheckInvalidConfig(t *testing.T) { + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte("not valid json")} + err := cmdCheck(args) + assertCNIError(t, err, 7, errInvalidCNIConfig) +} + +func TestCmdCheckMissingRoute(t *testing.T) { + // The VRF this termination's route should live in doesn't exist on the + // test host, so checkTerminationRoutes fails at the vrf.TableID lookup — + // the same "missing resources" shape internal/cni's own CHECK tests use. + conf := fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test","type":"galactic-route",`+ + `"vpc":%q,"vpcattachment":%q,`+ + `"terminations":[{"network":"fd00:2::/64","via":"fd00:1::1"}]}`, + testVPC, testAttachment, + ) + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte(conf)} + if err := cmdCheck(args); err == nil { + t.Fatal("expected error for missing VRF/route state") + } +} + +func TestCmdCheckNoTerminationsStillRequiresVRF(t *testing.T) { + // checkTerminationRoutes resolves the VRF table ID up front, before + // its per-termination loop — with zero terminations to check, an + // absent VRF still surfaces as an error rather than being skipped, the + // same order the original code (internal/cni/ops_check.go, before this + // split) already used. + args := &skel.CmdArgs{ + ContainerID: testContainerID, + StdinData: []byte(confJSON(testVPC, testAttachment, "")), + } + if err := cmdCheck(args); err == nil { + t.Fatal("expected error: VRF for this vpc/vpcAttachment does not exist on the test host") + } +} + +// ---- checkTerminationRoutes ------------------------------------------------- + +func TestCheckTerminationRoutesInvalidGateway(t *testing.T) { + err := checkTerminationRoutes(testVPC, testAttachment, []Termination{{Network: "fd00:2::/64", Via: "not-an-ip"}}) + if err == nil { + t.Fatal("expected error for invalid gateway") + } +} + +func TestCheckTerminationRoutesRequiresVRFEvenWithNone(t *testing.T) { + // vrf.TableID resolves before the (empty) per-termination loop runs, so + // a missing VRF still surfaces as an error here too. + if err := checkTerminationRoutes(testVPC, testAttachment, nil); err == nil { + t.Fatal("expected error: VRF for this vpc/vpcAttachment does not exist on the test host") + } +} + +// ---- cmdStatus -------------------------------------------------------------- + +func TestCmdStatusInvalidConfig(t *testing.T) { + args := &skel.CmdArgs{StdinData: []byte("not valid json")} + err := cmdStatus(args) + assertCNIError(t, err, 7, errInvalidCNIConfig) +} + +func TestCmdStatusReady(t *testing.T) { + args := &skel.CmdArgs{StdinData: []byte(`{"cniVersion":"1.0.0","type":"galactic-route"}`)} + if err := cmdStatus(args); err != nil { + t.Errorf("unexpected error: %v", err) + } +} + +// ---- resourceTracker -------------------------------------------------------- + +func TestResourceTrackerCleanupZeroValue(t *testing.T) { + tracker := &resourceTracker{} + tracker.cleanup() // should not panic; empty added slice, nothing to delete +} + +func TestResourceTrackerCleanupUnaddedRoute(t *testing.T) { + // A route that was never actually added (e.g. route.Add failed before + // tracker.added recorded it) should never be attempted here — this + // tracker only ever unwinds what it itself recorded as added. + tracker := &resourceTracker{vpc: testVPC, vpcAttachment: testAttachment, dev: testIfName} + tracker.cleanup() // no entries in added; should not panic or attempt anything +} diff --git a/internal/cniroute/config.go b/internal/cniroute/config.go new file mode 100644 index 00000000..777d810e --- /dev/null +++ b/internal/cniroute/config.go @@ -0,0 +1,237 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniroute + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + + "github.com/containernetworking/cni/pkg/types" + type100 "github.com/containernetworking/cni/pkg/types/100" + + "go.datum.net/galactic/internal/config" + "go.datum.net/galactic/internal/hostconf" +) + +var ConfFile = config.DefaultConfFile + +// cniConfig is the shared config resolver for env var resolution. +// Initialized by InitCNIConfig() (called from cmd/galactic-route/main.go). +// +// galactic-route has no k8s dependency, so — unlike every other binary in +// the chain — it never resolves NodeName or Kubeconfig. It uses +// config.CNIConfig purely for LogFile/LogLevel's env-var > conflist > +// default precedence, so logging behaves the same way here as everywhere +// else in the chain (see internal/hostconf's doc comment on the one +// static conflist file every binary shares). +var cniConfig *config.CNIConfig + +// InitCNIConfig initializes the shared config resolver for CNI env var +// resolution. Callers should invoke this once at process startup before any +// config lookups. +func InitCNIConfig() { + cniConfig = config.NewCNIConfig() +} + +const errInvalidCNIConfig = "invalid CNI config" + +const ( + errVPCRequired = "vpc is required and must be a non-empty base62 string" + errVPCAttachmentRequired = "vpcattachment is required and must be a non-empty base62 string" +) + +const sanitizeForErrorBinary = "" + +// isValidBase62 reports whether s contains only valid base62 characters +// ([0-9a-zA-Z]) and is non-empty. +func isValidBase62(s string) bool { + if len(s) == 0 { + return false + } + for _, c := range s { + if (c < '0' || c > '9') && (c < 'a' || c > 'z') && (c < 'A' || c > 'Z') { + return false + } + } + return true +} + +// loadHostConf loads node-local settings from the static per-node conflist. +// If the file is missing, it returns a zero-value HostConf (tolerating local +// test runs). +func loadHostConf(filePath string) (*HostConf, error) { + if filePath == "" { + filePath = config.DefaultConfFile + } + conf, err := hostconf.Load(filePath, hostconf.PluginType) + if err != nil { + if os.IsNotExist(unwrapPathError(err)) { + return &HostConf{}, nil + } + return nil, err + } + return conf, nil +} + +// unwrapPathError returns the innermost *os.PathError-shaped error wrapped +// by err, if any, so os.IsNotExist (which does not itself traverse %w +// wrapping) can still recognize a missing conflist file wrapped by +// hostconf.Load's fmt.Errorf("read conflist file %q: %w", ...). +func unwrapPathError(err error) error { + for { + unwrapped := errors.Unwrap(err) + if unwrapped == nil { + return err + } + err = unwrapped + } +} + +// parseLogLevel maps a config-supplied level name to a slog.Level. Matching +// is case-insensitive. An empty string resolves to config.DefaultLogLevel. +func parseLogLevel(s string) (slog.Level, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "": + return parseLogLevel(config.DefaultLogLevel) + case config.LogLevelDebug: + return slog.LevelDebug, nil + case config.DefaultLogLevel: + return slog.LevelInfo, nil + case config.LogLevelWarn, config.LogLevelWarning: + return slog.LevelWarn, nil + case config.LogLevelError: + return slog.LevelError, nil + default: + return slog.LevelInfo, fmt.Errorf("unknown log level %q (want %s, %s, %s, or %s)", + s, config.LogLevelDebug, config.DefaultLogLevel, config.LogLevelWarn, config.LogLevelError) + } +} + +// setupLogging configures the slog default logger to write to the specified +// path at the specified verbosity. +func setupLogging(logPath, logLevel string) { + if logPath == "" { + logPath = config.DefaultLogFile + } + level, err := parseLogLevel(logLevel) + if err != nil { + slog.Warn("Invalid log level, falling back to default", + "value", logLevel, "default", config.DefaultLogLevel, "err", err) + } + if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { + slog.Warn("Failed to create log directory", "path", filepath.Dir(logPath), "err", err) + return + } + file, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + slog.Warn("Failed to open log file, falling back to Stderr", "path", logPath, "err", err) + return + } + handler := slog.NewJSONHandler(file, &slog.HandlerOptions{Level: level}) + slog.SetDefault(slog.New(handler)) +} + +// statusConf holds the minimal CNI config fields needed for STATUS validation. +type statusConf struct { + CNIVersion string `json:"cniVersion"` + Type string `json:"type"` +} + +// parseStatusConf validates that the CNI config is parseable and contains +// the required top-level fields. galactic-route has no attachment-specific +// or API-server state to check — STATUS must succeed on a freshly started +// node before any ADD has run. +func parseStatusConf(data []byte) error { + var sc statusConf + if err := json.Unmarshal(data, &sc); err != nil { + return &types.Error{Code: 7, Msg: errInvalidCNIConfig, Details: err.Error()} + } + if sc.CNIVersion == "" { + return &types.Error{Code: 7, Msg: "cniVersion is required"} + } + if sc.Type == "" { + return &types.Error{Code: 7, Msg: "type is required"} + } + return nil +} + +// parseConf unmarshals the CNI configuration from stdin data (the same +// document the master plugin received), validates the base62-encoded +// identifier fields, and resolves logging. +func parseConf(data []byte) (*PluginConf, error) { + conf := &PluginConf{} + if err := json.Unmarshal(data, &conf); err != nil { + return nil, &types.Error{Code: 7, Msg: errInvalidCNIConfig, Details: err.Error()} + } + if !isValidBase62(conf.VPC) { + if len(conf.VPC) == 0 { + return nil, &types.Error{Code: 7, Msg: errVPCRequired} + } + return nil, &types.Error{ + Code: 7, + Msg: fmt.Sprintf("invalid base62 value for field 'vpc': %q", sanitizeForError(conf.VPC)), + } + } + if !isValidBase62(conf.VPCAttachment) { + if len(conf.VPCAttachment) == 0 { + return nil, &types.Error{Code: 7, Msg: errVPCAttachmentRequired} + } + return nil, &types.Error{ + Code: 7, + Msg: fmt.Sprintf("invalid base62 value for field 'vpcattachment': %q", sanitizeForError(conf.VPCAttachment)), + } + } + + hostConf, err := loadHostConf(ConfFile) + if err != nil { + return nil, fmt.Errorf("load host CNI config: %w", err) + } + + cniConfig.Resolve(&config.ConflistValues{ + LogFile: hostConf.LogFile, + LogLevel: hostConf.LogLevel, + }) + setupLogging(cniConfig.LogFile, cniConfig.LogLevel) + slog.Debug("CNI config received", "stdin", string(data)) + + if conf.PrevResult != nil { + if err := validatePrevResult(conf.PrevResult); err != nil { + return nil, &types.Error{Code: 6, Msg: fmt.Sprintf("invalid prevResult: %v", err)} + } + } + return conf, nil +} + +// validatePrevResult checks that the prevResult (from a preceding plugin in +// the CNI chain) is a valid, parseable CNI result. +func validatePrevResult(res types.Result) error { + if res == nil { + return nil + } + jsonBytes, err := json.Marshal(res) + if err != nil { + return fmt.Errorf("marshal prevResult: %w", err) + } + if _, err := type100.NewResult(jsonBytes); err != nil { + return fmt.Errorf("parse prevResult: %w", err) + } + return nil +} + +// sanitizeForError returns s unchanged if it contains only printable ASCII +// characters; otherwise returns "" to avoid corrupting log output. +func sanitizeForError(s string) string { + for _, c := range s { + if c < 0x20 || c > 0x7e { + return sanitizeForErrorBinary + } + } + return s +} diff --git a/internal/cniroute/ops_add.go b/internal/cniroute/ops_add.go new file mode 100644 index 00000000..a9750ab9 --- /dev/null +++ b/internal/cniroute/ops_add.go @@ -0,0 +1,88 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniroute + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + type100 "github.com/containernetworking/cni/pkg/types/100" + + "go.datum.net/galactic/internal/cni/route" + "go.datum.net/galactic/internal/plumbing/intf" +) + +// cmdAdd installs each of pluginConf's termination routes into the VRF +// routing table the preceding master plugin (galactic-cni/galactic-tap-cni) +// already created, then passes prevResult through unchanged — galactic- +// route adds no interfaces or IPs of its own, only kernel routes alongside +// whatever came before it in the chain. +func cmdAdd(args *skel.CmdArgs) (err error) { + pluginConf, err := parseConf(args.StdinData) + if err != nil { + return err + } + + prevResult, prevErr := parsePrevResult(pluginConf.RawPrevResult) + if prevErr != nil { + return &types.Error{Code: 6, Msg: fmt.Sprintf("parse prevResult: %v", prevErr)} + } + + slog.Info("ADD: starting", "containerID", args.ContainerID, + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment, + "terminations", len(pluginConf.Terminations)) + + // The host interface name is derived from (vpc, vpcAttachment) alone — + // identical for a veth master's host end and a tap master's tap device + // (both call intf.GenerateInterfaceNameHost), so galactic-route needs no + // interface-kind inference the way galactic-bgp does. + dev := intf.GenerateInterfaceNameHost(pluginConf.VPC, pluginConf.VPCAttachment) + + tracker := &resourceTracker{vpc: pluginConf.VPC, vpcAttachment: pluginConf.VPCAttachment, dev: dev} + defer func() { + if err != nil { + slog.Error("ADD: failed, rolling back created resources", "err", err, + "containerID", args.ContainerID, "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + tracker.cleanup() + } + }() + + for _, termination := range pluginConf.Terminations { + if err := route.Add(pluginConf.VPC, pluginConf.VPCAttachment, termination.Network, termination.Via, dev); err != nil { + return fmt.Errorf("add route %s: %w", termination.Network, err) + } + tracker.added = append(tracker.added, termination) + } + if len(tracker.added) > 0 { + slog.Debug("ADD: termination routes installed", "count", len(tracker.added), "dev", dev) + } + + return types.PrintResult(prevResult, pluginConf.CNIVersion) +} + +// parsePrevResult parses rawPrevResult (PluginConf.RawPrevResult; the typed +// PluginConf.PrevResult field is never populated by plain JSON unmarshal, +// per its "json:\"-\"" tag) into a versioned CNI result galactic-route can +// pass straight back through as its own ADD result. galactic-route is +// optional in the chain, but when present it must be chained after a +// master plugin, which always produces a prevResult. +func parsePrevResult(rawPrevResult map[string]interface{}) (types.Result, error) { + if rawPrevResult == nil { + return nil, errors.New("no prevResult: galactic-route must be chained after a master plugin") + } + jsonBytes, err := json.Marshal(rawPrevResult) + if err != nil { + return nil, fmt.Errorf("marshal prevResult: %w", err) + } + parsed, err := type100.NewResult(jsonBytes) + if err != nil { + return nil, fmt.Errorf("parse prevResult: %w", err) + } + return parsed, nil +} diff --git a/internal/cniroute/ops_check.go b/internal/cniroute/ops_check.go new file mode 100644 index 00000000..43edb7d8 --- /dev/null +++ b/internal/cniroute/ops_check.go @@ -0,0 +1,116 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniroute + +import ( + "fmt" + "log/slog" + "net" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/vishvananda/netlink" + + "go.datum.net/galactic/internal/plumbing/intf" + "go.datum.net/galactic/internal/plumbing/vrf" +) + +// cmdCheck verifies that the termination routes cmdAdd installed are still +// present in the VRF routing table. Per the plan's CHECK/STATUS +// distribution, this is galactic-route's entire CHECK story — moved +// unchanged from internal/cni/ops_check.go (also mirrored in +// internal/cnitap), since the underlying kernel state it verifies didn't +// change shape by moving which process installs it. +func cmdCheck(args *skel.CmdArgs) error { + pluginConf, err := parseConf(args.StdinData) + if err != nil { + return err + } + slog.Info("CHECK: starting", "containerID", args.ContainerID, + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + + if err := checkTerminationRoutes(pluginConf.VPC, pluginConf.VPCAttachment, pluginConf.Terminations); err != nil { + err = fmt.Errorf("CHECK failed: termination routes: %w", err) + slog.Error("CHECK: failed", "err", err, "containerID", args.ContainerID, + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + return err + } + slog.Info("CHECK: passed", "containerID", args.ContainerID, + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + return nil +} + +// cmdStatus implements the CNI spec STATUS operation. galactic-route has no +// API server or attachment-specific state to probe — it either parses a +// well-formed config or it doesn't, matching galactic-ipam's own trivial +// STATUS (see the plan's CHECK/STATUS distribution: implemented for +// uniformity across the chain, not skipped). +func cmdStatus(args *skel.CmdArgs) error { + if err := parseStatusConf(args.StdinData); err != nil { + return err + } + slog.Info("STATUS: ready") + return nil +} + +// checkTerminationRoutes verifies that all termination routes exist in the +// VRF table for the given VPC/VPCAttachment pair. +func checkTerminationRoutes(vpc, vpcAttachment string, terminations []Termination) error { + tableID, err := vrf.TableID(vpc, vpcAttachment) + if err != nil { + return fmt.Errorf("get VRF table ID: %w", err) + } + + handle, err := netlink.NewHandle() + if err != nil { + return fmt.Errorf("create netlink handle: %w", err) + } + defer handle.Close() //nolint:errcheck // netlink cleanup on teardown + + routes, err := handle.RouteListFiltered( + netlink.FAMILY_V6, + &netlink.Route{Table: int(tableID)}, + netlink.RT_FILTER_TABLE, + ) + if err != nil { + return fmt.Errorf("list routes: %w", err) + } + + dev := intf.GenerateInterfaceNameHost(vpc, vpcAttachment) + for _, term := range terminations { + // An empty Via is not an error: assembleRoute (route.go) installs a + // valid on-link route for it, device-scoped with no gateway, and + // cmdAdd installs it fine. Only reject a non-empty Via that fails to + // parse. + var viaIP net.IP + if term.Via != "" { + viaIP = net.ParseIP(term.Via) + if viaIP == nil { + return fmt.Errorf("invalid termination gateway %q", term.Via) + } + } + found := false + for _, r := range routes { + if r.Dst == nil || r.Dst.String() != term.Network || r.LinkIndex <= 0 { + continue + } + if viaIP != nil { + if r.Gw == nil || !r.Gw.Equal(viaIP) { + continue + } + } else if r.Gw != nil { + continue + } + // Verify the link name matches (defers to the veth/tap device). + if link, linkErr := handle.LinkByIndex(r.LinkIndex); linkErr == nil && link.Attrs().Name == dev { + found = true + break + } + } + if !found { + return fmt.Errorf("missing route %s via %q in VRF table %d", term.Network, term.Via, tableID) + } + } + return nil +} diff --git a/internal/cniroute/ops_del.go b/internal/cniroute/ops_del.go new file mode 100644 index 00000000..9bed975c --- /dev/null +++ b/internal/cniroute/ops_del.go @@ -0,0 +1,30 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniroute + +import ( + "log/slog" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + type100 "github.com/containernetworking/cni/pkg/types/100" +) + +// cmdDel is a no-op, same as every other binary in the chain: the +// termination routes this plugin's own ADD installed are keyed by (vpc, +// vpcAttachment) and may still be in use by another pod/VM sharing the +// same attachment. Deleting them here would race with a concurrent ADD +// during restarts. Cleanup is left entirely to galactic-router's GC +// controller — see internal/cni's own cmdDel for the full reasoning, +// identical here. This also matches the pre-split behavior: the old +// monolithic plugin's own DEL never deleted termination routes either, +// for the same reason. +func cmdDel(args *skel.CmdArgs) error { + slog.Info("DEL: skipping shared resource cleanup (handled by GC)", "containerID", args.ContainerID) + + result := &type100.Result{} + _ = types.PrintResult(result, "1.0.0") + return nil +} diff --git a/internal/cniroute/resource.go b/internal/cniroute/resource.go new file mode 100644 index 00000000..f3fa7781 --- /dev/null +++ b/internal/cniroute/resource.go @@ -0,0 +1,37 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cniroute + +import ( + "log/slog" + + "go.datum.net/galactic/internal/cni/route" +) + +// resourceTracker tracks routes created during cmdAdd for selective +// rollback. galactic-route's own ADD only ever creates termination +// routes — route-delete only, per the plan's decision that each binary's +// tracker unwinds exactly what its own ADD created (the VRF/veth-or-tap +// it runs alongside belongs to the master plugin's own tracker instead). +type resourceTracker struct { + vpc, vpcAttachment, dev string + added []Termination +} + +// cleanup deletes every route this tracker recorded as added. Errors are +// logged but never returned — the caller already has a failure. +func (rt *resourceTracker) cleanup() { + slog.Info("Selective rollback: cleaning up routes created during failed ADD", + "vpc", rt.vpc, "vpcAttachment", rt.vpcAttachment) + + for _, term := range rt.added { + if err := route.Delete(rt.vpc, rt.vpcAttachment, term.Network, term.Via, rt.dev); err != nil { + slog.Error("Rollback: failed to delete route", "err", err, + "network", term.Network, "via", term.Via, "dev", rt.dev) + } else { + slog.Debug("Rollback: deleted route", "network", term.Network, "via", term.Via, "dev", rt.dev) + } + } +} diff --git a/internal/cniroute/types.go b/internal/cniroute/types.go new file mode 100644 index 00000000..f24613f6 --- /dev/null +++ b/internal/cniroute/types.go @@ -0,0 +1,45 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package cniroute implements galactic-route, the termination-route plugin +// in the galactic CNI chain — chained after galactic-cni/galactic-tap-cni +// and before galactic-bgp per conflist order (optional: only present when +// an attachment has terminations to install). It installs kernel routes +// into the VRF routing table the preceding master plugin already created, +// then passes prevResult through unchanged, adding no interfaces or IPs of +// its own. +// +// Unlike every other binary in the chain, galactic-route has no Kubernetes +// dependency at all: it neither reads nor writes any CRD, and never needs +// a namespace. +package cniroute + +import ( + "github.com/containernetworking/cni/pkg/types" + + "go.datum.net/galactic/internal/hostconf" +) + +// Termination represents a network termination point with a destination +// CIDR and next-hop gateway address. +type Termination struct { + Network string `json:"network"` + Via string `json:"via,omitempty"` +} + +// PluginConf is the CNI plugin configuration passed via stdin on each +// invocation of galactic-route — the same document the master plugin +// itself received, since the CNI runtime passes each chain entry its own +// stanza plus prevResult. galactic-route only reads vpc/vpcattachment/ +// terminations out of it (mtu, ipam, namespace are the master's/ +// galactic-ipam's/galactic-bgp's own concerns). +type PluginConf struct { + types.PluginConf + VPC string `json:"vpc"` + VPCAttachment string `json:"vpcattachment"` + Terminations []Termination `json:"terminations,omitempty"` +} + +// HostConf holds node-local settings read from /etc/cni/net.d/10-galactic.conflist. +type HostConf = hostconf.HostConf diff --git a/internal/cnitap/cnitap.go b/internal/cnitap/cnitap.go new file mode 100644 index 00000000..a8c2ccd4 --- /dev/null +++ b/internal/cnitap/cnitap.go @@ -0,0 +1,30 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnitap + +import ( + "time" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/version" + + "go.datum.net/galactic/internal/metadata" +) + +const cniTimeout = 10 * time.Second + +// RunPlugin starts the CNI plugin, handling ADD, DEL, CHECK, and STATUS operations. +func RunPlugin() { + skel.PluginMainFuncs( + skel.CNIFuncs{ + Add: cmdAdd, + Check: cmdCheck, + Del: cmdDel, + Status: cmdStatus, + }, + version.All, + "CNI galactic-tap plugin "+metadata.Version, + ) +} diff --git a/internal/cnitap/cnitap_test.go b/internal/cnitap/cnitap_test.go new file mode 100644 index 00000000..dcf0c656 --- /dev/null +++ b/internal/cnitap/cnitap_test.go @@ -0,0 +1,289 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnitap + +import ( + "errors" + "fmt" + "net" + "os" + "strings" + "testing" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + + "go.datum.net/galactic/internal/cniipam" + "go.datum.net/galactic/internal/cnimaster" +) + +const ( + testVPC = "abc" + testAttachment = "def" + testContainerID = "test-container" + testInvalidBase62 = "abc-def" + testCNIVersion = "1.0.0" +) + +func TestMain(m *testing.M) { + _ = os.Setenv("GALACTIC_CNI_NODE_NAME", "test-node") + InitCNIConfig() + os.Exit(m.Run()) +} + +func assertCNIError(t *testing.T, err error, wantCode uint, wantMsg string) { + t.Helper() + var cniErr *types.Error + if !errors.As(err, &cniErr) { + t.Fatalf("expected *types.Error, got %T: %v", err, err) + } + if cniErr.Code != wantCode { + t.Fatalf("expected code %d, got %d (Msg: %q)", wantCode, cniErr.Code, cniErr.Msg) + } + if wantMsg != "" && !strings.Contains(cniErr.Msg, wantMsg) { + t.Fatalf("expected Msg to contain %q, got %q", wantMsg, cniErr.Msg) + } +} + +func mustParseCIDR(t *testing.T, cidr string) *net.IPNet { + t.Helper() + _, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + t.Fatalf("parse CIDR %q: %v", cidr, err) + } + return ipnet +} + +// ---- parseConf ----------------------------------------------------------- + +func TestParseConf(t *testing.T) { + tests := []struct { + name string + input string + wantVPC string + wantErr string + wantCode uint + }{ + { + name: "valid config", + input: fmt.Sprintf( + `{"cniVersion":"1.0.0","name":"test","type":"galactic-tap-cni","vpc":"%s","vpcattachment":"%s"}`, + testVPC, testAttachment, + ), + wantVPC: testVPC, + }, + {name: "invalid JSON", input: "not json", wantErr: "invalid CNI config", wantCode: 7}, + { + name: "missing vpc", + input: fmt.Sprintf(`{"cniVersion":"1.0.0","name":"test","type":"galactic-tap-cni","vpcattachment":"%s"}`, + testAttachment), + wantErr: "vpc is required and must be a non-empty base62 string", + wantCode: 7, + }, + { + name: "vpc with invalid char", + input: fmt.Sprintf(`{"cniVersion":"1.0.0","name":"test","type":"galactic-tap-cni","vpc":"%s","vpcattachment":"%s"}`, + testInvalidBase62, testAttachment), + wantErr: fmt.Sprintf("invalid base62 value for field 'vpc': %q", testInvalidBase62), + wantCode: 7, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conf, err := parseConf([]byte(tt.input)) + if tt.wantErr != "" { + if err == nil { + t.Fatalf("expected error containing %q, got nil", tt.wantErr) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("error %q does not contain %q", err, tt.wantErr) + } + if tt.wantCode > 0 { + assertCNIError(t, err, tt.wantCode, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if conf.VPC != tt.wantVPC { + t.Errorf("VPC = %q, want %q", conf.VPC, tt.wantVPC) + } + }) + } +} + +// ---- buildTapResult ------------------------------------------------------ + +func TestBuildTapResult(t *testing.T) { + subnet := mustParseCIDR(t, "fd00:10:ff01::1234/80") + gateway := net.ParseIP("fd00:10:ff01::1") + defaultRoute := mustParseCIDR(t, "::/0") + + conf := &PluginConf{ + PluginConf: types.PluginConf{CNIVersion: testCNIVersion}, + VPC: testVPC, + VPCAttachment: testAttachment, + } + + tests := []struct { + name string + ipRes *cniipam.IPAMResult + wantIPs int + wantRoutes int + }{ + { + name: "with IPAM config", + ipRes: &cniipam.IPAMResult{IPv6Subnet: subnet, IPv6Gateway: gateway, Routes: []*net.IPNet{defaultRoute}}, + wantIPs: 1, + wantRoutes: 1, + }, + {name: "without IPAM config", ipRes: nil, wantIPs: 0, wantRoutes: 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := buildTapResult(conf, tt.ipRes, "H0abc123", "aa:bb:cc:dd:ee:ff", 1500) + + if result.CNIVersion != testCNIVersion { + t.Errorf("CNIVersion = %q, want %q", result.CNIVersion, testCNIVersion) + } + if len(result.Interfaces) != 1 { + t.Fatalf("Interfaces count = %d, want 1", len(result.Interfaces)) + } + if result.Interfaces[0].Name != "H0abc123" { + t.Errorf("Interfaces[0].Name = %q, want %q", result.Interfaces[0].Name, "H0abc123") + } + if result.Interfaces[0].Sandbox != "" { + t.Errorf("Interfaces[0].Sandbox = %q, want empty", result.Interfaces[0].Sandbox) + } + if len(result.IPs) != tt.wantIPs { + t.Errorf("IPs count = %d, want %d", len(result.IPs), tt.wantIPs) + } + if tt.wantIPs > 0 { + if result.IPs[0].Interface == nil || *result.IPs[0].Interface != 0 { + t.Errorf("IPs[0].Interface = %v, want 0", result.IPs[0].Interface) + } + } + if len(result.Routes) != tt.wantRoutes { + t.Errorf("Routes count = %d, want %d", len(result.Routes), tt.wantRoutes) + } + }) + } +} + +// TestBuildTapResultIPv4Mask verifies that buildTapResult reports the IPv4 +// address with a /25 mask (matching the host gateway mask +// cnibgp.ConfigureHostGateway installs on the tap interface), not the /32 +// used for veth. +func TestBuildTapResultIPv4Mask(t *testing.T) { + ipv4Address := net.ParseIP("172.20.1.5") + ipv4Gateway := net.ParseIP("172.20.1.1") + ipv4Route := mustParseCIDR(t, "0.0.0.0/0") + + conf := &PluginConf{ + PluginConf: types.PluginConf{CNIVersion: testCNIVersion}, + VPC: testVPC, + VPCAttachment: testAttachment, + } + ipRes := &cniipam.IPAMResult{IPv4Address: ipv4Address, IPv4Gateway: ipv4Gateway, Routes: []*net.IPNet{ipv4Route}} + + result := buildTapResult(conf, ipRes, "H0abc123", "aa:bb:cc:dd:ee:ff", 1500) + + if len(result.IPs) != 1 { + t.Fatalf("IPs count = %d, want 1", len(result.IPs)) + } + wantIPv4Mask := net.CIDRMask(25, 32).String() + if result.IPs[0].Address.IP.String() != ipv4Address.String() || result.IPs[0].Address.Mask.String() != wantIPv4Mask { + t.Errorf("IPs[0].Address = %v, want %s/25", result.IPs[0].Address, ipv4Address) + } +} + +// TestBuildTapResultHostNetns verifies that the tap path produces a valid +// CNI result when args.Netns is the host network namespace. Kraftlet/ +// unikraft workloads pass the host netns because they don't have a Linux +// network namespace. main.go's own CNI_NETNS_OVERRIDE handles bypassing the +// CNI library's same-netns rejection check. The tap result must not +// reference a sandbox. +func TestBuildTapResultHostNetns(t *testing.T) { + subnet := mustParseCIDR(t, "fd00:10:ff01::1234/80") + gateway := net.ParseIP("fd00:10:ff01::1") + defaultRoute := mustParseCIDR(t, "::/0") + + conf := &PluginConf{ + PluginConf: types.PluginConf{CNIVersion: testCNIVersion}, + VPC: testVPC, + VPCAttachment: testAttachment, + } + ipRes := &cniipam.IPAMResult{IPv6Subnet: subnet, IPv6Gateway: gateway, Routes: []*net.IPNet{defaultRoute}} + + result := buildTapResult(conf, ipRes, "H0abc123", "aa:bb:cc:dd:ee:ff", 1500) + + if result.Interfaces[0].Sandbox != "" { + t.Errorf("Interfaces[0].Sandbox = %q, want empty (host netns, no sandbox)", result.Interfaces[0].Sandbox) + } +} + +// ---- cmdDel / cmdCheck / cmdStatus ---------------------------------------- + +func TestCmdDelIdempotent(t *testing.T) { + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte("not valid json")} + if err := cmdDel(args); err != nil { + t.Fatalf("cmdDel with invalid config returned error = %v, want nil", err) + } +} + +func TestCmdDelIdempotentMissingResources(t *testing.T) { + conf := fmt.Sprintf(`{"cniVersion":"1.0.0","name":"test","type":"galactic-tap-cni","vpc":"%s","vpcattachment":"%s"}`, + testVPC, testAttachment) + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte(conf)} + if err := cmdDel(args); err != nil { + t.Fatalf("cmdDel with missing resources returned error = %v, want nil", err) + } +} + +func TestCmdCheckInvalidConfig(t *testing.T) { + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte("not valid json")} + err := cmdCheck(args) + if err == nil || !strings.Contains(err.Error(), "invalid CNI config") { + t.Fatalf("expected 'invalid CNI config' error, got: %v", err) + } +} + +func TestCmdCheckValidConfigMissingResources(t *testing.T) { + conf := fmt.Sprintf(`{"cniVersion":"1.0.0","name":"test","type":"galactic-tap-cni","vpc":"%s","vpcattachment":"%s"}`, + testVPC, testAttachment) + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte(conf)} + err := cmdCheck(args) + if err == nil || !strings.Contains(err.Error(), "CHECK failed") { + t.Fatalf("expected 'CHECK failed', got: %v", err) + } +} + +func TestCmdStatusInvalidConfig(t *testing.T) { + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte("not valid json")} + err := cmdStatus(args) + assertCNIError(t, err, 7, "invalid CNI config") +} + +func TestCmdStatusAPIProbeFailure(t *testing.T) { + original := cnimaster.ProbeAPIServer + cnimaster.ProbeAPIServer = func() error { return errors.New("connection refused") } + defer func() { cnimaster.ProbeAPIServer = original }() + + conf := fmt.Sprintf(`{"cniVersion":"1.0.0","name":"test","type":"galactic-tap-cni","vpc":"%s","vpcattachment":"%s"}`, + testVPC, testAttachment) + args := &skel.CmdArgs{ContainerID: testContainerID, StdinData: []byte(conf)} + err := cmdStatus(args) + assertCNIError(t, err, 50, "API server health check failed") +} + +// ---- resourceTracker ------------------------------------------------------ + +func TestResourceTrackerCleanupZeroValue(t *testing.T) { + tracker := &resourceTracker{} + tracker.cleanup() // should not panic +} diff --git a/internal/cnitap/config.go b/internal/cnitap/config.go new file mode 100644 index 00000000..cf8c39cc --- /dev/null +++ b/internal/cnitap/config.go @@ -0,0 +1,32 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnitap + +import ( + "go.datum.net/galactic/internal/cnimaster" + "go.datum.net/galactic/internal/config" +) + +var ConfFile = config.DefaultConfFile + +// cniConfig is the shared config resolver for env var resolution. +// Initialized by InitCNIConfig() (called from cmd/galactic-tap-cni/main.go). +var cniConfig *config.CNIConfig + +// InitCNIConfig initializes the shared config resolver for CNI env var +// resolution. Callers should invoke this once at process startup before any +// config lookups. +func InitCNIConfig() { + cniConfig = config.NewCNIConfig() +} + +// parseConf unmarshals the CNI configuration from stdin data, validates the +// base62-encoded identifier fields, and resolves logging. The actual logic +// is shared with galactic-cni — see internal/cnimaster.ParseConf — since +// none of it is tap-specific; this is a thin wrapper binding it to this +// binary's own cniConfig/ConfFile. +func parseConf(data []byte) (*PluginConf, error) { + return cnimaster.ParseConf(data, cniConfig, ConfFile) +} diff --git a/internal/cnitap/ops_add.go b/internal/cnitap/ops_add.go new file mode 100644 index 00000000..e926f28d --- /dev/null +++ b/internal/cnitap/ops_add.go @@ -0,0 +1,142 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnitap + +import ( + "context" + "fmt" + "log/slog" + "os" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/plugins/pkg/ipam" + "github.com/vishvananda/netlink" + + "go.datum.net/galactic/internal/cni/hostgw" + "go.datum.net/galactic/internal/cni/tap" + "go.datum.net/galactic/internal/cniipam" + "go.datum.net/galactic/internal/cnimaster" + "go.datum.net/galactic/internal/nadpatch" + "go.datum.net/galactic/internal/plumbing/intf" + "go.datum.net/galactic/internal/plumbing/vrf" +) + +// cmdAdd mirrors internal/cni's own cmdAdd, minus everything specific to a +// guest-side netns: no host-device delegation, no guest interface, no +// netns IP configuration. The VM manages its own interface entirely. +// BGP/SRv6/eBPF publish is galactic-bgp's job, invoked next by the CNI +// runtime per conflist order, not by this process. +func cmdAdd(args *skel.CmdArgs) (err error) { + pluginConf, err := parseConf(args.StdinData) + if err != nil { + return err + } + + if pluginConf.PrevResult != nil { + if err := cnimaster.ValidatePrevResultAdd(pluginConf.PrevResult); err != nil { + return &types.Error{Code: 6, Msg: fmt.Sprintf("prevResult validation in ADD: %v", err)} + } + } + + nodeName := os.Getenv("NODE_NAME") + if nodeName == "" { + return &types.Error{Code: 4, Msg: "NODE_NAME environment variable is not set"} + } + + namespace := pluginConf.Namespace + + slog.Info("ADD: starting", + "containerID", args.ContainerID, "netns", args.Netns, "ifName", args.IfName, + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment, + "namespace", namespace, "nodeName", nodeName) + + tracker := &resourceTracker{ + vpc: pluginConf.VPC, + vpcAttachment: pluginConf.VPCAttachment, + } + // Record IPAM delegation intent up front, before the ExecAdd call + // below ever runs — see resourceTracker's ipamDelegated doc comment. + if pluginConf.IPAM != nil { + tracker.ipamDelegated = true + tracker.ipamType = pluginConf.IPAM.Type + tracker.ipamStdin = args.StdinData + } + + defer func() { + if err != nil { + slog.Error("ADD: failed, rolling back created resources", "err", err, + "containerID", args.ContainerID, "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + tracker.cleanup() + } + }() + + if err := vrf.Add(pluginConf.VPC, pluginConf.VPCAttachment); err != nil { + return fmt.Errorf("add VRF: %w", err) + } + tracker.vrfCreated = true + slog.Debug("ADD: VRF ready", "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + + if err := tap.Add(pluginConf.VPC, pluginConf.VPCAttachment, pluginConf.MTU); err != nil { + return fmt.Errorf("add tap: %w", err) + } + + hostName := intf.GenerateInterfaceNameHost(pluginConf.VPC, pluginConf.VPCAttachment) + hostLink, err := netlink.LinkByName(hostName) + if err != nil { + return fmt.Errorf("get host interface %q: %w", hostName, err) + } + hostMac := hostLink.Attrs().HardwareAddr.String() + hostMTU := hostLink.Attrs().MTU + slog.Debug("ADD: host interface ready", "name", hostName, "mac", hostMac, "mtu", hostMTU) + + k8sClient, err := cnimaster.NewK8sClient() + if err != nil { + return fmt.Errorf("create k8s client: %w", err) + } + podNamespace := nadpatch.ParsePodNamespace(args.Args) + nadCtx, nadCancel := context.WithTimeout(context.Background(), cniTimeout) + defer nadCancel() + if err := nadpatch.AnnotateNAD(nadCtx, k8sClient, pluginConf.Name, podNamespace, hostName); err != nil { + return fmt.Errorf("annotate NAD: %w", err) + } + + // Termination routes are galactic-route's job now — chained next after + // this plugin, when the attachment has any (see internal/cniroute). + + // Allocate IPAM for the tap interface via delegation (only if pluginConf + // carries an "ipam" block at all — see internal/cniipam's doc comment + // for the explicit contract). The VM manages its own guest interface; + // this plugin only configures the host side. + var ipamResult *cniipam.IPAMResult + if pluginConf.IPAM != nil { + cniResult, err := ipam.ExecAdd(pluginConf.IPAM.Type, args.StdinData) + if err != nil { + return fmt.Errorf("delegate to %s ADD: %w", pluginConf.IPAM.Type, err) + } + ipamResult, err = cniipam.ResultToIPAMResult(cniResult) + if err != nil { + return fmt.Errorf("convert IPAM result: %w", err) + } + } + if ipamResult != nil { + slog.Debug("ADD: IPAM allocated", "containerID", args.ContainerID, + "ipv6Subnet", ipamResult.IPv6Subnet, "ipv6Gateway", ipamResult.IPv6Gateway, + "ipv4Address", ipamResult.IPv4Address, "ipv4Gateway", ipamResult.IPv4Gateway) + } + + // Configure the gateway address on the host tap and install the VRF + // route — kernel-interface work this plugin owns (see + // internal/cni/hostgw's doc comment). + if err := hostgw.ConfigureHostGateway(pluginConf.VPC, pluginConf.VPCAttachment, ipamResult, nil); err != nil { + return err + } + if ipamResult != nil && ipamResult.IPv6Gateway != nil { + slog.Debug("ADD: host gateway configured", "name", hostName, "gateway", ipamResult.IPv6Gateway) + } + + result := buildTapResult(pluginConf, ipamResult, hostName, hostMac, hostMTU) + return types.PrintResult(result, pluginConf.CNIVersion) +} diff --git a/internal/cnitap/ops_check.go b/internal/cnitap/ops_check.go new file mode 100644 index 00000000..779f2198 --- /dev/null +++ b/internal/cnitap/ops_check.go @@ -0,0 +1,99 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnitap + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + + "github.com/containernetworking/cni/pkg/skel" + type100 "github.com/containernetworking/cni/pkg/types/100" + "github.com/containernetworking/plugins/pkg/ipam" + + "go.datum.net/galactic/internal/cnimaster" +) + +// cmdCheck validates that the node's tap-side networking state matches what +// was established during cmdAdd. Unlike internal/cni's own cmdCheck, there +// is no guest interface to verify — tap mode never enters a container +// netns. +func cmdCheck(args *skel.CmdArgs) error { + pluginConf, err := parseConf(args.StdinData) + if err != nil { + return err + } + slog.Info("CHECK: starting", "containerID", args.ContainerID, + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + + var errs []error + + hostName, nodeErrs := cnimaster.CheckNodeLevelState(pluginConf.VPC, pluginConf.VPCAttachment) + errs = append(errs, nodeErrs...) + + // Termination routes are galactic-route's own CHECK now (see + // internal/cniroute's checkTerminationRoutes) — this plugin's CHECK no + // longer verifies them. + + if pluginConf.RawPrevResult != nil { + if err := checkPrevResult(pluginConf.RawPrevResult, hostName); err != nil { + errs = append(errs, fmt.Errorf("prevResult validation: %w", err)) + } + } + + // Delegate CHECK to the IPAM plugin so a lost or corrupted allocation + // marker file is caught here too, not just at ADD/DEL time. + if pluginConf.IPAM != nil { + if err := ipam.ExecCheck(pluginConf.IPAM.Type, args.StdinData); err != nil { + errs = append(errs, fmt.Errorf("IPAM CHECK: %w", err)) + } + } + + if len(errs) > 0 { + err := fmt.Errorf("CHECK failed: %w", errors.Join(errs...)) + slog.Error("CHECK: failed", "err", err, "containerID", args.ContainerID, + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + return err + } + slog.Info("CHECK: passed", "containerID", args.ContainerID, + "vpc", pluginConf.VPC, "vpcAttachment", pluginConf.VPCAttachment) + return nil +} + +// cmdStatus implements the CNI spec STATUS operation — see +// internal/cnimaster.RunStatus for the full reasoning, shared verbatim with +// galactic-cni. +func cmdStatus(args *skel.CmdArgs) error { + return cnimaster.RunStatus(args.StdinData, cniConfig, ConfFile) +} + +// checkPrevResult validates that kernel state matches the host interface +// recorded in the prevResult returned by the most recent ADD. Tap mode has +// no guest-side interface or netns to validate against. +func checkPrevResult(rawPrevResult map[string]interface{}, _ string) error { + jsonBytes, err := json.Marshal(rawPrevResult) + if err != nil { + return fmt.Errorf("marshal prevResult: %w", err) + } + res, err := type100.NewResult(jsonBytes) + if err != nil { + return fmt.Errorf("parse prevResult: %w", err) + } + result, err := type100.GetResult(res) + if err != nil { + return fmt.Errorf("get prevResult: %w", err) + } + + for _, iface := range result.Interfaces { + if iface.Name == "" || iface.Sandbox != "" { + continue + } + if err := cnimaster.ValidateHostInterface(iface.Name, iface.Mac, iface.Mtu); err != nil { + return fmt.Errorf("interface %q (host): %w", iface.Name, err) + } + } + return nil +} diff --git a/internal/cnitap/ops_del.go b/internal/cnitap/ops_del.go new file mode 100644 index 00000000..0ff894c7 --- /dev/null +++ b/internal/cnitap/ops_del.go @@ -0,0 +1,52 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnitap + +import ( + "log/slog" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + type100 "github.com/containernetworking/cni/pkg/types/100" + "github.com/containernetworking/plugins/pkg/ipam" +) + +// cmdDel mirrors internal/cni's own cmdDel, minus everything guest-netns +// specific (no flushGuestNetnsConfig, no host-device DEL delegation — tap +// mode never touches a container netns at all). +func cmdDel(args *skel.CmdArgs) error { + // DEL is idempotent per the CNI spec: always return success. + slog.Info("DEL: starting", "containerID", args.ContainerID, "netns", args.Netns) + + pluginConf, parseErr := parseConf(args.StdinData) + if parseErr != nil { + slog.Error("DEL: failed to parse CNI config, skipping cleanup", "err", parseErr, + "containerID", args.ContainerID) + result := &type100.Result{} + _ = types.PrintResult(result, "1.0.0") + return nil + } + vpc, vpcAtt := pluginConf.VPC, pluginConf.VPCAttachment + + if pluginConf.IPAM != nil { + if err := ipam.ExecDel(pluginConf.IPAM.Type, args.StdinData); err != nil { + slog.Warn("DEL: IPAM delegation failed, allocation may not have been released", "err", err, + "containerID", args.ContainerID) + } + } + + // Shared resources (VRF, tap, routes, SRv6 ingress, BGPAdvertisement, + // BGPVRFInstance) are keyed by (vpc, vpcAttachment) and may still be in + // use by another VM. Deleting them here races with cmdAdd during + // restarts, so cleanup is left to galactic-router's GC controller — see + // internal/cni's own cmdDel for the full reasoning. + slog.Info("DEL: skipping shared resource cleanup (handled by GC)", + "containerID", args.ContainerID, "vpc", vpc, "vpcAttachment", vpcAtt) + + result := &type100.Result{} + _ = types.PrintResult(result, pluginConf.CNIVersion) + + return nil +} diff --git a/internal/cnitap/resource.go b/internal/cnitap/resource.go new file mode 100644 index 00000000..a5db5afb --- /dev/null +++ b/internal/cnitap/resource.go @@ -0,0 +1,51 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnitap + +import ( + "log/slog" + + "github.com/containernetworking/plugins/pkg/ipam" + + "go.datum.net/galactic/internal/cni/tap" + "go.datum.net/galactic/internal/cnimaster" +) + +// resourceTracker tracks resources created during cmdAdd for selective +// rollback. galactic-tap-cni's ADD only ever creates the VRF, the tap +// device, and (if delegated) an IPAM allocation — BGP/SRv6/eBPF publish is +// galactic-bgp's own, separately chain-invoked plugin now, with its own +// smaller tracker (internal/cnibgp); termination routes are galactic- +// route's own, with its own smaller tracker (internal/cniroute). +type resourceTracker struct { + vpc, vpcAttachment string + vrfCreated bool + + // ipamDelegated, ipamType, and ipamStdin record enough to release the + // IPAM allocation during rollback — see internal/cni's own + // resourceTracker for the full doc comment on why this fires + // unconditionally on "ipam" block presence rather than only after a + // confirmed ExecAdd. + ipamDelegated bool + ipamType string + ipamStdin []byte +} + +func (rt *resourceTracker) cleanup() { + // Release the IPAM allocation first — see internal/cni's own + // resourceTracker for the full doc comment on why this fires + // unconditionally on ipamDelegated alone. Interface/VRF cleanup is + // shared with galactic-cni's own tracker, so it lives in + // cnimaster.CleanupAttachment. + if rt.ipamDelegated { + if err := ipam.ExecDel(rt.ipamType, rt.ipamStdin); err != nil { + slog.Error("Rollback: failed to release IPAM allocation", "err", err, "ipamType", rt.ipamType) + } else { + slog.Debug("Rollback: released IPAM allocation", "ipamType", rt.ipamType) + } + } + + cnimaster.CleanupAttachment(rt.vpc, rt.vpcAttachment, "tap", tap.Delete) +} diff --git a/internal/cnitap/result.go b/internal/cnitap/result.go new file mode 100644 index 00000000..cb21b894 --- /dev/null +++ b/internal/cnitap/result.go @@ -0,0 +1,73 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cnitap + +import ( + "net" + + "github.com/containernetworking/cni/pkg/types" + type100 "github.com/containernetworking/cni/pkg/types/100" + + "go.datum.net/galactic/internal/cniipam" +) + +// buildTapResult constructs the CNI result for tap mode: a single host +// interface with optional IPAM data. The guest VM manages its own +// interface; the IP here describes the allocated subnet, which +// galactic-bgp (chained next by the runtime) reads back out of this same +// result to know what to advertise — see internal/cnibgp's doc comment. +// The IPv4 address is reported with a /25 mask, matching the mask +// hostgw.ConfigureHostGateway installs on the host side of the tap. +func buildTapResult( + pluginConf *PluginConf, + ipRes *cniipam.IPAMResult, + hostName, hostMac string, + hostMTU int, +) *type100.Result { + result := &type100.Result{ + CNIVersion: pluginConf.CNIVersion, + Interfaces: []*type100.Interface{ + { + Name: hostName, + Mac: hostMac, + Mtu: hostMTU, + Sandbox: "", + }, + }, + } + appendIPConfigs(result, ipRes, 0, net.CIDRMask(25, 32)) // index into Interfaces (host tap) + return result +} + +// appendIPConfigs adds one IPConfig per allocated address family in ipRes +// (IPv6, and IPv4 when present) plus any default routes, all pointing at +// the given Interfaces index. No-op when ipRes is nil. +func appendIPConfigs(result *type100.Result, ipRes *cniipam.IPAMResult, ifaceIndex int, ipv4Mask net.IPMask) { + if ipRes == nil { + return + } + if ipRes.IPv6Subnet != nil { + result.IPs = append(result.IPs, &type100.IPConfig{ + Address: *ipRes.IPv6Subnet, + Gateway: ipRes.IPv6Gateway, + Interface: type100.Int(ifaceIndex), + }) + } + if ipRes.IPv4Address != nil { + result.IPs = append(result.IPs, &type100.IPConfig{ + Address: net.IPNet{IP: ipRes.IPv4Address, Mask: ipv4Mask}, + Gateway: ipRes.IPv4Gateway, + Interface: type100.Int(ifaceIndex), + }) + } + if len(ipRes.Routes) > 0 { + result.Routes = make([]*types.Route, 0, len(ipRes.Routes)) + for _, dst := range ipRes.Routes { + result.Routes = append(result.Routes, &types.Route{ + Dst: *dst, + }) + } + } +} diff --git a/internal/cnitap/types.go b/internal/cnitap/types.go new file mode 100644 index 00000000..e5c80865 --- /dev/null +++ b/internal/cnitap/types.go @@ -0,0 +1,25 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package cnitap implements galactic-tap-cni, the tap master plugin for +// VM-based workloads (Kata, Firecracker, kraftlet/Unikraft). It mirrors +// internal/cni (the veth master, galactic-cni) but never delegates to +// host-device (no container netns to move anything into — the VM manages +// its own guest interface) and never configures a guest-side netns. +package cnitap + +import ( + "go.datum.net/galactic/internal/cnimaster" + "go.datum.net/galactic/internal/hostconf" +) + +// PluginConf is the CNI plugin configuration passed via stdin on each +// invocation of galactic-tap-cni. It's the same shape galactic-cni +// (internal/cni) uses — see internal/cnimaster's own doc comment — so both +// packages alias the one canonical definition rather than each declaring +// their own copy. +type PluginConf = cnimaster.PluginConf + +// HostConf holds node-local settings read from /etc/cni/net.d/10-galactic.conflist. +type HostConf = hostconf.HostConf diff --git a/internal/config/cni.go b/internal/config/cni.go index 0cd5dd8c..bd0f5ad5 100644 --- a/internal/config/cni.go +++ b/internal/config/cni.go @@ -6,7 +6,6 @@ package config import ( "os" - "strings" ) // --- CNI environment variable keys ----------------------------------------- @@ -15,7 +14,6 @@ const ( EnvCNINodeName = "GALACTIC_CNI_NODE_NAME" EnvCNIKubeconfig = "GALACTIC_CNI_KUBECONFIG" EnvCNIKubernetesConfig = "GALACTIC_CNI_KUBERNETES_CONFIG" - EnvCNIEnableLocalIPAM = "GALACTIC_CNI_ENABLE_LOCAL_IPAM" EnvLogLevel = "GALACTIC_CNI_LOG_LEVEL" EnvLogFile = "GALACTIC_CNI_LOG_FILE" EnvNamespace = "GALACTIC_CNI_NAMESPACE" @@ -105,10 +103,3 @@ type ConflistValues struct { LogFile string LogLevel string } - -// CNIGetEnableLocalIPAM reports whether local (in-memory) IPAM is enabled via -// environment variable. Returns false if the variable is unset or not "true". -func CNIGetEnableLocalIPAM() bool { - val := os.Getenv(EnvCNIEnableLocalIPAM) - return strings.EqualFold(val, "true") -} diff --git a/internal/config/cni_test.go b/internal/config/cni_test.go index ef749d1e..434d3d36 100644 --- a/internal/config/cni_test.go +++ b/internal/config/cni_test.go @@ -106,16 +106,3 @@ func TestCNIConfigNodeNameLegacyFallback(t *testing.T) { t.Errorf("NodeName = %q, want %q", cfg.NodeName, "legacy-node") } } - -func TestCNIGetEnableLocalIPAM(t *testing.T) { - t.Setenv(EnvCNIEnableLocalIPAM, "true") - if got := CNIGetEnableLocalIPAM(); !got { - t.Error("CNIGetEnableLocalIPAM() = false, want true") - } -} - -func TestCNIGetEnableLocalIPAMFalse(t *testing.T) { - if got := CNIGetEnableLocalIPAM(); got { - t.Error("CNIGetEnableLocalIPAM() = true, want false (env unset)") - } -} diff --git a/internal/config/ipam.go b/internal/config/ipam.go new file mode 100644 index 00000000..7c755b1c --- /dev/null +++ b/internal/config/ipam.go @@ -0,0 +1,31 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package config + +import ( + "os" + "strings" +) + +// EnvIPAMEnableLocalIPAM is galactic-ipam's own local-IPAM default-filler +// flag, read fresh on every invocation (mirrors CNIConfig's env-only +// resolution — galactic-ipam has no conflist/kubeconfig to read a +// middle-tier value from, since it has no k8s dependency at all). +// +// Renamed from the historical GALACTIC_CNI_ENABLE_LOCAL_IPAM now that the +// flag belongs entirely to galactic-ipam: it's no longer a trigger deciding +// whether IPAM runs at all (that's the "ipam" block's presence, decided by +// the master plugin before it ever delegates) — only a default-filler for +// when the ipam block is present but under-specified. See +// go.datum.net/galactic/internal/cniipam's own doc comment. +const EnvIPAMEnableLocalIPAM = "GALACTIC_IPAM_ENABLE_LOCAL_IPAM" + +// IPAMGetEnableLocalIPAM reports whether galactic-ipam's local-IPAM +// default-filler is enabled via environment variable. Returns false if the +// variable is unset or not "true". +func IPAMGetEnableLocalIPAM() bool { + val := os.Getenv(EnvIPAMEnableLocalIPAM) + return strings.EqualFold(val, "true") +} diff --git a/internal/crdnames/crdnames.go b/internal/crdnames/crdnames.go new file mode 100644 index 00000000..a39902b7 --- /dev/null +++ b/internal/crdnames/crdnames.go @@ -0,0 +1,83 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package crdnames is the shared vocabulary for naming and annotating the +// BGPVRFInstance/BGPAdvertisement CRDs a VPC attachment's chain of plugins +// cooperate on: galactic-bgp writes them, galactic-ipam's deallocation path +// (until its own local marker-file persistence lands) reads the subnet +// annotations back, and galactic-router's GC controller reads the netns +// annotation to decide liveness. Kept as one small leaf package — imported +// by internal/cni, internal/cnitap, internal/cniipam, and internal/cnibgp — +// so none of those need to import each other just to agree on a name. +package crdnames + +import "fmt" + +// AnnotationAllocatedSubnetIPv6 is the BGPAdvertisement annotation key prefix +// holding the allocated IPv6 pod subnet CIDR (the /96) for a container ID. +// The full key appends a truncated container ID; see SubnetKeyIPv6. +const AnnotationAllocatedSubnetIPv6 = "galactic.datum.net/allocated-subnet-ipv6" + +// AnnotationAllocatedSubnetIPv4 is the BGPAdvertisement annotation key prefix +// holding the allocated IPv4 pod address (the /32) for a container ID, when +// the attachment is dual-stack. The full key appends a truncated container +// ID; see SubnetKeyIPv4. +const AnnotationAllocatedSubnetIPv4 = "galactic.datum.net/allocated-subnet-ipv4" + +// AnnotationNetNS is the BGPAdvertisement annotation key prefix holding the +// CNI-provided network namespace path for a container ID. The GC controller +// checks whether this exact path still exists to decide if the container is +// still live — it cannot reconstruct the path from the container ID alone, +// since netns bind-mounts are named by the runtime's own convention (e.g. +// containerd's "cni-"), which is unrelated to the container ID. The +// full key appends a truncated container ID; see NetNSKey. +const AnnotationNetNS = "galactic.datum.net/netns" + +// containerIDLen is the number of characters used from a container ID in +// annotation keys. Kubernetes limits the name part of an annotation key to +// 63 bytes. The longest prefix sharing this constant is +// "allocated-subnet-ipv6." (or "-ipv4."), both 22 bytes, leaving 41 bytes for +// the container ID suffix — shorter prefixes ("netns.") just leave more room +// than they need. +const containerIDLen = 41 + +// truncate returns id, shortened to containerIDLen characters if longer. +func truncate(id string) string { + if len(id) > containerIDLen { + return id[:containerIDLen] + } + return id +} + +// SubnetKeyIPv6 returns the annotation key for storing the allocated IPv6 +// subnet for the given container ID. +func SubnetKeyIPv6(containerID string) string { + return fmt.Sprintf("%s.%s", AnnotationAllocatedSubnetIPv6, truncate(containerID)) +} + +// SubnetKeyIPv4 returns the annotation key for storing the allocated IPv4 +// address for the given container ID. +func SubnetKeyIPv4(containerID string) string { + return fmt.Sprintf("%s.%s", AnnotationAllocatedSubnetIPv4, truncate(containerID)) +} + +// NetNSKey returns the annotation key for storing the network namespace path +// used by the given container ID. +func NetNSKey(containerID string) string { + return fmt.Sprintf("%s.%s", AnnotationNetNS, truncate(containerID)) +} + +// BGPVRFInstanceName returns the deterministic name for a BGPVRFInstance. +// Each VPCAttachment is unique per interface across the cluster, so the +// (vpc, vpcAttachment) pair is a reliable 1:1 key. +func BGPVRFInstanceName(vpc, vpcAttachment string) string { + return fmt.Sprintf("%s-%s", vpc, vpcAttachment) +} + +// BGPAdvertisementName returns the deterministic name for a +// BGPAdvertisement. Each VPCAttachment is unique per interface across the +// cluster, so the (vpc, vpcAttachment) pair is a reliable 1:1 key. +func BGPAdvertisementName(vpc, vpcAttachment string) string { + return fmt.Sprintf("%s-%s", vpc, vpcAttachment) +} diff --git a/internal/crdnames/crdnames_test.go b/internal/crdnames/crdnames_test.go new file mode 100644 index 00000000..81db39ce --- /dev/null +++ b/internal/crdnames/crdnames_test.go @@ -0,0 +1,74 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package crdnames + +import ( + "strings" + "testing" +) + +func TestBGPVRFInstanceName(t *testing.T) { + tests := []struct{ vpc, attachment, want string }{ + {"abc", "def", "abc-def"}, + {"0000000jU", "00G", "0000000jU-00G"}, + } + for _, tt := range tests { + got := BGPVRFInstanceName(tt.vpc, tt.attachment) + if got != tt.want { + t.Errorf("BGPVRFInstanceName(%q, %q) = %q, want %q", tt.vpc, tt.attachment, got, tt.want) + } + } +} + +func TestBGPAdvertisementName(t *testing.T) { + tests := []struct{ vpc, attachment, want string }{ + {"abc", "def", "abc-def"}, + {"0000000jU", "00G", "0000000jU-00G"}, + } + for _, tt := range tests { + got := BGPAdvertisementName(tt.vpc, tt.attachment) + if got != tt.want { + t.Errorf("BGPAdvertisementName(%q, %q) = %q, want %q", tt.vpc, tt.attachment, got, tt.want) + } + } +} + +// TestAnnotationKeyNameLength verifies that every annotation key builder +// stays within Kubernetes' 63-byte limit on the "name" part of an +// annotation key (the segment after the last "/"), using a realistic +// 64-character container ID (containerd/Docker use full SHA256 hex +// digests). This guards against a real production incident: +// containerIDLen was sized for the old "allocated-subnet." prefix (17 +// bytes) and wasn't updated when the prefix grew by 5 bytes to +// "allocated-subnet-ipv6."/"-ipv4." — every BGPAdvertisement apply failed +// with "name part must be no more than 63 bytes" until fixed. +func TestAnnotationKeyNameLength(t *testing.T) { + const maxAnnotationNameLen = 63 + // A realistic full-length container ID (64 hex chars, as containerd/Docker use). + fullContainerID := strings.Repeat("a", 64) + + tests := []struct { + name string + key string + }{ + {"SubnetKeyIPv6", SubnetKeyIPv6(fullContainerID)}, + {"SubnetKeyIPv4", SubnetKeyIPv4(fullContainerID)}, + {"NetNSKey", NetNSKey(fullContainerID)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + slash := strings.LastIndex(tt.key, "/") + namePart := tt.key + if slash != -1 { + namePart = tt.key[slash+1:] + } + if len(namePart) > maxAnnotationNameLen { + t.Errorf("%s(%d-char containerID) name part %q is %d bytes, want <= %d", + tt.name, len(fullContainerID), namePart, len(namePart), maxAnnotationNameLen) + } + }) + } +} diff --git a/internal/gc/gc.go b/internal/gc/gc.go index 01594f1e..3333bccb 100644 --- a/internal/gc/gc.go +++ b/internal/gc/gc.go @@ -377,7 +377,7 @@ func SweepEBPFVRFTable(ctx context.Context, k8s client.Client, namespace, nodeNa if len(routers) == 0 { // A node with any live eBPF-registered attachment at all necessarily // has a BGPRouter targeting it -- registerEBPFDatapath requires one - // to run at all (internal/cni/bgp.go). Finding none here is + // to run at all (internal/cnibgp/bgp.go). Finding none here is // indistinguishable from a transient listing/cache hiccup or the // router having just been renamed/recreated, so it must not be // treated the same as "genuinely zero live attachments": doing so diff --git a/internal/hostconf/hostconf.go b/internal/hostconf/hostconf.go new file mode 100644 index 00000000..59eaa1c8 --- /dev/null +++ b/internal/hostconf/hostconf.go @@ -0,0 +1,154 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Package hostconf reads node-local settings (node name, kubeconfig, +// namespace, log file/level) from the static per-node CNI conflist written +// once by internal/installer.Bootstrap. Every binary in the galactic CNI +// plugin chain needs this same lookup, so it lives here rather than being +// duplicated per binary — internal/cni and internal/installer both used to +// carry their own near-identical copy, hardcoded to match a single plugin +// type ("galactic-cni"). +// +// The static conflist is not the per-attachment chain conflist the CNI +// runtime execs each plugin with (that one carries vpc/vpcattachment and is +// templated per VPCAttachment by the external companion operator) — it +// exists solely so any binary in the chain can find node-level settings by +// reading a well-known path off disk, independent of how it was actually +// invoked. Bootstrap only ever writes one entry, typed PluginType, so every +// caller in this repo passes that same constant. +package hostconf + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + + "github.com/vishvananda/netlink" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// PluginType is the "type" value Bootstrap always writes into the static +// conflist's single plugin entry, regardless of which binary is actually +// reading it. Every caller in the chain passes this to Load. +const PluginType = "galactic-cni" + +// HostConf holds node-local settings read from the static per-node conflist +// (default /etc/cni/net.d/10-galactic.conflist). +type HostConf struct { + NodeName string `json:"node_name"` + Kubeconfig string `json:"kubeconfig"` + Namespace string `json:"namespace"` + LogFile string `json:"log_file"` + LogLevel string `json:"log_level,omitempty"` +} + +// conflistEnvelope matches standard CNI conflist JSON structure. +type conflistEnvelope struct { + CNIVersion string `json:"cniVersion"` + Name string `json:"name"` + Plugins []json.RawMessage `json:"plugins"` +} + +// Load reads and parses the conflist at filePath and returns the HostConf +// carried by whichever plugin entry's "type" matches one of acceptedTypes. +// Returns an error wrapping fs.ErrNotExist (checkable via errors.Is) when +// filePath does not exist, so tolerant callers can fall back to defaults. +func Load(filePath string, acceptedTypes ...string) (*HostConf, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("read conflist file %q: %w", filePath, err) + } + + var env conflistEnvelope + if err := json.Unmarshal(data, &env); err != nil { + return nil, fmt.Errorf("parse conflist envelope: %w", err) + } + + accepted := make(map[string]bool, len(acceptedTypes)) + for _, t := range acceptedTypes { + accepted[t] = true + } + + for _, raw := range env.Plugins { + var meta struct { + Type string `json:"type"` + } + if err := json.Unmarshal(raw, &meta); err != nil { + continue + } + if accepted[meta.Type] { + var conf HostConf + if err := json.Unmarshal(raw, &conf); err != nil { + return nil, fmt.Errorf("parse host CNI config: %w", err) + } + return &conf, nil + } + } + + return nil, fmt.Errorf("conflist at %q does not contain a plugin with type in %v", filePath, acceptedTypes) +} + +// detectScheme returns a minimal scheme containing only corev1 types needed +// for node name detection. +func detectScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + return scheme +} + +// DetectNodeNameFromAPI queries the Kubernetes API and matches the node's +// InternalIP addresses against local interface addresses. Returns the first +// matching node name, or empty string with no error if detection fails +// (allowing callers to fall through to other resolution methods). Used as a +// fallback by any binary's config resolution when the static conflist is +// missing or doesn't carry a node name (e.g. hostPath mount issues in +// container-based test environments like Kind). +func DetectNodeNameFromAPI() (string, error) { + restCfg, err := ctrl.GetConfig() + if err != nil { + return "", fmt.Errorf("get kubeconfig: %w", err) + } + + k8sClient, err := client.New(restCfg, client.Options{ + Scheme: detectScheme(), + }) + if err != nil { + return "", fmt.Errorf("create k8s client: %w", err) + } + + var nodeList corev1.NodeList + if err := k8sClient.List(context.Background(), &nodeList, &client.ListOptions{ + Limit: 1000, + }); err != nil { + return "", fmt.Errorf("list nodes: %w", err) + } + + addrs, err := netlink.AddrList(nil, netlink.FAMILY_ALL) + if err != nil { + return "", fmt.Errorf("list local addresses: %w", err) + } + + localIPs := make(map[string]bool, len(addrs)) + for _, addr := range addrs { + localIPs[addr.IP.String()] = true + } + + for _, node := range nodeList.Items { + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP && localIPs[addr.Address] { + slog.Info("Auto-detected node name from Kubernetes API", + "nodeName", node.Name, "matchedIP", addr.Address) + return node.Name, nil + } + } + } + + return "", errors.New("no local interface address matched any node InternalIP") +} diff --git a/internal/hostconf/hostconf_test.go b/internal/hostconf/hostconf_test.go new file mode 100644 index 00000000..d2b59b04 --- /dev/null +++ b/internal/hostconf/hostconf_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 Datum Cloud, Inc. +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package hostconf + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestLoadMissingFile(t *testing.T) { + tmpDir := t.TempDir() + _, err := Load(filepath.Join(tmpDir, "does-not-exist.conflist"), PluginType) + if err == nil { + t.Fatal("expected error for missing file, got nil") + } + if !errors.Is(err, os.ErrNotExist) { + t.Errorf("expected error wrapping os.ErrNotExist, got %v", err) + } +} + +func TestLoadNoMatchingPluginType(t *testing.T) { + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "10-galactic.conflist") + content := `{"cniVersion":"1.0.0","name":"test","plugins":[{"type":"some-other-plugin"}]}` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("os.WriteFile: %v", err) + } + + if _, err := Load(path, PluginType); err == nil { + t.Fatal("expected error for missing plugin type, got nil") + } +} + +func TestLoadMatchingPluginType(t *testing.T) { + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "10-galactic.conflist") + content := `{ + "cniVersion": "1.0.0", + "name": "galactic", + "plugins": [ + { + "type": "galactic-cni", + "node_name": "test-worker", + "kubeconfig": "/etc/custom-kubeconfig", + "namespace": "custom-namespace", + "log_file": "/var/log/custom.log", + "log_level": "debug" + } + ] + }` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("os.WriteFile: %v", err) + } + + conf, err := Load(path, PluginType) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if conf.NodeName != "test-worker" { + t.Errorf("NodeName = %q, want %q", conf.NodeName, "test-worker") + } + if conf.Kubeconfig != "/etc/custom-kubeconfig" { + t.Errorf("Kubeconfig = %q, want %q", conf.Kubeconfig, "/etc/custom-kubeconfig") + } + if conf.Namespace != "custom-namespace" { + t.Errorf("Namespace = %q, want %q", conf.Namespace, "custom-namespace") + } + if conf.LogFile != "/var/log/custom.log" { + t.Errorf("LogFile = %q, want %q", conf.LogFile, "/var/log/custom.log") + } + if conf.LogLevel != "debug" { + t.Errorf("LogLevel = %q, want %q", conf.LogLevel, "debug") + } +} + +func TestLoadAcceptsAnyOfMultipleTypes(t *testing.T) { + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "10-galactic.conflist") + content := `{"cniVersion":"1.0.0","name":"test","plugins":[{"type":"galactic-tap-cni","node_name":"tap-node"}]}` + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatalf("os.WriteFile: %v", err) + } + + conf, err := Load(path, "galactic-cni", "galactic-tap-cni") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if conf.NodeName != "tap-node" { + t.Errorf("NodeName = %q, want %q", conf.NodeName, "tap-node") + } +} diff --git a/internal/installer/installer.go b/internal/installer/installer.go index 894a1194..2807e600 100644 --- a/internal/installer/installer.go +++ b/internal/installer/installer.go @@ -66,6 +66,10 @@ var ( HostEtcDir = "/host/var/lib/galactic" SADir = "/var/run/secrets/kubernetes.io/serviceaccount" SourceCNIBinary = "/galactic-cni" + SourceTapCNIBinary = "/galactic-tap-cni" + SourceIPAMBinary = "/galactic-ipam" + SourceBGPBinary = "/galactic-bgp" + SourceRouteBinary = "/galactic-route" SourceHostDeviceBinary = "/host-device" ) @@ -254,13 +258,28 @@ func Bootstrap(ctx context.Context, nodeName string) error { slog.Info("Starting CNI installer bootstrap", "nodeName", nodeName) - // 1. Copy CNI and host-device binaries to the host + // 1. Copy the CNI plugin chain's binaries to the host. Every binary in + // the chain ships in this same image and is staged here by this one + // init container, regardless of which master plugin(s) a given node's + // workloads actually use. if err := os.MkdirAll(HostBinDir, 0755); err != nil { return fmt.Errorf("create host CNI bin dir: %w", err) } if err := atomicCopyFile(SourceCNIBinary, filepath.Join(HostBinDir, "galactic-cni"), 0755); err != nil { return fmt.Errorf("copy galactic-cni binary: %w", err) } + if err := atomicCopyFile(SourceTapCNIBinary, filepath.Join(HostBinDir, "galactic-tap-cni"), 0755); err != nil { + return fmt.Errorf("copy galactic-tap-cni binary: %w", err) + } + if err := atomicCopyFile(SourceIPAMBinary, filepath.Join(HostBinDir, "galactic-ipam"), 0755); err != nil { + return fmt.Errorf("copy galactic-ipam binary: %w", err) + } + if err := atomicCopyFile(SourceBGPBinary, filepath.Join(HostBinDir, "galactic-bgp"), 0755); err != nil { + return fmt.Errorf("copy galactic-bgp binary: %w", err) + } + if err := atomicCopyFile(SourceRouteBinary, filepath.Join(HostBinDir, "galactic-route"), 0755); err != nil { + return fmt.Errorf("copy galactic-route binary: %w", err) + } if err := atomicCopyFile(SourceHostDeviceBinary, filepath.Join(HostBinDir, "host-device"), 0755); err != nil { return fmt.Errorf("copy host-device binary: %w", err) } diff --git a/internal/installer/installer_test.go b/internal/installer/installer_test.go index 6801cc44..ca164bac 100644 --- a/internal/installer/installer_test.go +++ b/internal/installer/installer_test.go @@ -71,6 +71,28 @@ func TestResolveLogLevel(t *testing.T) { } } +// assertBinaryCopied verifies that the binary at path exists and contains +// wantContent, factored out of TestBootstrap to keep its own cyclomatic +// complexity within golangci-lint's gocyclo budget. +func assertBinaryCopied(t *testing.T, path, wantContent string) { + t.Helper() + got, err := os.ReadFile(path) + if err != nil || string(got) != wantContent { + t.Fatalf("binary copy verification failed for %q: err=%v content=%q", path, err, got) + } +} + +// writeSourceBinary writes content to path, failing the test on error. +// Factored out alongside assertBinaryCopied for the same reason — each +// binary this test seeds would otherwise add its own branch to +// TestBootstrap's own cyclomatic complexity. +func writeSourceBinary(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0755); err != nil { + t.Fatalf("write source binary %q: %v", path, err) + } +} + func TestBootstrap(t *testing.T) { // Set up temporary directories for testing overrides tmpDir := t.TempDir() @@ -92,13 +114,17 @@ func TestBootstrap(t *testing.T) { // Create mock CNI source binary files SourceCNIBinary = filepath.Join(tmpDir, "source-galactic-cni") + SourceTapCNIBinary = filepath.Join(tmpDir, "source-galactic-tap-cni") + SourceIPAMBinary = filepath.Join(tmpDir, "source-galactic-ipam") + SourceBGPBinary = filepath.Join(tmpDir, "source-galactic-bgp") + SourceRouteBinary = filepath.Join(tmpDir, "source-galactic-route") SourceHostDeviceBinary = filepath.Join(tmpDir, "source-host-device") - if err := os.WriteFile(SourceCNIBinary, []byte("cni-content"), 0755); err != nil { - t.Fatalf("write SourceCNIBinary: %v", err) - } - if err := os.WriteFile(SourceHostDeviceBinary, []byte("host-device-content"), 0755); err != nil { - t.Fatalf("write SourceHostDeviceBinary: %v", err) - } + writeSourceBinary(t, SourceCNIBinary, "cni-content") + writeSourceBinary(t, SourceTapCNIBinary, "tap-cni-content") + writeSourceBinary(t, SourceIPAMBinary, "ipam-content") + writeSourceBinary(t, SourceBGPBinary, "bgp-content") + writeSourceBinary(t, SourceRouteBinary, "route-content") + writeSourceBinary(t, SourceHostDeviceBinary, "host-device-content") // Mock node object node := &corev1.Node{ @@ -151,10 +177,11 @@ func TestBootstrap(t *testing.T) { } // Verify binaries copied - cniContent, err := os.ReadFile(filepath.Join(HostBinDir, "galactic-cni")) - if err != nil || string(cniContent) != "cni-content" { - t.Fatalf("galactic-cni binary copy verification failed") - } + assertBinaryCopied(t, filepath.Join(HostBinDir, "galactic-cni"), "cni-content") + assertBinaryCopied(t, filepath.Join(HostBinDir, "galactic-tap-cni"), "tap-cni-content") + assertBinaryCopied(t, filepath.Join(HostBinDir, "galactic-ipam"), "ipam-content") + assertBinaryCopied(t, filepath.Join(HostBinDir, "galactic-bgp"), "bgp-content") + assertBinaryCopied(t, filepath.Join(HostBinDir, "galactic-route"), "route-content") // Verify conflist written conflist, err := loadHostConf(HostConflist) diff --git a/internal/cni/nad.go b/internal/nadpatch/nadpatch.go similarity index 62% rename from internal/cni/nad.go rename to internal/nadpatch/nadpatch.go index bdf4ee8a..8a10699a 100644 --- a/internal/cni/nad.go +++ b/internal/nadpatch/nadpatch.go @@ -2,7 +2,11 @@ // // SPDX-License-Identifier: AGPL-3.0-or-later -package cni +// Package nadpatch patches the NetworkAttachmentDefinition with the +// deterministic host-side interface name a master plugin (galactic-cni, +// galactic-tap-cni) just created — shared since NAD annotation is identical +// regardless of interface type. +package nadpatch import ( "context" @@ -17,10 +21,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// annotationHostInterface is the NAD annotation key that records the +// AnnotationHostInterface is the NAD annotation key that records the // deterministic host-side interface name created by the CNI plugin for // this VPC+VPCAttachment pair. -const annotationHostInterface = "k8s.v1.cni.cncf.io/host-interface" +const AnnotationHostInterface = "k8s.v1.cni.cncf.io/host-interface" // nadGVK is the GroupVersionKind for NetworkAttachmentDefinition. var nadGVK = schema.GroupVersionKind{ @@ -29,10 +33,11 @@ var nadGVK = schema.GroupVersionKind{ Kind: "NetworkAttachmentDefinition", } -// parsePodNamespace extracts the K8S_POD_NAMESPACE value from the CNI_ARGS -// environment variable string passed as args.Args by Multus. Returns an empty -// string when the value is not present (e.g. standalone CNI invocation). -func parsePodNamespace(cniArgs string) string { +// ParsePodNamespace extracts the K8S_POD_NAMESPACE value from the CNI_ARGS +// environment variable string passed as args.Args by Multus. Returns an +// empty string when the value is not present (e.g. standalone CNI +// invocation). +func ParsePodNamespace(cniArgs string) string { for _, part := range strings.Split(cniArgs, ";") { key, value, ok := strings.Cut(part, "=") if ok && key == "K8S_POD_NAMESPACE" { @@ -42,13 +47,13 @@ func parsePodNamespace(cniArgs string) string { return "" } -// annotateNAD patches the NetworkAttachmentDefinition with the host interface -// name. The NAD is expected to already exist (created by the external VPC -// operator before the CNI is invoked), so a not-found response is a hard -// failure rather than something to tolerate. A conflict response is the one -// case treated as non-fatal: it means the annotation was already applied by a -// previous invocation. -func annotateNAD(ctx context.Context, k8s client.Client, nadName, nadNamespace, hostInterface string) error { +// AnnotateNAD patches the NetworkAttachmentDefinition with the host +// interface name. The NAD is expected to already exist (created by the +// external VPC operator before the CNI is invoked), so a not-found response +// is a hard failure rather than something to tolerate. A conflict response +// is the one case treated as non-fatal: it means the annotation was already +// applied by a previous invocation. +func AnnotateNAD(ctx context.Context, k8s client.Client, nadName, nadNamespace, hostInterface string) error { if nadNamespace == "" { return nil } @@ -59,7 +64,7 @@ func annotateNAD(ctx context.Context, k8s client.Client, nadName, nadNamespace, nad.SetNamespace(nadNamespace) patch := fmt.Sprintf(`[{"op":"add","path":"/metadata/annotations","value":{"%s":"%s"}}]`, - annotationHostInterface, hostInterface) + AnnotationHostInterface, hostInterface) err := k8s.Patch(ctx, nad, client.RawPatch(types.JSONPatchType, []byte(patch))) if err != nil { diff --git a/internal/cni/nad_test.go b/internal/nadpatch/nadpatch_test.go similarity index 65% rename from internal/cni/nad_test.go rename to internal/nadpatch/nadpatch_test.go index 7bff7566..461f9207 100644 --- a/internal/cni/nad_test.go +++ b/internal/nadpatch/nadpatch_test.go @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0-or-later -package cni +package nadpatch import ( "context" @@ -10,25 +10,23 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +func fakeClient(objs ...client.Object) client.Client { + return fake.NewClientBuilder().WithScheme(runtime.NewScheme()).WithObjects(objs...).Build() +} + func TestParsePodNamespace(t *testing.T) { tests := []struct { name string cniArgs string expected string }{ - { - name: "empty string", - cniArgs: "", - expected: "", - }, - { - name: "namespace only", - cniArgs: "K8S_POD_NAMESPACE=default", - expected: "default", - }, + {name: "empty string", cniArgs: "", expected: ""}, + {name: "namespace only", cniArgs: "K8S_POD_NAMESPACE=default", expected: "default"}, { name: "full multus args", cniArgs: "K8S_POD_NAME=my-pod;K8S_POD_NAMESPACE=galactic-system;K8S_POD_INFRA_CONTAINER_ID=abc123", @@ -39,18 +37,14 @@ func TestParsePodNamespace(t *testing.T) { cniArgs: "K8S_POD_NAME=my-pod;K8S_POD_INFRA_CONTAINER_ID=abc123", expected: "", }, - { - name: "namespace with hyphens", - cniArgs: "K8S_POD_NAMESPACE=my-custom-namespace", - expected: "my-custom-namespace", - }, + {name: "namespace with hyphens", cniArgs: "K8S_POD_NAMESPACE=my-custom-namespace", expected: "my-custom-namespace"}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got := parsePodNamespace(tc.cniArgs) + got := ParsePodNamespace(tc.cniArgs) if got != tc.expected { - t.Errorf("parsePodNamespace(%q) = %q, want %q", tc.cniArgs, got, tc.expected) + t.Errorf("ParsePodNamespace(%q) = %q, want %q", tc.cniArgs, got, tc.expected) } }) } @@ -66,7 +60,7 @@ func TestAnnotateNAD(t *testing.T) { t.Run("NAD does not exist is a hard failure", func(t *testing.T) { k8s := fakeClient() - err := annotateNAD(context.Background(), k8s, nadName, nadNamespace, hostIface) + err := AnnotateNAD(context.Background(), k8s, nadName, nadNamespace, hostIface) if err == nil { t.Fatal("expected error when NAD does not exist, got nil") } @@ -82,8 +76,8 @@ func TestAnnotateNAD(t *testing.T) { nad.SetNamespace(nadNamespace) k8s := fakeClient(nad) - if err := annotateNAD(context.Background(), k8s, nadName, nadNamespace, hostIface); err != nil { - t.Fatalf("annotateNAD() = %v, want nil", err) + if err := AnnotateNAD(context.Background(), k8s, nadName, nadNamespace, hostIface); err != nil { + t.Fatalf("AnnotateNAD() = %v, want nil", err) } got := &unstructured.Unstructured{} @@ -91,16 +85,16 @@ func TestAnnotateNAD(t *testing.T) { if err := k8s.Get(context.Background(), client.ObjectKey{Name: nadName, Namespace: nadNamespace}, got); err != nil { t.Fatalf("get NAD after annotate: %v", err) } - if gotAnnotation := got.GetAnnotations()[annotationHostInterface]; gotAnnotation != hostIface { - t.Errorf("annotation %s = %q, want %q", annotationHostInterface, gotAnnotation, hostIface) + if gotAnnotation := got.GetAnnotations()[AnnotationHostInterface]; gotAnnotation != hostIface { + t.Errorf("annotation %s = %q, want %q", AnnotationHostInterface, gotAnnotation, hostIface) } }) t.Run("empty pod namespace is a no-op", func(t *testing.T) { k8s := fakeClient() - if err := annotateNAD(context.Background(), k8s, nadName, "", hostIface); err != nil { - t.Fatalf("annotateNAD() with empty namespace = %v, want nil", err) + if err := AnnotateNAD(context.Background(), k8s, nadName, "", hostIface); err != nil { + t.Fatalf("AnnotateNAD() with empty namespace = %v, want nil", err) } }) } diff --git a/internal/plumbing/ebpf/doc.go b/internal/plumbing/ebpf/doc.go index 058e6abe..65791409 100644 --- a/internal/plumbing/ebpf/doc.go +++ b/internal/plumbing/ebpf/doc.go @@ -32,13 +32,13 @@ // external event silently clearing the filter -- requires it. // - usidmap: the read/write API that populates and reconciles // locator_table/function_table/vrf_table, used by the CNI ADD path's -// registration call (internal/cni/bgp.go) and by the GC controller's +// registration call (internal/cnibgp/bgp.go) and by the GC controller's // sweep (internal/gc). // - metrics: Prometheus metrics and health-check event hooks spanning the // whole datapath (load/attach events, drops by reason, per-Argument // hit counters and Argument-space utilization). // -// internal/cni and internal/gc are the two callers outside this tree that +// internal/cnibgp and internal/gc are the two callers outside this tree that // drive usidmap's register/unregister/reconcile calls; internal/reconcile // and internal/plumbing/srv6's ComputeSID independently compute the same // SID this datapath decodes, for the BGP control-plane side of the same diff --git a/internal/plumbing/ebpf/usidmap/vrf.go b/internal/plumbing/ebpf/usidmap/vrf.go index 8914dc88..b94f43b0 100644 --- a/internal/plumbing/ebpf/usidmap/vrf.go +++ b/internal/plumbing/ebpf/usidmap/vrf.go @@ -102,7 +102,7 @@ func (t *VRFTable) Generation() uint64 { // because a repeat Register of the *same* key is not always a fresh // attachment lifecycle -- it is also, in the ordinary case, the CNI ADD // retry path re-registering after a transient k8s-op failure -// (internal/cni/bgp.go's retryK8sOps), which happens on an Argument that +// (internal/cnibgp/bgp.go's retryK8sOps), which happens on an Argument that // may already be carrying live traffic. R8's make-before-break migration // gate reads these counters to prove an Argument carried no traffic before // cutover; a blind overwrite that zeroed them on every retry would make a diff --git a/internal/plumbing/srv6/usid.go b/internal/plumbing/srv6/usid.go index 1b8718cb..d813fff4 100644 --- a/internal/plumbing/srv6/usid.go +++ b/internal/plumbing/srv6/usid.go @@ -20,7 +20,7 @@ import ( // no distinct wire code for a per-family variant anyway, design plan R3): // it is the only endpoint behavior the eBPF datapath's vrf_table ever // installs, regardless of pod-subnet address family (see -// internal/cni/bgp.go's registerEBPFDatapath/buildAdvertisementSpec). +// internal/cnibgp/bgp.go's registerEBPFDatapath/buildAdvertisementSpec). func functionNibble(fn bgpv1alpha1.SRv6Function) (uint8, error) { if fn == bgpv1alpha1.SRv6FunctionEndDT46 { return uformat.FunctionEndDT46, nil diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index c72ded88..1b3e0568 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -158,10 +158,19 @@ func TestKernelCapabilities(t *testing.T) { } } -// TestCNITapInterface exercises the galactic CNI plugin in tap interface mode. -// It creates a pod that invokes the CNI plugin with CNI_COMMAND=ADD and a tap -// config, then validates the CNI result JSON: a single host interface with an -// empty sandbox and the host-side gateway/subnet IPAM allocated for it. +// TestCNITapInterface exercises galactic-tap-cni, the tap master plugin in +// the galactic CNI chain (see internal/cnitap). It creates a pod that +// invokes the plugin with CNI_COMMAND=ADD and a tap config, then validates +// the CNI result JSON: a single host interface with an empty sandbox and +// the host-side gateway/subnet IPAM allocated for it. +// +// This exercises galactic-tap-cni's own ADD (VRF + tap creation, IPAM +// delegation to galactic-ipam) directly, then manually chains galactic-bgp +// after it (testChainedGalacticBGP below), feeding it the tap master's own +// CNI result as prevResult exactly as the CNI runtime would — the same +// manual-chaining approach used because a real conflist-driven chain would +// need a BGPRouter fixture and additional RBAC this test doesn't set up. +// It does not chain into galactic-route (this config has no terminations). // // This test requires a cluster node with VRF/tap kernel support (the same // prerequisites checked by TestKernelCapabilities). It will fail rather than @@ -173,29 +182,26 @@ func TestCNITapInterface(t *testing.T) { deletePod(t, name) // Start a shell so we can later exec the CNI plugin with stdin. - // The galactic-cni entrypoint is overridden to "sh" so the pod stays - // running and we can pipe the CNI config via kubectl exec -i. - // Run as the galactic-cni ServiceAccount so the CNI plugin's in-cluster - // client is bound by the galactic-cni ClusterRole (config/cni/rbac.yaml) - // when it lists/creates BGPRouter, BGPAdvertisement, and BGPVRFInstance. - // hostNetwork is required too: net.vrf.strict_mode (enabled on the Kind - // node in scripts/ci.sh) is per-netns, and the SEG6Local VRFTABLE route - // this test exercises needs it set in whichever netns the route lands in. - // The bpf-fs hostPath volume mirrors config/cni/daemonset.yaml's own - // bpf-fs mount: the eBPF uSID datapath's maps can only be pinned under - // attach.PinDir if the node's real bpffs (mounted onto the Kind node in - // scripts/ci.sh) is visible inside the pod -- a pod's own mount - // namespace can't create /sys/fs/bpf itself. The whole container spec - // (image, command, privileged) has to live in --overrides too, not the - // usual --image/--command/--privileged flags: kubectl run's overrides - // merge replaces the generated "containers" list wholesale rather than - // merging into it, so anything set only via those flags would otherwise - // be silently dropped the moment "containers" is also set here. + // The galactic-cni image's entrypoint is overridden to "sh" so the pod + // stays running and we can pipe the CNI config via kubectl exec -i. + // galactic-tap-cni ships in the same image (see containers/galactic-cni/ + // Dockerfile) alongside every other binary in the CNI chain, so no + // separate image is needed here. Run as the galactic-cni ServiceAccount: + // galactic-tap-cni's own ADD unconditionally builds an in-cluster k8s + // client for its NAD-annotation step (config/cni/rbac.yaml grants it), + // even though that step itself no-ops here (no CNI_ARGS, so + // nadpatch.ParsePodNamespace resolves an empty namespace). hostNetwork + // is required too, so the VRF/tap interfaces this test creates land in + // the same netns production's own hostNetwork DaemonSet would use. The + // whole container spec (image, command, privileged) has to live in + // --overrides too, not the usual --image/--command/--privileged flags: + // kubectl run's overrides merge replaces the generated "containers" + // list wholesale rather than merging into it, so anything set only via + // those flags would otherwise be silently dropped the moment + // "containers" is also set here. overrides := fmt.Sprintf(`{"spec":{"serviceAccountName":"galactic-cni","hostNetwork":true,`+ - `"volumes":[{"name":"bpf-fs","hostPath":{"path":"/sys/fs/bpf","type":"Directory"}}],`+ `"containers":[{"name":%q,"image":%q,"imagePullPolicy":"Never","command":["sleep","infinity"],`+ - `"securityContext":{"privileged":true},`+ - `"volumeMounts":[{"name":"bpf-fs","mountPath":"/sys/fs/bpf"}]}]}}`, name, image()) + `"securityContext":{"privileged":true}}]}}`, name, image()) runOut, err := kubectl( t.Context(), "run", name, @@ -212,45 +218,56 @@ func TestCNITapInterface(t *testing.T) { } // The eBPF uSID datapath is now the only forwarding path (see - // internal/cni/bgp.go's registerEBPFDatapath), so CNI ADD requires this - // node's locator_table/function_table/vrf_table maps to already be - // pinned under attach.PinDir. In production that's done ahead of time by - // the CNI DaemonSet's long-running "credential-refresh" container - // (config/cni/daemonset.yaml, `/galactic-cni run`); this test runs its - // own pod instead of relying on that DaemonSet, so it must start the - // same control daemon itself before exercising CNI ADD below. + // internal/cnibgp/bgp.go's registerEBPFDatapath, called from + // galactic-bgp's own cmdAdd — a separately chain-invoked binary since + // the CNI plugin-chain split, not inline in galactic-tap-cni's cmdAdd + // any more), so CNI ADD requires this node's locator_table/ + // function_table/vrf_table maps to already be pinned under + // attach.PinDir. In production that's done ahead of time by the CNI + // DaemonSet's long-running "credential-refresh" container (config/cni/ + // daemonset.yaml, `/galactic-cni run`); this test runs its own pod + // instead of relying on that DaemonSet, so it must start the same + // control daemon itself before exercising CNI ADD below. Required for + // testChainedGalacticBGP below too: registerEBPFDatapath's + // usidmap.OpenPinnedRegistry only opens already-pinned maps, it never + // loads/pins the eBPF program itself. startEBPFControlDaemon(t, name) // Write the CNI config to a file inside the pod, then run the plugin // with the config piped via stdin. The plugin reads config from stdin // (the CNI protocol) and CNI_NETNS from the environment. + // + // The "ipam" block's "type" names the delegated binary (galactic-ipam), + // not a pool-vs-static mode selector — presence of ipv6_subnet alone + // opts this config into pool IPAM (see internal/cniipam's doc comment + // and docs/cni/configuration.md). cniConf := `{ "cniVersion": "1.0.0", "name": "galactic", - "type": "galactic-cni", + "type": "galactic-tap-cni", "vpc": "1", "vpcattachment": "1", - "interface_type": "tap", - "srv6_locator": "2001:db8:ff01::/48", "ipam": { - "type": "pool" + "type": "galactic-ipam", + "ipv6_subnet": "fd00:e2e::/48" } }` // Step 1: write the CNI config and a wrapper script into the pod. - // Tap mode now runs IPAM allocation unconditionally (matching veth mode). - // GALACTIC_CNI_ENABLE_LOCAL_IPAM fills in default pool/subnet_len when - // omitted, but parseConf still requires an explicit "ipam" block to be - // present in the config (see docs/cni/configuration.md). + // CNI_PATH=/ lets IPAM delegation (galactic-tap-cni execs galactic-ipam + // via github.com/containernetworking/plugins/pkg/ipam.ExecAdd) find the + // delegate binary: every binary in the chain is copied to the image + // root by containers/galactic-cni/Dockerfile (not /opt/cni/bin — that + // path only exists on the real host once installer.Bootstrap's init + // container stages it there, which this test's pod never runs). script := `#!/bin/sh ip netns add e2e-tap-ns CNI_NETNS=/var/run/netns/e2e-tap-ns \ CNI_COMMAND=ADD \ CNI_CONTAINERID=e2e-tap-001 \ CNI_IFNAME=eth0 \ -CNI_PATH=/opt/cni/bin \ +CNI_PATH=/ \ NODE_NAME=` + nodeName() + ` \ -GALACTIC_CNI_ENABLE_LOCAL_IPAM=true \ - /galactic-cni < /tmp/cni.json + /galactic-tap-cni < /tmp/cni.json ` _, err = kubectl(t.Context(), "exec", name, "--", "sh", "-c", @@ -313,6 +330,109 @@ GALACTIC_CNI_ENABLE_LOCAL_IPAM=true \ if len(ips) != 1 { t.Errorf("ips count = %d, want 1", len(ips)) } + + // Step 3: chain galactic-bgp — the CNI runtime's next plugin in the + // conflist — after galactic-tap-cni, feeding it the tap master's own + // CNI result as prevResult, exactly as the runtime would. Everything + // up to this point only exercises galactic-tap-cni's own cmdAdd; + // BGPVRFInstance/BGPAdvertisement CRD creation and eBPF + // locator_table/function_table/vrf_table registration all moved into + // galactic-bgp's own cmdAdd with the CNI plugin-chain split (see + // internal/cnibgp's doc comment), so without this step the whole BGP + // publish path has no e2e coverage on the ADD path at all. + testChainedGalacticBGP(t, name, result) +} + +// testChainedGalacticBGP runs galactic-bgp's own ADD, chained after the tap +// master's ADD (tapResult is exactly what that ADD printed, fed through as +// prevResult — see internal/cnibgp/prevresult.go's inferFromPrevResult), +// then CHECK. It asserts the BGPVRFInstance/BGPAdvertisement CRDs exist +// after ADD, and — since checkEBPFEntry (internal/cnibgp/ops_check.go) reads +// back the locator_table, function_table, and vrf_table entries +// registerEBPFDatapath wrote and fails if any are missing or inconsistent — +// that CHECK succeeding is itself the assertion that eBPF registration +// happened correctly on ADD; there is no separate bpftool-style dump here. +func testChainedGalacticBGP(t *testing.T, podName string, tapResult map[string]any) { + t.Helper() + + const vpc, vpcAttachment = "1", "1" + crdName := vpc + "-" + vpcAttachment + t.Cleanup(func() { + //nolint:errcheck // best-effort cleanup, mirrors deletePod + kubectl(context.Background(), "delete", "bgpvrfinstance", crdName, "--ignore-not-found") + //nolint:errcheck // best-effort cleanup, mirrors deletePod + kubectl(context.Background(), "delete", "bgpadvertisement", crdName, "--ignore-not-found") + }) + + prevResultJSON, err := json.Marshal(tapResult) + if err != nil { + t.Fatalf("marshal tap ADD result for prevResult: %v", err) + } + bgpConf := fmt.Sprintf(`{ + "cniVersion": "1.0.0", + "name": "galactic", + "type": "galactic-bgp", + "vpc": %q, + "vpcattachment": %q, + "prevResult": %s +}`, vpc, vpcAttachment, prevResultJSON) + + // Reuses the same netns/containerID/ifname galactic-tap-cni's own step + // (above) already set up: a real chained plugin sees the identical + // values across every plugin invoked for one CNI ADD. + bgpScript := `#!/bin/sh +CNI_NETNS=/var/run/netns/e2e-tap-ns \ +CNI_COMMAND=$1 \ +CNI_CONTAINERID=e2e-tap-001 \ +CNI_IFNAME=eth0 \ +CNI_PATH=/ \ +NODE_NAME=` + nodeName() + ` \ + /galactic-bgp < /tmp/cni-bgp.json +` + _, err = kubectl(t.Context(), "exec", podName, "--", + "sh", "-c", + "echo '"+bgpConf+"' > /tmp/cni-bgp.json && "+ + "echo '"+bgpScript+"' > /tmp/run-bgp.sh && "+ + "chmod +x /tmp/run-bgp.sh", + ) + if err != nil { + t.Fatalf("write galactic-bgp config and script: %v", err) + } + + addOut, err := kubectl(t.Context(), "exec", podName, "-i", "--", "/tmp/run-bgp.sh", "ADD") + if err != nil { + t.Fatalf("galactic-bgp ADD failed: %v\noutput: %s", err, addOut) + } + + // galactic-bgp is the last plugin in the chain: its own result is + // prevResult passed through unchanged, not a new one it builds itself. + jsonStart := strings.Index(addOut, "{") + if jsonStart == -1 { + t.Fatalf("no JSON found in galactic-bgp ADD output:\n%s", addOut) + } + var bgpResult map[string]any + if err := json.Unmarshal([]byte(addOut[jsonStart:]), &bgpResult); err != nil { + t.Fatalf("galactic-bgp ADD output is not valid JSON: %v\noutput:\n%s", err, addOut) + } + if bgpIfaces, _ := bgpResult["interfaces"].([]any); len(bgpIfaces) != 1 { + t.Errorf("galactic-bgp ADD result interfaces count = %d, want 1 (passed through from prevResult unchanged)", + len(bgpIfaces)) + } + + if out, err := kubectl(t.Context(), "get", "bgpvrfinstance", crdName); err != nil { + t.Errorf("BGPVRFInstance %s not found after galactic-bgp ADD: %v\n%s", crdName, err, out) + } + if out, err := kubectl(t.Context(), "get", "bgpadvertisement", crdName); err != nil { + t.Errorf("BGPAdvertisement %s not found after galactic-bgp ADD: %v\n%s", crdName, err, out) + } + + // CHECK reads back the locator_table/function_table/vrf_table entries + // registerEBPFDatapath wrote on ADD (see ops_check.go's checkEBPFEntry) + // — its success is this test's assertion that eBPF registration + // actually happened, not just that the CRDs exist. + if checkOut, err := kubectl(t.Context(), "exec", podName, "-i", "--", "/tmp/run-bgp.sh", "CHECK"); err != nil { + t.Errorf("galactic-bgp CHECK failed: %v\noutput: %s", err, checkOut) + } } // startEBPFControlDaemon runs `/galactic-cni run` inside the already-running