diff --git a/cmd/galactic-cni/main.go b/cmd/galactic-cni/main.go
index f5968664..ded54112 100644
--- a/cmd/galactic-cni/main.go
+++ b/cmd/galactic-cni/main.go
@@ -5,10 +5,8 @@
package main
import (
- "encoding/json"
"errors"
"fmt"
- "io"
"log"
"os"
"strings"
@@ -108,27 +106,14 @@ func newRootCommand() *cobra.Command {
return nil
}
- // Read stdin once so we can inspect the CNI config before the
- // library runs its netns validation. We pipe the buffered bytes
- // back as os.Stdin so the CNI library can still read them.
- stdinData, _ := io.ReadAll(os.Stdin)
- r, w, _ := os.Pipe()
- go func() {
- _, _ = w.Write(stdinData)
- _ = w.Close()
- }()
- oldStdin := os.Stdin
- os.Stdin = r
-
- // Tap mode never enters a network namespace — all operations are
- // host-side. Set the override so the CNI library skips its same-
- // netns rejection check, which would otherwise reject kraftlet
- // workloads that pass the host netns.
- if isTapMode(stdinData) {
- _ = os.Setenv("CNI_NETNS_OVERRIDE", "true")
- }
-
- defer func() { os.Stdin = oldStdin }()
+ // galactic-cni is veth-only: it always moves an interface into
+ // the container's own netns, so it always needs the CNI
+ // library's normal same-netns rejection check — unlike
+ // galactic-tap-cni (which unconditionally sets
+ // CNI_NETNS_OVERRIDE, since tap workloads never enter a netns
+ // at all), there is no stdin-peeking tap-mode detection here
+ // anymore. Interface kind is which binary you invoke now, not a
+ // config field this process branches on.
cni.RunPlugin()
return nil
},
@@ -142,17 +127,6 @@ func newRootCommand() *cobra.Command {
return cmd
}
-// isTapMode returns true when the CNI config requests tap interface type.
-// Only a minimal JSON parse is needed — full validation happens later in
-// parseConf inside cmdAdd.
-func isTapMode(stdinData []byte) bool {
- var cfg struct {
- InterfaceType string `json:"interface_type"`
- }
- _ = json.Unmarshal(stdinData, &cfg)
- return cfg.InterfaceType == "tap"
-}
-
func main() {
if err := newRootCommand().Execute(); err != nil {
log.Fatalf("error: %v", err)
diff --git a/cmd/galactic-tap-cni/main.go b/cmd/galactic-tap-cni/main.go
index 1fa4a35c..023741bd 100644
--- a/cmd/galactic-tap-cni/main.go
+++ b/cmd/galactic-tap-cni/main.go
@@ -6,7 +6,6 @@ package main
import (
"fmt"
- "io"
"log"
"os"
"strings"
@@ -71,17 +70,10 @@ func newRootCommand() *cobra.Command {
// Tap mode never enters a network namespace — all operations
// are host-side. Set the override so the CNI library skips its
// same-netns rejection check, which would otherwise reject
- // kraftlet workloads that pass the host netns.
- stdinData, _ := io.ReadAll(os.Stdin)
- r, w, _ := os.Pipe()
- go func() {
- _, _ = w.Write(stdinData)
- _ = w.Close()
- }()
- oldStdin := os.Stdin
- os.Stdin = r
- defer func() { os.Stdin = oldStdin }()
-
+ // kraftlet workloads that pass the host netns. Unconditional
+ // here (unlike galactic-cni, which has no override logic at
+ // all): every invocation of this binary is tap mode, so there
+ // is no config content to peek at first.
_ = os.Setenv("CNI_NETNS_OVERRIDE", "true")
cnitap.RunPlugin()
diff --git a/deploy/containerlab/docs/tenants.md b/deploy/containerlab/docs/tenants.md
index 8ecb0915..cfdda621 100644
--- a/deploy/containerlab/docs/tenants.md
+++ b/deploy/containerlab/docs/tenants.md
@@ -9,11 +9,13 @@ pod-to-pod connectivity. Every one of them follows the same mechanism: Multus
attaches each `netshoot` pod to its VPC's `private` NetworkAttachmentDefinition
(via the `v1.multus-cni.io/default-network` annotation, which makes the VPC
interface the pod's `eth0` rather than an additional `net1` — there is no
-`k8s.v1.cni.cncf.io/networks` annotation in play here), which invokes
-`galactic-cni` to create a VRF, veth pair, SRv6 encapsulation route, and a
-`BGPAdvertisement` CRD. The `galactic-router` controller then advertises each
-pod's EVPN route to the route reflector, distributing reachability across
-sites.
+`k8s.v1.cni.cncf.io/networks` annotation in play here), which invokes the
+galactic CNI plugin chain: `galactic-cni` creates a VRF and veth pair, then
+`galactic-bgp` registers the attachment against the eBPF uSID datapath and
+writes a `BGPAdvertisement` CRD (see [docs/cni-cmd-sequence.md](../../../docs/cni-cmd-sequence.md)
+for the full per-binary ADD sequence). The `galactic-router` controller then
+advertises each pod's EVPN route to the route reflector, distributing
+reachability across sites.
They differ only in scope and addressing:
@@ -37,7 +39,7 @@ fabric. The low hextet of a pod's USID is `(Function << 12) | Argument`
`0xE` (`FunctionEndDT46`) for every plain L3 VRF attachment, and `Argument` is
a 12-bit value `galactic-router` allocates per-node as the lowest unused slot
in `[0x001, 0xFFF]` among that node's existing `BGPVRFInstance` CRDs
-(`allocateArgument`, `internal/cni/bgp.go`) — **not** a decode of the NAD's
+(`allocateArgument`, `internal/cnibgp/bgp.go`) — **not** a decode of the NAD's
`vpc`/`vpcattachment` values. Concretely, expect hextets in the
`0xe001`–`0xefff` range; the exact value depends on allocation order (`ns50`
is provisioned first in `task deploy`, then `ns10`, `ns20`, `ns30`, `ns40` in
diff --git a/deploy/containerlab/resources/tenants/ns10/dfw/nad.yaml b/deploy/containerlab/resources/tenants/ns10/dfw/nad.yaml
index d36c7be4..a57005bf 100644
--- a/deploy/containerlab/resources/tenants/ns10/dfw/nad.yaml
+++ b/deploy/containerlab/resources/tenants/ns10/dfw/nad.yaml
@@ -5,17 +5,27 @@ metadata:
name: private
namespace: ns10
spec:
- config: |
+ config: |-
{
"cniVersion": "1.0.0",
"name": "private",
- "type": "galactic-cni",
- "vpc": "10",
- "vpcattachment": "10",
- "namespace": "galactic-system",
- "ipam": {
- "type": "galactic-ipam",
- "ipv6_subnet": "fd20:10:ff01::/48",
- "address_families": ["ipv6"]
- }
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "10",
+ "vpcattachment": "10",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv6_subnet": "fd20:10:ff01::/48",
+ "address_families": ["ipv6"]
+ }
+ },
+ {
+ "type": "galactic-bgp",
+ "vpc": "10",
+ "vpcattachment": "10",
+ "namespace": "galactic-system"
+ }
+ ]
}
diff --git a/deploy/containerlab/resources/tenants/ns10/iad/nad.yaml b/deploy/containerlab/resources/tenants/ns10/iad/nad.yaml
index ea2b59e5..a8a80968 100644
--- a/deploy/containerlab/resources/tenants/ns10/iad/nad.yaml
+++ b/deploy/containerlab/resources/tenants/ns10/iad/nad.yaml
@@ -5,17 +5,27 @@ metadata:
name: private
namespace: ns10
spec:
- config: |
+ config: |-
{
"cniVersion": "1.0.0",
"name": "private",
- "type": "galactic-cni",
- "vpc": "10",
- "vpcattachment": "10",
- "namespace": "galactic-system",
- "ipam": {
- "type": "galactic-ipam",
- "ipv6_subnet": "fd20:10:ff03::/48",
- "address_families": ["ipv6"]
- }
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "10",
+ "vpcattachment": "10",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv6_subnet": "fd20:10:ff03::/48",
+ "address_families": ["ipv6"]
+ }
+ },
+ {
+ "type": "galactic-bgp",
+ "vpc": "10",
+ "vpcattachment": "10",
+ "namespace": "galactic-system"
+ }
+ ]
}
diff --git a/deploy/containerlab/resources/tenants/ns10/sjc/nad.yaml b/deploy/containerlab/resources/tenants/ns10/sjc/nad.yaml
index aa48fa98..2bdccbe3 100644
--- a/deploy/containerlab/resources/tenants/ns10/sjc/nad.yaml
+++ b/deploy/containerlab/resources/tenants/ns10/sjc/nad.yaml
@@ -5,17 +5,27 @@ metadata:
name: private
namespace: ns10
spec:
- config: |
+ config: |-
{
"cniVersion": "1.0.0",
"name": "private",
- "type": "galactic-cni",
- "vpc": "10",
- "vpcattachment": "10",
- "namespace": "galactic-system",
- "ipam": {
- "type": "galactic-ipam",
- "ipv6_subnet": "fd20:10:ff02::/48",
- "address_families": ["ipv6"]
- }
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "10",
+ "vpcattachment": "10",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv6_subnet": "fd20:10:ff02::/48",
+ "address_families": ["ipv6"]
+ }
+ },
+ {
+ "type": "galactic-bgp",
+ "vpc": "10",
+ "vpcattachment": "10",
+ "namespace": "galactic-system"
+ }
+ ]
}
diff --git a/deploy/containerlab/resources/tenants/ns20/dfw/nad.yaml b/deploy/containerlab/resources/tenants/ns20/dfw/nad.yaml
index 6d9f565e..58e96428 100644
--- a/deploy/containerlab/resources/tenants/ns20/dfw/nad.yaml
+++ b/deploy/containerlab/resources/tenants/ns20/dfw/nad.yaml
@@ -5,18 +5,28 @@ metadata:
name: private
namespace: ns20
spec:
- config: |
+ config: |-
{
"cniVersion": "1.0.0",
"name": "private",
- "type": "galactic-cni",
- "vpc": "20",
- "vpcattachment": "20",
- "namespace": "galactic-system",
- "ipam": {
- "type": "galactic-ipam",
- "ipv6_subnet": "fd20:20:ff01::/48",
- "ipv4_subnet": "172.21.1.0/24",
- "address_families": ["ipv6", "ipv4"]
- }
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "20",
+ "vpcattachment": "20",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv6_subnet": "fd20:20:ff01::/48",
+ "ipv4_subnet": "172.21.1.0/24",
+ "address_families": ["ipv6", "ipv4"]
+ }
+ },
+ {
+ "type": "galactic-bgp",
+ "vpc": "20",
+ "vpcattachment": "20",
+ "namespace": "galactic-system"
+ }
+ ]
}
diff --git a/deploy/containerlab/resources/tenants/ns20/iad/nad.yaml b/deploy/containerlab/resources/tenants/ns20/iad/nad.yaml
index 852c95fa..1ec302ec 100644
--- a/deploy/containerlab/resources/tenants/ns20/iad/nad.yaml
+++ b/deploy/containerlab/resources/tenants/ns20/iad/nad.yaml
@@ -5,18 +5,28 @@ metadata:
name: private
namespace: ns20
spec:
- config: |
+ config: |-
{
"cniVersion": "1.0.0",
"name": "private",
- "type": "galactic-cni",
- "vpc": "20",
- "vpcattachment": "20",
- "namespace": "galactic-system",
- "ipam": {
- "type": "galactic-ipam",
- "ipv6_subnet": "fd20:20:ff03::/48",
- "ipv4_subnet": "172.21.10.0/24",
- "address_families": ["ipv6", "ipv4"]
- }
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "20",
+ "vpcattachment": "20",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv6_subnet": "fd20:20:ff03::/48",
+ "ipv4_subnet": "172.21.10.0/24",
+ "address_families": ["ipv6", "ipv4"]
+ }
+ },
+ {
+ "type": "galactic-bgp",
+ "vpc": "20",
+ "vpcattachment": "20",
+ "namespace": "galactic-system"
+ }
+ ]
}
diff --git a/deploy/containerlab/resources/tenants/ns20/sjc/nad.yaml b/deploy/containerlab/resources/tenants/ns20/sjc/nad.yaml
index c82099db..ecdf720a 100644
--- a/deploy/containerlab/resources/tenants/ns20/sjc/nad.yaml
+++ b/deploy/containerlab/resources/tenants/ns20/sjc/nad.yaml
@@ -5,18 +5,28 @@ metadata:
name: private
namespace: ns20
spec:
- config: |
+ config: |-
{
"cniVersion": "1.0.0",
"name": "private",
- "type": "galactic-cni",
- "vpc": "20",
- "vpcattachment": "20",
- "namespace": "galactic-system",
- "ipam": {
- "type": "galactic-ipam",
- "ipv6_subnet": "fd20:20:ff02::/48",
- "ipv4_subnet": "172.21.20.0/24",
- "address_families": ["ipv6", "ipv4"]
- }
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "20",
+ "vpcattachment": "20",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv6_subnet": "fd20:20:ff02::/48",
+ "ipv4_subnet": "172.21.20.0/24",
+ "address_families": ["ipv6", "ipv4"]
+ }
+ },
+ {
+ "type": "galactic-bgp",
+ "vpc": "20",
+ "vpcattachment": "20",
+ "namespace": "galactic-system"
+ }
+ ]
}
diff --git a/deploy/containerlab/resources/tenants/ns30/dfw/nad.yaml b/deploy/containerlab/resources/tenants/ns30/dfw/nad.yaml
index 56b05256..ef3fba15 100644
--- a/deploy/containerlab/resources/tenants/ns30/dfw/nad.yaml
+++ b/deploy/containerlab/resources/tenants/ns30/dfw/nad.yaml
@@ -5,17 +5,27 @@ metadata:
name: private
namespace: ns30
spec:
- config: |
+ config: |-
{
"cniVersion": "1.0.0",
"name": "private",
- "type": "galactic-cni",
- "vpc": "30",
- "vpcattachment": "30",
- "namespace": "galactic-system",
- "ipam": {
- "type": "galactic-ipam",
- "ipv6_subnet": "fd20:30:ff01::/48",
- "address_families": ["ipv6"]
- }
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "30",
+ "vpcattachment": "30",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv6_subnet": "fd20:30:ff01::/48",
+ "address_families": ["ipv6"]
+ }
+ },
+ {
+ "type": "galactic-bgp",
+ "vpc": "30",
+ "vpcattachment": "30",
+ "namespace": "galactic-system"
+ }
+ ]
}
diff --git a/deploy/containerlab/resources/tenants/ns40/iad/nad.yaml b/deploy/containerlab/resources/tenants/ns40/iad/nad.yaml
index 425c2292..fd610037 100644
--- a/deploy/containerlab/resources/tenants/ns40/iad/nad.yaml
+++ b/deploy/containerlab/resources/tenants/ns40/iad/nad.yaml
@@ -5,17 +5,27 @@ metadata:
name: private
namespace: ns40
spec:
- config: |
+ config: |-
{
"cniVersion": "1.0.0",
"name": "private",
- "type": "galactic-cni",
- "vpc": "40",
- "vpcattachment": "40",
- "namespace": "galactic-system",
- "ipam": {
- "type": "galactic-ipam",
- "ipv4_subnet": "172.40.10.0/24",
- "address_families": ["ipv4"]
- }
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "40",
+ "vpcattachment": "40",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv4_subnet": "172.40.10.0/24",
+ "address_families": ["ipv4"]
+ }
+ },
+ {
+ "type": "galactic-bgp",
+ "vpc": "40",
+ "vpcattachment": "40",
+ "namespace": "galactic-system"
+ }
+ ]
}
diff --git a/deploy/containerlab/resources/tenants/ns50/dfw/nad.yaml b/deploy/containerlab/resources/tenants/ns50/dfw/nad.yaml
index 574a03bb..6a458e23 100644
--- a/deploy/containerlab/resources/tenants/ns50/dfw/nad.yaml
+++ b/deploy/containerlab/resources/tenants/ns50/dfw/nad.yaml
@@ -5,19 +5,29 @@ metadata:
name: private
namespace: ns50
spec:
- config: |
+ config: |-
{
"cniVersion": "1.0.0",
"name": "private",
- "type": "galactic-cni",
- "vpc": "50",
- "vpcattachment": "50",
- "namespace": "galactic-system",
- "ipam": {
- "type": "galactic-ipam",
- "ipv4_subnet": "172.20.1.0/24",
- "address_families": ["ipv4"]
- }
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "50",
+ "vpcattachment": "50",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv4_subnet": "172.20.1.0/24",
+ "address_families": ["ipv4"]
+ }
+ },
+ {
+ "type": "galactic-bgp",
+ "vpc": "50",
+ "vpcattachment": "50",
+ "namespace": "galactic-system"
+ }
+ ]
}
---
@@ -27,7 +37,7 @@ metadata:
name: igw
namespace: ns50
spec:
- config: |
+ config: |-
{
"cniVersion": "1.0.0",
"name": "public",
diff --git a/deploy/containerlab/resources/tenants/ns50/iad/nad.yaml b/deploy/containerlab/resources/tenants/ns50/iad/nad.yaml
index 8d197bfa..a13dc6b5 100644
--- a/deploy/containerlab/resources/tenants/ns50/iad/nad.yaml
+++ b/deploy/containerlab/resources/tenants/ns50/iad/nad.yaml
@@ -5,17 +5,27 @@ metadata:
name: private
namespace: ns50
spec:
- config: |
+ config: |-
{
"cniVersion": "1.0.0",
"name": "private",
- "type": "galactic-cni",
- "vpc": "50",
- "vpcattachment": "50",
- "namespace": "galactic-system",
- "ipam": {
- "type": "galactic-ipam",
- "ipv4_subnet": "172.20.10.0/24",
- "address_families": ["ipv4"]
- }
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "50",
+ "vpcattachment": "50",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv4_subnet": "172.20.10.0/24",
+ "address_families": ["ipv4"]
+ }
+ },
+ {
+ "type": "galactic-bgp",
+ "vpc": "50",
+ "vpcattachment": "50",
+ "namespace": "galactic-system"
+ }
+ ]
}
diff --git a/deploy/containerlab/resources/tenants/ns50/sjc/nad.yaml b/deploy/containerlab/resources/tenants/ns50/sjc/nad.yaml
index 8fd60ebd..b3d88dd8 100644
--- a/deploy/containerlab/resources/tenants/ns50/sjc/nad.yaml
+++ b/deploy/containerlab/resources/tenants/ns50/sjc/nad.yaml
@@ -5,17 +5,27 @@ metadata:
name: private
namespace: ns50
spec:
- config: |
+ config: |-
{
"cniVersion": "1.0.0",
"name": "private",
- "type": "galactic-cni",
- "vpc": "50",
- "vpcattachment": "50",
- "namespace": "galactic-system",
- "ipam": {
- "type": "galactic-ipam",
- "ipv4_subnet": "172.20.20.0/24",
- "address_families": ["ipv4"]
- }
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "50",
+ "vpcattachment": "50",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv4_subnet": "172.20.20.0/24",
+ "address_families": ["ipv4"]
+ }
+ },
+ {
+ "type": "galactic-bgp",
+ "vpc": "50",
+ "vpcattachment": "50",
+ "namespace": "galactic-system"
+ }
+ ]
}
diff --git a/docs/agents/ARCHITECTURE.md b/docs/agents/ARCHITECTURE.md
index 27c2bf36..3c8736df 100644
--- a/docs/agents/ARCHITECTURE.md
+++ b/docs/agents/ARCHITECTURE.md
@@ -1,22 +1,34 @@
# Architecture
-> Galactic is the SRv6 data plane for multi-cloud VPC networking, deployed as two
-> binaries on each Kubernetes node: a CNI plugin that attaches containers to VPC
-> networks, and a router that reconciles BGP CRDs and drives an embedded
+> Galactic is the SRv6 data plane for multi-cloud VPC networking, deployed on
+> each Kubernetes node as a CNI plugin chain that attaches containers/VMs to
+> VPC networks, and a router that reconciles BGP CRDs and drives an embedded
> GoBGP server to distribute EVPN (L2VPN/EVPN AFI/SAFI) paths between nodes.
-_Last updated: 2026-07-14_
+_Last updated: 2026-08-08_
---
## Overview
Galactic implements VPC isolation and cross-cluster reachability using Linux SRv6.
-When a pod is attached to a VPC, the CNI plugin creates the required kernel state
-(VRF, veth pair, SRv6 ingress route) and writes a `BGPAdvertisement` CRD.
-`galactic-router` watches that CRD and injects the EVPN path into the node-local
-GoBGP server. GoBGP distributes the path to a BGP route reflector, enabling pods
-on different nodes or clusters to reach each other via SRv6-encapsulated traffic.
+When a pod or VM is attached to a VPC, a chain of CNI plugins creates the required
+kernel state (VRF, veth pair or tap device, host-side routes) and writes a
+`BGPAdvertisement` CRD. `galactic-router` watches that CRD and injects the EVPN
+path into the node-local GoBGP server. GoBGP distributes the path to a BGP route
+reflector, enabling pods/VMs on different nodes or clusters to reach each other
+via SRv6-encapsulated traffic.
+
+The CNI attach side is a **chain of small binaries**, not one monolithic
+plugin: a master plugin (`galactic-cni` for containers, `galactic-tap-cni` for
+VM workloads) creates the VRF and host-side interface; an optional
+`galactic-route` installs static termination routes; `galactic-bgp` publishes
+the BGP/SRv6/eBPF state. IPAM is delegated to a separate `galactic-ipam`
+binary via the standard CNI IPAM delegation protocol, not chained. All five
+binaries (plus `vmtap-cni`, `host-device`) ship in the same
+`ghcr.io/datum-cloud/galactic-cni` image and are staged onto the host by the
+same init container — see [Repository Layout](#repository-layout) and
+[Module / Package Reference](#module--package-reference) below.
VPC and VPCAttachment CRDs are owned by a separate companion operator
(`go.datum.net/cloud`). Galactic receives pre-populated identifiers through the
@@ -26,22 +38,35 @@ CNI config and acts on them. `galactic-router` reconciles BGP CRDs
### SRv6 SID encoding
-
-Each container endpoint is assigned a /128 USID (Unique Local SID, RFC 8986 Section 3.2).
-There is no longer a companion-operator-injected `srv6_sid` NAD/config field: the CNI
-itself computes the SID in `resolveSRv6SID` (`internal/cni/bgp.go`) from the node's
-`BGPRouter.spec.srv6Locator` + `spec.nodeID` plus this attachment's VRFID (`srv6.ComputeSID`,
-`internal/plumbing/srv6/usid.go`), using the End.DT46 function. If the router lacks either
-`srv6Locator` or `nodeID`, SID resolution — and SRv6 ingress setup — is skipped entirely for
-that attachment. The CNI installs an END.DT46 decap route for the computed /128 and
-advertises it in a BGP Prefix-SID path attribute (RFC 9252 SRv6 L3 Service TLV,
-`internal/runtime/gobgp/paths.go`'s `prefixSIDAttr`) — not the EVPN Type 5 route's own
-Gateway IP field, which RFC 9136 requires to share the prefix's own address family and so
-cannot carry an IPv6 SID for an IPv4 VPC prefix.
+ (SID encoding/allocation, base62 interface naming, eBPF uSID datapath,
+ EVPN Type 5 path construction, worked ContainerLab example), does not
+ exist in this repository. Either write it, point this section
+ elsewhere, or remove this note — flagged for a human decision. -->
+
+Each attachment endpoint is assigned a /128 USID (Unique Local SID, RFC 8986
+Section 3.2). There is no companion-operator-injected `srv6_sid` NAD/config
+field: `galactic-bgp` (`internal/cnibgp/bgp.go`'s `registerEBPFDatapath`)
+derives the uSID `Block` from the node's `BGPRouter.spec.srv6Locator` via
+`uformat.Block`, and registers `locator_table`/`function_table`/`vrf_table`
+entries keyed on that `Block` plus this attachment's `Argument` (its
+`BGPVRFInstance`'s VRFID) in the eBPF uSID datapath's pinned maps
+(`internal/plumbing/ebpf/usidmap`) — no kernel seg6local route is installed
+per attachment anymore; the TC-BPF program (`internal/plumbing/ebpf/prog/usid.c`)
+is the only ingress/decap path (see Known Constraints below for the cutover
+history). If the router lacks either `srv6Locator` or `nodeID`, eBPF
+registration is skipped entirely for that attachment (`registerEBPFDatapath`
+returns `registered=false`, not an error).
+
+`galactic-router`'s own reconciler independently derives the *same* SID value
+from the same inputs (`srv6.ComputeSID`, `internal/plumbing/srv6/usid.go`,
+called from `internal/reconcile/reconcile.go`) for the BGP control-plane
+side: advertising it in a BGP Prefix-SID path attribute (RFC 9252 SRv6 L3
+Service TLV, `internal/runtime/gobgp/paths.go`'s `prefixSIDAttr`) — not the
+EVPN Type 5 route's own Gateway IP field, which RFC 9136 requires to share
+the prefix's own address family and so cannot carry an IPv6 SID for an IPv4
+VPC prefix. The CNI side and the router side compute the same bit layout via
+separate code paths (`uformat` is the single source of truth both build on),
+by design — see `internal/plumbing/ebpf/doc.go`.
All nodes in the same VPC derive the same BGP Route Target by truncating the
48-bit hex VPC identifier to its low 32 bits (`uint32(v)`), formatted as
@@ -56,8 +81,13 @@ Distinguisher and import/export Route Target.
```
galactic/
├── cmd/
-│ ├── galactic-cni/ # CNI binary
-│ └── galactic-router/ # Router binary (controller-runtime reconciler)
+│ ├── galactic-cni/ # veth master plugin binary
+│ ├── galactic-tap-cni/ # tap master plugin binary (VM workloads)
+│ ├── galactic-ipam/ # delegated CNI IPAM plugin binary
+│ ├── galactic-bgp/ # BGP/SRv6/eBPF publish plugin binary
+│ ├── galactic-route/ # termination-route plugin binary (optional chain stage)
+│ ├── galactic-router/ # Router binary (controller-runtime reconciler)
+│ └── vmtap-cni/ # Cilium chain-conflist patcher for VM tap interfaces
├── internal/
│ ├── controller/ # controller-runtime reconcilers (BGPRouter, BGPPeer,
│ │ # BGPAdvertisement, BGPVRFInstance, BGPPolicy, Secret,
@@ -70,39 +100,73 @@ galactic/
│ ├── model/ # DesiredRouter and family; re-exports BGP API enums
│ ├── hash/ # SHA-256 change detection over DesiredRouter
│ ├── metadata/ # Build-time version info (Version, GitCommit, etc.)
-│ ├── gc/ # Orphaned BGPAdvertisement/BGPVRFInstance CRD and
-│ │ # stale kernel VRF cleanup, driven by the GC controller
-│ ├── cni/ # CNI cmdAdd / cmdDel / cmdCheck, PluginConf parsing,
-│ │ # BGP CRD publish, built-in IPAM wiring
-│ │ ├── ipam/ # Built-in IPv6 pool + static IP allocators
+│ ├── gc/ # Orphaned BGPAdvertisement/BGPVRFInstance CRD, stale
+│ │ # kernel VRF, and eBPF vrf_table entry cleanup,
+│ │ # driven by the GC controller
+│ ├── cni/ # galactic-cni: veth master plugin (cmdAdd/cmdDel/
+│ │ │ # cmdCheck/cmdStatus, PluginConf parsing, NAD
+│ │ │ # annotation, host-device delegation)
+│ │ ├── hostconf/ # Shared static-conflist HostConf schema/loader,
+│ │ │ # read by every binary in the chain
+│ │ ├── hostgw/ # Host-side gateway address/route configuration,
+│ │ │ # called directly by both master plugins
+│ │ ├── crdnames/ # Deterministic BGPVRFInstance/BGPAdvertisement
+│ │ │ # CRD name derivation, shared by cnibgp/gc
+│ │ ├── nadpatch/ # NAD annotation patch, shared by cni/cnitap
+│ │ ├── ipam/ # Built-in IPv6/IPv4 pool + static IP allocators,
+│ │ │ # on-disk marker-file persistence
│ │ ├── route/ # Host-side static routes via netlink
│ │ ├── tap/ # Tap interface management (VM workloads)
│ │ └── veth/ # veth pair management
+│ ├── cnitap/ # galactic-tap-cni: tap master plugin (mirrors
+│ │ # internal/cni; no host-device delegation, no
+│ │ # guest netns)
+│ ├── cniipam/ # galactic-ipam: CNI IPAM delegation protocol
+│ │ # (cmdAdd/cmdDel/cmdCheck/cmdStatus); no k8s
+│ │ # dependency
+│ ├── cnibgp/ # galactic-bgp: BGP/SRv6/eBPF publish plugin;
+│ │ # learns interface kind + addresses from
+│ │ # prevResult alone, zero kernel-interface access
+│ ├── cniroute/ # galactic-route: termination-route plugin;
+│ │ # zero Kubernetes dependency
+│ ├── vmtap/ # vmtap-cni: patches Cilium's own chain conflist
+│ │ # to add a tap-interface stage for VM workloads
│ ├── installer/ # galactic-cni DaemonSet init/run logic: binary
-│ │ # staging, conflist templating, kubeconfig
-│ │ # refresh, gRPC health server
+│ │ # staging (all binaries above, one init
+│ │ # container/image), conflist templating,
+│ │ # kubeconfig refresh, gRPC health server
│ └── plumbing/ # Low-level kernel and network primitives
│ ├── intf/ # Interface naming, base62↔hex encoding
-│ ├── srv6/ # SRv6 ingress route add/del (END.DT46)
+│ ├── srv6/ # SID computation (ComputeSID) for the router's
+│ │ # own BGP Prefix-SID path attribute
+│ ├── ebpf/ # TC-BPF uSID datapath: preflight, uformat,
+│ │ # prog (usid.c), attach, usidmap, metrics —
+│ │ # see internal/plumbing/ebpf/doc.go
│ ├── sysctl/ # Interface sysctl helpers
│ └── vrf/ # Linux VRF create/delete/lookup
├── config/ # Kustomize-composed; `kubectl apply -k config/` deploys everything
-│ ├── system/ # galactic-system namespace (shared by both components)
+│ ├── system/ # galactic-system namespace (shared by all components)
│ ├── router/ # Shared RBAC/ServiceAccount, plus:
│ │ ├── base/ # common DaemonSet spec
│ │ ├── tenant/ # per-node role: base + node affinity excluding control-plane
│ │ │ # and tenant-control nodes
│ │ └── tenant-control/ # route-reflector role: base + GALACTIC_ROUTER_REFLECTOR=true,
│ │ # opt-in via the galactic.datumapis.com/node=control node label
-│ └── cni/ # hostNetwork DaemonSet: `init` container stages
-│ # galactic-cni/host-device into /opt/cni/bin
-│ # and writes the conflist + kubeconfig; `run`
-│ # container refreshes credentials and serves
-│ # gRPC health checks
+│ ├── cni/ # hostNetwork DaemonSet: `init` container stages
+│ │ # every chain binary + host-device into
+│ │ # /opt/cni/bin and writes the conflist +
+│ │ # kubeconfig; `run` container refreshes
+│ │ # credentials, manages the eBPF datapath, and
+│ │ # serves gRPC health checks
+│ ├── vmtap/ # vmtap-cni DaemonSet: stages vmtap-cni and patches
+│ │ # Cilium's chain conflist
+│ └── fabric/ # FRR underlay eBGP DaemonSet (not part of
+│ # `kubectl apply -k config/` — see below)
├── deploy/
│ └── containerlab/ # ContainerLab lab topology and scripts
└── containers/
- ├── galactic-cni/ # galactic-cni + host-device image (e2e test and production publish)
+ ├── galactic-cni/ # galactic-cni/-tap-cni/-ipam/-bgp/-route + vmtap-cni
+ │ # + host-device image (e2e test and production publish)
└── galactic-router/ # galactic-router production image
```
@@ -128,16 +192,20 @@ See [docs/agent-startup.md](../agent-startup.md) for the router startup sequence
| `internal/runtime/frr` | `galactic-router` | FRR stub (`--mode=fabric`) — returns "not implemented" for every method |
| `internal/model` | `galactic-router` | Internal BGP model types |
| `internal/hash` | `galactic-router` | Change detection |
-| `internal/metadata` | both | Build-time version info stamped via `-ldflags` |
-| `internal/gc` | `galactic-router` | Orphaned CRD/VRF cleanup, driven by the GC controller's ticker |
-| `internal/cni` | `galactic-cni` | CNI cmdAdd / cmdDel / cmdCheck; BGP CRD publish |
-| `internal/cni/ipam` | `galactic-cni` | Built-in IPv6 pool + static allocators |
-| `internal/cni/tap` | `galactic-cni` | Tap interface create/delete (VM workloads) |
-| `internal/installer` | `galactic-cni` | DaemonSet `init`/`run` logic: binary staging, conflist/kubeconfig templating, credential refresh, gRPC health server |
-| `internal/plumbing/intf` | both | Interface naming, base62↔hex encoding |
-| `internal/plumbing/srv6` | both | SRv6 ingress route add/del (END.DT46) |
-| `internal/plumbing/vrf` | both | Linux VRF create/delete/lookup |
-| `internal/plumbing/sysctl` | both | Interface sysctl helpers |
+| `internal/metadata` | every binary | Build-time version info stamped via `-ldflags` |
+| `internal/gc` | `galactic-router` | Orphaned CRD/VRF/eBPF-entry cleanup, driven by the GC controller's ticker |
+| `internal/cni` | `galactic-cni` | Veth master plugin: cmdAdd/cmdDel/cmdCheck/cmdStatus, PluginConf parsing, NAD annotation, host-device delegation |
+| `internal/cnitap` | `galactic-tap-cni` | Tap master plugin (mirrors `internal/cni`, no guest netns) |
+| `internal/cniipam` | `galactic-ipam` | Delegated CNI IPAM plugin (no k8s dependency) |
+| `internal/cnibgp` | `galactic-bgp` | BGP/SRv6/eBPF publish plugin (zero kernel-interface dependency) |
+| `internal/cniroute` | `galactic-route` | Termination-route plugin (no k8s dependency) |
+| `internal/vmtap` | `vmtap-cni` | Cilium chain-conflist patcher for VM tap interfaces |
+| `internal/installer` | `galactic-cni` | DaemonSet `init`/`run` logic: binary staging (every chain binary), conflist/kubeconfig templating, credential refresh, gRPC health server |
+| `internal/plumbing/intf` | every CNI-chain binary + router | Interface naming, base62↔hex encoding |
+| `internal/plumbing/srv6` | `galactic-router` | SID computation (`ComputeSID`) for the BGP Prefix-SID path attribute |
+| `internal/plumbing/ebpf` | `galactic-cni` (attach/metrics via `run`), `galactic-bgp` (registration), `galactic-router` (GC sweep) | TC-BPF uSID datapath: preflight, uformat, prog, attach, usidmap, metrics |
+| `internal/plumbing/vrf` | every CNI-chain binary + router | Linux VRF create/delete/lookup |
+| `internal/plumbing/sysctl` | `galactic-cni`, `galactic-tap-cni` | Interface sysctl helpers |
---
@@ -164,18 +232,65 @@ wrap `internal/installer`.
`parseConf()` resolves node name, kubeconfig, namespace, and log file on every
invocation (conflist → env vars → API auto-detect, in that precedence).
+`galactic-cni`'s own ADD creates the VRF, veth pair, and (if `"ipam"` is
+present) delegates IPAM and configures the host gateway — it prints its own
+CNI result and returns; BGP/SRv6/eBPF publish is `galactic-bgp`'s job,
+invoked next by the CNI runtime per conflist order, not by this process.
+
Two subcommands support the DaemonSet (see Known Constraints below for the manifest):
- `init` — `--node-name`/`-n` flag (or `GALACTIC_CNI_NODE_NAME`/`NODE_NAME` env),
- calls `installer.Bootstrap(ctx, nodeName)`: stages the `galactic-cni`/`host-device`
- binaries onto the host, does a one-shot dual-stack node-identity check against the
- Kubernetes API, and writes `ca.crt`/kubeconfig plus the static conflist.
+ calls `installer.Bootstrap(ctx, nodeName)`: stages every binary in the CNI chain
+ (`galactic-cni`, `galactic-tap-cni`, `galactic-ipam`, `galactic-bgp`,
+ `galactic-route`, `host-device`) onto the host, does a one-shot dual-stack
+ node-identity check against the Kubernetes API, and writes `ca.crt`/kubeconfig
+ plus the static conflist.
- `run` — `--grpc-health-port` flag (default `5180`), calls `installer.Run(ctx,
- grpcHealthPort)`: serves gRPC health checks and periodically refreshes the
- kubeconfig token and rotates the CNI log file.
+ grpcHealthPort)`: serves gRPC health checks, manages the eBPF uSID datapath's
+ load/attach lifecycle, and periodically refreshes the kubeconfig token and
+ rotates the CNI log file.
See [docs/cni-cmd-sequence.md](../cni-cmd-sequence.md) for the full ADD/DEL sequence.
+### `cmd/galactic-tap-cni/main.go` — tap master plugin
+
+Mirrors `cmd/galactic-cni/main.go`'s CNI-invocation role exactly (`internal/cnitap.RunPlugin()`),
+minus the `init`/`run` DaemonSet subcommands — those live only on `galactic-cni`, since
+there's exactly one init container per node regardless of how many workload types it serves.
+`internal/cnitap` mirrors `internal/cni` (VRF + tap creation, NAD annotation, IPAM
+delegation, host gateway configuration) but never delegates to host-device and never
+configures a guest netns — the VM hypervisor manages the tap fd directly.
+
+### `cmd/galactic-ipam/main.go` — delegated IPAM plugin
+
+The simplest binary in the chain: no DaemonSet subcommands, no Kubernetes client, no
+node-name/kubeconfig resolution. Invoked only via the CNI IPAM delegation protocol
+(`github.com/containernetworking/plugins/pkg/ipam.ExecAdd`/`ExecDel`/`ExecCheck`), never
+directly from a conflist. `internal/cniipam.RunPlugin()` implements `cmdAdd`/`cmdDel`/
+`cmdCheck`/`cmdStatus`; allocation state persists in on-disk marker files under this
+node's own filesystem (`internal/cni/ipam.DefaultLockDir`), so `cmdDel` never needs to
+read anything back from a CRD.
+
+### `cmd/galactic-bgp/main.go` — BGP/SRv6/eBPF publish plugin
+
+Chained after the master plugin (and, when present, `galactic-route`) per conflist
+order, never run standalone. `internal/cnibgp.RunPlugin()` learns everything it needs
+— which interface kind was created, what addresses were allocated — from `prevResult`
+alone (`RawPrevResult`, not the never-populated typed `PrevResult` field — see
+`internal/cnibgp/prevresult.go`), so it has zero kernel-interface dependency of its
+own. Unlike `galactic-ipam`/`galactic-route`, it does resolve node name/kubeconfig
+(reusing `internal/config.CNIConfig`, the same `GALACTIC_CNI_*` env vars every other
+k8s-talking binary in the chain uses) since it talks to the API server for
+`BGPVRFInstance`/`BGPAdvertisement` CRUD.
+
+### `cmd/galactic-route/main.go` — termination-route plugin
+
+Optional chain stage — present only for attachments with `terminations` to install.
+`internal/cniroute.RunPlugin()` installs each termination as a VRF-table route via
+`internal/cni/route`, deriving the host device name from `(vpc, vpcAttachment)` alone
+(identical for a veth or tap master's own host interface, so no interface-kind
+inference is needed). No Kubernetes dependency at all.
+
### `cmd/galactic-router/main.go` / `root.go` — Router daemon
`main.go` is a 3-line wrapper around `newRootCommand().Execute()`; all startup logic
@@ -222,47 +337,61 @@ lives in `root.go`'s `runCmd`:
See [docs/router/configuration.md](../router/configuration.md) for the full reference, including CLI flags and precedence.
-### galactic-cni CNI config fields (`PluginConf`)
-
-| Field | Type | Description |
-|-----------------|----------|-------------------------------------------------------------------------|
-| `vpc` | string | Base62-encoded 48-bit VPC identifier |
-| `vpcattachment` | string | Base62-encoded 16-bit VPCAttachment identifier |
-| `interface_type`| string | `veth` (default) or `tap`; tap mode omits guest-side/host-device config but still runs IPAM and SRv6/BGP publish (see the ADD result section below) |
-| `namespace` | string | Kubernetes namespace for BGP CRDs; resolution order is this field → `GALACTIC_CNI_NAMESPACE` → `HostConf.Namespace` (from the conflist) → `DefaultNamespace` (`galactic-system`) |
-| `mtu` | int | MTU for the host-side interface (veth pair or tap); 0 uses kernel default |
-| `terminations` | array | Static routes to install on the host-side interface (`network`, `via`) |
-| `ipam` | object | Built-in IPv6 pool/static allocator config (Galactic has no external IPAM delegation); used identically in `veth` and `tap` mode — `tap`'s `cmdAdd` calls `allocateIPAM()` unconditionally, so omitting this without `GALACTIC_CNI_ENABLE_LOCAL_IPAM` set is not safely tolerated in tap mode. See [docs/cni/configuration.md](../cni/configuration.md). |
-
-### galactic-cni environment variables
-
-There is no longer a `--node-name`/`--enable-local-ipam` CLI flag on the plugin
-invocation path (those flags now only exist on the `init`/`run` installer
-subcommands, and only `init`'s `--node-name` overlaps in purpose). `parseConf()`
-(`internal/cni/config.go`) resolves each setting below on every ADD/DEL/CHECK/STATUS
-call, in the listed precedence, and re-exports the result as a process env var for
-the rest of the invocation:
-
-| Variable | Resolution precedence (highest first) | Default |
-|------------------------------------|--------------------------------------------------------------------------------------------------------|---------|
-| Node name (`NODE_NAME`) | `GALACTIC_CNI_NODE_NAME` → `NODE_NAME` → `HostConf.NodeName` (conflist) → `detectNodeNameFromAPI()` (matches local interface addrs against Node `InternalIP`) | _(error if still empty)_ |
-| Kubeconfig (`KUBECONFIG`) | `GALACTIC_CNI_KUBECONFIG` → `HostConf.Kubeconfig` (conflist) | `/var/lib/galactic/kubeconfig` |
-| Namespace | `conf.Namespace` (CNI config JSON) → `GALACTIC_CNI_NAMESPACE` → `HostConf.Namespace` (conflist) | `galactic-system` |
-| Log file | `GALACTIC_CNI_LOG_FILE` → `HostConf.LogFile` (conflist) | `/var/log/galactic/galactic-cni.log` |
-| Log level | `GALACTIC_CNI_LOG_LEVEL` → `HostConf.LogLevel` (conflist) | `info` |
-| `GALACTIC_CNI_ENABLE_LOCAL_IPAM` | Read directly as an env var in `parseConf()` (no conflist or CLI-flag equivalent) | `false` |
+### CNI chain config fields
+
+There is no single `PluginConf` shape anymore — each binary in the chain reads
+only the fields its own JSON stanza carries. See
+[docs/cni/configuration.md](../cni/configuration.md) for the full per-binary
+field tables and seven example conflists; summary:
+
+| Field | Read by | Description |
+|-----------------|-------------------------------------------------------|--------------------------------------------------------------------------------------------------------------|
+| `vpc` | every binary | Base62-encoded 48-bit VPC identifier |
+| `vpcattachment` | every binary | Base62-encoded 16-bit VPCAttachment identifier |
+| `namespace` | `galactic-cni`/`galactic-tap-cni`/`galactic-bgp` | Kubernetes namespace for NAD/BGP CRD lookup; resolution order is this field → `GALACTIC_CNI_NAMESPACE` → `HostConf.Namespace` → `galactic-system` |
+| `mtu` | `galactic-cni`/`galactic-tap-cni` | MTU for the host-side interface (veth pair or tap); 0 uses kernel default |
+| `ipam` | `galactic-cni`/`galactic-tap-cni` (decides whether to delegate), `galactic-ipam` (reads the block's own sub-fields) | IPAM delegation block; `type` names the delegate binary (`galactic-ipam`). See [docs/cni/configuration.md](../cni/configuration.md#ipam-fields). |
+| `terminations` | `galactic-route` only | Static routes to install on the host-side interface (`network`, `via`) |
+
+There is no `interface_type` field anymore — which binary you invoke *is* the
+interface type (`galactic-cni` → veth, `galactic-tap-cni` → tap).
+
+### galactic-cni / galactic-tap-cni / galactic-bgp environment variables
+
+There is no `--node-name`/`--enable-local-ipam` CLI flag on any plugin invocation
+path (`--node-name` exists only on `galactic-cni`'s own `init` installer
+subcommand). Each binary's own `parseConf()` resolves the settings below on every
+ADD/DEL/CHECK/STATUS call, in the listed precedence, re-exporting the result as a
+process env var. `galactic-ipam` and `galactic-route` skip this table entirely —
+see [docs/cni/configuration.md](../cni/configuration.md#runtime-configuration).
+
+| Variable | Resolution precedence (highest first) | Default | Resolved by |
+|------------------------------------|--------------------------------------------------------------------------------------------------------|---------|-------------|
+| Node name (`NODE_NAME`) | `GALACTIC_CNI_NODE_NAME` → `NODE_NAME` → `HostConf.NodeName` (conflist) → `detectNodeNameFromAPI()` (matches local interface addrs against Node `InternalIP`) | _(error if still empty)_ | `galactic-cni`, `galactic-tap-cni`, `galactic-bgp` |
+| Kubeconfig (`KUBECONFIG`) | `GALACTIC_CNI_KUBECONFIG` → `HostConf.Kubeconfig` (conflist) | `/var/lib/galactic/kubeconfig` | `galactic-cni`, `galactic-tap-cni`, `galactic-bgp` |
+| Namespace | `conf.Namespace` (CNI config JSON) → `GALACTIC_CNI_NAMESPACE` → `HostConf.Namespace` (conflist) | `galactic-system` | `galactic-cni`, `galactic-tap-cni`, `galactic-bgp` |
+| Log file | `GALACTIC_CNI_LOG_FILE` → `HostConf.LogFile` (conflist) | `/var/log/galactic/galactic-cni.log` | every binary in the chain |
+| Log level | `GALACTIC_CNI_LOG_LEVEL` → `HostConf.LogLevel` (conflist) | `info` | every binary in the chain |
+| `GALACTIC_IPAM_ENABLE_LOCAL_IPAM` | Read directly as an env var by `galactic-ipam` only (`internal/config/ipam.go`); no conflist/CLI-flag equivalent, and it can no longer manufacture an `"ipam"` block that isn't already present | `false` | `galactic-ipam` only |
+
+`GALACTIC_CNI_ENABLE_LOCAL_IPAM` (the old, master-plugin-side predecessor of the
+row above) no longer exists at all — removed along with the dead
+`config.CNIGetEnableLocalIPAM()` it backed once IPAM's own env-var handling
+moved entirely into `galactic-ipam`.
`HostConf` (`node_name`, `kubeconfig`, `namespace`, `log_file`, `log_level`) is the JSON
-shape the `init` installer subcommand writes into the `galactic-cni`-typed plugin entry
-of the conflist at `--conf-file` (see `internal/installer/installer.go` and
-Entry Points above). `log_level` (`debug`/`info`/`warn`/`error`) controls how much detail
+shape the `init` installer subcommand writes into the static conflist at
+`--conf-file` — the same file every binary in the chain reads (see
+`internal/installer/installer.go` and Entry Points above). `log_level`
+(`debug`/`info`/`warn`/`error`) controls how much detail each binary's own
`setupLogging()` emits — `info` (the default) logs one line per operation for
start/outcome plus all warnings/errors; `debug` adds per-resource milestones. See
[docs/cni/configuration.md#log-verbosity](../cni/configuration.md#log-verbosity).
-### galactic-cni ADD result
+### CNI chain ADD result
-On a successful ADD, the plugin returns a CNI spec v1.0.0 result with the following structure:
+On a successful ADD, the master plugin (`galactic-cni`/`galactic-tap-cni`)
+returns a CNI spec v1.0.0 result with the following structure (veth shown):
```json
{
@@ -289,30 +418,37 @@ On a successful ADD, the plugin returns a CNI spec v1.0.0 result with the follow
The VRF dummy interface (`G{vpc}{att}V`) is **not** reported — it is pre-existing infrastructure created by the `vrf.Add()` plumbing function, not by the CNI attachment itself.
-This is the `veth`-mode result. In `tap` mode the result has a single interface (the
-host-side tap, empty sandbox, index `0`) — there is no guest interface entry since the fd
-is handed off to the VM hypervisor, not moved into a container netns. Tap mode is **not**
-"no IPAM, no BGP": `cmdAdd` (`internal/cni/ops_add.go`) calls `allocateIPAM()` to allocate
-a subnet/gateway and `configureHostGateway()` to assign it on the host tap, includes the
-resulting `ips`/`routes` in the tap result (`buildTapResult`, interface index `0`), and then
-calls `publishBGPStateK8s()` to create the SRv6 ingress route and
-`BGPVRFInstance`/`BGPAdvertisement` CRDs — the same BGP publish step veth mode uses. The
-only things tap mode skips are host-device delegation and guest-netns configuration, since
-there is no container network namespace to move an interface into.
-
-`configureHostGateway()` assigns the IPv4 gateway as a `/25` on the host tap (vs. `/32`
-everywhere else) so it looks like a real subnet to the VM guest, adding it with
-`IFA_F_NOPREFIXROUTE` to suppress the kernel's auto-created connected route for the wider
-mask — otherwise this would reintroduce the subnet-router-anycast hazard that `/32` avoids
-elsewhere. See [docs/cni/configuration.md](../cni/configuration.md) for details.
-
-The result is printed to Multus **before** SRv6 ingress setup and BGP CRD publish run
-(see [docs/cni-cmd-sequence.md](../cni-cmd-sequence.md)) — a successful ADD response does not
-guarantee the BGPAdvertisement/BGPVRFInstance CRDs exist yet.
-
-On DEL, the result contains only `cniVersion` (empty result; DEL only deallocates the
-pod's IPAM bookkeeping and does not attempt to unwind kernel/CRD state — see the
-`cmdDel` note in [docs/cni-cmd-sequence.md](../cni-cmd-sequence.md) and Known Constraints below).
+This is the veth-master result. `galactic-tap-cni`'s own result has a single
+interface (the host-side tap, empty sandbox, index `0`) — there is no guest
+interface entry since the fd is handed off to the VM hypervisor, not moved
+into a container netns. Both masters run IPAM identically (if `"ipam"` is
+present) and both configure the host gateway (`internal/cni/hostgw`) before
+printing their own result — see
+[Interface Types](../cni/configuration.md#master-plugin-fields-galactic-cni--galactic-tap-cni)
+in the CNI config doc.
+
+The host gateway's IPv4 address is assigned as a `/25` on the host tap (vs.
+`/32` everywhere else) so it looks like a real subnet to the VM guest, adding
+it with `IFA_F_NOPREFIXROUTE` to suppress the kernel's auto-created connected
+route for the wider mask — otherwise this would reintroduce the
+subnet-router-anycast hazard that `/32` avoids elsewhere. See
+[docs/cni/configuration.md](../cni/configuration.md) for details.
+
+**Every stage after the master plugin passes `prevResult` straight through as
+its own result, unchanged** — `galactic-route` and `galactic-bgp` add no
+interfaces or IPs of their own. This means the master's own printed result is
+the runtime's authoritative CNI result for the whole chain; a successful ADD
+response does not by itself guarantee `galactic-bgp` has even run yet, let
+alone that the `BGPAdvertisement`/`BGPVRFInstance` CRDs exist (see
+[docs/cni-cmd-sequence.md](../cni-cmd-sequence.md)).
+
+On DEL, every binary's result contains only `cniVersion` (empty result). Each
+binary's own DEL only cleans up what it itself created *and* is safe to
+release immediately for that specific container — IPAM deallocation
+(`galactic-ipam`, via its own on-disk marker file) and the guest-netns
+flush/host-device DEL (`galactic-cni` only). It does not attempt to unwind
+any shared, per-attachment kernel/CRD state — see the `cmdDel` note in
+[docs/cni-cmd-sequence.md](../cni-cmd-sequence.md) and Known Constraints below.
---
@@ -327,18 +463,28 @@ pod's IPAM bookkeeping and does not attempt to unwind kernel/CRD state — see t
| `internal/runtime/frr` | galactic-router | FRR stub — returns "not implemented" for every method | No |
| `internal/model` | both | `DesiredRouter`, `DesiredPeer`, `DesiredAdvertisement`, `DesiredPolicy`, `DesiredVRFInstance`, `RuntimeStatus`; re-exports BGP API enums | No |
| `internal/hash` | galactic-router | SHA-256 fingerprint of `DesiredRouter` for no-op suppression | No |
-| `internal/metadata` | both | Build-time vars (`Version`, `GitCommit`, `GitTreeState`, `BuildDate`) stamped via `-ldflags` | No |
-| `internal/gc` | galactic-router | Collects orphaned `BGPAdvertisement`/`BGPVRFInstance` CRDs and stale kernel VRFs; invoked by the GC controller's ticker | No |
-| `internal/cni` | galactic-cni | `cmdAdd` / `cmdDel` / `cmdCheck`; CNI PluginConf parsing; BGPVRFInstance/BGPAdvertisement lifecycle; delegates kernel work to plumbing | No |
-| `internal/cni/ipam` | galactic-cni | Built-in IPv6 pool allocator (in-memory, ephemeral) and static IP allocator | Yes (pool allocations) |
-| `internal/cni/route` | galactic-cni | Host-side static route add/delete via netlink | No |
-| `internal/cni/tap` | galactic-cni | Tap interface create/delete for VM workloads (Kata, Firecracker, QEMU) | No |
+| `internal/metadata` | every binary | Build-time vars (`Version`, `GitCommit`, `GitTreeState`, `BuildDate`) stamped via `-ldflags` | No |
+| `internal/gc` | galactic-router | Collects orphaned `BGPAdvertisement`/`BGPVRFInstance` CRDs, stale kernel VRFs, and stale eBPF `vrf_table` entries; invoked by the GC controller's ticker | No |
+| `internal/cni` | galactic-cni | Veth master plugin: `cmdAdd`/`cmdDel`/`cmdCheck`/`cmdStatus`; PluginConf parsing; NAD annotation; host-device delegation; delegates kernel work to plumbing | No |
+| `internal/cni/hostconf` | every CNI-chain binary | Shared `HostConf` schema + static-conflist loader, plus API-based node-name auto-detect | No |
+| `internal/cni/hostgw` | galactic-cni, galactic-tap-cni | Host-side gateway address/route configuration for a VPC attachment's allocated IPAM addresses | No |
+| `internal/cni/crdnames` | galactic-cni, galactic-bgp, galactic-router (gc) | Deterministic `BGPVRFInstance`/`BGPAdvertisement` CRD name + annotation-key derivation | No |
+| `internal/cni/nadpatch` | galactic-cni, galactic-tap-cni | NAD annotation patch (host interface name) + pod-namespace parsing from `CNI_ARGS` | No |
+| `internal/cni/ipam` | galactic-ipam | IPv6/IPv4 pool allocators + static IP allocator; on-disk marker-file persistence (flock-guarded, keyed by containerID) | Yes (pool allocations + marker files) |
+| `internal/cni/route` | galactic-route | Host-side static route add/delete via netlink | No |
+| `internal/cni/tap` | galactic-tap-cni | Tap interface create/delete for VM workloads (Kata, Firecracker, kraftlet/Unikraft) | No |
| `internal/cni/veth` | galactic-cni | veth pair create/delete | No |
-| `internal/installer` | galactic-cni | DaemonSet `init`/`run` support: binary staging, node-identity check, conflist/kubeconfig templating, credential refresh ticker, log rotation, gRPC health server | No |
-| `internal/plumbing/intf` | both | Deterministic interface naming (`G{vpc9}{att3}V/H/G`); base62↔hex encoding | No |
-| `internal/plumbing/srv6` | galactic-cni | SRv6 END.DT46 ingress route add/delete via netlink | No |
-| `internal/plumbing/vrf` | galactic-cni | Linux VRF create/delete/lookup via netlink | No |
-| `internal/plumbing/sysctl` | galactic-cni | Per-interface sysctl helpers | No |
+| `internal/cnitap` | galactic-tap-cni | Tap master plugin (mirrors `internal/cni`; no host-device delegation, no guest netns) | No |
+| `internal/cniipam` | galactic-ipam | CNI IPAM delegation protocol (`cmdAdd`/`cmdDel`/`cmdCheck`/`cmdStatus`); explicit `"ipam"`-block contract; no k8s dependency | No |
+| `internal/cnibgp` | galactic-bgp | BGP/SRv6/eBPF publish: SID/Argument allocation + collision detection, `registerEBPFDatapath`/`unregisterEBPFDatapath`, `BGPVRFInstance`/`BGPAdvertisement` CRUD with retry; learns everything from `prevResult` | No |
+| `internal/cniroute` | galactic-route | Termination-route plugin: installs/rolls-back VRF-table routes; no k8s dependency | No |
+| `internal/vmtap` | vmtap-cni | Patches Cilium's own chain conflist to add a tap-interface stage for VM workloads | No |
+| `internal/installer` | galactic-cni | DaemonSet `init`/`run` support: binary staging (every chain binary), node-identity check, conflist/kubeconfig templating, credential refresh ticker, log rotation, eBPF datapath lifecycle, gRPC health server | No |
+| `internal/plumbing/intf` | every CNI-chain binary + router | Deterministic interface naming (`G{vpc9}{att3}V/H/G`); base62↔hex encoding | No |
+| `internal/plumbing/srv6` | galactic-router | SID computation (`ComputeSID`) for the router's own BGP Prefix-SID path attribute | No |
+| `internal/plumbing/ebpf` | galactic-cni (attach/metrics via `run`), galactic-bgp (registration), galactic-router (gc sweep) | TC-BPF uSID datapath: kernel preflight, uFMT bit-layout codec, compiled program + bindings, load/pin/attach lifecycle, map read/write API, Prometheus metrics | Yes (pinned BPF maps) |
+| `internal/plumbing/vrf` | every CNI-chain binary + router | Linux VRF create/delete/lookup via netlink | No |
+| `internal/plumbing/sysctl` | galactic-cni, galactic-tap-cni | Per-interface sysctl helpers | No |
---
@@ -349,10 +495,10 @@ pod's IPAM bookkeeping and does not attempt to unwind kernel/CRD state — see t
| `github.com/osrg/gobgp/v4` | v4.7.0 | Embedded BGP server (tenant mode) |
| `go.datum.net/network` | bumped frequently | BGP CRD API types (BGPRouter, BGPPeer, BGPAdvertisement, BGPPolicy, BGPVRFInstance) |
| `sigs.k8s.io/controller-runtime` | v0.24.1 | Reconciler framework, manager, field indexes |
-| `github.com/spf13/cobra` | v1.10.2 | CLI command/flag handling for both binaries |
-| `github.com/spf13/viper` | v1.21.0 | Config resolution (flags/env/defaults) for `galactic-router` only; `galactic-cni` resolves config itself (conflist/env/API auto-detect in `internal/cni/config.go`) and does not import viper |
+| `github.com/spf13/cobra` | v1.10.2 | CLI command/flag handling for every binary |
+| `github.com/spf13/viper` | v1.21.0 | Config resolution (flags/env/defaults) for `galactic-router` only; every CNI-chain binary resolves config itself (conflist/env/API auto-detect) and does not import viper |
| `github.com/containernetworking/cni` | v1.3.0 | CNI plugin spec, skel, invoke |
-| `github.com/containernetworking/plugins` | v1.9.1 | `host-device` plugin, delegated to for moving the guest veth into the pod netns |
+| `github.com/containernetworking/plugins` | v1.9.1 | `pkg/ipam.ExecAdd`/`ExecDel`/`ExecCheck` (real IPAM delegation to `galactic-ipam`, used by `galactic-cni`/`galactic-tap-cni`); `host-device` plugin, delegated to by `galactic-cni` for moving the guest veth into the pod netns |
| `github.com/vishvananda/netlink` | pinned pseudo-version | Linux netlink: VRF, veth, SRv6 routes |
| `github.com/kenshaw/baseconv` | v0.1.1 | Base62↔hex conversion for interface names |
| `github.com/lorenzosaino/go-sysctl` | v0.3.1 | Interface sysctl helpers |
@@ -364,15 +510,15 @@ pod's IPAM bookkeeping and does not attempt to unwind kernel/CRD state — see t
## Key Design Decisions
-- **USID per endpoint, router-side computation.** Each (VPC, VPCAttachment) pair is assigned a unique /128 USID computed entirely by the CNI (`resolveSRv6SID`/`srv6.ComputeSID`) from the owning `BGPRouter`'s `srv6Locator` + `nodeID` plus this attachment's VRFID — there is no config-supplied SID field. The CNI installs an END.DT46 decap route for that /128. VPC identity is not encoded in the SID itself — VPC scoping comes from the BGPVRFInstance's route target instead.
+- **USID per endpoint, computed independently on both sides.** Each (VPC, VPCAttachment) pair is assigned a unique /128 USID from the owning `BGPRouter`'s `srv6Locator` + `nodeID` plus this attachment's VRFID — there is no config-supplied SID field. `galactic-bgp` registers the eBPF uSID datapath's map entries for it (`internal/cnibgp/bgp.go`'s `registerEBPFDatapath`); `galactic-router` independently derives the same value (`srv6.ComputeSID`) for the BGP Prefix-SID path attribute. See [SRv6 SID encoding](#srv6-sid-encoding) above. VPC identity is not encoded in the SID itself — VPC scoping comes from the BGPVRFInstance's route target instead.
- **Base62 interface names.** Kernel interface names use the format `G{9-char-vpc-base62}{3-char-att-base62}{suffix}` (suffix: `V` = VRF, `H` = host veth/tap, `G` = guest veth pre-move), fitting in the 15-character kernel limit. The hex form is used for BGP route targets; base62 for kernel interfaces.
- **GoBGP embedded, lazy-started.** GoBGP runs in-process (`--mode=tenant` only) and starts only when the first `BGPRouter` is reconciled for that router; `Apply` re-runs on every subsequent reconcile too (subject to hash-based no-op suppression), re-applying peers/VRFs/EVPN/policies each time. `listenPort` defaults to `179`; `-1` (outbound-only) is an operator choice for specific deployments, not the codebase default. ASN or RouterID changes trigger a full `Reconfigure` (fresh `BgpServer` — `StopBgp` is not called because it permanently terminates the v4 Serve loop).
- **Overlay BGP port.** galactic-router peers connect outbound on port `1790` by default (configurable per-peer via `BGPPeer.spec.remotePort`). Port `179` is occupied by the underlay FRR `bgpd` on every node, so the overlay uses a non-conflicting port. The `BGPPeer` CRD defaults `remotePort` to `179` (the IANA BGP port); galactic-router overrides this to `1790` when the field is unset, so existing CRDs without an explicit value continue to work. Set `remotePort: 179` explicitly when peering with external BGP speakers that listen on the standard port.
-- **VRF/route-target model via BGPVRFInstance.** The CNI creates a `BGPVRFInstance` (RouteDistinguisher + import/export Route Targets, all set to the derived RT) before the `BGPAdvertisement`; `galactic-router`'s GoBGP runtime applies VRFs (`applyVRFs`) before originating EVPN paths (`applyEVPN`).
-- **CRD-driven config, no sidecar gRPC.** `galactic-router` watches BGP CRDs directly via controller-runtime. The CNI writes `BGPVRFInstance`/`BGPAdvertisement` CRDs; the router reconciler picks them up. No in-node gRPC calls between the two binaries.
+- **VRF/route-target model via BGPVRFInstance.** `galactic-bgp` creates a `BGPVRFInstance` (RouteDistinguisher + import/export Route Targets, all set to the derived RT) before the `BGPAdvertisement`; `galactic-router`'s GoBGP runtime applies VRFs (`applyVRFs`) before originating EVPN paths (`applyEVPN`).
+- **CRD-driven config, no sidecar gRPC.** `galactic-router` watches BGP CRDs directly via controller-runtime. `galactic-bgp` writes `BGPVRFInstance`/`BGPAdvertisement` CRDs; the router reconciler picks them up. No in-node gRPC calls between any of the CNI-chain binaries and `galactic-router`.
- **Hash-based no-op suppression.** SHA-256 over the sorted `DesiredRouter` prevents redundant GoBGP Apply calls on every CRD event.
- **RuntimeFactory pattern.** `--mode=tenant` (`GALACTIC_ROUTER_ROUTER_MODE=tenant`) selects GoBGP; `--mode=fabric` selects the FRR stub; `--mode=transit` is accepted by validation but returns an error at startup (not yet implemented). The mode is selected at startup; no controller changes are needed to add a new mode.
-- **DEL is intentionally minimal; GC reclaims shared state asynchronously.** `cmdDel` only deallocates the pod's IPAM bookkeeping — it does not delete the VRF, veth/tap, routes, SRv6 ingress route, or `BGPAdvertisement`/`BGPVRFInstance` CRDs, because those are keyed by `(vpc, vpcAttachment)` and may be shared/reused by another pod (deleting them in DEL would race with a concurrent ADD during pod restarts). `galactic-router`'s GC controller (ticker-driven, default every 5m) reclaims orphaned CRDs and stale kernel VRFs once no live container still references them.
+- **DEL is intentionally minimal everywhere in the CNI chain; GC reclaims shared state asynchronously.** Every binary's own `cmdDel` only cleans up what it itself created *and* is safe to release immediately per-container (IPAM deallocation via `galactic-ipam`'s own on-disk marker file; guest-netns flush/host-device DEL in `galactic-cni`). None of them delete the VRF, veth/tap, routes, the eBPF `vrf_table` entry, or `BGPAdvertisement`/`BGPVRFInstance` CRDs — those are keyed by `(vpc, vpcAttachment)` and may be shared/reused by another pod/VM (deleting them in DEL would race with a concurrent ADD during restarts). `galactic-router`'s GC controller (ticker-driven, default every 5m) reclaims orphaned CRDs, stale kernel VRFs, and stale eBPF entries once no live container still references them.
- **gRPC health, configurable port.** Liveness and readiness probes use the gRPC health protocol (`google.golang.org/grpc/health`) on a configurable port (default `5000`). No HTTP health endpoint.
---
@@ -381,11 +527,11 @@ pod's IPAM bookkeeping and does not attempt to unwind kernel/CRD state — see t
| Layer | Command | Framework | Scope |
|------------|------------------|---------------------|------------------------------------------------------------------------|
-| Unit | `task test:unit` | `go test -race` | `internal/cni` (`cni_test.go`, `bgp_test.go`, `netns_test.go` — `buildResult`, `parseConf`, `routeTarget`, `lookupBGPRouter`), `internal/cni/{ipam,tap,veth}`, `internal/installer` (`installer_test.go` — `Bootstrap`/`Run` with mocked k8s client and netlink/host paths), `internal/plumbing/srv6`, `internal/gc`, `internal/reconcile`, `internal/controller`, `internal/plumbing/intf`, `internal/metadata`, `internal/runtime/gobgp` (partial), `internal/runtime/frr` |
-| E2E | `task test:e2e` | Kind + `go test` | Full BGPRouter lifecycle in a Kind cluster; builds and loads image |
+| Unit | `task test:unit` | `go test -race` | `internal/cni`, `internal/cnitap`, `internal/cniipam`, `internal/cnibgp`, `internal/cniroute` (`buildResult`/`buildVethResult`, `parseConf`, `routeTarget`, `lookupBGPRouter`, `inferFromPrevResult` — each package's own `cmdAdd`/`cmdDel`/`cmdCheck`/`cmdStatus`), `internal/cni/{hostconf,hostgw,crdnames,nadpatch,ipam,route,tap,veth}`, `internal/installer` (`installer_test.go` — `Bootstrap`/`Run` with mocked k8s client and netlink/host paths), `internal/plumbing/{srv6,ebpf}`, `internal/gc`, `internal/reconcile`, `internal/controller`, `internal/plumbing/intf`, `internal/metadata`, `internal/runtime/gobgp` (partial), `internal/runtime/frr` |
+| E2E | `task test:e2e` | Kind + `go test` | `galactic-tap-cni`'s own ADD (VRF + tap + IPAM delegation), kernel capability checks, CNI VERSION report. Does **not** exercise `galactic-route` or `galactic-bgp` (no BGPRouter fixture in the e2e suite) — see Known Constraints below. Full BGPRouter lifecycle coverage for `galactic-router` comes from this same Kind cluster's separate reconciler tests. |
| CI full | `task ci` | all of the above | lint → build → test:unit → test:e2e |
-`internal/plumbing/vrf` has no unit tests — it requires `CAP_NET_ADMIN` and a real kernel. `internal/cni` and `internal/plumbing/srv6` now have unit coverage for their pure-logic paths (this used to not be the case). `internal/plumbing/intf` is pure-function and fully unit-testable.
+`internal/plumbing/vrf` has no unit tests — it requires `CAP_NET_ADMIN` and a real kernel. `internal/cni/route` (wrapped by `internal/cniroute`) also has no unit tests of its own. `internal/plumbing/intf` is pure-function and fully unit-testable.
---
@@ -412,10 +558,12 @@ Runs on every PR and push to `main`. Two tiers:
- **GoBGP RIB is ephemeral.** All BGP state is in-process memory. On restart, sessions and paths must be re-established from CRD state; controller-runtime's reconcile loop handles this automatically.
- **EVPN Type 5 is implemented, not deferred.** `internal/runtime/gobgp/paths.go`'s `buildEVPNPaths` builds real `EVPNIPPrefixRoute` NLRIs, deriving the Route Distinguisher from `routerID + ":0"` (not from the CRD). The `BGPVRFInstance` CRD carries its own explicit `RouteDistinguisher` and import/export Route Targets (see Key Design Decisions above), applied via `internal/runtime/gobgp/runtime.go`'s `applyVRFs`. There is no `ErrMissingRouteDistinguisher` or similar rejection path in the current code.
-- **`cmdDel` does not tear down shared kernel/CRD state.** By design (see Key Design Decisions above) — cleanup of VRF, veth/tap, routes, SRv6 ingress, and BGP CRDs is deferred to `galactic-router`'s asynchronous GC controller, not performed synchronously in `cmdDel`.
-- **`internal/plumbing/vrf` has no unit tests.** It requires `CAP_NET_ADMIN` and a real kernel. `internal/cni` and `internal/plumbing/srv6` do now have unit coverage for their pure-logic paths. `internal/plumbing/intf` is fully unit-testable (pure functions only). Kernel-path coverage otherwise comes from the e2e suite (`task test:e2e`).
+- **No binary's `cmdDel` tears down shared kernel/CRD state.** By design (see Key Design Decisions above) — cleanup of VRF, veth/tap, routes, the eBPF `vrf_table` entry, and BGP CRDs is deferred to `galactic-router`'s asynchronous GC controller, not performed synchronously in any chain binary's `cmdDel`.
+- **`internal/plumbing/vrf` and `internal/cni/route` have no unit tests.** `vrf` requires `CAP_NET_ADMIN` and a real kernel; `route` (wrapped by `internal/cniroute`, which does have its own tests) was never backfilled with tests of its own when the CNI plugin-chain split moved its caller out of `internal/cni`. `internal/plumbing/intf` is fully unit-testable (pure functions only). Kernel-path coverage otherwise comes from the e2e suite (`task test:e2e`).
+- **The e2e suite doesn't cover `galactic-route` or `galactic-bgp`.** `TestCNITapInterface` (`tests/e2e/e2e_test.go`) only drives `galactic-tap-cni`'s own ADD — verifying BGP/SRv6/eBPF publish end-to-end would need a `BGPRouter` CRD fixture and additional RBAC the test doesn't set up. This gap predates the CNI plugin-chain split too: the monolithic `galactic-cni` this replaced was never e2e-verified past its own CNI result shape either.
+- **`docs/agents/ARCHITECTURE.md` and `docs/cni-cmd-sequence.md` describe the CNI chain; `vmtap-cni`/`internal/vmtap` (a separate, Cilium-chain-conflist-patching binary for VM tap interfaces, unrelated to the `galactic-cni`/`galactic-tap-cni`/`galactic-ipam`/`galactic-bgp`/`galactic-route` chain) has its own doc at `docs/vmtap-cni/configuration.md`** — cross-referenced from [Repository Layout](#repository-layout) and the [Module / Package Reference](#module--package-reference) table above, but not otherwise elaborated on in this document.
- **`--mode=transit` is unimplemented.** Accepted by CLI/env validation, but `runCmd` returns an error at startup ("mode=transit is not yet supported").
-- **`galactic-cni`'s install DaemonSet is a Go installer, not a shell script.** `config/cni/configmap.yaml`/`install.sh` were deleted; `config/cni/daemonset.yaml` now runs `hostNetwork: true` with an `install-cni` init container (`command: ["/galactic-cni", "init"]`, calling `installer.Bootstrap`) and a `credential-refresh` main container (`command: ["/galactic-cni", "run"]`, calling `installer.Run`), both on the same image (see CI/CD above). `Bootstrap` writes the CNI binaries to `/opt/cni/bin`, the static conflist to `/etc/cni/net.d/10-galactic.conflist`, and `ca.crt`/kubeconfig to `/var/lib/galactic` (chosen over `/etc/galactic` specifically so it lands under `/var`, the one path immutable-root distros like Talos allow hostPath writes to without a host-level `extraMounts` entry); `Run` refreshes the kubeconfig token every 300s and rotates the CNI log once it exceeds 10MB. `/opt/cni/bin` is fixed by the CNI/kubelet plugin-discovery convention and can't be relocated by this DaemonSet alone — on Talos it needs its own `extraMounts` entry in the machine config if it isn't writable by default. The `run` container also serves gRPC health checks on port `5180` (`livenessProbe`/`readinessProbe` in the DaemonSet spec), and `config/cni/rbac.yaml` grants `get` on `nodes` for `Bootstrap`'s node-identity check.
+- **`galactic-cni`'s install DaemonSet is a Go installer, not a shell script.** `config/cni/configmap.yaml`/`install.sh` were deleted; `config/cni/daemonset.yaml` now runs `hostNetwork: true` with an `install-cni` init container (`command: ["/galactic-cni", "init"]`, calling `installer.Bootstrap`) and a `credential-refresh` main container (`command: ["/galactic-cni", "run"]`, calling `installer.Run`), both on the same image (see CI/CD above). `Bootstrap` writes every binary in the CNI chain to `/opt/cni/bin`, the static conflist to `/etc/cni/net.d/10-galactic.conflist`, and `ca.crt`/kubeconfig to `/var/lib/galactic` (chosen over `/etc/galactic` specifically so it lands under `/var`, the one path immutable-root distros like Talos allow hostPath writes to without a host-level `extraMounts` entry); `Run` refreshes the kubeconfig token every 300s and rotates the CNI log once it exceeds 10MB. `/opt/cni/bin` is fixed by the CNI/kubelet plugin-discovery convention and can't be relocated by this DaemonSet alone — on Talos it needs its own `extraMounts` entry in the machine config if it isn't writable by default. The `run` container also serves gRPC health checks on port `5180` (`livenessProbe`/`readinessProbe` in the DaemonSet spec), and `config/cni/rbac.yaml` grants `get` on `nodes` for `Bootstrap`'s node-identity check.
- **The uSID TC-BPF datapath (`internal/plumbing/ebpf/prog/usid.c`) doesn't generate PMTUD ICMPv6 errors.** When `bpf_fib_lookup()` returns `BPF_FIB_LKUP_RET_FRAG_NEEDED` (egress route's MTU is smaller than the inner packet), the program counts `DROP_REASON_FIB_FRAG_NEEDED` and silently drops (`TC_ACT_SHOT`) rather than sending an ICMPv6 Packet Too Big back to the original sender — unlike the static-route `SEG6_LOCAL_ACTION_END_DT46` path this datapath replaces, where the kernel's own IPv6 stack emits that ICMP message. Accepted as a known cost of the TC-BPF cutover for this milestone; generating ICMPv6 PTB from the datapath itself is unscheduled future work, not planned for a specific milestone yet.
---
@@ -426,9 +574,15 @@ Runs on every PR and push to `main`. Two tiers:
| Concern | Start here |
|--------------------------------------------|--------------------------------------------------------------|
-| CNI attach/detach flow | `internal/cni/ops_add.go:cmdAdd`, `internal/cni/ops_del.go:cmdDel` (`internal/cni/cni.go` only holds `RunPlugin`) |
-| CNI runtime config resolution (conflist/env/API auto-detect) | `internal/cni/config.go:parseConf`, `loadHostConf`, `detectNodeNameFromAPI` |
-| BGP CRD publish (VRF + advertisement) | `internal/cni/bgp.go:publishBGPState` |
+| CNI master-plugin attach/detach flow (veth) | `internal/cni/ops_add.go:cmdAdd`, `internal/cni/ops_del.go:cmdDel` (`internal/cni/cni.go` only holds `RunPlugin`) |
+| CNI master-plugin attach/detach flow (tap) | `internal/cnitap/ops_add.go:cmdAdd`, `internal/cnitap/ops_del.go:cmdDel` (mirrors `internal/cni`) |
+| CNI runtime config resolution (conflist/env/API auto-detect) | `internal/cni/config.go:parseConf`, `loadHostConf`, `internal/cni/hostconf.DetectNodeNameFromAPI` |
+| IPAM delegation (master plugin side) | `internal/cni/result.go:configureIPAM` (`ipam.ExecAdd`), `internal/cni/ops_del.go:cmdDel` (`ipam.ExecDel`) |
+| IPAM delegation protocol (delegate side) | `internal/cniipam/ops.go:cmdAdd`/`cmdDel`, `internal/cniipam/allocate.go` |
+| Termination-route chain stage | `internal/cniroute/ops_add.go:cmdAdd`, `internal/cni/route/route.go` |
+| BGP CRD publish (VRF + advertisement) + eBPF registration | `internal/cnibgp/bgp.go:publishBGPState`, `registerEBPFDatapath`; entry point `internal/cnibgp/ops_add.go:cmdAdd` |
+| How `galactic-bgp`/`galactic-route` learn state without touching the kernel | `internal/cnibgp/prevresult.go:inferFromPrevResult` (reads `RawPrevResult`, not the dead `PrevResult` field) |
+| Host gateway address/route configuration (shared by both master plugins) | `internal/cni/hostgw/hostgw.go:ConfigureHostGateway` |
| CNI DaemonSet install/refresh | `internal/installer/installer.go:Bootstrap` (init container), `internal/installer/installer.go:Run` (long-running container) |
| CRD → BGP translation | `internal/reconcile/reconcile.go:BuildDesiredRouter` |
| BGP runtime application (GoBGP) | `internal/runtime/gobgp/runtime.go:Apply` |
@@ -453,6 +607,7 @@ Runs on every PR and push to `main`. Two tiers:
- `peerStatusRequeue = 30s` periodic requeue keeps BGPPeer session state current because BGP FSM transitions are not Kubernetes events.
- `annotationConfigHash` is persisted on the BGPRouter object (not just in memory) so no-op detection survives pod restarts without re-applying GoBGP config.
- GoBGP `Reconfigure()` calls `old.Stop()` then creates a fresh `BgpServer` — it does NOT call the BGP-level `StopBgp`/`StartBgp` on the old server, avoiding the v4 "Serve loop permanently dead" problem.
-- The CNI ADD result is printed to Multus **before** SRv6 ingress setup and BGP CRD publish run (`publishBGPState` is called after `PrintResult` inside `cmdAdd`) — a successful ADD response does not by itself guarantee the BGP CRDs exist yet.
-- `cmdDel` never deletes the VRF, veth/tap, routes, SRv6 ingress route, or `BGPAdvertisement`/`BGPVRFInstance` CRDs — only IPAM bookkeeping. Shared-resource cleanup is entirely the GC controller's job (`internal/gc`), to avoid racing a concurrent ADD during pod restarts.
-- Production images are published by `.github/workflows/publish.yaml` as two separate per-binary images (`galactic-cni`, `galactic-router`), not one shared image — see CI/CD above.
+- The master plugin's ADD result is the runtime's authoritative CNI result for the whole chain — `galactic-route`/`galactic-bgp` (chained after it) both pass `prevResult` straight through unchanged. A successful ADD response does not by itself guarantee `galactic-bgp` has even run yet, let alone that the BGP CRDs exist.
+- `types.PluginConf.PrevResult` (from `containernetworking/cni/pkg/types`) has JSON tag `"-"` and is **never populated** by plain `json.Unmarshal`; only the sibling `RawPrevResult map[string]interface{}` field actually receives the previous plugin's result. A pre-existing quirk of that library, not specific to this codebase — every CNI-chain package that reads prevResult (`internal/cni/ops_check.go`, `internal/cnibgp/prevresult.go`, `internal/cniroute/ops_add.go`) reads `RawPrevResult` for this reason.
+- No binary's `cmdDel` deletes the VRF, veth/tap, routes, the eBPF `vrf_table` entry, or `BGPAdvertisement`/`BGPVRFInstance` CRDs — each binary's own DEL only handles its own per-container bookkeeping (IPAM deallocation, guest-netns/host-device cleanup). Shared-resource cleanup is entirely the GC controller's job (`internal/gc`), to avoid racing a concurrent ADD during pod restarts.
+- Production images are published by `.github/workflows/publish.yaml` as two separate per-binary images (`galactic-cni`, `galactic-router`), not one shared image — see CI/CD above. `galactic-cni`'s own image carries all five CNI-chain binaries plus `vmtap-cni`/`host-device`, not just `galactic-cni` itself.
diff --git a/docs/cni-cmd-sequence.md b/docs/cni-cmd-sequence.md
index e810b623..3be37f74 100644
--- a/docs/cni-cmd-sequence.md
+++ b/docs/cni-cmd-sequence.md
@@ -1,8 +1,38 @@
# CNI cmdAdd / cmdDel Sequence Diagrams
-Per-interface-type sequence diagrams for the galactic-cni ADD and DEL paths.
-
-## cmdAdd — veth
+Per-binary sequence diagrams for the galactic CNI plugin chain's ADD and DEL
+paths.
+
+## Chain overview
+
+The chain is one master plugin, plus up to two optional plugins invoked
+after it in conflist order:
+
+1. **Master** — `galactic-cni` (veth, for containers) or `galactic-tap-cni`
+ (tap, for VM workloads: Kata, Firecracker, kraftlet/Unikraft). Creates
+ the VRF and host-side interface, annotates the NAD, delegates IPAM (if
+ an `"ipam"` block is present) and — for `galactic-cni` only —
+ host-device (to move the guest veth into the container netns), then
+ configures the host gateway address/route and prints the CNI result.
+2. **`galactic-route`** (optional — present only when the attachment has
+ `terminations`) — installs each termination as a route into the VRF
+ table, then passes `prevResult` through unchanged.
+3. **`galactic-bgp`** — publishes BGP/SRv6/eBPF state: `BGPVRFInstance`,
+ `BGPAdvertisement`, and (when this node's `BGPRouter` has SRv6
+ configured) the eBPF uSID datapath's `vrf_table` registration. Learns
+ everything it needs (interface kind, allocated addresses) from
+ `prevResult` alone — it never touches a kernel interface. Passes
+ `prevResult` through unchanged as the final CNI result.
+
+Every binary's own `cmdDel` is a no-op beyond binary-local, per-container
+cleanup (IPAM deallocation, guest netns flush, host-device DEL — all
+`galactic-cni`/`galactic-tap-cni` only). Shared node-level state (VRF,
+host interface, routes, SRv6/eBPF registration, `BGPVRFInstance`,
+`BGPAdvertisement`) is kept because it may still be in use by another
+pod/VM on the same `(vpc, vpcAttachment)`; `galactic-router`'s GC
+controller reclaims it once nothing references it anymore.
+
+## cmdAdd — veth (galactic-cni → galactic-route → galactic-bgp)
```mermaid
sequenceDiagram
@@ -11,7 +41,7 @@ sequenceDiagram
activate CNI
CNI->>CNI: parseConf()
- CNI->>CNI: resourceTracker{vpc, attachment, veth}
+ CNI->>CNI: resourceTracker{vpc, attachment}
CNI->>VRF: Add(vpc, attachment)
activate VRF
@@ -28,16 +58,8 @@ sequenceDiagram
Veth-->>CNI: ok
deactivate Veth
- loop terminations
- CNI->>Route: Add(vpc, attachment, network, via, dev)
- activate Route
- Route->>Route: netlink.RouteAdd in VRF table
- Route-->>CNI: ok
- deactivate Route
- end
-
- CNI->>CNI: buildVethResult()
- activate CNI
+ CNI->>K8s: newK8sClient()
+ CNI->>K8s: AnnotateNAD(name, podNamespace, hostName)
Note over CNI: host-device delegation
CNI->>HostDevice: ADD (move guest veth to container netns)
@@ -45,47 +67,72 @@ sequenceDiagram
HostDevice-->>CNI: ok
deactivate HostDevice
- CNI->>CNI: configureIPAM()
- CNI->>CNI: allocateIPAM() -> subnet + gateway
- CNI->>NetNS: configureInterfaceInNetns(guest, subnet, gw)
- activate NetNS
- NetNS->>NetNS: AddrAdd(subnet), LinkSetUp, RouteAdd(default via gw)
- NetNS-->>CNI: ok
- deactivate NetNS
+ opt "ipam" block present
+ CNI->>IPAM: ExecAdd(ipam.type, stdin)
+ activate IPAM
+ IPAM->>IPAM: allocate subnet/address (pool or static)
+ IPAM-->>CNI: CNI IPAM result
+ deactivate IPAM
+ CNI->>NetNS: configureInterfaceInNetns(guest, subnet, gw)
+ activate NetNS
+ NetNS->>NetNS: AddrAdd(subnet), LinkSetUp, RouteAdd(default via gw)
+ NetNS-->>CNI: ok
+ deactivate NetNS
+ end
CNI->>CNI: readGuestInterface(MAC, MTU)
+ CNI->>HostGW: ConfigureHostGateway(vpc, attachment, ipamResult, guestMAC)
+ activate HostGW
+ HostGW->>HostGW: AddrAdd(gateway) on host veth
+ HostGW->>HostGW: RouteAdd(pod subnet) in VRF table
+ HostGW->>HostGW: NeighSet(pod IP -> guest MAC), permanent
+ HostGW-->>CNI: ok
+ deactivate HostGW
+
CNI->>CNI: buildResult() + PrintResult()
+ CNI-->>Runtime: CNI result (JSON) — becomes next plugin's prevResult
deactivate CNI
- CNI->>CNI: publishBGPState()
- activate CNI
-
- CNI->>CNI: configureHostGateway(vpc, attachment, ipamResult)
- CNI->>CNI: AddrAdd(gateway/128) on host veth
- CNI->>CNI: RouteAdd(subnet to host veth) in VRF table
+ opt attachment has terminations
+ Runtime->>Route: ADD (stdin config, prevResult)
+ activate Route
+ Route->>Route: parseConf(); require prevResult
+ loop terminations
+ Route->>Route: route.Add(vpc, attachment, network, via, dev)
+ end
+ Route-->>Runtime: prevResult, unchanged
+ deactivate Route
+ end
- CNI->>CNI: decode VPC hex + VRFID
- CNI->>K8s: newK8sClient()
+ Runtime->>BGP: ADD (stdin config, prevResult)
+ activate BGP
+ BGP->>BGP: parseConf(); inferFromPrevResult(prevResult)
+ Note over BGP: interface kind + IPAM result inferred from
prevResult shape alone — no kernel access
+ BGP->>K8s: newK8sClient()
+
+ BGP->>BGP: publishBGPState() (retry loop)
+ activate BGP
+ BGP->>K8s: lookupBGPRouter(node)
+ BGP->>K8s: allocateArgument() -> VRFID
+ BGP->>K8s: CreateOrUpdate BGPVRFInstance
+ BGP->>K8s: checkArgumentCollision()
+ opt router has srv6Locator + nodeID
+ BGP->>eBPF: registerEBPFDatapath(block, argument, vrfTableID, egressKind)
+ activate eBPF
+ eBPF->>eBPF: locator_table / function_table / vrf_table entries
+ eBPF-->>BGP: ok
+ deactivate eBPF
+ end
+ BGP->>K8s: CreateOrUpdate BGPAdvertisement(prefixes, annotations)
+ deactivate BGP
- CNI->>CNI: publishBGPStateK8s() (retry loop)
- activate CNI
- CNI->>K8s: lookupBGPRouter(node)
- CNI->>CNI: resolveSRv6SID(locator, nodeID, vrfID)
- CNI->>SRv6: RouteIngressAdd(sid, vpc, attachment)
- activate SRv6
- SRv6->>SRv6: seg6local End.DT46 route
- SRv6-->>CNI: ok
- deactivate SRv6
- CNI->>K8s: CreateOrUpdate BGPVRFInstance
- CNI->>K8s: CreateOrUpdate BGPAdvertisement(prefix, annotations)
- CNI-->>Runtime: ok
- deactivate CNI
+ BGP-->>Runtime: prevResult, unchanged
+ deactivate BGP
Runtime-->>Runtime: CNI result (JSON)
- deactivate CNI
```
-## cmdAdd — tap
+## cmdAdd — tap (galactic-tap-cni → galactic-route → galactic-bgp)
```mermaid
sequenceDiagram
@@ -94,7 +141,7 @@ sequenceDiagram
activate CNI
CNI->>CNI: parseConf()
- CNI->>CNI: resourceTracker{vpc, attachment, tap}
+ CNI->>CNI: resourceTracker{vpc, attachment}
CNI->>VRF: Add(vpc, attachment)
activate VRF
@@ -111,48 +158,76 @@ sequenceDiagram
Tap-->>CNI: ok
deactivate Tap
- loop terminations
- CNI->>Route: Add(vpc, attachment, network, via, dev)
- activate Route
- Route->>Route: netlink.RouteAdd in VRF table
- Route-->>CNI: ok
- deactivate Route
- end
+ CNI->>K8s: newK8sClient()
+ CNI->>K8s: AnnotateNAD(name, podNamespace, hostName)
- Note over CNI: tap branch - no host-device, no guest netns
+ Note over CNI: tap branch — no host-device, no guest netns
- CNI->>CNI: allocateIPAM() -> subnet + gateway
- CNI->>CNI: configureHostGateway(vpc, attachment, ipamResult)
- CNI->>CNI: AddrAdd(gateway/128) on host tap
- CNI->>CNI: RouteAdd(subnet to host tap) in VRF table
+ opt "ipam" block present
+ CNI->>IPAM: ExecAdd(ipam.type, stdin)
+ activate IPAM
+ IPAM->>IPAM: allocate subnet/address (pool or static)
+ IPAM-->>CNI: CNI IPAM result
+ deactivate IPAM
+ end
+
+ CNI->>HostGW: ConfigureHostGateway(vpc, attachment, ipamResult, nil)
+ activate HostGW
+ HostGW->>HostGW: AddrAdd(gateway, /25 + NOPREFIXROUTE) on host tap
+ HostGW->>HostGW: RouteAdd(pod subnet) in VRF table
+ Note over HostGW: no guest MAC in tap mode — no neighbor entry installed
+ HostGW-->>CNI: ok
+ deactivate HostGW
CNI->>CNI: buildTapResult(ipamResult) + PrintResult()
+ CNI-->>Runtime: CNI result (JSON) — becomes next plugin's prevResult
+ deactivate CNI
- CNI->>CNI: decode VPC hex + VRFID
- CNI->>K8s: newK8sClient()
+ opt attachment has terminations
+ Runtime->>Route: ADD (stdin config, prevResult)
+ activate Route
+ Route->>Route: parseConf(); require prevResult
+ loop terminations
+ Route->>Route: route.Add(vpc, attachment, network, via, dev)
+ end
+ Route-->>Runtime: prevResult, unchanged
+ deactivate Route
+ end
- CNI->>CNI: publishBGPStateK8s() (retry loop)
- activate CNI
- CNI->>K8s: lookupBGPRouter(node)
- CNI->>CNI: resolveSRv6SID(locator, nodeID, vrfID)
- CNI->>SRv6: RouteIngressAdd(sid, vpc, attachment)
- activate SRv6
- SRv6->>SRv6: seg6local End.DT46 route
- SRv6-->>CNI: ok
- deactivate SRv6
- CNI->>K8s: CreateOrUpdate BGPVRFInstance
- CNI->>K8s: CreateOrUpdate BGPAdvertisement(prefix, annotations)
- CNI-->>Runtime: ok
- deactivate CNI
+ Runtime->>BGP: ADD (stdin config, prevResult)
+ activate BGP
+ BGP->>BGP: parseConf(); inferFromPrevResult(prevResult)
+ Note over BGP: single interface, empty sandbox -> ifaceType = tap
+ BGP->>K8s: newK8sClient()
+
+ BGP->>BGP: publishBGPState() (retry loop)
+ activate BGP
+ BGP->>K8s: lookupBGPRouter(node)
+ BGP->>K8s: allocateArgument() -> VRFID
+ BGP->>K8s: CreateOrUpdate BGPVRFInstance
+ BGP->>K8s: checkArgumentCollision()
+ opt router has srv6Locator + nodeID
+ BGP->>eBPF: registerEBPFDatapath(block, argument, vrfTableID, egressKind)
+ activate eBPF
+ eBPF->>eBPF: locator_table / function_table / vrf_table entries
+ eBPF-->>BGP: ok
+ deactivate eBPF
+ end
+ BGP->>K8s: CreateOrUpdate BGPAdvertisement(prefixes, annotations)
+ deactivate BGP
+
+ BGP-->>Runtime: prevResult, unchanged
+ deactivate BGP
Runtime-->>Runtime: CNI result (JSON)
- deactivate CNI
```
-## cmdDel — veth / tap (shared)
+## cmdDel — every binary in the chain
-Both interface types share the same DEL path. Per the CNI spec, DEL is
-idempotent — missing resources are never errors.
+Per the CNI spec, DEL is idempotent — missing resources are never errors.
+The runtime calls DEL on every chain entry that had a successful ADD, in
+reverse order; every one of those calls is independently idempotent, so
+the order doesn't matter for correctness.
```mermaid
sequenceDiagram
@@ -166,16 +241,17 @@ sequenceDiagram
CNI-->>Runtime: nil
deactivate CNI
else parse succeeds
- alt hasIPAM
- CNI->>K8s: newK8sClient()
- alt k8s client OK
- CNI->>CNI: deallocateIPAM()
- CNI->>K8s: Get BGPAdvertisement -> read subnet annotation
- CNI->>IPAM: PoolAllocator.Deallocate(subnet)
- end
+ alt "ipam" block present (galactic-cni/galactic-tap-cni only)
+ CNI->>IPAM: ExecDel(ipam.type, stdin)
+ activate IPAM
+ IPAM->>IPAM: look up this containerID's own marker file, delete it
+ IPAM-->>CNI: ok
+ deactivate IPAM
end
- Note over CNI: Shared resources (VRF, interface, routes, SRv6,
BGPAdvertisement, BGPVRFInstance) are NOT deleted here.
They may be in use by another pod on the same (vpc, attachment).
The GC controller collects orphans periodically.
+ Note over CNI: galactic-cni only: flush the guest netns'
address/route, then forward DEL to host-device
+
+ Note over CNI,BGP: Shared resources (VRF, host interface, routes,
eBPF vrf_table entry, BGPAdvertisement, BGPVRFInstance) are
NOT deleted by any binary's DEL — they may be in use by
another pod/VM on the same (vpc, attachment).
galactic-router's GC controller collects orphans periodically.
CNI->>CNI: slog.Info("DEL: skipping shared resource cleanup (handled by GC)")
CNI->>CNI: print empty result
diff --git a/docs/cni/configuration.md b/docs/cni/configuration.md
index e29507aa..5313bbd9 100644
--- a/docs/cni/configuration.md
+++ b/docs/cni/configuration.md
@@ -1,24 +1,74 @@
# CNI Configuration
-`galactic-cni` is configured through the CNI JSON configuration passed by Multus
-(or any CNI manager), plus node-local settings resolved at runtime from the
-conflist, environment variables, and (as a last resort) the Kubernetes API.
+The galactic CNI plugin chain is configured through the CNI JSON conflist passed
+by Multus (or any CNI manager), plus node-local settings resolved at runtime from
+the static conflist, environment variables, and (for `galactic-cni`/`galactic-tap-cni`/
+`galactic-bgp`, as a last resort) the Kubernetes API.
-> Last verified: 2026-07-28 against the current working tree of `internal/cni/config.go`,
-> `internal/cni/ipam_ops.go`, and `internal/installer/installer.go`.
+> Last verified: 2026-08-08 against the current working tree of `internal/cni/`,
+> `internal/cnitap/`, `internal/cniipam/`, `internal/cnibgp/`, `internal/cniroute/`,
+> and `internal/installer/installer.go`.
+
+## Chain structure
+
+Each `NetworkAttachmentDefinition` (or other Multus-driven config) supplies a
+full CNI conflist — a `"plugins"` array, not a single plugin object — because
+BGP/SRv6/eBPF publish now runs as its own chained binary rather than inline
+inside the master plugin. A real-world attachment's conflist has this shape:
+
+```json
+{
+ "cniVersion": "1.0.0",
+ "name": "private",
+ "plugins": [
+ { "type": "galactic-cni", "...": "..." },
+ { "type": "galactic-route", "...": "..." },
+ { "type": "galactic-bgp", "...": "..." }
+ ]
+}
+```
+
+- **`galactic-cni`** (veth, containers) or **`galactic-tap-cni`** (tap, VM
+ workloads) — always first. Creates the VRF and host-side interface,
+ annotates the NAD, delegates IPAM (if an `"ipam"` block is present) and
+ — `galactic-cni` only — host-device (to move the guest veth into the
+ container netns), configures the host gateway, and prints the CNI
+ result.
+- **`galactic-route`** — optional; include only when the attachment has
+ `terminations` to install. Installs each as a VRF-table route, then
+ passes `prevResult` through unchanged.
+- **`galactic-bgp`** — publishes `BGPVRFInstance`/`BGPAdvertisement` and
+ (when this node's `BGPRouter` has SRv6 configured) the eBPF uSID
+ datapath's `vrf_table` registration. Learns everything it needs
+ (interface kind, allocated addresses) from `prevResult` alone. In
+ practice this stage is not optional — without it the attachment is
+ never BGP-advertised and stays unreachable from other nodes — but
+ nothing enforces its presence at the CNI-config level; that's the
+ conflist author's responsibility (the companion operator, cross-repo,
+ out of scope here).
+
+Every binary's own JSON stanza carries only the fields that binary itself
+reads (`vpc`/`vpcattachment` are duplicated across every stanza; nothing
+else is). See [docs/cni-cmd-sequence.md](../cni-cmd-sequence.md) for the
+full ADD/DEL sequence across all three stages.
## Runtime Configuration
-There is no `--node-name` or `--enable-local-ipam` CLI flag on the `galactic-cni`
-plugin invocation itself. Instead, `parseConf()` (`internal/cni/config.go`) resolves
-node name, kubeconfig, namespace, log file, and log level on every ADD/DEL/CHECK/STATUS
-call, reading (in order) the CNI config JSON, environment variables, and a `HostConf`
-block parsed out of the conflist at `--conf-file` (default
-`/etc/cni/net.d/10-galactic.conflist`). `HostConf` is written by the `galactic-cni init`
-subcommand (`internal/installer.Bootstrap`), which runs as the CNI DaemonSet's init
-container — see [docs/agents/ARCHITECTURE.md](../agents/ARCHITECTURE.md#known-constraints)
+There is no `--node-name` or similar CLI flag on any plugin's own invocation.
+Instead, each binary's own `parseConf()` resolves node-level settings on every
+ADD/DEL/CHECK/STATUS call, reading (in order) the CNI config JSON, environment
+variables, and a `HostConf` block parsed out of the conflist at `--conf-file`
+(default `/etc/cni/net.d/10-galactic.conflist` — this is the one *static*,
+node-level conflist every binary shares; not the per-attachment `plugins[]`
+conflist described above). `HostConf` is written by the `galactic-cni init`
+subcommand (`internal/installer.Bootstrap`), which runs as the CNI DaemonSet's
+init container — see [docs/agents/ARCHITECTURE.md](../agents/ARCHITECTURE.md#known-constraints)
for how the DaemonSet stages it.
+`galactic-ipam` and `galactic-route` have no Kubernetes dependency at all, so
+they resolve only `LogFile`/`LogLevel` from `HostConf` — never `NodeName` or
+`Kubeconfig`, and never fall back to the Kubernetes API.
+
### `HostConf` fields (written into the conflist by `galactic-cni init`)
| Field | Description |
@@ -26,89 +76,88 @@ for how the DaemonSet stages it.
| `NodeName` | The Kubernetes node name the installer bootstrapped on. |
| `Kubeconfig` | Path to the kubeconfig `Bootstrap`/`Run` maintain (`/var/lib/galactic/kubeconfig` by default). |
| `Namespace` | Kubernetes namespace for BGP CRDs (`galactic-system` by default). |
-| `LogFile` | Path the plugin logs to (`/var/log/galactic/galactic-cni.log` by default). |
+| `LogFile` | Path the plugin logs to (`/var/log/galactic/galactic-cni.log` by default, shared across every binary in the chain). |
| `LogLevel` | Verbosity of plugin logging: `debug`, `info`, `warn`, or `error` (`info` by default). See [Log verbosity](#log-verbosity) below. |
### Resolution precedence
-| Setting | Precedence (highest first) | Default (if nothing resolves) |
-| ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ |
-| Node name | `GALACTIC_CNI_NODE_NAME` env → `NODE_NAME` env → `HostConf.NodeName` → auto-detect via the Kubernetes API (`detectNodeNameFromAPI`: lists Nodes, matches local interface addresses against `status.addresses[].type=InternalIP`) | _(error: "node name is required")_ |
-| Kubeconfig | `GALACTIC_CNI_KUBECONFIG` env → `HostConf.Kubeconfig` | `/var/lib/galactic/kubeconfig` |
-| Namespace | `namespace` field in the CNI config JSON → `GALACTIC_CNI_NAMESPACE` env → `HostConf.Namespace` | `galactic-system` |
-| Log file | `GALACTIC_CNI_LOG_FILE` env → `HostConf.LogFile` | `/var/log/galactic/galactic-cni.log` |
-| Log level | `GALACTIC_CNI_LOG_LEVEL` env → `HostConf.LogLevel` | `info` |
-| Enable local IPAM | `GALACTIC_CNI_ENABLE_LOCAL_IPAM` env only (no conflist field, no CLI flag) | `false` |
-
-The resolved node name is re-exported as the `NODE_NAME` process environment variable
-and the resolved kubeconfig as `KUBECONFIG`, since other code in `internal/cni` reads
-those directly. Auto-detection exists to tolerate environments (e.g. Kind-based e2e)
-where the conflist's hostPath mount isn't populated yet.
+| Setting | Precedence (highest first) | Default (if nothing resolves) | Resolved by |
+| ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------- | -------------------------------------------------- |
+| Node name | `GALACTIC_CNI_NODE_NAME` env → `NODE_NAME` env → `HostConf.NodeName` → auto-detect via the Kubernetes API (`detectNodeNameFromAPI`: lists Nodes, matches local interface addresses against `status.addresses[].type=InternalIP`) | _(error: "node name is required")_ | `galactic-cni`, `galactic-tap-cni`, `galactic-bgp` |
+| Kubeconfig | `GALACTIC_CNI_KUBECONFIG` env → `HostConf.Kubeconfig` | `/var/lib/galactic/kubeconfig` | `galactic-cni`, `galactic-tap-cni`, `galactic-bgp` |
+| Namespace | `namespace` field in the CNI config JSON → `GALACTIC_CNI_NAMESPACE` env → `HostConf.Namespace` | `galactic-system` | `galactic-cni`, `galactic-tap-cni`, `galactic-bgp` |
+| Log file | `GALACTIC_CNI_LOG_FILE` env → `HostConf.LogFile` | `/var/log/galactic/galactic-cni.log` | every binary in the chain |
+| Log level | `GALACTIC_CNI_LOG_LEVEL` env → `HostConf.LogLevel` | `info` | every binary in the chain |
+
+`GALACTIC_CNI_*` env var names are shared as-is across every binary that
+resolves node-level settings — there's no per-binary prefix for these, since
+they're the same physical node's settings regardless of which chain binary
+reads them (unlike `GALACTIC_IPAM_ENABLE_LOCAL_IPAM` below, which is a
+domain-specific knob belonging entirely to `galactic-ipam`).
+
+The resolved node name is re-exported as the `NODE_NAME` process environment
+variable and the resolved kubeconfig as `KUBECONFIG` (`galactic-cni`/
+`galactic-tap-cni`/`galactic-bgp` only), since other code in those packages
+reads those directly. Auto-detection exists to tolerate environments (e.g.
+Kind-based e2e) where the conflist's hostPath mount isn't populated yet.
### Log verbosity
-`setupLogging()` (`internal/cni/config.go`) builds a JSON `slog` handler at the
-resolved level. Since each CNI invocation is a fresh, short-lived process, this
-level is re-resolved on every ADD/DEL/CHECK/STATUS call — there's no persistent
-daemon to reconfigure at runtime.
+Each binary's own `setupLogging()` builds a JSON `slog` handler at the
+resolved level. Since every CNI invocation is a fresh, short-lived process,
+this level is re-resolved on every ADD/DEL/CHECK/STATUS call, in every
+binary — there's no persistent daemon to reconfigure at runtime, and every
+binary shares the same log file by default so a single chain invocation's
+log lines interleave in call order.
-| Level | What's logged |
-| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `debug` | Everything: per-resource milestones (VRF/interface/route/IPAM ready, BGP CRDs applied, kernel-level veth/tap/route operations) in addition to `info` and above. Use this when troubleshooting a specific ADD/DEL failure. |
-| `info` (default) | One line per operation marking start and outcome (`ADD: starting` / `ADD: BGP state published`, `DEL: starting` / `DEL: skipping shared resource cleanup`, `CHECK: starting` / `CHECK: passed`/`failed`, `STATUS: probing API server reachability` / `STATUS: ready`), plus all `warn`/`error` events. |
-| `warn` | Recoverable anomalies only: stale-state repairs (leftover veth/tap from a prior failed ADD), iptables-missing fallback, k8s API retries. |
-| `error` | Failures only. |
+| Level | What's logged |
+| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| `debug` | Everything: per-resource milestones (VRF/interface/route/IPAM ready, BGP CRDs applied, kernel-level operations) in addition to `info` and above. Use this when troubleshooting a specific ADD/DEL failure. |
+| `info` (default) | One line per operation marking start and outcome (`ADD: starting` / `ADD: BGP state published`, `DEL: starting` / `DEL: skipping shared resource cleanup`, `CHECK: starting` / `CHECK: passed`/`failed`, `STATUS: ready`), plus all `warn`/`error` events. |
+| `warn` | Recoverable anomalies only: stale-state repairs, k8s API retries. |
+| `error` | Failures only. |
An unrecognized `log_level` value does not fail the CNI operation — it logs a
warning and falls back to `info`.
-### `GALACTIC_CNI_ENABLE_LOCAL_IPAM`
-
-When enabled, the plugin performs IP allocation using a built-in IPv6 pool
-allocator even when no explicit `ipam` block is present in the CNI config.
-This is useful for simple deployments that do not need an external IPAM
-plugin.
+### `GALACTIC_IPAM_ENABLE_LOCAL_IPAM`
-When local IPAM is active but the config does not specify pool parameters,
-the following defaults are used:
+Read only by `galactic-ipam` (`internal/config/ipam.go`) — renamed from the
+historical `GALACTIC_CNI_ENABLE_LOCAL_IPAM`, which no longer exists at all.
+The old name could manufacture an `"ipam"` block out of thin air even when
+the master plugin's own config had none; the new one can't; it only fills
+in a default IPv6 pool CIDR when an `"ipam"` block is present but specifies
+neither `static_ip` nor a subnet:
| Parameter | Default |
-| ------------- | ---------------------------------------- |
-| Pool CIDR | `fd00:10:ff01::/48` |
-| Subnet length | `/96` |
-| Gateway | First usable address in the pool (`::1`) |
+| ------------- | ----------------------------------------- |
+| Pool CIDR | `fd00:10:ff01::/64` |
+| Subnet length | `/96` |
+| Gateway | First usable address in the pool (`::1`) |
-If an explicit `ipam` block is present in the CNI config, it takes precedence
-and this environment variable has no effect on the allocation behavior.
+Whether IPAM runs at all is decided **solely** by whether `"ipam"` is present
+in the master plugin's own stanza — no environment variable, on either side
+of this rename, can trigger or suppress that decision. See [IPAM Fields](#ipam-fields)
+below and `internal/cniipam`'s package doc comment for the full explicit
+contract.
**Type:** bool
**Default:** `false`
-## CNI Configuration JSON
-
-The CNI configuration is a JSON object passed at pod creation time. It extends
-the standard CNI `PluginConf` with Galactic-specific fields.
-
-### Top-Level Fields
+## Master Plugin Fields (`galactic-cni` / `galactic-tap-cni`)
-| Field | Required | Type | Description |
-| ---------------- | -------- | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `vpc` | **Yes** | `string` | Base62-encoded VPC identifier (48-bit value). Used to derive VRF names, interface names, and BGP route targets. |
-| `vpcattachment` | **Yes** | `string` | Base62-encoded VPC attachment identifier (16-bit value). Paired with `vpc` for deterministic VRF/BGP naming. |
-| `interface_type` | No | `string` | Interface mode: `"veth"` (default, for containers) or `"tap"` (for VMs such as Kata, Firecracker, QEMU). Both modes run IPAM and SRv6/BGP publish; `tap` mode only skips host-device delegation and guest-netns configuration (see the Tap mode section below). |
-| `mtu` | No | `int` | MTU for the host-side interface. For `veth` mode this applies to both veth endpoints; for `tap` mode it applies to the tap interface. |
-| `namespace` | No | `string` | Kubernetes namespace used to look up the `BGPRouter` CRD. Resolution order: this field → `GALACTIC_CNI_NAMESPACE` env → `HostConf.Namespace` (conflist) → `galactic-system`. See [Runtime Configuration](#runtime-configuration) above. |
-| `ipam` | No* | `IPAM` | Legacy static-IP / local-IPAM configuration block (see IPAM sub-fields below). Only `type: "static"` still drives its own allocation path; `type: "pool"` is otherwise superseded by `ipv6_subnet`/`ipv4_subnet` below. *Required unless `GALACTIC_CNI_ENABLE_LOCAL_IPAM`, `ipv6_subnet`, or `ipv4_subnet` is set — applies identically in `veth` and `tap` mode. In `tap` mode `cmdAdd` (`internal/cni/ops_add.go`) calls `allocateIPAM` unconditionally (unlike `veth` mode, which checks first), so a config satisfying none of those currently produces a nil-pointer panic in `tap` mode rather than a clean validation error — always set one of them for tap. |
-| `ipv6_subnet` | No* | `string` | Region IPv6 pool CIDR for the NAD-driven pool-IPAM path; endpoints allocate a `/96` from it by default. Setting this field or `ipv4_subnet` (or both) opts a config into pool IPAM directly — no `ipam` block needed. See [Pool IPAM via `ipv6_subnet`/`ipv4_subnet`](#pool-ipam-via-ipv6_subnetipv4_subnet) below. |
-| `ipv4_subnet` | No | `string` | Optional site IPv4 pool CIDR; endpoints allocate a `/32` host address from it. May be set alone (IPv4-only), alongside `ipv6_subnet` (dual-stack), or omitted entirely (IPv6-only, given `ipv6_subnet` is set). |
-| `address_families` | No | `[]string` | Families to record as in-use: any of `"ipv6"`, `"ipv4"`. Defaults to `["ipv6"]` when omitted. Validated at parse time, but the families actually allocated are driven by which of `ipv6_subnet`/`ipv4_subnet` are set — keep this field consistent with those. |
+| Field | Required | Type | Description |
+| --------------- | -------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `vpc` | **Yes** | `string` | Base62-encoded VPC identifier (48-bit value). Used to derive VRF names, interface names, and BGP route targets. |
+| `vpcattachment` | **Yes** | `string` | Base62-encoded VPC attachment identifier (16-bit value). Paired with `vpc` for deterministic VRF/BGP naming. |
+| `mtu` | No | `int` | MTU for the host-side interface. For `galactic-cni` this applies to both veth endpoints; for `galactic-tap-cni` it applies to the tap interface. |
+| `namespace` | No | `string` | Kubernetes namespace used for NAD lookup (and, for `galactic-bgp`'s own stanza, `BGPRouter`/BGP CRD lookup). Resolution order: this field → `GALACTIC_CNI_NAMESPACE` env → `HostConf.Namespace` → `galactic-system`. |
+| `ipam` | No | `IPAM` | IPAM delegation block (see [IPAM Fields](#ipam-fields) below). Presence alone decides whether IPAM runs at all — no env var or sibling field can trigger or suppress it. |
Standard CNI fields (`cniVersion`, `name`, `dns`, `runtimeConfig`) are also
-supported via the embedded `types.PluginConf`. `galactic-cni` declares support
+supported via the embedded `types.PluginConf`. Both binaries declare support
for the full CNI spec range (`version.All`, from `github.com/containernetworking/cni`
-v1.3.0 in `go.mod`) and returns CNI Result `1.0.0` (`type100`); generated
-configs (the installer's default conflist, Multus `NetworkAttachmentDefinition`
-manifests) use `"cniVersion": "1.0.0"`.
+v1.3.0 in `go.mod`) and return CNI Result `1.0.0` (`type100`).
Despite that broad declared range, `cniVersion` must be `"1.0.0"` or `"1.1.0"`
in practice: `galactic-bgp`, chained after the master plugin, reconstructs
@@ -120,38 +169,32 @@ ADD fail for every attachment in the chain. Every config in this doc already
uses `"1.0.0"`; keep it that way for any config authored outside these
examples.
-`terminations` (static routes to add on the host side) is **not** a
-`galactic-cni`/`galactic-tap-cni` field — it belongs to `galactic-route`, the
-chained plugin invoked after the master plugin per conflist order (see
-[Termination Fields](#termination-fields) below). Putting a `terminations`
-array in the master's own stanza does nothing: `galactic-cni`'s slimmer
-`PluginConf` silently drops the unknown field on unmarshal.
+There is no `interface_type` field anymore: which binary you invoke *is* the
+interface type. `galactic-cni` always creates a veth pair; `galactic-tap-cni`
+always creates a tap device. There is likewise no `terminations` field on
+either master plugin's own stanza anymore — that field now lives entirely on
+`galactic-route`'s own stanza (see [Termination Fields](#termination-fields)
+below).
-### Interface Types
-
-#### `veth` (default)
+### `galactic-cni` (veth)
Creates a veth pair: one endpoint stays in the host namespace (named
`GH`, e.g. `G0000000010010H`)
and the other is moved into the container via the host-device CNI plugin
(renamed to the `CNI_IFNAME` value, typically `eth0`). The guest interface
-receives an IP address from IPAM and a default route via the pool gateway.
+receives an IP address from IPAM (if configured) and a default route via the
+pool gateway.
-#### `tap`
+### `galactic-tap-cni` (tap)
Creates a tap interface in the host namespace (same naming pattern as the veth
host endpoint: `GH`) and enslaves it to the VRF. No
-interface is moved into the container — the tap fd is managed directly by the
-guest VM hypervisor, so `tap` mode skips host-device delegation and guest-netns
-configuration. Unlike an earlier version of this plugin, `tap` mode is **not**
-"no IPAM, no BGP": `cmdAdd` calls `allocateIPAM` to allocate a subnet/gateway,
-`configureHostGateway` to assign the gateway on the host tap and install the pod
-subnet route into the VRF table, includes the resulting `ips`/`routes` in the CNI
-result (interface index `0`, the host tap — there is no guest interface entry),
-and then `publishBGPStateK8s` to create the SRv6 ingress route and
-`BGPVRFInstance`/`BGPAdvertisement` CRDs, exactly as `veth` mode does. The guest
-VM still configures its own IP addresses independently (the CNI-allocated
-subnet/gateway describe only the host-side BGP-advertised state).
+interface is moved into a container — the tap fd is managed directly by the
+guest VM hypervisor (Kata, Firecracker, kraftlet/Unikraft), so this binary
+never delegates to host-device and never configures a guest netns. It still
+runs IPAM (if `"ipam"` is present) and configures the host gateway exactly as
+`galactic-cni` does; the CNI result carries a single interface (the host tap,
+empty sandbox) since there's no guest-side interface entry.
The IPv4 gateway address on the host tap is a `/25`, not the `/32` used
everywhere else (veth's host/guest gateways, and the pod's own address in both
@@ -160,34 +203,51 @@ bare host route. Because a wider mask would normally make the kernel
auto-install a connected route for the whole `/25` in the VRF table — exactly
the subnet-router-anycast hazard the `/32` choice exists to avoid elsewhere —
the address is added with `IFA_F_NOPREFIXROUTE`, which suppresses that
-auto-created route. The explicit pod-subnet `/32` route `configureHostGateway`
+auto-created route. The explicit pod-subnet `/32` route `hostgw.ConfigureHostGateway`
installs remains the only route governing delivery to the VM's address.
-> **Note:** Tap mode is intended for VM-based workloads (Kata, Firecracker,
-> QEMU) where the hypervisor opens the tap fd and handles guest networking.
-
-### IPAM Fields
-
-| Field | Required | Type | Description |
-| ------------ | ---------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| `type` | Conditionally required | `string` | `"pool"` or `"static"`. Required whenever an `ipam` block is present — an `ipam` block with an empty `type` is a hard error. `"static"` drives its own allocation path (below); `"pool"` is otherwise vestigial now that pool allocation is driven by `ipv6_subnet`/`ipv4_subnet` (see below) — its former `pool`/`gateway`/`subnet_len` sub-fields have been retired. |
-| `static_ip` | Conditionally required | `string` | A single IPv6 address to assign when `type` is `"static"`. |
-
-#### IPAM `type=static`
-
-Validates and assigns a single IPv6 address with a `/64` mask. No deallocation
-needed, and no IPv4 address is ever allocated alongside it — static IPAM is a
-single fixed address, not a dual-stack pool.
+## IPAM Fields
+
+`"ipam"`'s `type` names the **delegated binary** (per the CNI IPAM delegation
+protocol, `github.com/containernetworking/plugins/pkg/ipam.ExecAdd`/`ExecDel`/
+`ExecCheck`) — currently only `galactic-ipam` exists, so this is always
+`"galactic-ipam"` in practice. It is not a pool-vs-static mode selector: mode
+is decided entirely by which of the fields below are present.
+
+| Field | Required | Type | Description |
+| ------------------ | -------- | ---------- | ---------------------------------------------------------------------------------------------------------------------- |
+| `type` | **Yes** | `string` | Names the delegated CNI IPAM binary. Always `"galactic-ipam"` today. |
+| `static_ip` | No | `string` | A single IPv6 address to assign. Presence selects the static allocation path (below); mutually exclusive in practice with the subnet fields. |
+| `ipv6_subnet` | No | `string` | Region IPv6 pool CIDR; endpoints allocate a `/96` from it by default. |
+| `ipv4_subnet` | No | `string` | Site IPv4 pool CIDR; endpoints allocate a `/32` host address from it. |
+| `address_families` | No | `[]string` | Families to record as in-use: any of `"ipv6"`, `"ipv4"`. Defaults to `["ipv6"]`. Validated at parse time — keep this consistent with which of `ipv6_subnet`/`ipv4_subnet` are set. |
+| `routes` | No | `[]Route` | Declared on the `IPAM` struct (`dst`, `gw`) but not read by any current allocation path — vestigial. |
+| `addresses` | No | `[]Address`| Declared on the `IPAM` struct (`address`) but not read by any current allocation path — vestigial. |
+
+Whether IPAM runs at all is decided **solely** by `"ipam"` block presence in
+the master plugin's own stanza — no environment variable can trigger or
+suppress that (see [`GALACTIC_IPAM_ENABLE_LOCAL_IPAM`](#galactic_ipam_enable_local_ipam)
+above, which only fills a default when the block is present but
+under-specified). Once delegated to, `static_ip` presence selects the static
+path; otherwise `ipv6_subnet`/`ipv4_subnet` (either alone, or both) select the
+pool path. See `internal/cniipam`'s package doc comment for the full explicit
+contract.
+
+### IPAM `static_ip`
+
+Validates and assigns a single IPv6 address with a `/64` mask. No IPv4 address
+is ever allocated alongside it — static IPAM is a single fixed address, not a
+dual-stack pool.
### Pool IPAM via `ipv6_subnet`/`ipv4_subnet`
-This is the NAD-driven path most VPC attachments use (`wantsIPAM`/`allocatePoolIPAM`
-in `internal/cni/ipam_ops.go`). Either `ipv6_subnet` or `ipv4_subnet` alone is
-sufficient to opt a config into pool IPAM — an `ipam` block is not required, and
-neither field depends on the other being set:
+This is the NAD-driven path most VPC attachments use (`allocatePool` in
+`internal/cniipam/allocate.go`, backed by `internal/cni/ipam`'s pool
+allocators). Either field alone is sufficient — neither depends on the other
+being set:
-- **IPv6-only:** set `ipv6_subnet`, omit `ipv4_subnet`. Allocates a `/96` subnet
- from the region pool; no IPv4 address is allocated.
+- **IPv6-only:** set `ipv6_subnet`, omit `ipv4_subnet`. Allocates a `/96`
+ subnet from the region pool; no IPv4 address is allocated.
- **IPv4-only:** set `ipv4_subnet`, omit `ipv6_subnet`. Allocates a `/32` host
address from the site pool; no IPv6 subnet is allocated, and the resulting
`BGPAdvertisement` carries only the IPv4 `/32` prefix.
@@ -195,55 +255,71 @@ neither field depends on the other being set:
`BGPAdvertisement` carries both prefixes and the CNI result carries both
`IPConfig`/route entries.
-Allocation is in-memory and thread-safe per pool; allocations are ephemeral
-(lost on process restart). `cmdDel` looks up each family's allocated address
-independently from its own `BGPAdvertisement` CRD annotation, so cleanup of
-one family never depends on the other having been allocated.
+Allocation state persists in on-disk marker files under `galactic-ipam`'s own
+lock directory (`internal/cni/ipam.DefaultLockDir`, flock-guarded, keyed by
+containerID — both address families). `galactic-ipam` never needs a
+Kubernetes client for this: `cmdDel` looks up and removes its own
+containerID's marker file directly, with no dependency on a `BGPAdvertisement`
+CRD annotation (that coupling existed before this allocator gained its own
+persistence and has since been removed).
-When neither `ipv6_subnet` nor `ipv4_subnet` is set and `GALACTIC_CNI_ENABLE_LOCAL_IPAM`
+When neither `ipv6_subnet` nor `ipv4_subnet` is set and `GALACTIC_IPAM_ENABLE_LOCAL_IPAM`
is enabled, allocation falls back to the built-in default IPv6 pool CIDR (see
-[`GALACTIC_CNI_ENABLE_LOCAL_IPAM`](#galactic_cni_enable_local_ipam) above) —
+[`GALACTIC_IPAM_ENABLE_LOCAL_IPAM`](#galactic_ipam_enable_local_ipam) above) —
this fallback is IPv6-only; there is no default IPv4 pool.
-### Termination Fields
+## Termination Fields (`galactic-route`)
+
+`galactic-route`'s own stanza carries only `vpc`, `vpcattachment`, and
+`terminations` — no `namespace` field, since this binary has no Kubernetes
+dependency at all. Include this stanza in the chain only for attachments that
+actually need static routes; it's the one stage in the chain that's genuinely
+optional.
-`terminations` is a field of `galactic-route`'s own conflist stanza — the
-chained CNI plugin invoked after `galactic-cni`/`galactic-tap-cni` and before
-`galactic-bgp` per conflist order, present only for attachments that need
-static routes installed on the host side. Each entry in the array has the
-following fields:
+Each entry in `terminations` has:
| Field | Required | Type | Description |
| --------- | -------- | -------- | ------------------------------------------------------------------------------------------ |
| `network` | **Yes** | `string` | CIDR prefix for a static route (e.g. `"fd00::/48"`). |
| `via` | No | `string` | Next-hop gateway IP. If omitted, a link-local route is installed via the host-side device. |
-`galactic-route`'s `cmdAdd` installs routes into the VRF table for each
-termination entry. `cmdDel` is a no-op — routes are keyed by
-`(vpc, vpcattachment)` and may still be in use by another pod/VM sharing the
-same attachment, so cleanup is left entirely to `galactic-router`'s GC
-controller.
+Used in `cmdAdd` to install routes into the VRF table for each termination
+entry, via the host-side interface name derived from `(vpc, vpcAttachment)`
+alone — identical whether the preceding master plugin was `galactic-cni` or
+`galactic-tap-cni`. `cmdDel` is a no-op: like every other shared, per-attachment
+resource in the chain, termination routes may still be in use by another pod/VM
+on the same attachment, so cleanup is left to `galactic-router`'s GC controller.
+
+## BGP Publish Fields (`galactic-bgp`)
+
+`galactic-bgp`'s own stanza carries only `vpc`, `vpcattachment`, and
+`namespace` — nothing else. It learns which interface kind was created and
+what addresses were allocated entirely from `prevResult` (the accumulated
+result of every preceding plugin in the chain), never from its own config or
+a kernel call.
## Example Configurations
+Every example below is a full conflist (a `NetworkAttachmentDefinition`'s
+`spec.config`, or an equivalent standalone conflist file) — not a single
+plugin object — per [Chain structure](#chain-structure) above.
+
### Minimal configuration (overlay)
```json
{
"cniVersion": "1.0.0",
"name": "galactic",
- "type": "galactic-cni",
- "vpc": "1",
- "vpcattachment": "1"
+ "plugins": [
+ { "type": "galactic-cni", "vpc": "1", "vpcattachment": "1" },
+ { "type": "galactic-bgp", "vpc": "1", "vpcattachment": "1" }
+ ]
}
```
-Omits `namespace` (defaults to `galactic-system`) and `ipam`, and has no
-`galactic-route` chain entry (see [Configuration with terminations](#configuration-with-terminations)
-below) since there are no static routes to install. Without
-`GALACTIC_CNI_ENABLE_LOCAL_IPAM` set, no IP address is assigned to the
-guest interface. With `GALACTIC_CNI_ENABLE_LOCAL_IPAM` set, a subnet is allocated
-from the built-in pool.
+Omits `namespace` (defaults to `galactic-system`), `ipam`, and a
+`galactic-route` stage. Without `GALACTIC_IPAM_ENABLE_LOCAL_IPAM` set, no IP
+address is assigned to the guest interface.
### Pool IPAM, IPv6-only (testvpc)
@@ -251,55 +327,75 @@ from the built-in pool.
{
"cniVersion": "1.0.0",
"name": "testvpc",
- "type": "galactic-cni",
- "vpc": "10",
- "vpcattachment": "10",
- "namespace": "galactic-system",
- "ipv6_subnet": "fd00:10:ff02::/48",
- "address_families": ["ipv6"]
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "10",
+ "vpcattachment": "10",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv6_subnet": "fd00:10:ff02::/48",
+ "address_families": ["ipv6"]
+ }
+ },
+ { "type": "galactic-bgp", "vpc": "10", "vpcattachment": "10", "namespace": "galactic-system" }
+ ]
}
```
-No `ipam` block needed — `ipv6_subnet` alone opts the config into pool IPAM.
-
### Pool IPAM, dual-stack
```json
{
"cniVersion": "1.0.0",
"name": "vpc21",
- "type": "galactic-cni",
- "vpc": "21",
- "vpcattachment": "21",
- "namespace": "galactic-system",
- "ipv6_subnet": "fd00:10:ff03::/48",
- "ipv4_subnet": "172.21.1.0/24",
- "address_families": ["ipv6", "ipv4"]
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "21",
+ "vpcattachment": "21",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv6_subnet": "fd00:10:ff03::/48",
+ "ipv4_subnet": "172.21.1.0/24",
+ "address_families": ["ipv6", "ipv4"]
+ }
+ },
+ { "type": "galactic-bgp", "vpc": "21", "vpcattachment": "21", "namespace": "galactic-system" }
+ ]
}
```
Allocates from both pools independently; the `BGPAdvertisement` carries both
the IPv6 `/96` and IPv4 `/32` prefixes.
-### Pool IPAM, IPv4-only
+### Pool IPAM, IPv4-only (tap)
```json
{
"cniVersion": "1.0.0",
"name": "vpc20",
- "type": "galactic-cni",
- "vpc": "20",
- "vpcattachment": "20",
- "interface_type": "tap",
- "namespace": "galactic-system",
- "ipv4_subnet": "172.20.1.0/24",
- "address_families": ["ipv4"]
+ "plugins": [
+ {
+ "type": "galactic-tap-cni",
+ "vpc": "20",
+ "vpcattachment": "20",
+ "namespace": "galactic-system",
+ "ipam": {
+ "type": "galactic-ipam",
+ "ipv4_subnet": "172.20.1.0/24",
+ "address_families": ["ipv4"]
+ }
+ },
+ { "type": "galactic-bgp", "vpc": "20", "vpcattachment": "20", "namespace": "galactic-system" }
+ ]
}
```
`ipv4_subnet` alone opts the config into pool IPAM with no IPv6 allocation at
-all — no `ipv6_subnet` is required, and the resulting `BGPAdvertisement`
-carries only the IPv4 `/32` prefix.
+all; the resulting `BGPAdvertisement` carries only the IPv4 `/32` prefix.
### Static IP configuration
@@ -307,13 +403,15 @@ carries only the IPv4 `/32` prefix.
{
"cniVersion": "1.0.0",
"name": "galactic",
- "type": "galactic-cni",
- "vpc": "1",
- "vpcattachment": "1",
- "ipam": {
- "type": "static",
- "static_ip": "fd00:1::1"
- }
+ "plugins": [
+ {
+ "type": "galactic-cni",
+ "vpc": "1",
+ "vpcattachment": "1",
+ "ipam": { "type": "galactic-ipam", "static_ip": "fd00:1::1" }
+ },
+ { "type": "galactic-bgp", "vpc": "1", "vpcattachment": "1" }
+ ]
}
```
@@ -331,10 +429,7 @@ carries only the IPv4 `/32` prefix.
"type": "galactic-cni",
"vpc": "1",
"vpcattachment": "1",
- "ipam": {
- "type": "pool",
- "pool": "fd00:1:ff01::/48"
- }
+ "ipam": { "type": "galactic-ipam", "ipv6_subnet": "fd00:1:ff01::/48" }
},
{
"type": "galactic-route",
@@ -344,13 +439,15 @@ carries only the IPv4 `/32` prefix.
{ "network": "fd00::/48", "via": "fe80::1" },
{ "network": "fd01::/48" }
]
- }
+ },
+ { "type": "galactic-bgp", "vpc": "1", "vpcattachment": "1" }
]
}
```
The first termination installs a specific next-hop route; the second installs
-an on-link route via the host-side device.
+an on-link route via the host-side device. `galactic-route` runs between the
+master plugin and `galactic-bgp`.
### Tap interface configuration (VM workloads)
@@ -358,23 +455,23 @@ an on-link route via the host-side device.
{
"cniVersion": "1.0.0",
"name": "galactic-tap",
- "type": "galactic-cni",
- "vpc": "1",
- "vpcattachment": "1",
- "interface_type": "tap",
- "mtu": 9000,
- "ipam": {
- "type": "pool",
- "pool": "fd00:10:ff03::/48"
- }
+ "plugins": [
+ {
+ "type": "galactic-tap-cni",
+ "vpc": "1",
+ "vpcattachment": "1",
+ "mtu": 9000,
+ "ipam": { "type": "galactic-ipam", "ipv6_subnet": "fd00:10:ff03::/48" }
+ },
+ { "type": "galactic-bgp", "vpc": "1", "vpcattachment": "1" }
+ ]
}
```
-Tap mode creates a tap interface in the host namespace, enslaves it to the
-VRF, and applies forwarding sysctls. It then runs IPAM (allocating the subnet
-shown above) and SRv6/BGP publish exactly as `veth` mode does — see the `tap`
-description under Interface Types above. Only host-device delegation and
-guest-netns configuration are skipped; the guest VM still configures its own
-IP addresses independently once the hypervisor (Kata, Firecracker, QEMU) opens
-the tap fd at runtime. The `ipam` block (or `GALACTIC_CNI_ENABLE_LOCAL_IPAM`)
-is required here for the same reason it is in `veth` mode.
+`galactic-tap-cni` creates a tap interface in the host namespace, enslaves it
+to the VRF, and applies forwarding sysctls. It then runs IPAM (allocating the
+subnet shown above) and configures the host gateway exactly as `galactic-cni`
+does — see [`galactic-tap-cni` (tap)](#galactic-tap-cni-tap) above. Only
+host-device delegation and guest-netns configuration are skipped; the guest VM
+still configures its own IP addresses independently once the hypervisor
+(Kata, Firecracker, kraftlet/Unikraft) opens the tap fd at runtime.
diff --git a/internal/config/cni.go b/internal/config/cni.go
index 0cd5dd8c..bd0f5ad5 100644
--- a/internal/config/cni.go
+++ b/internal/config/cni.go
@@ -6,7 +6,6 @@ package config
import (
"os"
- "strings"
)
// --- CNI environment variable keys -----------------------------------------
@@ -15,7 +14,6 @@ const (
EnvCNINodeName = "GALACTIC_CNI_NODE_NAME"
EnvCNIKubeconfig = "GALACTIC_CNI_KUBECONFIG"
EnvCNIKubernetesConfig = "GALACTIC_CNI_KUBERNETES_CONFIG"
- EnvCNIEnableLocalIPAM = "GALACTIC_CNI_ENABLE_LOCAL_IPAM"
EnvLogLevel = "GALACTIC_CNI_LOG_LEVEL"
EnvLogFile = "GALACTIC_CNI_LOG_FILE"
EnvNamespace = "GALACTIC_CNI_NAMESPACE"
@@ -105,10 +103,3 @@ type ConflistValues struct {
LogFile string
LogLevel string
}
-
-// CNIGetEnableLocalIPAM reports whether local (in-memory) IPAM is enabled via
-// environment variable. Returns false if the variable is unset or not "true".
-func CNIGetEnableLocalIPAM() bool {
- val := os.Getenv(EnvCNIEnableLocalIPAM)
- return strings.EqualFold(val, "true")
-}
diff --git a/internal/config/cni_test.go b/internal/config/cni_test.go
index ef749d1e..434d3d36 100644
--- a/internal/config/cni_test.go
+++ b/internal/config/cni_test.go
@@ -106,16 +106,3 @@ func TestCNIConfigNodeNameLegacyFallback(t *testing.T) {
t.Errorf("NodeName = %q, want %q", cfg.NodeName, "legacy-node")
}
}
-
-func TestCNIGetEnableLocalIPAM(t *testing.T) {
- t.Setenv(EnvCNIEnableLocalIPAM, "true")
- if got := CNIGetEnableLocalIPAM(); !got {
- t.Error("CNIGetEnableLocalIPAM() = false, want true")
- }
-}
-
-func TestCNIGetEnableLocalIPAMFalse(t *testing.T) {
- if got := CNIGetEnableLocalIPAM(); got {
- t.Error("CNIGetEnableLocalIPAM() = true, want false (env unset)")
- }
-}
diff --git a/internal/gc/gc.go b/internal/gc/gc.go
index 01594f1e..3333bccb 100644
--- a/internal/gc/gc.go
+++ b/internal/gc/gc.go
@@ -377,7 +377,7 @@ func SweepEBPFVRFTable(ctx context.Context, k8s client.Client, namespace, nodeNa
if len(routers) == 0 {
// A node with any live eBPF-registered attachment at all necessarily
// has a BGPRouter targeting it -- registerEBPFDatapath requires one
- // to run at all (internal/cni/bgp.go). Finding none here is
+ // to run at all (internal/cnibgp/bgp.go). Finding none here is
// indistinguishable from a transient listing/cache hiccup or the
// router having just been renamed/recreated, so it must not be
// treated the same as "genuinely zero live attachments": doing so
diff --git a/internal/plumbing/ebpf/doc.go b/internal/plumbing/ebpf/doc.go
index 058e6abe..65791409 100644
--- a/internal/plumbing/ebpf/doc.go
+++ b/internal/plumbing/ebpf/doc.go
@@ -32,13 +32,13 @@
// external event silently clearing the filter -- requires it.
// - usidmap: the read/write API that populates and reconciles
// locator_table/function_table/vrf_table, used by the CNI ADD path's
-// registration call (internal/cni/bgp.go) and by the GC controller's
+// registration call (internal/cnibgp/bgp.go) and by the GC controller's
// sweep (internal/gc).
// - metrics: Prometheus metrics and health-check event hooks spanning the
// whole datapath (load/attach events, drops by reason, per-Argument
// hit counters and Argument-space utilization).
//
-// internal/cni and internal/gc are the two callers outside this tree that
+// internal/cnibgp and internal/gc are the two callers outside this tree that
// drive usidmap's register/unregister/reconcile calls; internal/reconcile
// and internal/plumbing/srv6's ComputeSID independently compute the same
// SID this datapath decodes, for the BGP control-plane side of the same
diff --git a/internal/plumbing/ebpf/usidmap/vrf.go b/internal/plumbing/ebpf/usidmap/vrf.go
index 8914dc88..b94f43b0 100644
--- a/internal/plumbing/ebpf/usidmap/vrf.go
+++ b/internal/plumbing/ebpf/usidmap/vrf.go
@@ -102,7 +102,7 @@ func (t *VRFTable) Generation() uint64 {
// because a repeat Register of the *same* key is not always a fresh
// attachment lifecycle -- it is also, in the ordinary case, the CNI ADD
// retry path re-registering after a transient k8s-op failure
-// (internal/cni/bgp.go's retryK8sOps), which happens on an Argument that
+// (internal/cnibgp/bgp.go's retryK8sOps), which happens on an Argument that
// may already be carrying live traffic. R8's make-before-break migration
// gate reads these counters to prove an Argument carried no traffic before
// cutover; a blind overwrite that zeroed them on every retry would make a
diff --git a/internal/plumbing/srv6/usid.go b/internal/plumbing/srv6/usid.go
index 1b8718cb..d813fff4 100644
--- a/internal/plumbing/srv6/usid.go
+++ b/internal/plumbing/srv6/usid.go
@@ -20,7 +20,7 @@ import (
// no distinct wire code for a per-family variant anyway, design plan R3):
// it is the only endpoint behavior the eBPF datapath's vrf_table ever
// installs, regardless of pod-subnet address family (see
-// internal/cni/bgp.go's registerEBPFDatapath/buildAdvertisementSpec).
+// internal/cnibgp/bgp.go's registerEBPFDatapath/buildAdvertisementSpec).
func functionNibble(fn bgpv1alpha1.SRv6Function) (uint8, error) {
if fn == bgpv1alpha1.SRv6FunctionEndDT46 {
return uformat.FunctionEndDT46, nil
diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go
index 1294aec6..af60b950 100644
--- a/tests/e2e/e2e_test.go
+++ b/tests/e2e/e2e_test.go
@@ -158,10 +158,19 @@ func TestKernelCapabilities(t *testing.T) {
}
}
-// TestCNITapInterface exercises galactic-tap-cni, the tap master plugin.
-// It creates a pod that invokes the plugin with CNI_COMMAND=ADD and a tap
-// config, then validates the CNI result JSON: a single host interface with an
-// empty sandbox and the host-side gateway/subnet IPAM allocated for it.
+// TestCNITapInterface exercises galactic-tap-cni, the tap master plugin in
+// the galactic CNI chain (see internal/cnitap). It creates a pod that
+// invokes the plugin with CNI_COMMAND=ADD and a tap config, then validates
+// the CNI result JSON: a single host interface with an empty sandbox and
+// the host-side gateway/subnet IPAM allocated for it.
+//
+// This exercises galactic-tap-cni's own ADD (VRF + tap creation, IPAM
+// delegation to galactic-ipam) directly, then manually chains galactic-bgp
+// after it (testChainedGalacticBGP below), feeding it the tap master's own
+// CNI result as prevResult exactly as the CNI runtime would — the same
+// manual-chaining approach used because a real conflist-driven chain would
+// need a BGPRouter fixture and additional RBAC this test doesn't set up.
+// It does not chain into galactic-route (this config has no terminations).
//
// This test requires a cluster node with VRF/tap kernel support (the same
// prerequisites checked by TestKernelCapabilities). It will fail rather than
@@ -173,24 +182,29 @@ func TestCNITapInterface(t *testing.T) {
deletePod(t, name)
// Start a shell so we can later exec the CNI plugin with stdin.
- // The galactic-cni entrypoint is overridden to "sh" so the pod stays
- // running and we can pipe the CNI config via kubectl exec -i.
- // Run as the galactic-cni ServiceAccount so the CNI plugin's in-cluster
- // client is bound by the galactic-cni ClusterRole (config/cni/rbac.yaml)
- // when it lists/creates BGPRouter, BGPAdvertisement, and BGPVRFInstance.
- // hostNetwork is required too: net.vrf.strict_mode (enabled on the Kind
- // node in scripts/ci.sh) is per-netns, and the SEG6Local VRFTABLE route
- // this test exercises needs it set in whichever netns the route lands in.
- // The bpf-fs hostPath volume mirrors config/cni/daemonset.yaml's own
- // bpf-fs mount: the eBPF uSID datapath's maps can only be pinned under
- // attach.PinDir if the node's real bpffs (mounted onto the Kind node in
- // scripts/ci.sh) is visible inside the pod -- a pod's own mount
- // namespace can't create /sys/fs/bpf itself. The whole container spec
- // (image, command, privileged) has to live in --overrides too, not the
- // usual --image/--command/--privileged flags: kubectl run's overrides
- // merge replaces the generated "containers" list wholesale rather than
- // merging into it, so anything set only via those flags would otherwise
- // be silently dropped the moment "containers" is also set here.
+ // The galactic-cni image's entrypoint is overridden to "sh" so the pod
+ // stays running and we can pipe the CNI config via kubectl exec -i.
+ // galactic-tap-cni ships in the same image (see containers/galactic-cni/
+ // Dockerfile) alongside every other binary in the CNI chain, so no
+ // separate image is needed here. Run as the galactic-cni ServiceAccount:
+ // galactic-tap-cni's own ADD unconditionally builds an in-cluster k8s
+ // client for its NAD-annotation step (config/cni/rbac.yaml grants it),
+ // even though that step itself no-ops here (no CNI_ARGS, so
+ // nadpatch.ParsePodNamespace resolves an empty namespace). hostNetwork
+ // is required too, so the VRF/tap interfaces this test creates land in
+ // the same netns production's own hostNetwork DaemonSet would use. The
+ // bpf-fs hostPath volume mirrors config/cni/daemonset.yaml's own bpf-fs
+ // mount: this test chains galactic-bgp (testChainedGalacticBGP below),
+ // which registers the eBPF uSID datapath, and its maps can only be
+ // pinned under attach.PinDir if the node's real bpffs (mounted onto the
+ // Kind node in scripts/ci.sh) is visible inside the pod -- a pod's own
+ // mount namespace can't create /sys/fs/bpf itself. The whole container
+ // spec (image, command, privileged) has to live in --overrides too, not
+ // the usual --image/--command/--privileged flags: kubectl run's
+ // overrides merge replaces the generated "containers" list wholesale
+ // rather than merging into it, so anything set only via those flags
+ // would otherwise be silently dropped the moment "containers" is also
+ // set here.
overrides := fmt.Sprintf(`{"spec":{"serviceAccountName":"galactic-cni","hostNetwork":true,`+
`"volumes":[{"name":"bpf-fs","hostPath":{"path":"/sys/fs/bpf","type":"Directory"}}],`+
`"containers":[{"name":%q,"image":%q,"imagePullPolicy":"Never","command":["sleep","infinity"],`+
@@ -221,20 +235,20 @@ func TestCNITapInterface(t *testing.T) {
// DaemonSet's long-running "credential-refresh" container (config/cni/
// daemonset.yaml, `/galactic-cni run`); this test runs its own pod
// instead of relying on that DaemonSet, so it must start the same
- // control daemon itself before exercising CNI ADD below.
+ // control daemon itself before exercising CNI ADD below. Required for
+ // testChainedGalacticBGP below too: registerEBPFDatapath's
+ // usidmap.OpenPinnedRegistry only opens already-pinned maps, it never
+ // loads/pins the eBPF program itself.
startEBPFControlDaemon(t, name)
// Write the CNI config to a file inside the pod, then run the plugin
// with the config piped via stdin. The plugin reads config from stdin
// (the CNI protocol) and CNI_NETNS from the environment.
//
- // The "ipam" block's "type" now names the delegated binary
- // (galactic-ipam), not a pool-vs-static mode selector -- this step
- // rewired IPAM from an in-process call into real CNI IPAM delegation
- // (github.com/containernetworking/plugins/pkg/ipam.ExecAdd), so
- // "pool" is no longer a valid type value; presence of ipv6_subnet
- // alone opts this config into pool IPAM (see internal/cniipam's doc
- // comment and docs/cni/configuration.md).
+ // The "ipam" block's "type" names the delegated binary (galactic-ipam),
+ // not a pool-vs-static mode selector — presence of ipv6_subnet alone
+ // opts this config into pool IPAM (see internal/cniipam's doc comment
+ // and docs/cni/configuration.md).
cniConf := `{
"cniVersion": "1.0.0",
"name": "galactic",
@@ -248,11 +262,11 @@ func TestCNITapInterface(t *testing.T) {
}`
// Step 1: write the CNI config and a wrapper script into the pod.
// CNI_PATH=/ lets IPAM delegation (galactic-tap-cni execs galactic-ipam
- // via ipam.ExecAdd) find the delegate binary: every binary in the
- // chain is copied to the image root by containers/galactic-cni/
- // Dockerfile (not /opt/cni/bin -- that path only exists on the real
- // host once installer.Bootstrap's init container stages it there,
- // which this test's pod never runs).
+ // via github.com/containernetworking/plugins/pkg/ipam.ExecAdd) find the
+ // delegate binary: every binary in the chain is copied to the image
+ // root by containers/galactic-cni/Dockerfile (not /opt/cni/bin — that
+ // path only exists on the real host once installer.Bootstrap's init
+ // container stages it there, which this test's pod never runs).
script := `#!/bin/sh
ip netns add e2e-tap-ns
CNI_NETNS=/var/run/netns/e2e-tap-ns \