docs: align the NVIDIA foundation with generic GGUF evidence

README.md

@@ -6,7 +6,7 @@ The reusable base workflows live in `pkg/edgebase`. New operator workflows shoul
 
 This is Layer 1 of the `Edge-Computing-LLM` platform. It owns the local Linux + k3s + NVIDIA substrate. `llm-observability-stack` is Layer 2 and must not install GPU Operator, NVIDIA device plugin, or DCGM exporter in the main local NVIDIA path.
 
-[`qwen-gguf-observability`](https://github.com/Edge-Computing-LLM/qwen-gguf-observability)
+[`gguf-observability`](https://github.com/Edge-Computing-LLM/gguf-observability)
 may read the resulting node, RuntimeClass, and GPU-capacity status as a runtime
 evidence companion. It does not install or manage this infrastructure layer.
 
@@ -33,7 +33,7 @@ GPU layer is mandatory and fallback would hide an infrastructure problem.
 - [Bundled Helm chart](charts/k3s-nvidia-edge/README.md)
 - [Contributing](CONTRIBUTING.md)
 - [Security](SECURITY.md)
-- [Qwen GGUF runtime evidence companion](https://github.com/Edge-Computing-LLM/qwen-gguf-observability)
+- [Multi-model GGUF runtime evidence companion](https://github.com/Edge-Computing-LLM/gguf-observability)
 
 The default profile matches the working local Xubuntu 24 setup:
 

cmd/k3s-nvidia-edge/main.go

@@ -37,7 +37,7 @@ Global flags:
   --yes                 execute mutating commands; otherwise mutating commands are dry-run
   --sudo                use sudo for host-level commands (default true)
   --verbose             print command output while commands run
-  --reference-root DIR  root containing Project-Rancher-K3S, Project-Nvidia, etc.
+  --reference-root DIR  root containing Project-Rancher, Project-Nvidia, etc.
   --min-cuda-version    minimum host CUDA toolkit version for doctor/install
   --require-host-cuda   require host CUDA toolkit, not only driver/container CUDA
 

docs/architecture.md

@@ -10,9 +10,9 @@ metrics and schedule Ollama with `RuntimeClass/nvidia`, but it must not install
 GPU Operator, NVIDIA device plugin, Node Feature Discovery, or DCGM exporter in
 the main local NVIDIA path.
 
-`qwen-gguf-observability` is not Layer 3 and owns no infrastructure. It is a
+`gguf-observability` is not Layer 3 and owns no infrastructure. It is a
 read-only evidence consumer that verifies selected outputs of Layer 1 together
-with the Qwen runtime deployed by Layer 2.
+with the selected GGUF model runtime deployed by Layer 2.
 
 ## Components
 

docs/live-validation-2026-07-19.md

@@ -0,0 +1,25 @@
+# Live validation: 2026-07-19
+
+The repository gates passed on Ubuntu 24.04 with Go 1.26.5, Helm 4.2.3, k3s
+v1.36.2+k3s1, and NVIDIA GPU Operator v26.3.3:
+
+- `go mod verify`, formatting, unit tests, race tests, vet, and build
+- `govulncheck ./...` with no reachable vulnerabilities
+- Helm lint and chart rendering
+- host `nvidia-smi`, RuntimeClass discovery, and GPU capacity inspection
+
+The live cluster audit correctly found a post-reboot network fault. k3s still
+advertised an InternalIP and Flannel interface that were no longer assigned to
+the host. Pod traffic to the Kubernetes API therefore failed with `no route to
+host`, causing GPU Operator, Node Feature Discovery, metrics-server,
+kube-state-metrics, and node-exporter readiness failures. GPU allocation and
+the active Ollama workload remained available, which demonstrated why capacity
+alone is not a sufficient health signal.
+
+The CLI now checks both invariants before CUDA validation:
+
+1. The advertised k3s InternalIP must exist on a current host interface.
+2. GPU Operator pods must be Running and ready, or successfully Completed.
+
+No host addresses, kubeconfig data, Secrets, prompts, responses, or pod logs
+are recorded in this document.

docs/troubleshooting.md

@@ -59,7 +59,19 @@ kubectl get endpoints kubernetes -n default -o yaml
 kubectl get node -o wide
 ```
 
-Pin `node-ip` in `/etc/rancher/k3s/config.yaml` when the machine regularly moves between interfaces.
+If the configured address or interface no longer exists, remove stale
+`node-ip` and `flannel-iface` entries so a single-interface workstation can
+auto-detect its active network again. If the pinning is intentional, update
+both values together. Then restart k3s and verify that the node InternalIP is
+assigned to a current host interface:
+
+```bash
+sudo systemctl restart k3s
+bin/k3s-nvidia-edge doctor --require-host-cuda=false
+```
+
+Do not copy an address from an old `kubectl get nodes -o wide` result back into
+the configuration; confirm it against `ip -o -4 address show` first.
 
 ## Validation Pod Stuck
 

pkg/edgebase/options.go

@@ -25,7 +25,7 @@ type Options struct {
 func DefaultOptions() Options {
 	return Options{
 		Sudo:                   true,
-		ReferenceRoot:          "/media/waqasm86/External1/Waqas-Projects",
+		ReferenceRoot:          "/media/waqasm86/External1/Waqas-Projects/Project-Linux-Kubernetes-Nvidia",
 		GPUOperatorVersion:     "v26.3.3",
 		CUDATestImage:          "nvidia/cuda:12.8.1-base-ubuntu24.04",
 		MinCUDAVersion:         "12.8",

pkg/edgebase/shell.go

@@ -63,6 +63,39 @@ while true; do
 done`
 }
 
+func GPUOperatorHealthCheck() string {
+	return `set -euo pipefail
+pods="$(kubectl get pods -n gpu-operator --no-headers)"
+if [ -z "$pods" ]; then
+  echo "no GPU Operator pods found"
+  exit 1
+fi
+bad="$(printf '%s\n' "$pods" | awk '$3!="Running" && $3!="Completed" {print}')"
+notready="$(printf '%s\n' "$pods" | awk '$3=="Running" {split($2,a,"/"); if (a[1] != a[2]) print}')"
+if [ -n "$bad" ] || [ -n "$notready" ]; then
+  echo "GPU Operator has unhealthy pods"
+  [ -z "$bad" ] || printf '%s\n' "$bad"
+  [ -z "$notready" ] || printf '%s\n' "$notready"
+  exit 1
+fi
+echo "GPU Operator pods are healthy"`
+}
+
+func NodeAddressHealthCheck() string {
+	return `set -euo pipefail
+node_ip="$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')"
+if [ -z "$node_ip" ]; then
+  echo "k3s node has no InternalIP"
+  exit 1
+fi
+if ! ip -o -4 address show | awk '{split($4,a,"/"); print a[1]}' | grep -Fxq "$node_ip"; then
+  echo "k3s node InternalIP is not assigned to a current host interface"
+  echo "check node-ip and flannel-iface in /etc/rancher/k3s/config.yaml, then restart k3s"
+  exit 1
+fi
+echo "k3s node InternalIP matches a current host interface"`
+}
+
 func GPUCapacityCheck() string {
 	return `set -euo pipefail
 gpu="$(kubectl get nodes -o jsonpath='{.items[0].status.allocatable.nvidia\.com/gpu}')"

pkg/edgebase/workflows.go

@@ -13,10 +13,12 @@ func Doctor(ctx context.Context, r *Runner, opts Options) error {
 		{Name: "Kernel", Command: "uname -a"},
 		{Name: "NVIDIA driver and GPU", Command: "nvidia-smi"},
 		{Name: "CUDA toolkit version", Command: HostCUDACheck(opts.MinCUDAVersion, opts.RequireHostCUDA)},
-		{Name: "Required commands", Command: "missing=0; for c in curl apt-get systemctl kubectl helm jq grep awk sort sed; do command -v $c >/dev/null && echo \"$c: $(command -v $c)\" || { echo \"$c: missing\"; missing=1; }; done; exit $missing"},
+		{Name: "Required commands", Command: "missing=0; for c in curl apt-get systemctl kubectl helm jq grep awk sort sed ip; do command -v $c >/dev/null && echo \"$c: $(command -v $c)\" || { echo \"$c: missing\"; missing=1; }; done; exit $missing"},
 		{Name: "NVIDIA host packages", Command: PackageInventoryCommand()},
 		{Name: "k3s service", Command: "systemctl is-active k3s"},
+		{Name: "k3s node address", Command: NodeAddressHealthCheck()},
 		{Name: "cluster nodes", Command: "kubectl get nodes -o wide || true"},
+		{Name: "GPU Operator health", Command: GPUOperatorHealthCheck()},
 		{Name: "GPU capacity", Command: "kubectl get nodes -o json | jq '.items[] | {name:.metadata.name, capacity_gpu:.status.capacity[\"nvidia.com/gpu\"], allocatable_gpu:.status.allocatable[\"nvidia.com/gpu\"]}' || true"},
 		{Name: "GPU Operator values", Command: "helm get values gpu-operator -n gpu-operator -o yaml || true"},
 	}
@@ -77,6 +79,15 @@ func Status(ctx context.Context, r *Runner, opts Options) error {
 }
 
 func Validate(ctx context.Context, r *Runner, opts Options) error {
+	for _, step := range []Step{
+		{Name: "k3s node address", Command: NodeAddressHealthCheck()},
+		{Name: "GPU Operator health", Command: GPUOperatorHealthCheck()},
+		{Name: "GPU capacity", Command: GPUCapacityCheck()},
+	} {
+		if err := r.Run(ctx, step); err != nil {
+			return err
+		}
+	}
 	return r.Run(ctx, Step{
 		Name:     "CUDA validation pod",
 		Mutating: true,
@@ -117,7 +128,7 @@ func Uninstall(ctx context.Context, r *Runner, opts Options) error {
 
 func Repos(ctx context.Context, r *Runner, opts Options) error {
 	root := opts.ReferenceRoot
-	command := fmt.Sprintf(`for d in %s/Project-Kubernetes-Sigs/* %s/Project-CoreDNS/* %s/Project-Rancher-K3S/* %s/Project-Nvidia/* %s/Project-Cloudflare/*; do
+	command := fmt.Sprintf(`for d in %s/Project-Kubernetes-Sigs/* %s/Project-CoreDNS/* %s/Project-Rancher/* %s/Project-Nvidia/* %s/Project-Cloudflare/*; do
   [ -d "$d/.git" ] || continue
   printf 'repo: %%s\n' "$d"
   printf 'origin: '; git -C "$d" remote get-url origin || true
@@ -291,8 +302,8 @@ func CleanupLegacySteps(opts Options) []Step {
 
 func LocalRepoPaths(root string) []string {
 	return []string{
-		filepath.Join(root, "Project-Rancher-K3S", "k3s"),
-		filepath.Join(root, "Project-Rancher-K3S", "local-path-provisioner"),
+		filepath.Join(root, "Project-Rancher", "k3s"),
+		filepath.Join(root, "Project-Rancher", "local-path-provisioner"),
 		filepath.Join(root, "Project-CoreDNS", "coredns"),
 		filepath.Join(root, "Project-Kubernetes-Sigs", "node-feature-discovery"),
 		filepath.Join(root, "Project-Kubernetes-Sigs", "dra-driver-nvidia-gpu"),

pkg/edgebase/workflows_test.go

@@ -92,6 +92,24 @@ func TestBundledChartsStepChecksAllCharts(t *testing.T) {
 	}
 }
 
+func TestReferenceRepositoryPathsMatchLocalLayout(t *testing.T) {
+	opts := DefaultOptions()
+	if !contains(opts.ReferenceRoot, "Project-Linux-Kubernetes-Nvidia") {
+		t.Fatalf("reference root does not target the project collection: %s", opts.ReferenceRoot)
+	}
+	paths := LocalRepoPaths(opts.ReferenceRoot)
+	all := ""
+	for _, path := range paths {
+		all += path + "\n"
+	}
+	if !contains(all, "/Project-Rancher/k3s") {
+		t.Fatalf("reference paths do not use Project-Rancher: %s", all)
+	}
+	if contains(all, "Project-Rancher-K3S") {
+		t.Fatalf("reference paths retain obsolete Project-Rancher-K3S layout: %s", all)
+	}
+}
+
 func TestValidateWaitsForSucceededPod(t *testing.T) {
 	manifest := CUDATestManifest(DefaultOptions().CUDATestImage)
 	if !contains(manifest, "runtimeClassName: nvidia") {
@@ -104,6 +122,22 @@ func TestValidateWaitsForSucceededPod(t *testing.T) {
 	}
 }
 
+func TestHealthChecksDetectStaleNetworkingAndUnreadyPods(t *testing.T) {
+	network := NodeAddressHealthCheck()
+	for _, want := range []string{"InternalIP", "ip -o -4 address show", "flannel-iface"} {
+		if !contains(network, want) {
+			t.Fatalf("node address health check missing %q", want)
+		}
+	}
+
+	operator := GPUOperatorHealthCheck()
+	for _, want := range []string{"gpu-operator", "Completed", "notready", "exit 1"} {
+		if !contains(operator, want) {
+			t.Fatalf("GPU Operator health check missing %q", want)
+		}
+	}
+}
+
 func TestCleanupStepsAvoidArchivedPackages(t *testing.T) {
 	opts := DefaultOptions()
 	all := joinStepCommands(CleanupLegacySteps(opts))