diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 848f2e81f..254dc8419 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -302,7 +302,7 @@ jobs: matrix: include: ${{ fromJson(needs.changes.outputs.e2e-matrix) }} runs-on: ${{ matrix.os }} - # Includes ExporterSet QEMU coverage (TCG); flash/boot still skipped until #924. + # Includes ExporterSet QEMU coverage (TCG), including flash/boot. # Job > Ginkgo suite timeout (60m in e2e/lib/common.sh) for setup/log upload. timeout-minutes: 70 steps: diff --git a/controller/hack/sample-x86_64-kind.yaml b/controller/hack/sample-x86_64-kind.yaml index 53228d2c1..e6265f73f 100644 --- a/controller/hack/sample-x86_64-kind.yaml +++ b/controller/hack/sample-x86_64-kind.yaml @@ -8,6 +8,12 @@ # acceleration, which is fine for a functional/architecture smoke test # but much slower than a real x86_64+KVM cluster. # +# Guest disk uses emptyDir sized from parameters.resources.storage (no +# parameters.storage.storageClassName). The provisioner also sets +# ephemeral-storage requests/limits so the scheduler accounts for local +# disk. To use a PVC instead, set parameters.storage.storageClassName +# on the VirtualTargetClass (ExporterSet parameters deep-merge over it). +# # For real deployments with KVM acceleration, use sample-x86_64.yaml # on a cluster that exposes /dev/kvm via the kubevirt device plugin # and taints its KVM-capable nodes with jumpstarter.dev/kvm. diff --git a/controller/hack/sample-x86_64.yaml b/controller/hack/sample-x86_64.yaml index 3e78fbb00..a52eec461 100644 --- a/controller/hack/sample-x86_64.yaml +++ b/controller/hack/sample-x86_64.yaml @@ -7,6 +7,12 @@ # This creates: # - A VirtualTargetClass for x86_64 QEMU VMs with KVM acceleration # - An ExporterSet that manages a pool of virtual exporters +# +# Guest disk: sized emptyDir at /disk from parameters.resources.storage +# (with ephemeral-storage accounting). To use a StorageClass, set +# parameters.storage.storageClassName — the provisioner renders a generic +# ephemeral PVC whose lifetime follows the Pod (ExitAndReplace). +# ExporterSet.spec.parameters.storage deep-merges over the class. --- apiVersion: virtualtarget.jumpstarter.dev/v1alpha1 kind: VirtualTargetClass @@ -37,6 +43,9 @@ spec: cpu: 2 memory: 2Gi storage: 20Gi + # storage: + # storageClassName: "your-storage-class" + # accessModes: ["ReadWriteOnce"] --- apiVersion: virtualtarget.jumpstarter.dev/v1alpha1 kind: ExporterSet @@ -50,6 +59,9 @@ spec: scaleDownCooldown: 5m recycleStrategy: ExitAndReplace virtualTargetClassName: qemu-x86-64 + # parameters: + # storage: + # storageClassName: "override-sc" # override class; "" forces emptyDir selector: matchLabels: board: x86-64-virtual diff --git a/controller/internal/exporterset/disk/disk.go b/controller/internal/exporterset/disk/disk.go new file mode 100644 index 000000000..884d280a4 --- /dev/null +++ b/controller/internal/exporterset/disk/disk.go @@ -0,0 +1,206 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package disk provides shared helpers for guest-disk volume provisioning +// used by ExporterSet provisioners. +// +// Size comes from parameters.resources.storage (JEP-0014). Kubernetes +// backend config comes from parameters.storage: +// +// parameters: +// resources: +// storage: 20Gi +// storage: +// storageClassName: gp3 # omit or "" → sized emptyDir +// accessModes: ["ReadWriteOnce"] +// +// When storageClassName is set, the volume is a generic ephemeral PVC +// (volumeClaimTemplate) so its lifetime follows the Pod (ExitAndReplace). +package disk + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +const ( + // VolumeName is the Pod volume name for guest disk storage. + VolumeName = "disk" + + // MountPath is where guest disk storage is mounted in exporter and runtime. + MountPath = "/disk" + + // DefaultSize is used when parameters.resources.storage is unset. + DefaultSize = "10Gi" +) + +// Spec is the resolved guest-disk volume configuration. +type Spec struct { + Size resource.Quantity + StorageClassName string + AccessModes []corev1.PersistentVolumeAccessMode +} + +// UsePVC reports whether the disk is backed by an ephemeral PVC. +func (s Spec) UsePVC() bool { + return s.StorageClassName != "" +} + +// Mount returns the VolumeMount for /disk. +func Mount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: VolumeName, + MountPath: MountPath, + } +} + +// FromParameters reads disk size and optional storage backend from merged +// ExporterSet/VirtualTargetClass parameters. +func FromParameters(params map[string]interface{}) (Spec, error) { + size, err := SizeFromParameters(params) + if err != nil { + return Spec{}, err + } + spec := Spec{ + Size: size, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + } + if params == nil { + return spec, nil + } + storage, ok := params["storage"].(map[string]interface{}) + if !ok { + if params["storage"] != nil { + return Spec{}, fmt.Errorf("parameters.storage must be an object, got %T", params["storage"]) + } + return spec, nil + } + + switch v := storage["storageClassName"].(type) { + case string: + spec.StorageClassName = v + case nil: + // omit → emptyDir + default: + return Spec{}, fmt.Errorf("parameters.storage.storageClassName must be a string, got %T", v) + } + + if raw, exists := storage["accessModes"]; exists && raw != nil { + modes, err := parseAccessModes(raw) + if err != nil { + return Spec{}, err + } + spec.AccessModes = modes + } + return spec, nil +} + +// SizeFromParameters reads parameters.resources.storage, defaulting to DefaultSize. +func SizeFromParameters(params map[string]interface{}) (resource.Quantity, error) { + raw := DefaultSize + if params != nil { + if resources, ok := params["resources"].(map[string]interface{}); ok { + switch v := resources["storage"].(type) { + case string: + if v != "" { + raw = v + } + case float64: + return resource.Quantity{}, fmt.Errorf("parameters.resources.storage must be a string quantity (e.g. \"10Gi\"), got number %v", v) + case nil: + // use default + default: + return resource.Quantity{}, fmt.Errorf("parameters.resources.storage must be a string quantity (e.g. \"10Gi\"), got %T", v) + } + } + } + + qty, err := resource.ParseQuantity(raw) + if err != nil { + return resource.Quantity{}, fmt.Errorf("parse parameters.resources.storage %q: %w", raw, err) + } + return qty, nil +} + +// Volume builds the guest-disk Pod volume from spec. +func Volume(spec Spec) corev1.Volume { + vol := corev1.Volume{Name: VolumeName} + if spec.UsePVC() { + sc := spec.StorageClassName + vol.VolumeSource = corev1.VolumeSource{ + Ephemeral: &corev1.EphemeralVolumeSource{ + VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{ + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: spec.AccessModes, + StorageClassName: &sc, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: spec.Size, + }, + }, + }, + }, + }, + } + return vol + } + size := spec.Size.DeepCopy() + vol.VolumeSource = corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + SizeLimit: &size, + }, + } + return vol +} + +// SetEphemeralStorage ensures requests and limits include ephemeral-storage +// equal to size (used when guest disk is backed by emptyDir). +func SetEphemeralStorage(resources *corev1.ResourceRequirements, size resource.Quantity) { + if resources.Requests == nil { + resources.Requests = corev1.ResourceList{} + } + if resources.Limits == nil { + resources.Limits = corev1.ResourceList{} + } + // Only set when unset so explicit scheduling.resources win. + if _, ok := resources.Requests[corev1.ResourceEphemeralStorage]; !ok { + resources.Requests[corev1.ResourceEphemeralStorage] = size.DeepCopy() + } + if _, ok := resources.Limits[corev1.ResourceEphemeralStorage]; !ok { + resources.Limits[corev1.ResourceEphemeralStorage] = size.DeepCopy() + } +} + +func parseAccessModes(v interface{}) ([]corev1.PersistentVolumeAccessMode, error) { + items, ok := v.([]interface{}) + if !ok { + return nil, fmt.Errorf("parameters.storage.accessModes must be a list of strings, got %T", v) + } + if len(items) == 0 { + return nil, fmt.Errorf("parameters.storage.accessModes must not be empty") + } + out := make([]corev1.PersistentVolumeAccessMode, 0, len(items)) + for _, item := range items { + s, ok := item.(string) + if !ok || s == "" { + return nil, fmt.Errorf("parameters.storage.accessModes must be a list of strings, got %T", item) + } + out = append(out, corev1.PersistentVolumeAccessMode(s)) + } + return out, nil +} diff --git a/controller/internal/exporterset/disk/disk_test.go b/controller/internal/exporterset/disk/disk_test.go new file mode 100644 index 000000000..a06482af5 --- /dev/null +++ b/controller/internal/exporterset/disk/disk_test.go @@ -0,0 +1,149 @@ +/* +Copyright 2026 The Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package disk + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestFromParameters_defaults(t *testing.T) { + spec, err := FromParameters(nil) + if err != nil { + t.Fatalf("nil params: %v", err) + } + if !spec.Size.Equal(resource.MustParse(DefaultSize)) { + t.Errorf("size = %v, want %s", spec.Size, DefaultSize) + } + if spec.UsePVC() { + t.Error("expected emptyDir when storageClassName is unset") + } + if len(spec.AccessModes) != 1 || spec.AccessModes[0] != corev1.ReadWriteOnce { + t.Errorf("accessModes = %v, want [ReadWriteOnce]", spec.AccessModes) + } +} + +func TestFromParameters_storageClassAndSize(t *testing.T) { + spec, err := FromParameters(map[string]interface{}{ + "resources": map[string]interface{}{"storage": "15Gi"}, + "storage": map[string]interface{}{ + "storageClassName": "gp3", + "accessModes": []interface{}{"ReadWriteOnce", "ReadWriteMany"}, + }, + }) + if err != nil { + t.Fatalf("FromParameters: %v", err) + } + if !spec.Size.Equal(resource.MustParse("15Gi")) { + t.Errorf("size = %v, want 15Gi", spec.Size) + } + if spec.StorageClassName != "gp3" { + t.Errorf("storageClassName = %q, want gp3", spec.StorageClassName) + } + if !spec.UsePVC() { + t.Error("expected PVC when storageClassName is set") + } + if len(spec.AccessModes) != 2 { + t.Fatalf("accessModes = %v", spec.AccessModes) + } +} + +func TestFromParameters_emptyStorageClassForcesEmptyDir(t *testing.T) { + spec, err := FromParameters(map[string]interface{}{ + "storage": map[string]interface{}{ + "storageClassName": "", + }, + }) + if err != nil { + t.Fatalf("FromParameters: %v", err) + } + if spec.UsePVC() { + t.Error("empty storageClassName should force emptyDir") + } +} + +func TestFromParameters_rejectsNumericStorage(t *testing.T) { + _, err := FromParameters(map[string]interface{}{ + "resources": map[string]interface{}{"storage": 10.0}, + }) + if err == nil { + t.Fatal("expected error for numeric storage") + } +} + +func TestVolume_emptyDir(t *testing.T) { + spec := Spec{Size: resource.MustParse("7Gi")} + vol := Volume(spec) + if vol.Name != VolumeName { + t.Errorf("name = %q, want %s", vol.Name, VolumeName) + } + if vol.EmptyDir == nil || vol.EmptyDir.SizeLimit == nil { + t.Fatalf("expected sized emptyDir, got %#v", vol) + } + if !vol.EmptyDir.SizeLimit.Equal(spec.Size) { + t.Errorf("SizeLimit = %v, want %v", vol.EmptyDir.SizeLimit, spec.Size) + } +} + +func TestVolume_ephemeralPVC(t *testing.T) { + sc := "fast-ssd" + spec := Spec{ + Size: resource.MustParse("20Gi"), + StorageClassName: sc, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + } + vol := Volume(spec) + if vol.Ephemeral == nil || vol.Ephemeral.VolumeClaimTemplate == nil { + t.Fatalf("expected ephemeral volumeClaimTemplate, got %#v", vol) + } + claim := vol.Ephemeral.VolumeClaimTemplate.Spec + if claim.StorageClassName == nil || *claim.StorageClassName != sc { + t.Errorf("StorageClassName = %v, want %s", claim.StorageClassName, sc) + } + got := claim.Resources.Requests[corev1.ResourceStorage] + if !got.Equal(spec.Size) { + t.Errorf("storage request = %v, want %v", got, spec.Size) + } +} + +func TestSetEphemeralStorage(t *testing.T) { + size := resource.MustParse("10Gi") + res := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + }, + } + SetEphemeralStorage(&res, size) + if !res.Requests[corev1.ResourceEphemeralStorage].Equal(size) { + t.Errorf("request = %v, want %v", res.Requests[corev1.ResourceEphemeralStorage], size) + } + if !res.Limits[corev1.ResourceEphemeralStorage].Equal(size) { + t.Errorf("limit = %v, want %v", res.Limits[corev1.ResourceEphemeralStorage], size) + } + if !res.Requests[corev1.ResourceCPU].Equal(resource.MustParse("1")) { + t.Error("cpu request should be preserved") + } + + custom := resource.MustParse("1Gi") + res.Requests[corev1.ResourceEphemeralStorage] = custom + SetEphemeralStorage(&res, size) + if !res.Requests[corev1.ResourceEphemeralStorage].Equal(custom) { + t.Errorf("should preserve explicit ephemeral-storage, got %v", res.Requests[corev1.ResourceEphemeralStorage]) + } +} diff --git a/controller/internal/exporterset/provisioners/qemu/qemu.go b/controller/internal/exporterset/provisioners/qemu/qemu.go index 9043e1d26..7c6c90a44 100644 --- a/controller/internal/exporterset/provisioners/qemu/qemu.go +++ b/controller/internal/exporterset/provisioners/qemu/qemu.go @@ -30,6 +30,7 @@ import ( jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1" virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/disk" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -154,7 +155,10 @@ func (p *Provisioner) resolveImageSpec(spec *virtualtargetv1alpha1.ImageSpec, de // Kubernetes terminates sidecars and the Pod completes. // Pod restartPolicy is Never so a clean exporter exit is not // restarted in-place (ExporterSet replaces the instance instead). -// - Shared emptyDir for Unix sockets (QMP, serial, launcher) and disk. +// - Shared emptyDir for Unix sockets (QMP, serial, launcher). +// - Guest disk volume at /disk (ephemeral PVC when +// parameters.storage.storageClassName is set, otherwise sized +// emptyDir with ephemeral-storage requests/limits). // // The caller (reconciler) is responsible for setting // OwnerReferences on the Pod and injecting the config volume. @@ -172,6 +176,11 @@ func (p *Provisioner) RenderPod( runAsExporter := exporterNonRootUID exporterNonRoot := true + diskSpec, err := disk.FromParameters(mergedParameters) + if err != nil { + return nil, err + } + var exporterSpec, runtimeSpec *virtualtargetv1alpha1.ImageSpec if images != nil { exporterSpec = images.Exporter @@ -194,6 +203,8 @@ func (p *Provisioner) RenderPod( }) } + diskMount := disk.Mount() + podMeta := metav1.ObjectMeta{ Namespace: exporterSet.Namespace, Labels: maps.Clone(exporterSet.Spec.Template.Metadata.Labels), @@ -248,6 +259,7 @@ func (p *Provisioner) RenderPod( Name: sharedVolumeName, MountPath: sharedMountPath, }, + diskMount, }, }, }, @@ -275,6 +287,7 @@ func (p *Provisioner) RenderPod( Name: sharedVolumeName, MountPath: sharedMountPath, }, + diskMount, }, }, }, @@ -291,6 +304,8 @@ func (p *Provisioner) RenderPod( }, } + pod.Spec.Volumes = append(pod.Spec.Volumes, disk.Volume(diskSpec)) + // Apply scheduling from VirtualTargetClass. // Clone maps and slices to avoid mutating the VTC's fields. if vtc.Spec.Scheduling != nil { @@ -311,6 +326,23 @@ func (p *Provisioner) RenderPod( } } + if diskSpec.UsePVC() { + // fsGroup so the non-root exporter can write the ephemeral claim. + if pod.Spec.SecurityContext == nil { + pod.Spec.SecurityContext = &corev1.PodSecurityContext{} + } + pod.Spec.SecurityContext.FSGroup = &runAsExporter + } else { + // emptyDir guest disks consume node ephemeral storage — ensure the + // scheduler and kubelet account for it on containers that mount /disk. + disk.SetEphemeralStorage(&pod.Spec.Containers[0].Resources, diskSpec.Size) + for i := range pod.Spec.InitContainers { + if pod.Spec.InitContainers[i].Name == runtimeContainerName { + disk.SetEphemeralStorage(&pod.Spec.InitContainers[i].Resources, diskSpec.Size) + } + } + } + return pod, nil } diff --git a/controller/internal/exporterset/provisioners/qemu/qemu_test.go b/controller/internal/exporterset/provisioners/qemu/qemu_test.go index 8edf567e5..da6e3c395 100644 --- a/controller/internal/exporterset/provisioners/qemu/qemu_test.go +++ b/controller/internal/exporterset/provisioners/qemu/qemu_test.go @@ -22,6 +22,7 @@ import ( jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1" virtualtargetv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/virtualtarget/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter/controller/internal/exporterset/disk" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -98,17 +99,43 @@ func assertRenderPodMetadata(t *testing.T, pod *corev1.Pod, exporterSet *virtual func assertRenderPodSharedVolume(t *testing.T, pod *corev1.Pod) { t.Helper() - // Only shared volume — config volume is injected by the reconciler. - if len(pod.Spec.Volumes) != 1 { - t.Fatalf("expected 1 volume (shared), got %d", len(pod.Spec.Volumes)) + // Shared emptyDir + guest disk emptyDir (default size). Config volume is + // injected by the reconciler. + if len(pod.Spec.Volumes) != 2 { + t.Fatalf("expected 2 volumes (shared + disk), got %d", len(pod.Spec.Volumes)) } - if pod.Spec.Volumes[0].EmptyDir == nil { - t.Fatal("expected shared emptyDir volume at index 0") + if pod.Spec.Volumes[0].Name != sharedVolumeName || pod.Spec.Volumes[0].EmptyDir == nil { + t.Fatalf("expected shared emptyDir volume at index 0, got %#v", pod.Spec.Volumes[0]) } wantLimit := resource.MustParse(sharedVolumeSizeLimit) if pod.Spec.Volumes[0].EmptyDir.SizeLimit == nil || !pod.Spec.Volumes[0].EmptyDir.SizeLimit.Equal(wantLimit) { - t.Errorf("SizeLimit = %v, want %v", pod.Spec.Volumes[0].EmptyDir.SizeLimit, wantLimit) + t.Errorf("shared SizeLimit = %v, want %v", pod.Spec.Volumes[0].EmptyDir.SizeLimit, wantLimit) + } + if pod.Spec.Volumes[1].Name != disk.VolumeName || pod.Spec.Volumes[1].EmptyDir == nil { + t.Fatalf("expected disk emptyDir volume at index 1, got %#v", pod.Spec.Volumes[1]) + } + wantDisk := resource.MustParse("10Gi") + if pod.Spec.Volumes[1].EmptyDir.SizeLimit == nil || + !pod.Spec.Volumes[1].EmptyDir.SizeLimit.Equal(wantDisk) { + t.Errorf("disk SizeLimit = %v, want %v", pod.Spec.Volumes[1].EmptyDir.SizeLimit, wantDisk) + } + + ephemeral := pod.Spec.Containers[0].Resources.Requests[corev1.ResourceEphemeralStorage] + if !ephemeral.Equal(wantDisk) { + t.Errorf("exporter ephemeral-storage request = %v, want %v", ephemeral, wantDisk) + } + runtimeEphemeral := pod.Spec.InitContainers[1].Resources.Requests[corev1.ResourceEphemeralStorage] + if !runtimeEphemeral.Equal(wantDisk) { + t.Errorf("runtime ephemeral-storage request = %v, want %v", runtimeEphemeral, wantDisk) + } + + assertDiskMount(t, runtimeContainerName, pod.Spec.InitContainers[1].VolumeMounts) + assertDiskMount(t, "exporter", pod.Spec.Containers[0].VolumeMounts) + for _, m := range pod.Spec.InitContainers[0].VolumeMounts { + if m.Name == disk.VolumeName { + t.Error("copy-jumpstarter-exec should not mount /disk") + } } } @@ -165,6 +192,16 @@ func assertSharedMount(t *testing.T, name string, mounts []corev1.VolumeMount) { t.Errorf("%s missing VolumeMount %s -> %s; got %#v", name, sharedVolumeName, sharedMountPath, mounts) } +func assertDiskMount(t *testing.T, name string, mounts []corev1.VolumeMount) { + t.Helper() + for _, m := range mounts { + if m.Name == disk.VolumeName && m.MountPath == disk.MountPath { + return + } + } + t.Errorf("%s missing VolumeMount %s -> %s; got %#v", name, disk.VolumeName, disk.MountPath, mounts) +} + func TestRenderPod_clonesSchedulingFromVTC(t *testing.T) { cpu := resource.MustParse("500m") mem := resource.MustParse("512Mi") @@ -421,3 +458,78 @@ func TestRenderPod_partialImageOverride(t *testing.T) { t.Errorf("exporter image = %q, want %q", pod.Spec.Containers[0].Image, wantExporter) } } + +func TestRenderPod_diskEphemeralWhenStorageClassSet(t *testing.T) { + exporterSet := &virtualtargetv1alpha1.ExporterSet{ + ObjectMeta: metav1.ObjectMeta{Name: "demo-set", Namespace: "default"}, + } + vtc := &virtualtargetv1alpha1.VirtualTargetClass{ + Spec: virtualtargetv1alpha1.VirtualTargetClassSpec{ + Provisioner: ProvisionerName, + }, + } + params := map[string]interface{}{ + "resources": map[string]interface{}{ + "storage": "20Gi", + }, + "storage": map[string]interface{}{ + "storageClassName": "fast-ssd", + }, + } + + pod, err := New("dev").RenderPod(context.Background(), exporterSet, vtc, params, nil, nil) + if err != nil { + t.Fatalf("RenderPod() error = %v", err) + } + + var diskVol *corev1.Volume + for i := range pod.Spec.Volumes { + if pod.Spec.Volumes[i].Name == disk.VolumeName { + diskVol = &pod.Spec.Volumes[i] + break + } + } + if diskVol == nil || diskVol.Ephemeral == nil || diskVol.Ephemeral.VolumeClaimTemplate == nil { + t.Fatalf("expected disk ephemeral volume, got %#v", diskVol) + } + claim := diskVol.Ephemeral.VolumeClaimTemplate.Spec + if claim.StorageClassName == nil || *claim.StorageClassName != "fast-ssd" { + t.Errorf("StorageClassName = %v, want fast-ssd", claim.StorageClassName) + } + want := resource.MustParse("20Gi") + if !claim.Resources.Requests[corev1.ResourceStorage].Equal(want) { + t.Errorf("storage request = %v, want %v", claim.Resources.Requests[corev1.ResourceStorage], want) + } + if _, ok := pod.Spec.Containers[0].Resources.Requests[corev1.ResourceEphemeralStorage]; ok { + t.Error("PVC mode should not set ephemeral-storage for guest disk") + } + if pod.Spec.SecurityContext == nil || pod.Spec.SecurityContext.FSGroup == nil || + *pod.Spec.SecurityContext.FSGroup != exporterNonRootUID { + t.Errorf("FSGroup = %v, want %d", pod.Spec.SecurityContext, exporterNonRootUID) + } +} + +func TestRenderPod_diskEmptyDirUsesParamSize(t *testing.T) { + exporterSet := &virtualtargetv1alpha1.ExporterSet{ + ObjectMeta: metav1.ObjectMeta{Name: "demo-set", Namespace: "default"}, + } + vtc := &virtualtargetv1alpha1.VirtualTargetClass{ + Spec: virtualtargetv1alpha1.VirtualTargetClassSpec{Provisioner: ProvisionerName}, + } + params := map[string]interface{}{ + "resources": map[string]interface{}{ + "storage": "7Gi", + }, + } + + pod, err := New("dev").RenderPod(context.Background(), exporterSet, vtc, params, nil, nil) + if err != nil { + t.Fatalf("RenderPod() error = %v", err) + } + + want := resource.MustParse("7Gi") + diskVol := pod.Spec.Volumes[1] + if diskVol.EmptyDir == nil || diskVol.EmptyDir.SizeLimit == nil || !diskVol.EmptyDir.SizeLimit.Equal(want) { + t.Errorf("disk SizeLimit = %v, want %v", diskVol.EmptyDir, want) + } +} diff --git a/docs/source/contributing/jeps/JEP-0014-virtual-scalable-exporters.md b/docs/source/contributing/jeps/JEP-0014-virtual-scalable-exporters.md index cc80c5da4..af871b27a 100644 --- a/docs/source/contributing/jeps/JEP-0014-virtual-scalable-exporters.md +++ b/docs/source/contributing/jeps/JEP-0014-virtual-scalable-exporters.md @@ -164,10 +164,10 @@ spec: cpu: 4 memory: 4Gi storage: 16Gi + # storage: + # storageClassName: gp3 # omit → sized emptyDir at /disk ``` -**Example: ExporterSet (generic scaling resource)** - ```yaml apiVersion: virtualtarget.jumpstarter.dev/v1alpha1 kind: ExporterSet @@ -296,21 +296,32 @@ spec: initContainers: - name: copy-jumpstarter-exec # one-shot: stage binary onto shared volume image: quay.io/jumpstarter-dev/jumpstarter:latest + volumeMounts: + - name: shared + mountPath: /shared - name: target-runtime # native sidecar (starts before exporter) restartPolicy: Always image: quay.io/jumpstarter-dev/virtual/qemu-runtime:latest volumeMounts: - name: shared mountPath: /shared + - name: disk + mountPath: /disk containers: - name: exporter # main — default logs; exit tears Pod down image: quay.io/jumpstarter-dev/jumpstarter:latest volumeMounts: - name: shared mountPath: /shared + - name: disk + mountPath: /disk volumes: - name: shared - emptyDir: {} + emptyDir: + sizeLimit: 100Mi # sockets / cidata / jumpstarter-exec + - name: disk + emptyDir: + sizeLimit: 16Gi # guest disk; or ephemeral volumeClaimTemplate ``` Benefits: @@ -582,8 +593,8 @@ with env() as client: **Exporter actions:** -- `storage.flash` writes the image to shared storage (or tells QEMU runtime via - QMP/`blockdev-add`). +- `storage.flash` writes the image to `/disk` on the guest-disk volume (or tells + QEMU runtime via QMP/`blockdev-add`). - `power.on` sends QEMU start via QMP or launcher socket on shared volume. - Serial/network drivers proxy to the QEMU runtime sidecar. @@ -954,6 +965,28 @@ firmware: # unchanged — set did not specify firmware digest: sha256:abc... ``` +**Guest disk (`qemu.jumpstarter.dev`):** `parameters.resources.storage` is the +guest disk size (also mapped to the QEMU driver's `disk_size`). Optional +`parameters.storage` selects the Kubernetes volume backend: + +```yaml +parameters: + resources: + storage: 16Gi + storage: + storageClassName: gp3 # omit or "" → sized emptyDir + accessModes: ["ReadWriteOnce"] # default ReadWriteOnce +``` + +When `storageClassName` is set, the provisioner attaches a [generic ephemeral +volume](https://kubernetes.io/docs/concepts/storage/ephemeral-volumes/#generic-ephemeral-volumes) +(`volumeClaimTemplate`) at `/disk` so the claim is created and deleted with the +Pod (ExitAndReplace). When it is omitted, `/disk` is a sized `emptyDir` and +the provisioner sets `ephemeral-storage` requests/limits for scheduler +accounting. Unix sockets stay on the separate 100Mi `/shared` volume. +ExporterSet `parameters.storage` deep-merges over the class; an empty +`storageClassName` on the set forces emptyDir. + **Status subresource (ExporterSet):** ```yaml @@ -1467,9 +1500,9 @@ Unit tests should meet the project test coverage requirements. - Kind e2e suite labeled `exporterset-qemu` (`e2e/test/exporterset_qemu_test.go`, part of `make e2e-run` / CI `e2e-tests`): apply kind-friendly `VirtualTargetClass` + `ExporterSet`, wait for Exporter/Pod Ready, lease, - flash Alpine UEFI tiny, expect a serial-console boot marker. Flash/boot is - skipped until guest-disk capacity lands (#924); control-plane coverage runs - today. Use `make e2e-exporterset-qemu` for a focused local run. + flash Alpine UEFI tiny, expect a serial-console boot marker. Guest disk is a + sized emptyDir (or ephemeral PVC when `parameters.storage.storageClassName` is + set). Use `make e2e-exporterset-qemu` for a focused local run. - Mixed physical/virtual lease orchestration - Provisioner failure and recovery scenarios - Parameter deep-merge and provisioner-side validation @@ -1663,7 +1696,7 @@ flash-at-lease workflow (DD-7). - [ ] Watch Leases and Exporters for scaling decisions - [ ] Add `exporterSets` section to `Jumpstarter` operator CR - [x] Integration test: deploy `ExporterSet`, lease, flash, boot, release, - observe scaling (`exporterset-qemu` e2e; flash/boot gated on #924 storage) + observe scaling (`exporterset-qemu` e2e) ### Phase 3: External / off-cluster provisioning @@ -1721,6 +1754,9 @@ claim CRDs. - 2026-07-28: Made `DriverConfig.name` mandatory (no longer derived from type); removed `wait-for-binary.sh` script in favor of direct `jumpstarter-exec` entrypoint in `qemu-runtime` container; updated all examples +- 2026-09-01: Guest disk at `/disk` via `parameters.resources.storage` (size) + and optional `parameters.storage.storageClassName` (generic ephemeral PVC or + sized emptyDir); sockets remain on `/shared` ## References diff --git a/e2e/README.md b/e2e/README.md index e3e656ab4..d8d858702 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -163,7 +163,7 @@ arch detected via `qemu-guest-arch.sh`; Alpine guest image ensured via | Test Name | Steps | Pass Check | |---|---|---| | brings an Exporter Online with a Ready Pod | wait for ExporterSet-created Exporter, wait Online/Registered/Available, wait Pod Running+Ready | Pod ready; `target-runtime` container has the expected `qemu-system-*` binary | -| leases, flashes Alpine, and boots to a console login marker | (skipped if shared emptyDir `sizeLimit` is empty or `100Mi`, pending #924) run `qemu_flash_boot.py` under `jmp shell --duration 1h` | script output contains "OK: matched marker" | +| leases, flashes Alpine, and boots to a console login marker | run `qemu_flash_boot.py` under `jmp shell --duration 1h` | script output contains "OK: matched marker" | | power cycles QEMU then rotates the Pod/Exporter and stays responsive | record old Pod name/UID; `j qemu power on/off` inside shell, verify qemu binary is the running process; wait old Pod+Exporter deleted, one new Pod/Exporter running w/ new UID; re-run `j qemu power on/off` | ExitAndReplace produced exactly one new, ready, differently-UID'd Pod/Exporter that still answers power commands | --- diff --git a/e2e/test/exporterset_qemu_test.go b/e2e/test/exporterset_qemu_test.go index 36975de64..5cc6dd1bf 100644 --- a/e2e/test/exporterset_qemu_test.go +++ b/e2e/test/exporterset_qemu_test.go @@ -159,24 +159,6 @@ var _ = Describe("ExporterSet QEMU E2E Tests", Label("exporterset-qemu"), Ordere }) It("leases, flashes Alpine, and boots to a console login marker", func() { - By("waiting for a Running pod so we can read shared volume SizeLimit") - Eventually(func() string { - return KubectlQuery("-n", ns, "get", "pod", - "-l", guest.Selector, - "--field-selector=status.phase=Running", - "-o", "jsonpath={.items[0].metadata.name}") - }, 2*time.Minute, qemuPollPeriod).ShouldNot(BeEmpty()) - - sizeLimit := KubectlQuery("-n", ns, "get", "pod", - "-l", guest.Selector, - "--field-selector=status.phase=Running", - "-o", "jsonpath={.items[0].spec.volumes[?(@.name==\"shared\")].emptyDir.sizeLimit}") - // Without the storage follow-up (#924), SizeLimit stays at 100Mi and - // flashing Alpine evicts the Pod. Skip until capacity is available. - if sizeLimit == "" || sizeLimit == "100Mi" { - Skip(fmt.Sprintf("shared emptyDir SizeLimit=%q is too small for Alpine flash; needs #924 storage work", sizeLimit)) - } - By("running flash+boot helper under jmp shell") // Long timeout: Kind uses TCG emulation without KVM. cmd := JmpCmd( diff --git a/python/packages/jumpstarter-driver-qemu/jumpstarter_driver_qemu/driver.py b/python/packages/jumpstarter-driver-qemu/jumpstarter_driver_qemu/driver.py index b346a17e1..2cba7ecc4 100644 --- a/python/packages/jumpstarter-driver-qemu/jumpstarter_driver_qemu/driver.py +++ b/python/packages/jumpstarter-driver-qemu/jumpstarter_driver_qemu/driver.py @@ -285,13 +285,15 @@ async def on(self) -> None: # noqa: C901 for device in devices: cmdline += ["-device", device] - if bios.exists(): + if bios.exists() or self.parent._runtime_firmware_path(bios): cmdline += [ "-bios", str(bios), ] - if ovmf_code.exists() and ovmf_vars.exists(): + if (ovmf_code.exists() or self.parent._runtime_firmware_path(ovmf_code)) and ( + ovmf_vars.exists() or self.parent._runtime_firmware_path(ovmf_vars) + ): cmdline += [ "-drive", f"file={ovmf_code},if=pflash,format=raw,unit=0,readonly=on", @@ -482,6 +484,7 @@ def __post_init__(self): @property def _work_dir(self) -> str: + """Directory for sockets and jumpstarter-exec in sidecar mode.""" if self.launcher_socket: # Sidecar: QEMU only sees the shared volume. Derive from the # socket path so production (/shared/launcher.sock) and tests @@ -489,6 +492,18 @@ def _work_dir(self) -> str: return str(Path(self.launcher_socket).parent) return self._tmp_dir.name + @property + def _disk_dir(self) -> str: + """Directory for flashable guest disk images (root, bios, …).""" + if self.launcher_socket: + work = Path(self._work_dir) + # Production sidecar: sockets on /shared, guest disk on /disk. + # Tests use a tmp shared dir — keep disks next to sockets. + if work == Path("/shared"): + return "/disk" + return str(work) + return self._tmp_dir.name + @property def _pty(self) -> str: return str(Path(self._work_dir) / "pty") @@ -518,6 +533,10 @@ def _wrap_command(self, cmd: list[str]) -> list[str]: def _cid(self) -> int: return randbits(32) + def _runtime_firmware_path(self, path: Path) -> bool: + """True when path is a default firmware path that lives in the runtime image.""" + return self.launcher_socket is not None and path in self.default_partitions.values() + def validate_partition( self, partition: str | None = None, @@ -525,13 +544,13 @@ def validate_partition( ) -> Path: match partition: case "root" | None: - path = Path(self._work_dir) / "root" + path = Path(self._disk_dir) / "root" case "OVMF_CODE.fd": - path = Path(self._work_dir) / "OVMF_CODE.fd" + path = Path(self._disk_dir) / "OVMF_CODE.fd" case "OVMF_VARS.fd": - path = Path(self._work_dir) / "OVMF_VARS.fd" + path = Path(self._disk_dir) / "OVMF_VARS.fd" case "bios": - path = Path(self._work_dir) / "bios" + path = Path(self._disk_dir) / "bios" case _: raise ValueError(f"invalid partition name: {partition}") diff --git a/python/packages/jumpstarter-driver-qemu/jumpstarter_driver_qemu/driver_test.py b/python/packages/jumpstarter-driver-qemu/jumpstarter_driver_qemu/driver_test.py index bb7e133b0..777595825 100644 --- a/python/packages/jumpstarter-driver-qemu/jumpstarter_driver_qemu/driver_test.py +++ b/python/packages/jumpstarter-driver-qemu/jumpstarter_driver_qemu/driver_test.py @@ -206,6 +206,23 @@ def test_set_memory_size_invalid(): driver.set_memory_size("invalid") +def test_disk_dir_uses_tmp_by_default(): + driver = Qemu() + assert driver._disk_dir == driver._tmp_dir.name + + +def test_disk_dir_stays_with_shared_in_tests(tmp_path): + shared = tmp_path / "shared" + shared.mkdir() + driver = Qemu(launcher_socket=str(shared / "launcher.sock")) + assert driver._disk_dir == str(shared) + + +def test_disk_dir_is_slash_disk_in_production_sidecar(): + driver = Qemu(launcher_socket="/shared/launcher.sock") + assert driver._disk_dir == "/disk" + + def test_cidata_uses_tmp_by_default(): """Local mode keeps cloud-init vvfat content under a system temp dir.""" driver = Qemu()