diff --git a/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go b/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go index 63ef1f274..e07befef6 100644 --- a/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go +++ b/controller/deploy/operator/api/v1alpha1/jumpstarter_types.go @@ -46,6 +46,9 @@ const ( // provisioner controller deployments are available ConditionTypeExporterSetControllersReady = "ExporterSetControllersReady" + // ConditionTypeTelemetryDeploymentReady indicates whether the telemetry deployment is available + ConditionTypeTelemetryDeploymentReady = "TelemetryDeploymentReady" + // ConditionTypeReady indicates whether the overall Jumpstarter system is ready ConditionTypeReady = "Ready" ) @@ -201,6 +204,13 @@ type JumpstarterSpec struct { // Deprecated labels configuration for warning users about label keys that should no longer be used. // +optional DeprecatedLabels DeprecatedLabelsConfig `json:"deprecatedLabels,omitempty"` + + // Telemetry configuration for the optional telemetry service. + // When enabled, the operator deploys a jumpstarter-telemetry service that receives + // structured log entries from exporters via gRPC. The controller advertises the + // telemetry endpoint to exporters so they can push logs without cluster credentials. + // +optional + Telemetry *TelemetryConfig `json:"telemetry,omitempty"` } // HiddenLabelsConfig defines label keys to hide from exporter listings by default. @@ -266,6 +276,55 @@ type DeprecatedLabelsConfig struct { Keys map[string]string `json:"keys,omitempty"` } +// TelemetryConfig defines configuration for the telemetry service deployment. +// When enabled, the operator creates a Deployment and ClusterIP Service for +// jumpstarter-telemetry, which receives structured log entries from exporters. +type TelemetryConfig struct { + // Enable the telemetry service deployment. + // When enabled, the operator deploys a jumpstarter-telemetry pod and a ClusterIP + // Service, and configures the controller to advertise the endpoint to exporters. + // +kubebuilder:default=false + Enabled bool `json:"enabled,omitempty"` + + // Container image for the telemetry pod in 'registry/repository/image:tag' format. + // +kubebuilder:default="quay.io/jumpstarter-dev/jumpstarter-telemetry:latest" + Image string `json:"image,omitempty"` + + // Image pull policy for the telemetry container. + // +kubebuilder:default="IfNotPresent" + // +kubebuilder:validation:Enum=Always;IfNotPresent;Never + ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` + + // Number of telemetry replicas to run. + // Multiple replicas provide HA; each exporter connects to exactly one replica + // via a persistent MetricsStream, so Prometheus sum-by queries across replicas + // yield exact totals without double-counting (see JEP-0013 DD-8). + // +kubebuilder:default=1 + // +kubebuilder:validation:Minimum=1 + Replicas *int32 `json:"replicas,omitempty"` + + // Resource requirements for the telemetry pod. + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // Logging configuration for the telemetry log ingestion path. + Logging TelemetryLoggingConfig `json:"logging,omitempty"` +} + +// TelemetryLoggingConfig configures the log push path to the telemetry service. +type TelemetryLoggingConfig struct { + // Filter controls which log entries are forwarded to the telemetry service. + Filter TelemetryLoggingFilterConfig `json:"filter,omitempty"` +} + +// TelemetryLoggingFilterConfig controls which log entries are forwarded to the telemetry service. +type TelemetryLoggingFilterConfig struct { + // Minimum log severity to forward. + // Accepted values: debug, info, warning, error, critical. Defaults to "info". + // +kubebuilder:default="info" + // +kubebuilder:validation:Enum=debug;info;warning;error;critical + MinSeverity string `json:"minSeverity,omitempty"` +} + // LeasePolicyConfig defines policy constraints for leases. type LeasePolicyConfig struct { // Maximum number of user-defined tags allowed per lease. diff --git a/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go b/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go index 9e78d6bf2..b542068db 100644 --- a/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/controller/deploy/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -524,6 +524,11 @@ func (in *JumpstarterSpec) DeepCopyInto(out *JumpstarterSpec) { (*in).DeepCopyInto(*out) } in.DeprecatedLabels.DeepCopyInto(&out.DeprecatedLabels) + if in.Telemetry != nil { + in, out := &in.Telemetry, &out.Telemetry + *out = new(TelemetryConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JumpstarterSpec. @@ -885,3 +890,56 @@ func (in *TLSConfig) DeepCopy() *TLSConfig { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TelemetryConfig) DeepCopyInto(out *TelemetryConfig) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + in.Resources.DeepCopyInto(&out.Resources) + out.Logging = in.Logging +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryConfig. +func (in *TelemetryConfig) DeepCopy() *TelemetryConfig { + if in == nil { + return nil + } + out := new(TelemetryConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TelemetryLoggingConfig) DeepCopyInto(out *TelemetryLoggingConfig) { + *out = *in + out.Filter = in.Filter +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryLoggingConfig. +func (in *TelemetryLoggingConfig) DeepCopy() *TelemetryLoggingConfig { + if in == nil { + return nil + } + out := new(TelemetryLoggingConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TelemetryLoggingFilterConfig) DeepCopyInto(out *TelemetryLoggingFilterConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetryLoggingFilterConfig. +func (in *TelemetryLoggingFilterConfig) DeepCopy() *TelemetryLoggingFilterConfig { + if in == nil { + return nil + } + out := new(TelemetryLoggingFilterConfig) + in.DeepCopyInto(out) + return out +} diff --git a/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml b/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml index 47a74f86e..5dc9cf8d4 100644 --- a/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml +++ b/controller/deploy/operator/config/crd/bases/operator.jumpstarter.dev_jumpstarters.yaml @@ -2090,6 +2090,125 @@ spec: type: object type: array type: object + telemetry: + description: |- + Telemetry configuration for the optional telemetry service. + When enabled, the operator deploys a jumpstarter-telemetry service that receives + structured log entries from exporters via gRPC. The controller advertises the + telemetry endpoint to exporters so they can push logs without cluster credentials. + properties: + enabled: + default: false + description: |- + Enable the telemetry service deployment. + When enabled, the operator deploys a jumpstarter-telemetry pod and a ClusterIP + Service, and configures the controller to advertise the endpoint to exporters. + type: boolean + image: + default: quay.io/jumpstarter-dev/jumpstarter-telemetry:latest + description: Container image for the telemetry pod in 'registry/repository/image:tag' + format. + type: string + imagePullPolicy: + default: IfNotPresent + description: Image pull policy for the telemetry container. + enum: + - Always + - IfNotPresent + - Never + type: string + logging: + description: Logging configuration for the telemetry log ingestion + path. + properties: + filter: + description: Filter controls which log entries are forwarded + to the telemetry service. + properties: + minSeverity: + default: info + description: |- + Minimum log severity to forward. + Accepted values: debug, info, warning, error, critical. Defaults to "info". + enum: + - debug + - info + - warning + - error + - critical + type: string + type: object + type: object + replicas: + default: 1 + description: |- + Number of telemetry replicas to run. + Multiple replicas provide HA; each exporter connects to exactly one replica + via a persistent MetricsStream, so Prometheus sum-by queries across replicas + yield exact totals without double-counting (see JEP-0013 DD-8). + format: int32 + minimum: 1 + type: integer + resources: + description: Resource requirements for the telemetry pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + type: object type: object status: description: |- diff --git a/controller/deploy/operator/internal/controller/jumpstarter/certificates.go b/controller/deploy/operator/internal/controller/jumpstarter/certificates.go index 32183aec5..47c0c6d78 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/certificates.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/certificates.go @@ -114,6 +114,13 @@ func (r *JumpstarterReconciler) reconcileCertificates(ctx context.Context, js *o } } + // Create telemetry certificate if telemetry is enabled + if js.Spec.Telemetry != nil && js.Spec.Telemetry.Enabled { + if err := r.reconcileTelemetryCertificate(ctx, js, issuerRef); err != nil { + return fmt.Errorf("failed to reconcile telemetry certificate: %w", err) + } + } + // Reconcile CA ConfigMap AFTER certificates are created // This ensures cert-manager has had a chance to create the CA secret // which we need to populate the ConfigMap for the login service @@ -369,6 +376,28 @@ func (r *JumpstarterReconciler) reconcileRouterCertificate(ctx context.Context, return r.reconcileServerCertificate(ctx, js, issuerRef, certName, "router", dnsNames, extraLabels) } +// reconcileTelemetryCertificate creates the TLS certificate for the telemetry service. +func (r *JumpstarterReconciler) reconcileTelemetryCertificate(ctx context.Context, js *operatorv1alpha1.Jumpstarter, issuerRef cmmeta.ObjectReference) error { + certName := getTelemetryCertSecretName(js) + includeInternalNames := !isExternalIssuer(js) + dnsNames := r.collectTelemetryDNSNames(js, includeInternalNames) + return r.reconcileServerCertificate(ctx, js, issuerRef, certName, "telemetry", dnsNames, nil) +} + +// collectTelemetryDNSNames collects all DNS names for the telemetry certificate. +func (r *JumpstarterReconciler) collectTelemetryDNSNames(js *operatorv1alpha1.Jumpstarter, includeInternalNames bool) []string { + var dnsNames []string + if includeInternalNames { + dnsNames = append(dnsNames, + telemetryServiceName, + fmt.Sprintf("%s.%s", telemetryServiceName, js.Namespace), + fmt.Sprintf("%s.%s.svc", telemetryServiceName, js.Namespace), + fmt.Sprintf("%s.%s.svc.cluster.local", telemetryServiceName, js.Namespace), + ) + } + return dnsNames +} + // collectControllerDNSNames collects all DNS names for the controller certificate. // When includeInternalNames is false, internal Kubernetes service DNS names are // omitted so that external issuers (e.g. ACME/Let's Encrypt) don't attempt to diff --git a/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go b/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go index 21c5c2147..db92b2943 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/jumpstarter_controller.go @@ -210,12 +210,24 @@ func (r *JumpstarterReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - // Reconcile Services + // Reconcile Telemetry Deployment (Service is reconciled below in the networking stage) + if err := r.reconcileTelemetryDeploymentStage(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile Telemetry deployment") + return ctrl.Result{}, err + } + + // Reconcile Services (controller, router, login endpoints, and telemetry ClusterIP) if err := r.reconcileServices(ctx, &jumpstarter); err != nil { log.Error(err, "Failed to reconcile Services") return ctrl.Result{}, err } + // Reconcile Telemetry ClusterIP Service (part of the networking stage) + if err := r.reconcileTelemetryServiceStage(ctx, &jumpstarter); err != nil { + log.Error(err, "Failed to reconcile Telemetry service") + return ctrl.Result{}, err + } + // Reconcile ConfigMaps (after deployments and services, before secrets) if err := r.reconcileConfigMaps(ctx, &jumpstarter, desiredConfigMap); err != nil { log.Error(err, "Failed to reconcile ConfigMaps") @@ -1292,6 +1304,20 @@ func (r *JumpstarterReconciler) buildConfig(ctx context.Context, jumpstarter *op Keys: jumpstarter.Spec.DeprecatedLabels.Keys, } + // Telemetry configuration. + // Certificate is intentionally omitted until the telemetry binary supports TLS serving. + if jumpstarter.Spec.Telemetry != nil && jumpstarter.Spec.Telemetry.Enabled { + t := jumpstarter.Spec.Telemetry + telemetryCfg := &config.Telemetry{ + Enabled: true, + Endpoint: telemetryEndpointFor(jumpstarter.Namespace), + } + if t.Logging.Filter.MinSeverity != "" { + telemetryCfg.Logging.Filter.MinSeverity = t.Logging.Filter.MinSeverity + } + cfg.Telemetry = telemetryCfg + } + // gRPC keepalive configuration if jumpstarter.Spec.Controller.GRPC.Keepalive != nil { ka := &cfg.Grpc.Keepalive diff --git a/controller/deploy/operator/internal/controller/jumpstarter/status.go b/controller/deploy/operator/internal/controller/jumpstarter/status.go index 95ad1df51..cbe15429f 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/status.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/status.go @@ -111,6 +111,21 @@ func (r *JumpstarterReconciler) updateStatus(ctx context.Context, js *operatorv1 messages = append(messages, routersMsg) } + // Check telemetry deployment readiness (only if enabled), clear stale condition when disabled. + if js.Spec.Telemetry != nil && js.Spec.Telemetry.Enabled { + telReady, telMsg := r.checkTelemetryDeploymentReady(ctx, js) + setCondition(js, operatorv1alpha1.ConditionTypeTelemetryDeploymentReady, + telReady, + conditionReason(telReady, "DeploymentAvailable", "DeploymentNotAvailable"), + telMsg) + if !telReady { + allReady = false + messages = append(messages, telMsg) + } + } else { + meta.RemoveStatusCondition(&js.Status.Conditions, operatorv1alpha1.ConditionTypeTelemetryDeploymentReady) + } + // Check ExporterSet controller deployments readiness (only if configured) if js.Spec.ExporterSets != nil && hasEnabledProvisioners(js.Spec.ExporterSets.Provisioners) { esReady, esMsg := r.checkExporterSetControllersReady(ctx, js) @@ -465,6 +480,32 @@ func (r *JumpstarterReconciler) checkExporterSetControllersReady(ctx context.Con return false, fmt.Sprintf("ExporterSet controller deployments not available for provisioners: %v", notReady) } +// checkTelemetryDeploymentReady checks if the telemetry deployment is available. +func (r *JumpstarterReconciler) checkTelemetryDeploymentReady(ctx context.Context, js *operatorv1alpha1.Jumpstarter) (bool, string) { + log := logf.FromContext(ctx) + deploymentName := fmt.Sprintf("%s-telemetry", js.Name) + deployment := &appsv1.Deployment{} + err := r.Get(ctx, types.NamespacedName{Name: deploymentName, Namespace: js.Namespace}, deployment) + if err != nil { + if errors.IsNotFound(err) { + return false, fmt.Sprintf("Telemetry deployment %s not found", deploymentName) + } + log.Error(err, "failed to get telemetry deployment", "deployment", deploymentName) + return false, "error querying telemetry deployment; check operator logs for details" + } + + for _, cond := range deployment.Status.Conditions { + if cond.Type == appsv1.DeploymentAvailable { + if cond.Status == corev1.ConditionTrue { + return true, fmt.Sprintf("Telemetry deployment %s is available", deploymentName) + } + return false, fmt.Sprintf("Telemetry deployment %s not available: %s", deploymentName, cond.Message) + } + } + + return false, fmt.Sprintf("Telemetry deployment %s has no Available condition", deploymentName) +} + // hasEnabledProvisioners returns true if at least one provisioner is enabled. func hasEnabledProvisioners(provisioners []operatorv1alpha1.ProvisionerConfig) bool { for _, p := range provisioners { diff --git a/controller/deploy/operator/internal/controller/jumpstarter/suite_test.go b/controller/deploy/operator/internal/controller/jumpstarter/suite_test.go index 677f91e80..29de33345 100644 --- a/controller/deploy/operator/internal/controller/jumpstarter/suite_test.go +++ b/controller/deploy/operator/internal/controller/jumpstarter/suite_test.go @@ -70,7 +70,9 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "..", "config", "crd", "bases"), + }, ErrorIfCRDPathMissing: true, } diff --git a/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go b/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go new file mode 100644 index 000000000..c1dbcc532 --- /dev/null +++ b/controller/deploy/operator/internal/controller/jumpstarter/telemetry.go @@ -0,0 +1,480 @@ +/* +Copyright 2026 by the Jumpstarter Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jumpstarter + +import ( + "context" + "fmt" + + certmanagerv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/deploy/operator/api/v1alpha1" +) + +const ( + telemetryPort = 9093 + telemetryCertSuffix = "-telemetry-tls" + telemetryServiceName = "jumpstarter-telemetry" + telemetryComponentApp = "jumpstarter-telemetry" + telemetrySASuffix = "-telemetry" + grpcPortName = "grpc" +) + +// reconcileTelemetryDeploymentStage reconciles only the telemetry Deployment (and cleanup). +// It is called from the Deployment stage of the reconcile loop, before Services. +// Certificate reconciliation is handled in certificates.go alongside other certs. +func (r *JumpstarterReconciler) reconcileTelemetryDeploymentStage(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + if jumpstarter.Spec.Telemetry == nil || !jumpstarter.Spec.Telemetry.Enabled { + return r.cleanupTelemetry(ctx, jumpstarter) + } + + if err := r.reconcileTelemetryServiceAccount(ctx, jumpstarter); err != nil { + return fmt.Errorf("failed to reconcile telemetry service account: %w", err) + } + + if err := r.reconcileTelemetryDeployment(ctx, jumpstarter); err != nil { + return fmt.Errorf("failed to reconcile telemetry deployment: %w", err) + } + + return nil +} + +// reconcileTelemetryServiceAccount creates a dedicated, no-RBAC ServiceAccount for +// the telemetry pod. The telemetry binary has no need for Kubernetes API access; +// giving it the controller-manager SA would grant it far more privilege than required. +func (r *JumpstarterReconciler) reconcileTelemetryServiceAccount(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + saName := jumpstarter.Name + telemetrySASuffix + + desired := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + Namespace: jumpstarter.Namespace, + Labels: telemetryLabels(jumpstarter), + }, + } + + existing := &corev1.ServiceAccount{} + existing.Name = saName + existing.Namespace = jumpstarter.Namespace + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existing, func() error { + existing.Labels = desired.Labels + return controllerutil.SetControllerReference(jumpstarter, existing, r.Scheme) + }) + if err != nil { + return err + } + + log.V(1).Info("Telemetry ServiceAccount reconciled", "name", saName, "operation", op) + return nil +} + +// reconcileTelemetryServiceStage reconciles only the telemetry ClusterIP Service. +// It is called from the Services/networking stage of the reconcile loop. +func (r *JumpstarterReconciler) reconcileTelemetryServiceStage(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + if jumpstarter.Spec.Telemetry == nil || !jumpstarter.Spec.Telemetry.Enabled { + return r.cleanupTelemetryService(ctx, jumpstarter) + } + + if err := r.reconcileTelemetryService(ctx, jumpstarter); err != nil { + return fmt.Errorf("failed to reconcile telemetry service: %w", err) + } + + return nil +} + +// cleanupTelemetryService removes only the telemetry Service when telemetry is disabled. +// This provides symmetric cleanup so the Service isn't orphaned if the reconcile loop +// order changes (e.g. services reconciled before deployments). +func (r *JumpstarterReconciler) cleanupTelemetryService(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + + svc := &corev1.Service{} + svc.Name = telemetryServiceName + svc.Namespace = jumpstarter.Namespace + if err := r.Delete(ctx, svc); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete telemetry service: %w", err) + } else if err == nil { + log.Info("Deleted telemetry service", "name", telemetryServiceName) + } + + return nil +} + +// reconcileTelemetryDeployment creates or updates the telemetry Deployment. +func (r *JumpstarterReconciler) reconcileTelemetryDeployment(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + desiredDeployment := createTelemetryDeployment(jumpstarter) + + existingDeployment := &appsv1.Deployment{} + existingDeployment.Name = desiredDeployment.Name + existingDeployment.Namespace = desiredDeployment.Namespace + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingDeployment, func() error { + if existingDeployment.CreationTimestamp.IsZero() { + existingDeployment.Labels = desiredDeployment.Labels + existingDeployment.Annotations = desiredDeployment.Annotations + existingDeployment.Spec = desiredDeployment.Spec + return controllerutil.SetControllerReference(jumpstarter, existingDeployment, r.Scheme) + } + + desiredDeployment.Spec.Template.Spec.DeprecatedServiceAccount = existingDeployment.Spec.Template.Spec.DeprecatedServiceAccount + desiredDeployment.Spec.Template.Spec.SchedulerName = existingDeployment.Spec.Template.Spec.SchedulerName + + if !deploymentNeedsUpdate(existingDeployment, desiredDeployment) { + log.V(1).Info("Telemetry deployment is up to date", "name", existingDeployment.Name) + // Always ensure owner reference is set even if no other updates needed + return controllerutil.SetControllerReference(jumpstarter, existingDeployment, r.Scheme) + } + + diff, diffErr := generateDiff(existingDeployment, desiredDeployment) + if diffErr != nil { + log.V(1).Info("Failed to generate deployment diff", "error", diffErr) + } else if diff != "" { + log.V(1).Info("Telemetry deployment differences detected", + "name", existingDeployment.Name, + "namespace", existingDeployment.Namespace, + "diff", diff) + } + + existingDeployment.Labels = desiredDeployment.Labels + existingDeployment.Annotations = desiredDeployment.Annotations + existingDeployment.Spec.Replicas = desiredDeployment.Spec.Replicas + existingDeployment.Spec.Selector = desiredDeployment.Spec.Selector + existingDeployment.Spec.Template = desiredDeployment.Spec.Template + return controllerutil.SetControllerReference(jumpstarter, existingDeployment, r.Scheme) + }) + + if err != nil { + log.Error(err, "Failed to reconcile telemetry deployment", + "name", desiredDeployment.Name, "namespace", desiredDeployment.Namespace) + return err + } + + switch op { + case controllerutil.OperationResultNone: + log.V(1).Info("Telemetry deployment is up to date", + "name", existingDeployment.Name, "namespace", existingDeployment.Namespace) + default: + log.Info("Telemetry deployment reconciled", + "name", existingDeployment.Name, "namespace", existingDeployment.Namespace, "operation", op) + } + + switch op { + case controllerutil.OperationResultCreated: + r.emitEventf(jumpstarter, corev1.EventTypeNormal, "TelemetryDeploymentCreated", + "Telemetry deployment created: name=%s namespace=%s", + existingDeployment.Name, existingDeployment.Namespace) + case controllerutil.OperationResultUpdated: + r.emitEventf(jumpstarter, corev1.EventTypeNormal, "TelemetryDeploymentUpdated", + "Telemetry deployment updated: name=%s namespace=%s", + existingDeployment.Name, existingDeployment.Namespace) + } + + return nil +} + +// reconcileTelemetryService creates or updates the telemetry ClusterIP Service. +func (r *JumpstarterReconciler) reconcileTelemetryService(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + + labels := telemetryLabels(jumpstarter) + desiredService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: telemetryServiceName, + Namespace: jumpstarter.Namespace, + Labels: labels, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: labels, + Ports: []corev1.ServicePort{ + { + Name: grpcPortName, + Port: int32(telemetryPort), + TargetPort: intstr.FromInt(telemetryPort), + Protocol: corev1.ProtocolTCP, + }, + }, + }, + } + + existingService := &corev1.Service{} + existingService.Name = desiredService.Name + existingService.Namespace = desiredService.Namespace + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, existingService, func() error { + existingService.Labels = desiredService.Labels + existingService.Spec.Type = desiredService.Spec.Type + existingService.Spec.Selector = desiredService.Spec.Selector + existingService.Spec.Ports = desiredService.Spec.Ports + return controllerutil.SetControllerReference(jumpstarter, existingService, r.Scheme) + }) + + if err != nil { + log.Error(err, "Failed to reconcile telemetry service", "name", desiredService.Name) + return err + } + + switch op { + case controllerutil.OperationResultNone: + log.V(1).Info("Telemetry service is up to date", + "name", existingService.Name, "namespace", existingService.Namespace) + case controllerutil.OperationResultCreated: + log.Info("Telemetry service reconciled", + "name", existingService.Name, "namespace", existingService.Namespace, "operation", op) + r.emitEventf(jumpstarter, corev1.EventTypeNormal, "TelemetryServiceCreated", + "Telemetry service created: name=%s namespace=%s", + existingService.Name, existingService.Namespace) + case controllerutil.OperationResultUpdated: + log.Info("Telemetry service reconciled", + "name", existingService.Name, "namespace", existingService.Namespace, "operation", op) + r.emitEventf(jumpstarter, corev1.EventTypeNormal, "TelemetryServiceUpdated", + "Telemetry service updated: name=%s namespace=%s", + existingService.Name, existingService.Namespace) + } + + return nil +} + +// createTelemetryDeployment builds the desired Deployment for the telemetry service. +func createTelemetryDeployment(jumpstarter *operatorv1alpha1.Jumpstarter) *appsv1.Deployment { + t := jumpstarter.Spec.Telemetry + labels := telemetryLabels(jumpstarter) + + replicas := int32(1) + if t.Replicas != nil { + replicas = *t.Replicas + } + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-telemetry", jumpstarter.Name), + Namespace: jumpstarter.Namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + ProgressDeadlineSeconds: ptr.To(int32(600)), + RevisionHistoryLimit: ptr.To(int32(10)), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxSurge: &intstr.IntOrString{Type: intstr.String, StrVal: "25%"}, + MaxUnavailable: &intstr.IntOrString{Type: intstr.String, StrVal: "25%"}, + }, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyAlways, + DNSPolicy: corev1.DNSClusterFirst, + TerminationGracePeriodSeconds: ptr.To(int64(30)), + Containers: []corev1.Container{ + { + Name: "telemetry", + Image: t.Image, + ImagePullPolicy: t.ImagePullPolicy, + Command: []string{"/telemetry"}, + Args: []string{ + fmt.Sprintf("--grpc-bind=:%d", telemetryPort), + }, + Env: []corev1.EnvVar{ + { + Name: "CONTROLLER_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "jumpstarter-controller-secret", + }, + Key: "key", + }, + }, + }, + }, + Ports: []corev1.ContainerPort{ + { + ContainerPort: int32(telemetryPort), + Name: grpcPortName, + Protocol: corev1.ProtocolTCP, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(telemetryPort), + }, + }, + InitialDelaySeconds: 10, + PeriodSeconds: 20, + TimeoutSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(telemetryPort), + }, + }, + InitialDelaySeconds: 5, + PeriodSeconds: 10, + TimeoutSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 3, + }, + Resources: defaultTelemetryResources(t.Resources), + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: corev1.TerminationMessageReadFile, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + }, + }, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + ServiceAccountName: jumpstarter.Name + telemetrySASuffix, + }, + }, + }, + } +} + +// cleanupTelemetry removes telemetry resources when telemetry is disabled. +// GC cannot remove the cert-manager Certificate while the Jumpstarter CR still +// exists (GC only fires when the owner is deleted), so it is deleted explicitly. +func (r *JumpstarterReconciler) cleanupTelemetry(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) error { + log := logf.FromContext(ctx) + + deploymentName := fmt.Sprintf("%s-telemetry", jumpstarter.Name) + dep := &appsv1.Deployment{} + dep.Name = deploymentName + dep.Namespace = jumpstarter.Namespace + if err := r.Delete(ctx, dep); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete telemetry deployment: %w", err) + } else if err == nil { + log.Info("Deleted telemetry deployment", "name", deploymentName) + r.emitEventf(jumpstarter, corev1.EventTypeNormal, "TelemetryDeploymentDeleted", + "Telemetry deployment deleted: name=%s", deploymentName) + } + + certName := getTelemetryCertSecretName(jumpstarter) + cert := &certmanagerv1.Certificate{} + cert.Name = certName + cert.Namespace = jumpstarter.Namespace + if err := r.Delete(ctx, cert); err != nil && !errors.IsNotFound(err) && !meta.IsNoMatchError(err) { + return fmt.Errorf("failed to delete telemetry certificate: %w", err) + } else if err == nil { + log.Info("Deleted telemetry certificate", "name", certName) + } + + saName := jumpstarter.Name + telemetrySASuffix + sa := &corev1.ServiceAccount{} + sa.Name = saName + sa.Namespace = jumpstarter.Namespace + if err := r.Delete(ctx, sa); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete telemetry service account: %w", err) + } else if err == nil { + log.Info("Deleted telemetry service account", "name", saName) + } + + return nil +} + +// getTelemetryCertSecretName returns the name of the telemetry TLS secret. +func getTelemetryCertSecretName(js *operatorv1alpha1.Jumpstarter) string { + return js.Name + telemetryCertSuffix +} + +// resolveTelemetryCA reads the CA certificate that exporters need to verify the +// telemetry TLS connection. For self-signed CA mode, the cert is in the CA secret; +// for external issuers, the user-provided caBundle is used. +func (r *JumpstarterReconciler) resolveTelemetryCA(ctx context.Context, jumpstarter *operatorv1alpha1.Jumpstarter) (string, error) { + if jumpstarter.Spec.CertManager.Server != nil && jumpstarter.Spec.CertManager.Server.IssuerRef != nil { + if len(jumpstarter.Spec.CertManager.Server.IssuerRef.CABundle) > 0 { + return string(jumpstarter.Spec.CertManager.Server.IssuerRef.CABundle), nil + } + return "", nil + } + + // Self-signed CA mode — read from the CA secret created by cert-manager + caSecretName := jumpstarter.Name + caCertificateSuffix + caSecret := &corev1.Secret{} + if err := r.Get(ctx, client.ObjectKey{Name: caSecretName, Namespace: jumpstarter.Namespace}, caSecret); err != nil { + return "", fmt.Errorf("CA secret %s not found: %w", caSecretName, err) + } + if cert, ok := caSecret.Data["tls.crt"]; ok { + return string(cert), nil + } + return "", fmt.Errorf("CA secret %s missing tls.crt", caSecretName) +} + +// telemetryEndpointFor returns the in-cluster gRPC endpoint for the telemetry service. +func telemetryEndpointFor(namespace string) string { + return fmt.Sprintf("%s.%s.svc:%d", telemetryServiceName, namespace, telemetryPort) +} + +// telemetryLabels returns the standard labels for telemetry resources. +func telemetryLabels(jumpstarter *operatorv1alpha1.Jumpstarter) map[string]string { + return map[string]string{ + "component": "telemetry", + "app": telemetryComponentApp, + "controller": jumpstarter.Name, + } +} + +// defaultTelemetryResources returns sensible defaults for the telemetry pod if no +// explicit resource requirements are provided. +func defaultTelemetryResources(spec corev1.ResourceRequirements) corev1.ResourceRequirements { + if len(spec.Requests) == 0 && len(spec.Limits) == 0 && len(spec.Claims) == 0 { + return corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + } + } + return spec +} diff --git a/controller/deploy/operator/internal/controller/jumpstarter/telemetry_test.go b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_test.go new file mode 100644 index 000000000..66e86d113 --- /dev/null +++ b/controller/deploy/operator/internal/controller/jumpstarter/telemetry_test.go @@ -0,0 +1,752 @@ +/* +Copyright 2026. The Jumpstarter Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jumpstarter + +import ( + "context" + "fmt" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/deploy/operator/api/v1alpha1" + "github.com/jumpstarter-dev/jumpstarter/controller/deploy/operator/internal/controller/jumpstarter/endpoints" +) + +var _ = Describe("Telemetry Lifecycle", func() { + const crName = "test-telemetry" + + var crNamespace string + ctx := context.Background() + + makeJumpstarterSpec := func() operatorv1alpha1.JumpstarterSpec { + return operatorv1alpha1.JumpstarterSpec{ + BaseDomain: "example.com", + CertManager: operatorv1alpha1.CertManagerConfig{ + Enabled: false, + }, + Controller: operatorv1alpha1.ControllerConfig{ + Image: "quay.io/jumpstarter/jumpstarter:latest", + Replicas: 1, + GRPC: operatorv1alpha1.GRPCConfig{ + Endpoints: []operatorv1alpha1.Endpoint{{Address: "controller"}}, + }, + }, + Routers: operatorv1alpha1.RoutersConfig{ + Image: "quay.io/jumpstarter/jumpstarter:latest", + Replicas: 1, + GRPC: operatorv1alpha1.GRPCConfig{ + Endpoints: []operatorv1alpha1.Endpoint{{Address: "router"}}, + }, + }, + } + } + + newReconciler := func() *JumpstarterReconciler { + return &JumpstarterReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + EndpointReconciler: endpoints.NewReconciler(k8sClient, k8sClient.Scheme(), cfg), + } + } + + doReconcile := func() { + _, err := newReconciler().Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: crName, Namespace: crNamespace}, + }) + Expect(err).NotTo(HaveOccurred()) + } + + getConfigData := func() string { + cm := &corev1.ConfigMap{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: "jumpstarter-controller", + Namespace: crNamespace, + }, cm) + Expect(err).NotTo(HaveOccurred()) + return cm.Data["config"] + } + + BeforeEach(func() { + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "telemetry-test-"}} + Expect(k8sClient.Create(ctx, ns)).To(Succeed()) + crNamespace = ns.Name + }) + + AfterEach(func() { + _ = k8sClient.Delete(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: crNamespace}, + }) + }) + + It("creates Deployment and Service when telemetry is enabled", func() { + By("creating a Jumpstarter CR with telemetry enabled") + spec := makeJumpstarterSpec() + spec.Telemetry = &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + ImagePullPolicy: corev1.PullIfNotPresent, + } + Expect(k8sClient.Create(ctx, &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: spec, + })).To(Succeed()) + + By("reconciling") + doReconcile() + + By("verifying the Deployment exists with correct settings") + deployment := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: crName + "-telemetry", + Namespace: crNamespace, + }, deployment)).To(Succeed()) + + Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(1)) + container := deployment.Spec.Template.Spec.Containers[0] + Expect(container.Name).To(Equal("telemetry")) + Expect(container.Image).To(Equal("quay.io/jumpstarter-dev/jumpstarter-telemetry:latest")) + Expect(container.ImagePullPolicy).To(Equal(corev1.PullIfNotPresent)) + Expect(container.Command).To(Equal([]string{"/telemetry"})) + Expect(container.Args).To(ContainElement(fmt.Sprintf("--grpc-bind=:%d", telemetryPort))) + + By("verifying labels") + Expect(deployment.Labels).To(HaveKeyWithValue("component", "telemetry")) + Expect(deployment.Labels).To(HaveKeyWithValue("app", "jumpstarter-telemetry")) + Expect(deployment.Labels).To(HaveKeyWithValue("controller", crName)) + + By("verifying replicas default to 1") + Expect(*deployment.Spec.Replicas).To(Equal(int32(1))) + + By("verifying security context") + Expect(container.SecurityContext.AllowPrivilegeEscalation).NotTo(BeNil()) + Expect(*container.SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) + Expect(container.SecurityContext.Capabilities.Drop).To(ContainElement(corev1.Capability("ALL"))) + Expect(deployment.Spec.Template.Spec.SecurityContext.RunAsNonRoot).NotTo(BeNil()) + Expect(*deployment.Spec.Template.Spec.SecurityContext.RunAsNonRoot).To(BeTrue()) + + By("verifying the Service exists") + svc := &corev1.Service{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: telemetryServiceName, + Namespace: crNamespace, + }, svc)).To(Succeed()) + Expect(svc.Spec.Type).To(Equal(corev1.ServiceTypeClusterIP)) + Expect(svc.Spec.Ports).To(HaveLen(1)) + Expect(svc.Spec.Ports[0].Port).To(Equal(int32(telemetryPort))) + Expect(svc.Spec.Ports[0].Name).To(Equal("grpc")) + Expect(svc.Spec.Selector).To(HaveKeyWithValue("app", "jumpstarter-telemetry")) + }) + + It("includes liveness and readiness probes on the telemetry container", func() { + By("creating a Jumpstarter CR with telemetry enabled") + spec := makeJumpstarterSpec() + spec.Telemetry = &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + } + Expect(k8sClient.Create(ctx, &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: spec, + })).To(Succeed()) + + doReconcile() + + deployment := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: crName + "-telemetry", + Namespace: crNamespace, + }, deployment)).To(Succeed()) + + container := deployment.Spec.Template.Spec.Containers[0] + + By("verifying liveness probe uses TCP on the gRPC port") + Expect(container.LivenessProbe).NotTo(BeNil()) + Expect(container.LivenessProbe.TCPSocket).NotTo(BeNil()) + Expect(container.LivenessProbe.TCPSocket.Port.IntValue()).To(Equal(telemetryPort)) + + By("verifying readiness probe uses TCP on the gRPC port") + Expect(container.ReadinessProbe).NotTo(BeNil()) + Expect(container.ReadinessProbe.TCPSocket).NotTo(BeNil()) + Expect(container.ReadinessProbe.TCPSocket.Port.IntValue()).To(Equal(telemetryPort)) + }) + + It("respects the replicas field", func() { + By("creating a Jumpstarter CR with telemetry replicas=3") + spec := makeJumpstarterSpec() + spec.Telemetry = &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + Replicas: ptr.To(int32(3)), + } + Expect(k8sClient.Create(ctx, &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: spec, + })).To(Succeed()) + + doReconcile() + + deployment := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: crName + "-telemetry", + Namespace: crNamespace, + }, deployment)).To(Succeed()) + Expect(*deployment.Spec.Replicas).To(Equal(int32(3))) + }) + + It("does not create telemetry resources when telemetry is disabled", func() { + By("creating a Jumpstarter CR without telemetry") + spec := makeJumpstarterSpec() + Expect(k8sClient.Create(ctx, &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: spec, + })).To(Succeed()) + + doReconcile() + + By("verifying no telemetry Deployment exists") + deployment := &appsv1.Deployment{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: crName + "-telemetry", + Namespace: crNamespace, + }, deployment) + Expect(errors.IsNotFound(err)).To(BeTrue(), "telemetry deployment should not exist") + + By("verifying no telemetry Service exists") + svc := &corev1.Service{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: telemetryServiceName, + Namespace: crNamespace, + }, svc) + Expect(errors.IsNotFound(err)).To(BeTrue(), "telemetry service should not exist") + + By("verifying no telemetry ServiceAccount exists") + sa := &corev1.ServiceAccount{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: crName + telemetrySASuffix, + Namespace: crNamespace, + }, sa) + Expect(errors.IsNotFound(err)).To(BeTrue(), "telemetry service account should not exist") + }) + + It("cleans up telemetry resources when telemetry is disabled after being enabled", func() { + By("creating a Jumpstarter CR with telemetry enabled") + spec := makeJumpstarterSpec() + spec.Telemetry = &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + } + Expect(k8sClient.Create(ctx, &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: spec, + })).To(Succeed()) + + By("first reconcile — resources should be created") + doReconcile() + + deployment := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: crName + "-telemetry", + Namespace: crNamespace, + }, deployment)).To(Succeed()) + + svc := &corev1.Service{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: telemetryServiceName, + Namespace: crNamespace, + }, svc)).To(Succeed()) + + By("disabling telemetry") + js := &operatorv1alpha1.Jumpstarter{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: crName, Namespace: crNamespace}, js)).To(Succeed()) + js.Spec.Telemetry.Enabled = false + Expect(k8sClient.Update(ctx, js)).To(Succeed()) + + By("second reconcile — resources should be cleaned up") + doReconcile() + + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: crName + "-telemetry", + Namespace: crNamespace, + }, deployment) + Expect(errors.IsNotFound(err)).To(BeTrue(), "telemetry deployment should be deleted") + + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: telemetryServiceName, + Namespace: crNamespace, + }, svc) + Expect(errors.IsNotFound(err)).To(BeTrue(), "telemetry service should be deleted") + + By("verifying the dedicated ServiceAccount is also deleted") + sa := &corev1.ServiceAccount{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: crName + telemetrySASuffix, + Namespace: crNamespace, + }, sa) + Expect(errors.IsNotFound(err)).To(BeTrue(), "telemetry service account should be deleted") + + By("verifying telemetry is absent from the controller ConfigMap") + configData := getConfigData() + Expect(configData).NotTo(ContainSubstring("telemetry"), + "telemetry should not appear in ConfigMap after disabling") + }) + + It("propagates telemetry config into the controller ConfigMap", func() { + By("creating a Jumpstarter CR with telemetry and a custom minSeverity") + spec := makeJumpstarterSpec() + spec.Telemetry = &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + Logging: operatorv1alpha1.TelemetryLoggingConfig{ + Filter: operatorv1alpha1.TelemetryLoggingFilterConfig{ + MinSeverity: "warning", + }, + }, + } + Expect(k8sClient.Create(ctx, &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: spec, + })).To(Succeed()) + + doReconcile() + + configData := getConfigData() + Expect(configData).To(ContainSubstring("telemetry")) + Expect(configData).To(ContainSubstring("enabled: true")) + Expect(configData).To(ContainSubstring(telemetryServiceName)) + Expect(configData).To(ContainSubstring("warning")) + }) + + It("does not include telemetry in ConfigMap when disabled", func() { + By("creating a Jumpstarter CR without telemetry") + spec := makeJumpstarterSpec() + Expect(k8sClient.Create(ctx, &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: spec, + })).To(Succeed()) + + doReconcile() + + configData := getConfigData() + Expect(configData).NotTo(ContainSubstring("telemetry")) + }) + + It("sets TelemetryDeploymentReady status condition", func() { + By("creating a Jumpstarter CR with telemetry enabled") + spec := makeJumpstarterSpec() + spec.Telemetry = &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + } + Expect(k8sClient.Create(ctx, &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: spec, + })).To(Succeed()) + + By("first reconcile — deployment exists but not Available yet") + doReconcile() + + js := &operatorv1alpha1.Jumpstarter{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: crName, Namespace: crNamespace}, js)).To(Succeed()) + cond := meta.FindStatusCondition(js.Status.Conditions, operatorv1alpha1.ConditionTypeTelemetryDeploymentReady) + Expect(cond).NotTo(BeNil(), "TelemetryDeploymentReady condition should be set") + Expect(cond.Status).To(Equal(metav1.ConditionFalse), + "should be False while deployment is not Available") + + By("marking the telemetry deployment as Available") + deployment := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: crName + "-telemetry", + Namespace: crNamespace, + }, deployment)).To(Succeed()) + deployment.Status.Conditions = []appsv1.DeploymentCondition{ + { + Type: appsv1.DeploymentAvailable, + Status: corev1.ConditionTrue, + Reason: "MinimumReplicasAvailable", + }, + } + Expect(k8sClient.Status().Update(ctx, deployment)).To(Succeed()) + + By("second reconcile — condition should become True") + doReconcile() + + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: crName, Namespace: crNamespace}, js)).To(Succeed()) + cond = meta.FindStatusCondition(js.Status.Conditions, operatorv1alpha1.ConditionTypeTelemetryDeploymentReady) + Expect(cond).NotTo(BeNil()) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal("DeploymentAvailable")) + }) + + It("does not mount TLS certs even when cert-manager is enabled (TLS serving not yet supported by the binary)", func() { + // EXTERNAL_CERT_PEM/EXTERNAL_KEY_PEM and the tls-certs volume are intentionally + // omitted until the telemetry binary is updated to serve TLS. + // CONTROLLER_KEY is always set for token validation (not TLS-related). + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: "test-tls", Namespace: "default"}, + Spec: operatorv1alpha1.JumpstarterSpec{ + CertManager: operatorv1alpha1.CertManagerConfig{Enabled: true}, + Telemetry: &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + ImagePullPolicy: corev1.PullIfNotPresent, + }, + }, + } + + dep := createTelemetryDeployment(js) + + container := dep.Spec.Template.Spec.Containers[0] + // CONTROLLER_KEY should be set for token validation + Expect(container.Env).To(HaveLen(1)) + Expect(container.Env[0].Name).To(Equal("CONTROLLER_KEY")) + // TLS-related env vars should NOT be set + for _, env := range container.Env { + Expect(env.Name).NotTo(Equal("EXTERNAL_CERT_PEM")) + Expect(env.Name).NotTo(Equal("EXTERNAL_KEY_PEM")) + } + Expect(container.VolumeMounts).To(BeNil()) + Expect(dep.Spec.Template.Spec.Volumes).To(BeNil()) + }) + + It("uses a dedicated service account separate from the controller", func() { + spec := makeJumpstarterSpec() + spec.Telemetry = &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:latest", + } + Expect(k8sClient.Create(ctx, &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: spec, + })).To(Succeed()) + + doReconcile() + + telemetryDep := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: crName + "-telemetry", + Namespace: crNamespace, + }, telemetryDep)).To(Succeed()) + + expectedSA := crName + telemetrySASuffix + Expect(telemetryDep.Spec.Template.Spec.ServiceAccountName).To(Equal(expectedSA)) + + By("verifying the dedicated ServiceAccount exists with no RBAC bindings") + sa := &corev1.ServiceAccount{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: expectedSA, + Namespace: crNamespace, + }, sa)).To(Succeed()) + }) + + It("updates the deployment when the CR spec changes", func() { + By("creating a Jumpstarter CR with telemetry enabled") + spec := makeJumpstarterSpec() + spec.Telemetry = &operatorv1alpha1.TelemetryConfig{ + Enabled: true, + Image: "quay.io/jumpstarter-dev/jumpstarter-telemetry:v1", + } + Expect(k8sClient.Create(ctx, &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: crName, Namespace: crNamespace}, + Spec: spec, + })).To(Succeed()) + + doReconcile() + + By("verifying initial image") + deployment := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: crName + "-telemetry", + Namespace: crNamespace, + }, deployment)).To(Succeed()) + Expect(deployment.Spec.Template.Spec.Containers[0].Image).To( + Equal("quay.io/jumpstarter-dev/jumpstarter-telemetry:v1")) + + By("updating the image in the CR") + js := &operatorv1alpha1.Jumpstarter{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: crName, Namespace: crNamespace}, js)).To(Succeed()) + js.Spec.Telemetry.Image = "quay.io/jumpstarter-dev/jumpstarter-telemetry:v2" + Expect(k8sClient.Update(ctx, js)).To(Succeed()) + + By("reconciling again") + doReconcile() + + By("verifying image is updated") + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: crName + "-telemetry", + Namespace: crNamespace, + }, deployment)).To(Succeed()) + Expect(deployment.Spec.Template.Spec.Containers[0].Image).To( + Equal("quay.io/jumpstarter-dev/jumpstarter-telemetry:v2")) + }) +}) + +var _ = Describe("telemetryEndpointFor", func() { + It("returns the correct in-cluster endpoint format", func() { + endpoint := telemetryEndpointFor("my-namespace") + Expect(endpoint).To(Equal(fmt.Sprintf("%s.my-namespace.svc:%d", telemetryServiceName, telemetryPort))) + }) +}) + +var _ = Describe("telemetryLabels", func() { + It("returns correct labels", func() { + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: "my-js"}, + } + labels := telemetryLabels(js) + Expect(labels).To(HaveKeyWithValue("component", "telemetry")) + Expect(labels).To(HaveKeyWithValue("app", telemetryComponentApp)) + Expect(labels).To(HaveKeyWithValue("controller", "my-js")) + }) +}) + +var _ = Describe("getTelemetryCertSecretName", func() { + It("returns the correct secret name", func() { + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: "jumpstarter"}, + } + Expect(getTelemetryCertSecretName(js)).To(Equal("jumpstarter-telemetry-tls")) + }) +}) + +var _ = Describe("defaultTelemetryResources", func() { + It("should return defaults when spec is empty", func() { + result := defaultTelemetryResources(corev1.ResourceRequirements{}) + + Expect(result.Requests).To(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("50m"))) + Expect(result.Requests).To(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("128Mi"))) + Expect(result.Limits).To(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("500m"))) + Expect(result.Limits).To(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("256Mi"))) + }) + + It("should return user-specified resources when requests are set", func() { + custom := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, + } + + result := defaultTelemetryResources(custom) + + Expect(result.Requests).To(HaveKeyWithValue(corev1.ResourceCPU, resource.MustParse("100m"))) + Expect(result.Limits).To(BeNil()) + }) + + It("should return user-specified resources when limits are set", func() { + custom := corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + } + + result := defaultTelemetryResources(custom) + + Expect(result.Limits).To(HaveKeyWithValue(corev1.ResourceMemory, resource.MustParse("512Mi"))) + Expect(result.Requests).To(BeNil()) + }) + + It("should preserve claims-only input without applying defaults", func() { + custom := corev1.ResourceRequirements{ + Claims: []corev1.ResourceClaim{{Name: "gpu"}}, + } + + result := defaultTelemetryResources(custom) + + Expect(result.Claims).To(HaveLen(1)) + Expect(result.Claims[0].Name).To(Equal("gpu")) + Expect(result.Requests).To(BeNil()) + Expect(result.Limits).To(BeNil()) + }) +}) + +var _ = Describe("collectTelemetryDNSNames", func() { + It("includes internal DNS names when includeInternalNames is true", func() { + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "my-ns"}, + } + r := &JumpstarterReconciler{} + dnsNames := r.collectTelemetryDNSNames(js, true) + + Expect(dnsNames).To(HaveLen(4)) + Expect(dnsNames).To(ContainElement(telemetryServiceName)) + Expect(dnsNames).To(ContainElement(telemetryServiceName + ".my-ns")) + Expect(dnsNames).To(ContainElement(telemetryServiceName + ".my-ns.svc")) + Expect(dnsNames).To(ContainElement(telemetryServiceName + ".my-ns.svc.cluster.local")) + }) + + It("returns empty when includeInternalNames is false", func() { + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "my-ns"}, + } + r := &JumpstarterReconciler{} + dnsNames := r.collectTelemetryDNSNames(js, false) + + Expect(dnsNames).To(BeEmpty()) + }) +}) + +var _ = Describe("resolveTelemetryCA", func() { + var crNamespace string + ctx := context.Background() + + BeforeEach(func() { + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "tel-ca-test-"}} + Expect(k8sClient.Create(ctx, ns)).To(Succeed()) + crNamespace = ns.Name + }) + + AfterEach(func() { + _ = k8sClient.Delete(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: crNamespace}, + }) + }) + + It("returns the caBundle from an external issuer when provided", func() { + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: "test-ca", Namespace: crNamespace}, + Spec: operatorv1alpha1.JumpstarterSpec{ + CertManager: operatorv1alpha1.CertManagerConfig{ + Enabled: true, + Server: &operatorv1alpha1.ServerCertConfig{ + IssuerRef: &operatorv1alpha1.IssuerReference{ + Name: "my-issuer", + Kind: "ClusterIssuer", + CABundle: []byte("fake-ca-bundle"), + }, + }, + }, + }, + } + + r := &JumpstarterReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + ca, err := r.resolveTelemetryCA(ctx, js) + Expect(err).NotTo(HaveOccurred()) + Expect(ca).To(Equal("fake-ca-bundle")) + }) + + It("reads the CA from the self-signed CA secret", func() { + By("creating the CA secret") + caSecretName := "test-ca-self" + caCertificateSuffix + Expect(k8sClient.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: caSecretName, Namespace: crNamespace}, + Data: map[string][]byte{"tls.crt": []byte(testPEM)}, + })).To(Succeed()) + + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: "test-ca-self", Namespace: crNamespace}, + Spec: operatorv1alpha1.JumpstarterSpec{ + CertManager: operatorv1alpha1.CertManagerConfig{Enabled: true}, + }, + } + + r := &JumpstarterReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + ca, err := r.resolveTelemetryCA(ctx, js) + Expect(err).NotTo(HaveOccurred()) + Expect(ca).To(ContainSubstring("BEGIN CERTIFICATE")) + }) + + It("returns an error when the CA secret does not exist", func() { + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: "test-ca-missing", Namespace: crNamespace}, + Spec: operatorv1alpha1.JumpstarterSpec{ + CertManager: operatorv1alpha1.CertManagerConfig{Enabled: true}, + }, + } + + r := &JumpstarterReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + _, err := r.resolveTelemetryCA(ctx, js) + Expect(err).To(HaveOccurred()) + Expect(strings.ToLower(err.Error())).To(ContainSubstring("not found")) + }) + + It("returns an error when the CA secret exists but is missing tls.crt", func() { + By("creating a CA secret without tls.crt key") + caSecretName := "test-ca-no-cert" + caCertificateSuffix + Expect(k8sClient.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: caSecretName, Namespace: crNamespace}, + Data: map[string][]byte{"other-key": []byte("some-data")}, + })).To(Succeed()) + + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: "test-ca-no-cert", Namespace: crNamespace}, + Spec: operatorv1alpha1.JumpstarterSpec{ + CertManager: operatorv1alpha1.CertManagerConfig{Enabled: true}, + }, + } + + r := &JumpstarterReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + _, err := r.resolveTelemetryCA(ctx, js) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("missing tls.crt")) + }) + + It("returns ('', nil) when an external IssuerRef has a nil CABundle", func() { + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: "test-ca-no-bundle", Namespace: crNamespace}, + Spec: operatorv1alpha1.JumpstarterSpec{ + CertManager: operatorv1alpha1.CertManagerConfig{ + Enabled: true, + Server: &operatorv1alpha1.ServerCertConfig{ + IssuerRef: &operatorv1alpha1.IssuerReference{ + Name: "my-issuer", + Kind: "ClusterIssuer", + // CABundle intentionally nil + }, + }, + }, + }, + } + + r := &JumpstarterReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + ca, err := r.resolveTelemetryCA(ctx, js) + Expect(err).NotTo(HaveOccurred()) + Expect(ca).To(BeEmpty()) + }) + + It("returns ('', nil) when an external IssuerRef has an empty CABundle", func() { + js := &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{Name: "test-ca-empty-bundle", Namespace: crNamespace}, + Spec: operatorv1alpha1.JumpstarterSpec{ + CertManager: operatorv1alpha1.CertManagerConfig{ + Enabled: true, + Server: &operatorv1alpha1.ServerCertConfig{ + IssuerRef: &operatorv1alpha1.IssuerReference{ + Name: "my-issuer", + Kind: "ClusterIssuer", + CABundle: []byte{}, + }, + }, + }, + }, + } + + r := &JumpstarterReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + ca, err := r.resolveTelemetryCA(ctx, js) + Expect(err).NotTo(HaveOccurred()) + Expect(ca).To(BeEmpty()) + }) +}) diff --git a/controller/internal/config/types.go b/controller/internal/config/types.go index 890f10ac2..5fe43a5e5 100644 --- a/controller/internal/config/types.go +++ b/controller/internal/config/types.go @@ -61,7 +61,7 @@ type TelemetryLogging struct { type TelemetryLoggingFilter struct { // MinSeverity is the minimum log severity to forward. // Accepted values: debug, info, warning, error, critical. Defaults to "info" when empty. - MinSeverity string `json:"min_severity,omitempty" yaml:"min_severity,omitempty"` + MinSeverity string `json:"minSeverity,omitempty" yaml:"minSeverity,omitempty"` } var validSeverities = map[string]struct{}{