diff --git a/apis/quay/v1/quayregistry_types.go b/apis/quay/v1/quayregistry_types.go index 651d0f9ba..484bbd093 100644 --- a/apis/quay/v1/quayregistry_types.go +++ b/apis/quay/v1/quayregistry_types.go @@ -238,6 +238,8 @@ const ( ConditionReasonComponentOverrideInvalid ConditionReason = "ComponentOverrideInvalid" ConditionReasonPVCPending ConditionReason = "PVCPending" ConditionReasonPVCProvisioningFailed ConditionReason = "PVCProvisioningFailed" + ConditionReasonCredentialRequestNotProvisioned ConditionReason = "CredentialRequestNotProvisioned" + ConditionReasonConflictingCredentials ConditionReason = "ConflictingCredentials" ) // Condition is a single condition of a QuayRegistry. diff --git a/bundle/manifests/quay-operator.clusterserviceversion.yaml b/bundle/manifests/quay-operator.clusterserviceversion.yaml index ab2eb2c06..ed986b168 100644 --- a/bundle/manifests/quay-operator.clusterserviceversion.yaml +++ b/bundle/manifests/quay-operator.clusterserviceversion.yaml @@ -42,7 +42,7 @@ metadata: features.operators.openshift.io/fips-compliant: "true" features.operators.openshift.io/proxy-aware: "true" features.operators.openshift.io/tls-profiles: "false" - features.operators.openshift.io/token-auth-aws: "false" + features.operators.openshift.io/token-auth-aws: "true" features.operators.openshift.io/token-auth-azure: "false" features.operators.openshift.io/token-auth-gcp: "false" name: quay-operator.v3.99.0-dev @@ -168,6 +168,17 @@ spec: value: quay.io/sclorg/postgresql-13-c9s:latest - name: RELATED_IMAGE_COMPONENT_REDIS value: quay.io/sclorg/redis-7-c9s:latest + volumeMounts: + - name: bound-sa-token + mountPath: /var/run/secrets/openshift/serviceaccount + readOnly: true + volumes: + - name: bound-sa-token + projected: + sources: + - serviceAccountToken: + path: token + audience: openshift serviceAccountName: quay-operator permissions: - rules: @@ -252,6 +263,33 @@ spec: verbs: - get serviceAccountName: quay-operator + clusterPermissions: + - rules: + - apiGroups: + - config.openshift.io + resources: + - infrastructures + verbs: + - get + - apiGroups: + - operator.openshift.io + resources: + - cloudcredentials + verbs: + - get + - apiGroups: + - cloudcredential.openshift.io + resources: + - credentialsrequests + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + serviceAccountName: quay-operator strategy: deployment installModes: - supported: true diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ea9902270..662cb9e7a 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -68,10 +68,23 @@ rules: - patch - update - watch +- apiGroups: + - cloudcredential.openshift.io + resources: + - credentialsrequests + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - config.openshift.io resources: - apiservers + - infrastructures verbs: - get - apiGroups: @@ -99,6 +112,12 @@ rules: - patch - update - watch +- apiGroups: + - operator.openshift.io + resources: + - cloudcredentials + verbs: + - get - apiGroups: - quay.redhat.com resources: diff --git a/controllers/quay/features.go b/controllers/quay/features.go index e93d54a7b..3ccb382f8 100644 --- a/controllers/quay/features.go +++ b/controllers/quay/features.go @@ -16,6 +16,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" @@ -192,6 +193,7 @@ func (r *QuayRegistryReconciler) checkManagedTLS( } var errRouteProbeInProgress = fmt.Errorf("route probe in progress, awaiting ingress status") +var errCredentialRequestPending = fmt.Errorf("CredentialRequest not yet provisioned by CCO") // checkExternalTLSSecret validates and reads TLS cert/key from an external Secret referenced // by the TLS component's secretRef. Populates TLSCert, TLSKey, and TLSSecretHash on the context. @@ -669,3 +671,77 @@ func getCertificatesPEM(address string) ([]byte, error) { return b.Bytes(), nil } + +func (r *QuayRegistryReconciler) checkSTSCapability( + ctx context.Context, qctx *quaycontext.QuayRegistryContext, quay *v1.QuayRegistry, +) error { + if r.STSRoleARN == "" { + return nil + } + + if v1.ComponentIsManaged(quay.Spec.Components, v1.ComponentObjectStorage) { + r.Log.Info("ROLEARN is set but ObjectStorage is managed, skipping STS path") + return nil + } + + var infra unstructured.Unstructured + infra.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "config.openshift.io", + Version: "v1", + Kind: "Infrastructure", + }) + if err := r.Get(ctx, types.NamespacedName{Name: "cluster"}, &infra); err != nil { + if errors.IsNotFound(err) || meta.IsNoMatchError(err) { + r.Log.Info("Infrastructure API not available, skipping STS path") + return nil + } + return fmt.Errorf("unable to get Infrastructure: %w", err) + } + + platformType, _, _ := unstructured.NestedString(infra.Object, "status", "platformStatus", "type") + if platformType != "AWS" { + r.Log.Info("cluster platform is not AWS, skipping STS path", "platform", platformType) + return nil + } + + var cco unstructured.Unstructured + cco.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "operator.openshift.io", + Version: "v1", + Kind: "CloudCredential", + }) + if err := r.Get(ctx, types.NamespacedName{Name: "cluster"}, &cco); err != nil { + if errors.IsNotFound(err) || meta.IsNoMatchError(err) { + r.Log.Info("CloudCredential API not available, skipping STS path") + return nil + } + return fmt.Errorf("unable to get CloudCredential: %w", err) + } + + credentialsMode, _, _ := unstructured.NestedString(cco.Object, "spec", "credentialsMode") + if credentialsMode == "Mint" || credentialsMode == "Passthrough" { + r.Log.Info("CCO credentials mode is not STS-compatible, skipping STS path", "mode", credentialsMode) + return nil + } + + var crList unstructured.UnstructuredList + crList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "cloudcredential.openshift.io", + Version: "v1", + Kind: "CredentialsRequestList", + }) + if err := r.List(ctx, &crList, client.InNamespace(quay.GetNamespace())); err != nil { + r.Log.Info("CredentialsRequest CRD not available, skipping STS path") + return nil + } + + qctx.STSEnabled = true + qctx.STSRoleARN = r.STSRoleARN + qctx.STSCredentialRequestName = fmt.Sprintf("%s-quay-app", quay.GetName()) + qctx.STSCredentialSecretName = fmt.Sprintf("%s-quay-app-aws", quay.GetName()) + r.Log.Info("STS capability detected", + "roleARN", r.STSRoleARN, + "credentialRequestName", qctx.STSCredentialRequestName, + ) + return nil +} diff --git a/controllers/quay/features_test.go b/controllers/quay/features_test.go index f4f3245b1..3b88d4e8a 100644 --- a/controllers/quay/features_test.go +++ b/controllers/quay/features_test.go @@ -10,7 +10,9 @@ import ( routev1 "github.com/openshift/api/route/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -538,3 +540,235 @@ func Test_checkExternalTLSSecret(t *testing.T) { }) } } + +func TestCheckSTSCapability(t *testing.T) { + makeInfra := func(platform string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "config.openshift.io", + Version: "v1", + Kind: "Infrastructure", + }) + obj.SetName("cluster") + _ = unstructured.SetNestedField(obj.Object, platform, "status", "platformStatus", "type") + return obj + } + + makeCCO := func(mode string) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "operator.openshift.io", + Version: "v1", + Kind: "CloudCredential", + }) + obj.SetName("cluster") + if mode != "" { + _ = unstructured.SetNestedField(obj.Object, mode, "spec", "credentialsMode") + } + return obj + } + + makeCRDScheme := func() *runtime.Scheme { + s := runtime.NewScheme() + _ = v1.AddToScheme(s) + + infraGVK := schema.GroupVersionKind{Group: "config.openshift.io", Version: "v1", Kind: "Infrastructure"} + ccoGVK := schema.GroupVersionKind{Group: "operator.openshift.io", Version: "v1", Kind: "CloudCredential"} + crGVK := schema.GroupVersionKind{Group: "cloudcredential.openshift.io", Version: "v1", Kind: "CredentialsRequest"} + + s.AddKnownTypeWithName(infraGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(ccoGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(crGVK, &unstructured.Unstructured{}) + s.AddKnownTypeWithName(schema.GroupVersionKind{Group: "cloudcredential.openshift.io", Version: "v1", Kind: "CredentialsRequestList"}, &unstructured.UnstructuredList{}) + + return s + } + + quay := &v1.QuayRegistry{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "ns", + }, + Spec: v1.QuayRegistrySpec{ + Components: []v1.Component{ + {Kind: v1.ComponentObjectStorage, Managed: false}, + }, + }, + } + + quayManaged := &v1.QuayRegistry{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "ns", + }, + Spec: v1.QuayRegistrySpec{ + Components: []v1.Component{ + {Kind: v1.ComponentObjectStorage, Managed: true}, + }, + }, + } + + for _, tt := range []struct { + name string + roleARN string + quay *v1.QuayRegistry + objects []client.Object + expectEnabled bool + expectErr bool + }{ + { + name: "ROLEARN empty", + roleARN: "", + quay: quay, + expectEnabled: false, + }, + { + name: "ObjectStorage managed", + roleARN: "arn:aws:iam::123456789012:role/test", + quay: quayManaged, + expectEnabled: false, + }, + { + name: "non-AWS platform", + roleARN: "arn:aws:iam::123456789012:role/test", + quay: quay, + objects: []client.Object{ + makeInfra("GCP"), + makeCCO(""), + }, + expectEnabled: false, + }, + { + name: "CCO Mint mode", + roleARN: "arn:aws:iam::123456789012:role/test", + quay: quay, + objects: []client.Object{ + makeInfra("AWS"), + makeCCO("Mint"), + }, + expectEnabled: false, + }, + { + name: "CCO Passthrough mode", + roleARN: "arn:aws:iam::123456789012:role/test", + quay: quay, + objects: []client.Object{ + makeInfra("AWS"), + makeCCO("Passthrough"), + }, + expectEnabled: false, + }, + { + name: "STS capable cluster", + roleARN: "arn:aws:iam::123456789012:role/test", + quay: quay, + objects: []client.Object{ + makeInfra("AWS"), + makeCCO(""), + }, + expectEnabled: true, + }, + { + name: "Infrastructure API unavailable", + roleARN: "arn:aws:iam::123456789012:role/test", + quay: quay, + objects: []client.Object{}, + expectEnabled: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + s := makeCRDScheme() + cli := fake.NewClientBuilder().WithScheme(s).WithObjects(tt.objects...).Build() + + r := &QuayRegistryReconciler{ + Client: cli, + Log: logf.Log.WithName("test"), + STSRoleARN: tt.roleARN, + } + + qctx := quaycontext.NewQuayRegistryContext() + err := r.checkSTSCapability(context.Background(), qctx, tt.quay) + + if tt.expectErr { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if qctx.STSEnabled != tt.expectEnabled { + t.Errorf("STSEnabled = %v, want %v", qctx.STSEnabled, tt.expectEnabled) + } + if tt.expectEnabled { + if qctx.STSRoleARN != tt.roleARN { + t.Errorf("STSRoleARN = %q, want %q", qctx.STSRoleARN, tt.roleARN) + } + if qctx.STSCredentialRequestName != "test-quay-app" { + t.Errorf("STSCredentialRequestName = %q, want %q", qctx.STSCredentialRequestName, "test-quay-app") + } + if qctx.STSCredentialSecretName != "test-quay-app-aws" { + t.Errorf("STSCredentialSecretName = %q, want %q", qctx.STSCredentialSecretName, "test-quay-app-aws") + } + } + }) + } +} + +func TestCheckSTSCredentialConflict(t *testing.T) { + for _, tt := range []struct { + name string + usercfg map[string]interface{} + expectErr bool + }{ + { + name: "no DISTRIBUTED_STORAGE_CONFIG", + usercfg: map[string]interface{}{}, + expectErr: false, + }, + { + name: "storage without static keys", + usercfg: map[string]interface{}{ + "DISTRIBUTED_STORAGE_CONFIG": map[string]interface{}{ + "default": []interface{}{ + "S3Storage", + map[string]interface{}{ + "s3_bucket": "my-bucket", + "storage_path": "/datastorage/registry", + "s3_region": "us-east-1", + "host": "s3.amazonaws.com", + }, + }, + }, + }, + expectErr: false, + }, + { + name: "storage with s3_access_key", + usercfg: map[string]interface{}{ + "DISTRIBUTED_STORAGE_CONFIG": map[string]interface{}{ + "default": []interface{}{ + "S3Storage", + map[string]interface{}{ + "s3_access_key": "AKIAIOSFODNN7EXAMPLE", + "s3_secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "s3_bucket": "my-bucket", + }, + }, + }, + }, + expectErr: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + err := checkSTSCredentialConflict(tt.usercfg) + if tt.expectErr && err == nil { + t.Fatal("expected error, got nil") + } + if !tt.expectErr && err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} diff --git a/controllers/quay/quayregistry_controller.go b/controllers/quay/quayregistry_controller.go index 7a5e1e6f2..df0a3a577 100644 --- a/controllers/quay/quayregistry_controller.go +++ b/controllers/quay/quayregistry_controller.go @@ -82,6 +82,7 @@ type QuayRegistryReconciler struct { WatchNamespace string Requeue ctrl.Result SkipResourceRequests bool + STSRoleARN string // Cached route discovery results (write-once, read-many) supportsRoutes bool @@ -533,6 +534,9 @@ func (r *QuayRegistryReconciler) quayAppDeploymentRolledOut( // +kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheusrules;servicemonitors,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=config.openshift.io,resources=apiservers,verbs=get +// +kubebuilder:rbac:groups=config.openshift.io,resources=infrastructures,verbs=get +// +kubebuilder:rbac:groups=operator.openshift.io,resources=cloudcredentials,verbs=get +// +kubebuilder:rbac:groups=cloudcredential.openshift.io,resources=credentialsrequests,verbs=create;delete;get;list;patch;update;watch // Reconcile is called every time an update happens in a QuayRegistry object. It attempts to // create all needed objects to get a quay instance running. @@ -747,6 +751,17 @@ func (r *QuayRegistryReconciler) Reconcile(ctx context.Context, req ctrl.Request ) } + if err := r.checkSTSCapability(ctx, quayContext, updatedQuay); err != nil { + return r.reconcileWithCondition( + ctx, + &quay, + v1.ConditionTypeRolloutBlocked, + metav1.ConditionTrue, + v1.ConditionReasonConfigInvalid, + fmt.Sprintf("error checking STS capability: %s", err), + ) + } + if err = v1.EnsureDefaultComponents(quayContext, updatedQuay); err != nil { log.Error(err, "could not ensure default `spec.components`") return r.Requeue, err @@ -798,6 +813,34 @@ func (r *QuayRegistryReconciler) Reconcile(ctx context.Context, req ctrl.Request ) } + if quayContext.STSEnabled { + if err := checkSTSCredentialConflict(usercfg); err != nil { + return r.reconcileWithCondition( + ctx, + &quay, + v1.ConditionTypeRolloutBlocked, + metav1.ConditionTrue, + v1.ConditionReasonConflictingCredentials, + err.Error(), + ) + } + + if err := r.ensureCredentialRequest(ctx, quayContext, updatedQuay); err != nil { + if goerrors.Is(err, errCredentialRequestPending) { + log.Info("STS CredentialRequest not yet provisioned, requeueing") + return r.Requeue, nil + } + return r.reconcileWithCondition( + ctx, + &quay, + v1.ConditionTypeRolloutBlocked, + metav1.ConditionTrue, + v1.ConditionReasonCredentialRequestNotProvisioned, + fmt.Sprintf("STS CredentialRequest error: %s", err), + ) + } + } + log.Info("inflating QuayRegistry into Kubernetes objects") deploymentObjects, err := kustomize.Inflate( quayContext, updatedQuay, cbundle, log, r.SkipResourceRequests, @@ -1609,3 +1652,123 @@ func filterDeferredWorkloads(objects []client.Object, deferWorkloads bool) []cli } return filtered } + +func checkSTSCredentialConflict(usercfg map[string]interface{}) error { + dsc, ok := usercfg["DISTRIBUTED_STORAGE_CONFIG"] + if !ok { + return nil + } + + storageMap, ok := dsc.(map[string]interface{}) + if !ok { + return nil + } + + for name, entry := range storageMap { + arr, ok := entry.([]interface{}) + if !ok || len(arr) < 2 { + continue + } + + cfg, ok := arr[1].(map[string]interface{}) + if !ok { + continue + } + + if _, has := cfg["s3_access_key"]; has { + return fmt.Errorf( + "storage entry %q contains s3_access_key; remove static AWS "+ + "credentials from configBundleSecret when using STS", name, + ) + } + if _, has := cfg["s3_secret_key"]; has { + return fmt.Errorf( + "storage entry %q contains s3_secret_key; remove static AWS "+ + "credentials from configBundleSecret when using STS", name, + ) + } + } + return nil +} + +func (r *QuayRegistryReconciler) ensureCredentialRequest( + ctx context.Context, qctx *quaycontext.QuayRegistryContext, quay *v1.QuayRegistry, +) error { + cr := &unstructured.Unstructured{} + cr.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "cloudcredential.openshift.io", + Version: "v1", + Kind: "CredentialsRequest", + }) + cr.SetName(qctx.STSCredentialRequestName) + cr.SetNamespace(quay.GetNamespace()) + + providerSpec := map[string]interface{}{ + "apiVersion": "cloudcredential.openshift.io/v1", + "kind": "AWSProviderSpec", + "stsIAMRoleARN": qctx.STSRoleARN, + "statementEntries": []interface{}{ + map[string]interface{}{ + "effect": "Allow", + "action": []interface{}{ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:ListBucketMultipartUploads", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + }, + "resource": "*", + }, + }, + } + + if err := unstructured.SetNestedField(cr.Object, providerSpec, "spec", "providerSpec"); err != nil { + return fmt.Errorf("unable to set providerSpec: %w", err) + } + + secretRef := map[string]interface{}{ + "name": qctx.STSCredentialSecretName, + "namespace": quay.GetNamespace(), + } + if err := unstructured.SetNestedField(cr.Object, secretRef, "spec", "secretRef"); err != nil { + return fmt.Errorf("unable to set secretRef: %w", err) + } + + serviceAccountNames := []interface{}{fmt.Sprintf("%s-quay-app", quay.GetName())} + if err := unstructured.SetNestedSlice(cr.Object, serviceAccountNames, "spec", "serviceAccountNames"); err != nil { + return fmt.Errorf("unable to set serviceAccountNames: %w", err) + } + + if err := unstructured.SetNestedField(cr.Object, "/var/run/secrets/openshift/serviceaccount/token", "spec", "cloudTokenPath"); err != nil { + return fmt.Errorf("unable to set cloudTokenPath: %w", err) + } + + isController := true + cr.SetOwnerReferences([]metav1.OwnerReference{ + { + APIVersion: v1.GroupVersion.String(), + Kind: "QuayRegistry", + Name: quay.Name, + UID: quay.UID, + Controller: &isController, + }, + }) + + if err := r.Patch(ctx, cr, client.Apply, client.FieldOwner("quay-operator"), client.ForceOwnership); err != nil { + return fmt.Errorf("unable to apply CredentialRequest: %w", err) + } + + r.Log.Info("CredentialRequest applied", "name", qctx.STSCredentialRequestName) + + provisioned, _, _ := unstructured.NestedBool(cr.Object, "status", "provisioned") + if !provisioned { + return errCredentialRequestPending + } + + qctx.STSCredentialProvisioned = true + r.Log.Info("CredentialRequest provisioned", "name", qctx.STSCredentialRequestName) + return nil +} diff --git a/main.go b/main.go index 5d8e53d10..596a8c927 100644 --- a/main.go +++ b/main.go @@ -153,6 +153,7 @@ func main() { WatchNamespace: namespace, Requeue: ctrl.Result{RequeueAfter: 10 * time.Second}, SkipResourceRequests: skipres, + STSRoleARN: os.Getenv("ROLEARN"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "QuayRegistry") os.Exit(1) diff --git a/pkg/context/context.go b/pkg/context/context.go index 0888030f5..fc146807a 100644 --- a/pkg/context/context.go +++ b/pkg/context/context.go @@ -53,6 +53,13 @@ type QuayRegistryContext struct { // Clair integration SecurityScannerV4PSK string + + // STS/CCO + STSEnabled bool + STSRoleARN string + STSCredentialRequestName string + STSCredentialSecretName string + STSCredentialProvisioned bool } // NewQuayRegistryContext returns a fresh context for reconciling a `QuayRegistry`. diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index 0e3d27a3f..18bc9f832 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -224,6 +224,33 @@ func Process(quay *v1.QuayRegistry, qctx *quaycontext.QuayRegistryContext, obj c return dep, nil } + if qctx.STSEnabled && qctx.STSCredentialProvisioned && strings.HasSuffix(dep.Name, "quay-app") { + volName := "sts-credentials" + dep.Spec.Template.Spec.Volumes = append(dep.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: volName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: qctx.STSCredentialSecretName, + }, + }, + }) + for i := range dep.Spec.Template.Spec.Containers { + if dep.Spec.Template.Spec.Containers[i].Name != "quay-app" { + continue + } + ref := &dep.Spec.Template.Spec.Containers[i] + ref.VolumeMounts = append(ref.VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: "/var/run/secrets/cloud", + ReadOnly: true, + }) + UpsertContainerEnv(ref, corev1.EnvVar{ + Name: "AWS_SHARED_CREDENTIALS_FILE", + Value: "/var/run/secrets/cloud/credentials", + }) + } + } + fgns, err := v1.FieldGroupNamesForManagedComponents(quay) if err != nil { return nil, err diff --git a/pkg/middleware/middleware_test.go b/pkg/middleware/middleware_test.go index bbb7a9b79..1f8ce3bc2 100644 --- a/pkg/middleware/middleware_test.go +++ b/pkg/middleware/middleware_test.go @@ -866,3 +866,91 @@ func TestProcessJobSecurityContextOverride(t *testing.T) { }) } } + +func TestSTSVolumeInjection(t *testing.T) { + quay := &v1.QuayRegistry{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "ns"}, + Spec: v1.QuayRegistrySpec{ + Components: []v1.Component{ + {Kind: v1.ComponentQuay, Managed: true}, + }, + }, + } + + dep := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-quay-app", + Annotations: map[string]string{"quay-component": "quay-app"}, + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "quay-app"}, + }, + }, + }, + }, + } + + t.Run("STS disabled leaves deployment unchanged", func(t *testing.T) { + qctx := quaycontext.NewQuayRegistryContext() + result, err := Process(quay, qctx, dep.DeepCopy(), false) + assert.NoError(t, err) + + d := result.(*appsv1.Deployment) + for _, c := range d.Spec.Template.Spec.Containers { + for _, env := range c.Env { + assert.NotEqual(t, "AWS_SHARED_CREDENTIALS_FILE", env.Name) + } + } + for _, vol := range d.Spec.Template.Spec.Volumes { + assert.NotEqual(t, "sts-credentials", vol.Name) + } + }) + + t.Run("STS enabled and provisioned injects volume and env", func(t *testing.T) { + qctx := quaycontext.NewQuayRegistryContext() + qctx.STSEnabled = true + qctx.STSCredentialProvisioned = true + qctx.STSCredentialSecretName = "test-quay-app-aws" + + result, err := Process(quay, qctx, dep.DeepCopy(), false) + assert.NoError(t, err) + + d := result.(*appsv1.Deployment) + + foundVol := false + for _, vol := range d.Spec.Template.Spec.Volumes { + if vol.Name == "sts-credentials" { + foundVol = true + assert.Equal(t, "test-quay-app-aws", vol.Secret.SecretName) + } + } + assert.True(t, foundVol, "expected sts-credentials volume") + + for _, c := range d.Spec.Template.Spec.Containers { + foundMount := false + for _, vm := range c.VolumeMounts { + if vm.Name == "sts-credentials" { + foundMount = true + assert.Equal(t, "/var/run/secrets/cloud", vm.MountPath) + assert.True(t, vm.ReadOnly) + } + } + assert.True(t, foundMount, "expected sts-credentials volume mount") + + foundEnv := false + for _, env := range c.Env { + if env.Name == "AWS_SHARED_CREDENTIALS_FILE" { + foundEnv = true + assert.Equal(t, "/var/run/secrets/cloud/credentials", env.Value) + } + } + assert.True(t, foundEnv, "expected AWS_SHARED_CREDENTIALS_FILE env var") + } + }) +} diff --git a/test/chainsaw/Makefile b/test/chainsaw/Makefile index 4ebee96f6..7b4f418f2 100644 --- a/test/chainsaw/Makefile +++ b/test/chainsaw/Makefile @@ -11,7 +11,7 @@ CHAINSAW_REPORT_PATH ?= CHAINSAW_REPORT_NAME ?= chainsaw-report CHAINSAW_REPORT_FLAGS := $(if $(CHAINSAW_REPORT_FORMAT),--report-format $(CHAINSAW_REPORT_FORMAT) --report-name $(CHAINSAW_REPORT_NAME) --report-path $(or $(CHAINSAW_REPORT_PATH),.)) -.PHONY: chainsaw test-e2e test-e2e-destructive test-e2e-hpa test-e2e-kind +.PHONY: chainsaw test-e2e test-e2e-destructive test-e2e-hpa test-e2e-kind test-e2e-sts chainsaw: $(CHAINSAW) ## Download chainsaw locally if necessary. $(CHAINSAW): $(LOCALBIN) @@ -19,11 +19,14 @@ $(CHAINSAW): $(LOCALBIN) @test -s $(LOCALBIN)/chainsaw && $(LOCALBIN)/chainsaw version 2>/dev/null | grep -q $(CHAINSAW_VERSION) || \ { curl -sL "https://github.com/kyverno/chainsaw/releases/download/$(CHAINSAW_VERSION)/chainsaw_linux_amd64.tar.gz" | tar -xz -C $(LOCALBIN) chainsaw; } -test-e2e: chainsaw ## Run Chainsaw e2e tests (excludes destructive tests) - $(CHAINSAW) test --config $(CHAINSAW_CONFIG) --test-dir $(CHAINSAW_TEST_DIR) --values $(CHAINSAW_VALUES_OPENSHIFT) --parallel $(CHAINSAW_PARALLEL) --exclude-test-regex "/ca.rotation" $(CHAINSAW_REPORT_FLAGS) $(CHAINSAW_EXTRA_ARGS) +test-e2e: chainsaw ## Run Chainsaw e2e tests (excludes destructive and STS tests) + $(CHAINSAW) test --config $(CHAINSAW_CONFIG) --test-dir $(CHAINSAW_TEST_DIR) --values $(CHAINSAW_VALUES_OPENSHIFT) --parallel $(CHAINSAW_PARALLEL) --exclude-test-regex "/ca.rotation|/sts.cco" $(CHAINSAW_REPORT_FLAGS) $(CHAINSAW_EXTRA_ARGS) test-e2e-destructive: chainsaw ## Run destructive Chainsaw e2e tests (ca-rotation, OpenShift only) $(CHAINSAW) test --config $(CHAINSAW_CONFIG) --test-dir $(CHAINSAW_TEST_DIR) --values $(CHAINSAW_VALUES_OPENSHIFT) --parallel $(CHAINSAW_PARALLEL) --include-test-regex "/ca.rotation" $(CHAINSAW_REPORT_FLAGS) $(CHAINSAW_EXTRA_ARGS) test-e2e-kind: chainsaw ## Run Chainsaw e2e tests (KinD) - $(CHAINSAW) test --config $(CHAINSAW_CONFIG) --test-dir $(CHAINSAW_TEST_DIR) --values $(CHAINSAW_VALUES_KIND) --parallel $(CHAINSAW_PARALLEL) --exclude-test-regex "/ca.rotation|/hpa|/tls.security.profile" $(CHAINSAW_REPORT_FLAGS) $(CHAINSAW_EXTRA_ARGS) + $(CHAINSAW) test --config $(CHAINSAW_CONFIG) --test-dir $(CHAINSAW_TEST_DIR) --values $(CHAINSAW_VALUES_KIND) --parallel $(CHAINSAW_PARALLEL) --exclude-test-regex "/ca.rotation|/hpa|/tls.security.profile|/sts.cco" $(CHAINSAW_REPORT_FLAGS) $(CHAINSAW_EXTRA_ARGS) + +test-e2e-sts: chainsaw ## Run STS/CCO e2e tests (ROSA only, requires ROLEARN + STS_S3_BUCKET) + $(CHAINSAW) test --config $(CHAINSAW_CONFIG) --test-dir $(CHAINSAW_TEST_DIR) --values $(CHAINSAW_VALUES_OPENSHIFT) --parallel 1 --include-test-regex "/sts.cco" $(CHAINSAW_REPORT_FLAGS) $(CHAINSAW_EXTRA_ARGS) diff --git a/test/chainsaw/sts_cco/00-assert-credential-request.yaml b/test/chainsaw/sts_cco/00-assert-credential-request.yaml new file mode 100644 index 000000000..19f12b6b8 --- /dev/null +++ b/test/chainsaw/sts_cco/00-assert-credential-request.yaml @@ -0,0 +1,13 @@ +apiVersion: cloudcredential.openshift.io/v1 +kind: CredentialsRequest +metadata: + name: test-quay-app + ownerReferences: + - kind: QuayRegistry + name: test +spec: + secretRef: + name: test-quay-app-aws + serviceAccountNames: + - test-quay-app + cloudTokenPath: /var/run/secrets/openshift/serviceaccount/token diff --git a/test/chainsaw/sts_cco/00-create-quay-registry.yaml b/test/chainsaw/sts_cco/00-create-quay-registry.yaml new file mode 100644 index 000000000..279714415 --- /dev/null +++ b/test/chainsaw/sts_cco/00-create-quay-registry.yaml @@ -0,0 +1,29 @@ +apiVersion: quay.redhat.com/v1 +kind: QuayRegistry +metadata: + name: test +spec: + configBundleSecret: quay-config-bundle + components: + - kind: objectstorage + managed: false + - kind: route + managed: true + - kind: tls + managed: true + - kind: postgres + managed: true + - kind: redis + managed: true + - kind: clair + managed: true + - kind: clairpostgres + managed: true + - kind: mirror + managed: true + - kind: monitoring + managed: true + - kind: quay + managed: true + - kind: horizontalpodautoscaler + managed: false diff --git a/test/chainsaw/sts_cco/01-assert-provisioned.yaml b/test/chainsaw/sts_cco/01-assert-provisioned.yaml new file mode 100644 index 000000000..940e7c1c1 --- /dev/null +++ b/test/chainsaw/sts_cco/01-assert-provisioned.yaml @@ -0,0 +1,6 @@ +apiVersion: cloudcredential.openshift.io/v1 +kind: CredentialsRequest +metadata: + name: test-quay-app +status: + provisioned: true diff --git a/test/chainsaw/sts_cco/02-assert-deployment.yaml b/test/chainsaw/sts_cco/02-assert-deployment.yaml new file mode 100644 index 000000000..9e5943a5d --- /dev/null +++ b/test/chainsaw/sts_cco/02-assert-deployment.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-quay-app +spec: + template: + spec: + containers: + - name: quay-app + env: + - name: AWS_SHARED_CREDENTIALS_FILE + value: /var/run/secrets/cloud/credentials + volumeMounts: + - name: sts-credentials + mountPath: /var/run/secrets/cloud + readOnly: true + volumes: + - name: sts-credentials + secret: + secretName: test-quay-app-aws diff --git a/test/chainsaw/sts_cco/03-assert-status.yaml b/test/chainsaw/sts_cco/03-assert-status.yaml new file mode 100644 index 000000000..ff5605525 --- /dev/null +++ b/test/chainsaw/sts_cco/03-assert-status.yaml @@ -0,0 +1,8 @@ +apiVersion: quay.redhat.com/v1 +kind: QuayRegistry +metadata: + name: test +status: + conditions: + - type: RolloutBlocked + status: "False" diff --git a/test/chainsaw/sts_cco/chainsaw-test.yaml b/test/chainsaw/sts_cco/chainsaw-test.yaml new file mode 100644 index 000000000..17cddcc6e --- /dev/null +++ b/test/chainsaw/sts_cco/chainsaw-test.yaml @@ -0,0 +1,57 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + creationTimestamp: null + name: sts-cco +spec: + concurrent: false + description: > + Validates the STS/CCO credential flow for unmanaged S3 storage on an + STS-enabled OpenShift cluster (ROSA). Requires ROLEARN and STS_S3_BUCKET + environment variables set on the operator Deployment. Not run on KinD. + steps: + - name: create-registry-with-sts + try: + - script: + shell: /bin/bash + content: | + set -euo pipefail + + STS_S3_BUCKET="${STS_S3_BUCKET:?STS_S3_BUCKET must be set}" + STS_S3_REGION="${STS_S3_REGION:-us-east-1}" + + cat <