From adf9fdc4ba08e7239767d34c24c9baa77da3215f Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sun, 22 Mar 2026 09:24:09 +0100 Subject: [PATCH 01/54] Add TLS termination and ingress passthrough support to Barbican Add pod-level TLS on top of the Apache migration. Barbican now serves on two ports: - Port 9311 (HTTP): internal traffic, protected by Linkerd mTLS (unchanged) - Port 443 (HTTPS): external traffic via ingress TLS passthrough (new) New file: - _tls-hardening.conf.tpl: TLS 1.3/1.2 cipher suite hardening with brainpool curve preference, AEAD-only, PFS-only cipher suites Modified files: - _wsgi-barbican.conf.tpl: add conditional HTTPS VirtualHost on 443 with SSL - api-deployment.yaml: TLS init container (copies key from K8s Secret to tmpfs at /mnt/secrets), TLS volumes, port 443, hash annotation, Stakater reloader - etc-configmap.yaml: add tls-hardening.conf entry when TLS enabled - service.yaml: add port 443 when TLS enabled - ingress.yaml: ssl-passthrough mode with Linkerd skip-inbound-ports annotation - values.yaml: tls section with all configuration options When tls.enabled is false, all behavior is identical to the Apache migration. Envelope encryption defaults to noop (ready for vault-transit via config flag). Co-authored-by: Rajiv Mucheli --- .../barbican/templates/api-deployment.yaml | 48 ++++++++++++++++++- .../barbican/templates/etc-configmap.yaml | 4 ++ .../templates/etc/_tls-hardening.conf.tpl | 33 +++++++++++++ .../templates/etc/_wsgi-barbican.conf.tpl | 27 +++++++++++ openstack/barbican/templates/ingress.yaml | 11 ++++- openstack/barbican/templates/service.yaml | 5 ++ openstack/barbican/values.yaml | 19 ++++++++ 7 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 openstack/barbican/templates/etc/_tls-hardening.conf.tpl diff --git a/openstack/barbican/templates/api-deployment.yaml b/openstack/barbican/templates/api-deployment.yaml index fc6b81ed2f2..1a609e1e5d6 100644 --- a/openstack/barbican/templates/api-deployment.yaml +++ b/openstack/barbican/templates/api-deployment.yaml @@ -12,7 +12,7 @@ metadata: type: api component: barbican annotations: - secret.reloader.stakater.com/reload: "{{ .Release.Name }}-secrets" + secret.reloader.stakater.com/reload: "{{ .Release.Name }}-secrets{{- if .Values.tls.enabled }},{{ .Values.tls.secretName | default "barbican-tls" }}{{- end }}" deployment.reloader.stakater.com/pause-period: "60s" spec: replicas: {{ required ".Values.api.replicas is missing" .Values.api.replicas }} @@ -59,6 +59,29 @@ spec: {{- if .Values.proxysql.native_sidecar }} {{- include "utils.proxysql.container" . | indent 6 }} {{- end }} + {{- if .Values.tls.enabled }} + - name: tls-init + image: {{ required ".Values.global.registry is missing" .Values.global.registry }}/loci-barbican:{{ required "Values.imageVersionBarbicanApi is missing" .Values.imageVersionBarbicanApi }} + command: + - bash + - -c + - | + set -ex + # Copy TLS key material from K8s Secret to tmpfs + # In noop envelope-encryption mode, the key is plaintext in the Secret. + # When vault-transit or TPM is enabled, this step would decrypt first. + cp /tls-input/tls.key /mnt/secrets/tls.key + cp /tls-input/tls.crt /mnt/secrets/tls.crt + chmod 0440 /mnt/secrets/tls.key + chmod 0444 /mnt/secrets/tls.crt + echo "TLS key material placed on tmpfs" + volumeMounts: + - name: tls-input + mountPath: /tls-input + readOnly: true + - name: tls-secrets + mountPath: /mnt/secrets + {{- end }} containers: - name: barbican-api image: {{required ".Values.global.registry is missing" .Values.global.registry }}/loci-barbican:{{required "Values.imageVersionBarbicanApi is missing" .Values.imageVersionBarbicanApi}} @@ -115,6 +138,10 @@ spec: ports: - name: barbican-api containerPort: {{.Values.api_port_internal}} + {{- if .Values.tls.enabled }} + - name: barbican-tls + containerPort: 443 + {{- end }} volumeMounts: - name: etcbarbican mountPath: /etc/barbican @@ -138,6 +165,15 @@ spec: mountPath: /etc/barbican/logging.ini subPath: logging.ini readOnly: true + {{- if .Values.tls.enabled }} + - name: tls-secrets + mountPath: /mnt/secrets + readOnly: true + - name: barbican-etc + mountPath: /etc/apache2/conf-enabled/tls-hardening.conf + subPath: tls-hardening.conf + readOnly: true + {{- end }} - mountPath: /etc/barbican/barbican.conf.d name: barbican-etc-confd readOnly: true @@ -334,3 +370,13 @@ spec: {{- end }} {{- include "utils.proxysql.volumes" . | indent 8 }} {{- include "utils.trust_bundle.volumes" . | indent 8 }} + {{- if .Values.tls.enabled }} + - name: tls-input + secret: + secretName: {{ .Values.tls.secretName | default "barbican-tls" }} + defaultMode: 0440 + - name: tls-secrets + emptyDir: + medium: Memory + sizeLimit: 1Mi + {{- end }} diff --git a/openstack/barbican/templates/etc-configmap.yaml b/openstack/barbican/templates/etc-configmap.yaml index e735e25c787..6d93bcce19a 100644 --- a/openstack/barbican/templates/etc-configmap.yaml +++ b/openstack/barbican/templates/etc-configmap.yaml @@ -11,6 +11,10 @@ data: {{ include (print .Template.BasePath "/etc/_barbican-api-paste.ini.tpl") . | indent 4 }} barbican.conf: | {{ include (print .Template.BasePath "/etc/_barbican.conf.tpl") . | indent 4 }} +{{- if .Values.tls.enabled }} + tls-hardening.conf: | +{{ include (print .Template.BasePath "/etc/_tls-hardening.conf.tpl") . | indent 4 }} +{{- end }} policy.yaml: | {{ include (print .Template.BasePath "/etc/_barbican-policy.yaml.tpl") . | indent 4 }} logging.ini: | diff --git a/openstack/barbican/templates/etc/_tls-hardening.conf.tpl b/openstack/barbican/templates/etc/_tls-hardening.conf.tpl new file mode 100644 index 00000000000..5598d9dd0ce --- /dev/null +++ b/openstack/barbican/templates/etc/_tls-hardening.conf.tpl @@ -0,0 +1,33 @@ +{{/* +Apache TLS protocol hardening configuration. +Configures TLS 1.3/1.2 with compliant cipher suites, PFS, AEAD modes, +and brainpool curve preference. Shared between Keystone and Barbican. +*/}} + +# TLS Protocol Versions: only 1.3 and 1.2 +SSLProtocol -all +TLSv1.3 +TLSv1.2 + +# TLS 1.2 Cipher Suites (ECDHE + AEAD only, PFS required) +SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256 + +# TLS 1.3 Cipher Suites +SSLCipherSuite TLSv1.3 TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + +# Server chooses cipher order +SSLHonorCipherOrder on + +# ECDHE Curve Preference: brainpool first, NIST as fallback +SSLOpenSSLConfCmd Curves brainpoolP256r1:brainpoolP384r1:prime256v1:secp384r1 + +# Signature algorithms preference +SSLOpenSSLConfCmd SignatureAlgorithms ecdsa_secp256r1_sha256:ecdsa_secp384r1_sha384:rsa_pss_rsae_sha256:rsa_pss_rsae_sha384 + +# Session cache +SSLSessionCache shmcb:/run/apache2/ssl_scache(512000) +SSLSessionCacheTimeout 300 + +# Disable session tickets (for PFS) +SSLSessionTickets off + +# Strict SNI +SSLStrictSNIVHostCheck on diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 2a4441db1a5..8e66f25096a 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -16,6 +16,33 @@ WSGIDaemonProcess barbican-api processes={{ .Values.api.processes | default 1 }} Listen 0.0.0.0:{{ .Values.api_port_internal }} +{{- if .Values.tls.enabled }} +# External HTTPS endpoint (via Ingress TLS passthrough) +Listen 0.0.0.0:443 + + + SSLEngine on + SSLCertificateFile /mnt/secrets/tls.crt + SSLCertificateKeyFile /mnt/secrets/tls.key + + Include /etc/apache2/conf-enabled/tls-hardening.conf + + WSGIDaemonProcess barbican-api-tls processes={{ .Values.api.processes | default 1 }} threads=1 user=barbican group=barbican display-name=%{GROUP} + WSGIProcessGroup barbican-api-tls + WSGIScriptAlias / /var/www/cgi-bin/barbican/barbican-wsgi-api + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + + = 2.4> + ErrorLogFormat "%{cu}t %M" + + ErrorLog /dev/stderr + + KeepAliveTimeout 61 + +{{- end }} + +# Internal HTTP endpoint (protected by Linkerd mTLS at the network layer) ServerName {{ include "barbican_api_endpoint_host_public" . }} diff --git a/openstack/barbican/templates/ingress.yaml b/openstack/barbican/templates/ingress.yaml index 20ebb94a242..59cf32b7e98 100644 --- a/openstack/barbican/templates/ingress.yaml +++ b/openstack/barbican/templates/ingress.yaml @@ -13,7 +13,12 @@ metadata: type: api component: barbican annotations: - {{- if .Values.tlsacme }} + {{- if and .Values.tls.enabled .Values.tls.passthrough }} + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + config.linkerd.io/skip-inbound-ports: "443" + disco: "true" + {{- else if .Values.tlsacme }} kubernetes.io/tls-acme: "true" disco: "true" {{- end }} @@ -38,4 +43,8 @@ spec: service: name: barbican-api port: + {{- if and .Values.tls.enabled .Values.tls.passthrough }} + number: 443 + {{- else }} number: {{.Values.api_port_internal}} + {{- end }} diff --git a/openstack/barbican/templates/service.yaml b/openstack/barbican/templates/service.yaml index 9e9f6ca2899..2616d5b6e6b 100644 --- a/openstack/barbican/templates/service.yaml +++ b/openstack/barbican/templates/service.yaml @@ -23,3 +23,8 @@ spec: ports: - name: barbican-api port: {{.Values.api_port_internal}} + {{- if .Values.tls.enabled }} + - name: barbican-tls + port: 443 + targetPort: 443 + {{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index 2993252f39f..bc008c51f27 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -489,6 +489,25 @@ ingress: enableConnectionsLimit: true limitConnections: 20 +# Pod-level TLS termination in Apache +tls: + enabled: false + # Name of the K8s Secret containing tls.key and tls.crt (created by TLS bootstrap Job) + secretName: barbican-tls + # Enable TLS passthrough on the ingress (sends encrypted stream directly to pod) + passthrough: false + # Envelope encryption mode for the private key: "none" (plaintext) or "vault-transit" + envelopeEncryption: none + # TLS lifecycle manager binary for bootstrap Job and rotation CronJob + lifecycle: + image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager + imageTag: latest + # cert-manager issuer configuration + issuerName: digicert-issuer + issuerKind: ClusterDigicertIssuer + issuerGroup: certmanager.cloud.sap + keyAlgorithm: ECDSAP256 + # sapcc/openstack-rate-limit-middleware sapcc_rate_limit: enabled: true From 93e66df4ef7c250825ad1ce933d450505c36d9fa Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sun, 22 Mar 2026 09:28:20 +0100 Subject: [PATCH 02/54] Add TLS termination and ingress passthrough support to Keystone Dual-port Apache: HTTPS on 443 (external) + HTTP on 5000 (internal). TLS hardening config (shared with Barbican): brainpool curves, PFS, AEAD. Init container copies key from K8s Secret to tmpfs at /mnt/secrets. Ingress passthrough mode with source-IP affinity and Linkerd skip-inbound-ports. SAML federation Location blocks included in the HTTPS VirtualHost. When tls.enabled is false, all behavior is identical to current deployment. --- .../keystone/templates/configmap-etc.yaml | 4 ++ .../keystone/templates/deployment-api.yaml | 46 ++++++++++++++++++- .../templates/etc/_tls-hardening.conf.tpl | 33 +++++++++++++ .../templates/etc/_wsgi-keystone.conf.tpl | 40 ++++++++++++++++ openstack/keystone/templates/ingress-api.yaml | 13 +++++- openstack/keystone/templates/service-api.yaml | 6 +++ openstack/keystone/values.yaml | 14 ++++++ 7 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 openstack/keystone/templates/etc/_tls-hardening.conf.tpl diff --git a/openstack/keystone/templates/configmap-etc.yaml b/openstack/keystone/templates/configmap-etc.yaml index 810015e3084..175d0caa2bb 100644 --- a/openstack/keystone/templates/configmap-etc.yaml +++ b/openstack/keystone/templates/configmap-etc.yaml @@ -21,6 +21,10 @@ data: {{ include (print .Template.BasePath "/etc/_sso_callback_template.html.tpl") . | indent 4 }} wsgi-keystone.conf: | {{ include (print .Template.BasePath "/etc/_wsgi-keystone.conf.tpl") . | indent 4 }} +{{- if .Values.tls.enabled }} + tls-hardening.conf: | +{{ include (print .Template.BasePath "/etc/_tls-hardening.conf.tpl") . | indent 4 }} +{{- end }} {{- if .Values.watcher.enabled }} watcher.yaml: | {{ include (print .Template.BasePath "/etc/_watcher.yaml.tpl") . | indent 4 }} diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 0b7718e5486..4cb6b2b304b 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -12,7 +12,7 @@ metadata: component: keystone type: api annotations: - secret.reloader.stakater.com/reload: "{{ .Release.Name }}-secrets,{{ .Release.Name }}-federation{{- if .Values.federation.saml.enabled }},{{ .Release.Name }}-federation-saml,{{ .Values.federation.saml.idp.spKeySecretName | default "keystone-saml-sp" }}{{- end }}" + secret.reloader.stakater.com/reload: "{{ .Release.Name }}-secrets,{{ .Release.Name }}-federation{{- if .Values.federation.saml.enabled }},{{ .Release.Name }}-federation-saml,{{ .Values.federation.saml.idp.spKeySecretName | default "keystone-saml-sp" }}{{- end }}{{- if .Values.tls.enabled }},{{ .Values.tls.secretName | default "keystone-tls" }}{{- end }}" {{- if .Values.federation.saml.enabled }} configmap.reloader.stakater.com/reload: "keystone-saml-shibboleth-config,keystone-saml-federation-saml-tenants,{{ .Values.federation.saml.idp.metadataConfigMap }}" {{- end }} @@ -54,6 +54,9 @@ spec: {{- if .Values.federation.saml.enabled }} federation-saml-hash: {{ include (print $.Template.BasePath "/federation-saml.yaml") . | sha256sum }} {{- end }} + {{- if .Values.tls.enabled }} + tls-secret-hash: {{ .Values.tls.secretName | default "keystone-tls" }} + {{- end }} {{- if .Values.api.metrics.enabled }} prometheus.io/scrape: "true" prometheus.io/targets: {{ required ".Values.alerts.prometheus missing" .Values.alerts.prometheus | quote }} @@ -107,6 +110,24 @@ spec: - name: saml-sp-tmpfs mountPath: /saml-sp-keys {{- end }} + {{- if .Values.tls.enabled }} + - name: tls-init + image: {{ required ".Values.global.dockerHubMirror is missing" .Values.global.dockerHubMirror }}/library/busybox + command: ["sh", "-c"] + args: + - | + cp /tls-input/tls.key /mnt/secrets/tls.key + cp /tls-input/tls.crt /mnt/secrets/tls.crt + chmod 0440 /mnt/secrets/tls.key + chmod 0444 /mnt/secrets/tls.crt + echo "TLS key material placed on tmpfs" + volumeMounts: + - name: tls-input + mountPath: /tls-input + readOnly: true + - name: tls-secrets + mountPath: /mnt/secrets + {{- end }} containers: - name: keystone-api image: {{ required ".Values.global.registryAlternateRegion is missing" .Values.global.registryAlternateRegion }}/{{ .Values.api.image }}:{{ required ".Values.api.imageTag is missing" .Values.api.imageTag }} @@ -156,6 +177,10 @@ spec: ports: - name: public containerPort: 5000 + {{- if .Values.tls.enabled }} + - name: public-tls + containerPort: 443 + {{- end }} lifecycle: preStop: exec: @@ -244,6 +269,15 @@ spec: mountPath: /etc/shibboleth/metadata readOnly: true {{- end }} + {{- if .Values.tls.enabled }} + - name: tls-secrets + mountPath: /mnt/secrets + readOnly: true + - name: keystone-etc + mountPath: /etc/apache2/conf-enabled/tls-hardening.conf + subPath: tls-hardening.conf + readOnly: true + {{- end }} - name: keystone-etc mountPath: /etc/apache2/mods-available/mpm_event.conf subPath: mpm_event.conf @@ -367,3 +401,13 @@ spec: name: keystone-sentry-filter-rules {{- include "utils.proxysql.volumes" . | indent 8 }} {{- include "utils.trust_bundle.volumes" . | indent 8 }} + {{- if .Values.tls.enabled }} + - name: tls-input + secret: + secretName: {{ .Values.tls.secretName | default "keystone-tls" }} + defaultMode: 0440 + - name: tls-secrets + emptyDir: + medium: Memory + sizeLimit: 1Mi + {{- end }} diff --git a/openstack/keystone/templates/etc/_tls-hardening.conf.tpl b/openstack/keystone/templates/etc/_tls-hardening.conf.tpl new file mode 100644 index 00000000000..5598d9dd0ce --- /dev/null +++ b/openstack/keystone/templates/etc/_tls-hardening.conf.tpl @@ -0,0 +1,33 @@ +{{/* +Apache TLS protocol hardening configuration. +Configures TLS 1.3/1.2 with compliant cipher suites, PFS, AEAD modes, +and brainpool curve preference. Shared between Keystone and Barbican. +*/}} + +# TLS Protocol Versions: only 1.3 and 1.2 +SSLProtocol -all +TLSv1.3 +TLSv1.2 + +# TLS 1.2 Cipher Suites (ECDHE + AEAD only, PFS required) +SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256 + +# TLS 1.3 Cipher Suites +SSLCipherSuite TLSv1.3 TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + +# Server chooses cipher order +SSLHonorCipherOrder on + +# ECDHE Curve Preference: brainpool first, NIST as fallback +SSLOpenSSLConfCmd Curves brainpoolP256r1:brainpoolP384r1:prime256v1:secp384r1 + +# Signature algorithms preference +SSLOpenSSLConfCmd SignatureAlgorithms ecdsa_secp256r1_sha256:ecdsa_secp384r1_sha384:rsa_pss_rsae_sha256:rsa_pss_rsae_sha384 + +# Session cache +SSLSessionCache shmcb:/run/apache2/ssl_scache(512000) +SSLSessionCacheTimeout 300 + +# Disable session tickets (for PFS) +SSLSessionTickets off + +# Strict SNI +SSLStrictSNIVHostCheck on diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 94b960cece6..6b20198807d 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -73,6 +73,46 @@ CustomLog /dev/stdout proxy env=forwarded WSGIServerMetrics On {{- end }} +{{- if .Values.tls.enabled }} +# External HTTPS endpoint (via Ingress TLS passthrough) +Listen 0.0.0.0:443 + + + ServerName {{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + + SSLEngine on + SSLCertificateFile /mnt/secrets/tls.crt + SSLCertificateKeyFile /mnt/secrets/tls.key + Include /etc/apache2/conf-enabled/tls-hardening.conf + + WSGIDaemonProcess keystone-tls processes=8 threads=1 user=keystone group=keystone display-name=%{GROUP} + WSGIProcessGroup keystone-tls + WSGIScriptAlias / /var/www/cgi-bin/keystone/keystone-wsgi-public + WSGIApplicationGroup %{GLOBAL} + WSGIPassAuthorization On + LimitRequestBody 114688 + LimitRequestFieldSize 16380 + = 2.4> + ErrorLogFormat "%{cu}t %M" + + ErrorLog /dev/stdout + + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout combined env=!forwarded + CustomLog /dev/stdout proxy env=forwarded + + KeepAliveTimeout 61 + + {{- if .Values.federation.saml.enabled }} + + SetHandler shib + + IncludeOptional /etc/apache2/conf-enabled/federation-saml.conf + {{- end }} + +{{- end }} + +# Internal HTTP endpoint (protected by Linkerd mTLS at the network layer) ServerName {{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} WSGIDaemonProcess keystone-public processes=8 threads=1 user=keystone group=keystone display-name=%{GROUP} diff --git a/openstack/keystone/templates/ingress-api.yaml b/openstack/keystone/templates/ingress-api.yaml index 22b68557f49..20a05f30764 100644 --- a/openstack/keystone/templates/ingress-api.yaml +++ b/openstack/keystone/templates/ingress-api.yaml @@ -46,7 +46,12 @@ metadata: component: keystone type: api annotations: - {{- if or .Values.federation.oidc.enabled .Values.federation.saml.enabled }} + {{- if and .Values.tls.enabled .Values.tls.passthrough }} + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + config.linkerd.io/skip-inbound-ports: "443" + nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr" + {{- else if or .Values.federation.oidc.enabled .Values.federation.saml.enabled }} # the affinity bits are required for federation; they do not affect the # cases when the API is used, but will affect browser-based workflows ingress.kubernetes.io/affinity: cookie @@ -79,7 +84,7 @@ metadata: ingress.kubernetes.io/auth-tls-verify-depth: "3" nginx.ingress.kubernetes.io/auth-tls-verify-depth: "3" {{- end }} - {{- if .Values.services.ingress.tlsacme }} + {{- if and .Values.services.ingress.tlsacme (not (and .Values.tls.enabled .Values.tls.passthrough)) }} kubernetes.io/tls-acme: "true" {{- end }} {{- if .Values.global.is_global_region }} @@ -123,5 +128,9 @@ spec: service: name: "{{if .Values.global.is_global_region }}keystone-global{{else}}keystone{{end}}" port: + {{- if and .Values.tls.enabled .Values.tls.passthrough }} + number: 443 + {{- else }} number: 5000 + {{- end }} {{- end }} diff --git a/openstack/keystone/templates/service-api.yaml b/openstack/keystone/templates/service-api.yaml index 15f7d815eb0..baf7e0d6226 100644 --- a/openstack/keystone/templates/service-api.yaml +++ b/openstack/keystone/templates/service-api.yaml @@ -30,6 +30,12 @@ spec: protocol: "TCP" port: 5000 targetPort: 5000 + {{- if .Values.tls.enabled }} + - name: public-tls + protocol: "TCP" + port: 443 + targetPort: 443 + {{- end }} {{- if .Values.global.keystone_internal_ip }} externalIPs: - {{ .Values.global.keystone_internal_ip | quote }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 117193f9df4..48165fab39e 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -57,6 +57,20 @@ osprofiler-redis: alerts: support_group: identity +# Pod-level TLS termination in Apache +tls: + enabled: false + secretName: keystone-tls + passthrough: false + envelopeEncryption: none + lifecycle: + image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager + imageTag: latest + issuerName: digicert-issuer + issuerKind: ClusterDigicertIssuer + issuerGroup: certmanager.cloud.sap + keyAlgorithm: ECDSAP256 + federation: oidc: enabled: false From be8050df5b945de92c49cefa2d6916894fb06390 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sun, 22 Mar 2026 09:29:07 +0100 Subject: [PATCH 03/54] Add TLS bootstrap Job and weekly rotation CronJob to Keystone Bootstrap Job (Helm pre-install/pre-upgrade hook, weight -5) ensures a valid TLS certificate exists before pods start. Rotation CronJob runs weekly to renew certificates 30 days before expiry. Both use the shared tls-lifecycle-manager Go binary with noop envelope encryption. RBAC: ServiceAccount + Role for Secrets, CertificateRequests, Deployments. --- .../templates/cronjob-tls-rotation.yaml | 41 +++++++++++ .../keystone/templates/job-tls-bootstrap.yaml | 69 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 openstack/keystone/templates/cronjob-tls-rotation.yaml create mode 100644 openstack/keystone/templates/job-tls-bootstrap.yaml diff --git a/openstack/keystone/templates/cronjob-tls-rotation.yaml b/openstack/keystone/templates/cronjob-tls-rotation.yaml new file mode 100644 index 00000000000..1c71c0acbff --- /dev/null +++ b/openstack/keystone/templates/cronjob-tls-rotation.yaml @@ -0,0 +1,41 @@ +{{- if .Values.tls.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-tls-rotation + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: keystone + type: tls-rotation +spec: + schedule: "0 3 * * 1" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + serviceAccountName: {{ .Release.Name }}-tls-manager + containers: + - name: tls-rotation + image: {{ .Values.tls.lifecycle.image }}:{{ .Values.tls.lifecycle.imageTag }} + imagePullPolicy: IfNotPresent + args: + - --mode=rotation + - --namespace={{ .Release.Namespace }} + - --secret-name={{ .Values.tls.secretName | default "keystone-tls" }} + - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} + - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} + - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + - --renewal-threshold-days=30 + - --deployment-name={{ .Release.Name }}-api + - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + backoffLimit: 3 +{{- end }} diff --git a/openstack/keystone/templates/job-tls-bootstrap.yaml b/openstack/keystone/templates/job-tls-bootstrap.yaml new file mode 100644 index 00000000000..8d3115e038c --- /dev/null +++ b/openstack/keystone/templates/job-tls-bootstrap.yaml @@ -0,0 +1,69 @@ +{{- if .Values.tls.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-tls-bootstrap + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: keystone + type: tls-bootstrap + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + template: + spec: + restartPolicy: Never + serviceAccountName: {{ .Release.Name }}-tls-manager + containers: + - name: tls-bootstrap + image: {{ .Values.tls.lifecycle.image }}:{{ .Values.tls.lifecycle.imageTag }} + imagePullPolicy: IfNotPresent + args: + - --mode=bootstrap + - --namespace={{ .Release.Namespace }} + - --secret-name={{ .Values.tls.secretName | default "keystone-tls" }} + - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} + - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} + - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + backoffLimit: 3 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-tls-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-tls-manager +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update"] + - apiGroups: ["cert-manager.io"] + resources: ["certificaterequests"] + verbs: ["create", "get", "delete", "watch"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-tls-manager +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-tls-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-tls-manager +{{- end }} From 3ca5746c833f79a086ae19297daa363a1c0927e4 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sun, 22 Mar 2026 09:29:16 +0100 Subject: [PATCH 04/54] Add TLS bootstrap Job and weekly rotation CronJob to Barbican Same pattern as Keystone. Bootstrap Job runs at weight -5 (before the existing migration-job at weight -1). Rotation CronJob runs weekly. Co-authored-by: Rajiv Mucheli --- .../templates/cronjob-tls-rotation.yaml | 41 +++++++++++ .../barbican/templates/job-tls-bootstrap.yaml | 69 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 openstack/barbican/templates/cronjob-tls-rotation.yaml create mode 100644 openstack/barbican/templates/job-tls-bootstrap.yaml diff --git a/openstack/barbican/templates/cronjob-tls-rotation.yaml b/openstack/barbican/templates/cronjob-tls-rotation.yaml new file mode 100644 index 00000000000..2b25309f0d7 --- /dev/null +++ b/openstack/barbican/templates/cronjob-tls-rotation.yaml @@ -0,0 +1,41 @@ +{{- if .Values.tls.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }}-tls-rotation + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: barbican + type: tls-rotation +spec: + schedule: "0 3 * * 1" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + serviceAccountName: {{ .Release.Name }}-tls-manager + containers: + - name: tls-rotation + image: {{ .Values.tls.lifecycle.image }}:{{ .Values.tls.lifecycle.imageTag }} + imagePullPolicy: IfNotPresent + args: + - --mode=rotation + - --namespace={{ .Release.Namespace }} + - --secret-name={{ .Values.tls.secretName | default "barbican-tls" }} + - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} + - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} + - --common-name={{ include "barbican_api_endpoint_host_public" . }} + - --san-dns={{ include "barbican_api_endpoint_host_public" . }} + - --renewal-threshold-days=30 + - --deployment-name=barbican-api + - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + backoffLimit: 3 +{{- end }} diff --git a/openstack/barbican/templates/job-tls-bootstrap.yaml b/openstack/barbican/templates/job-tls-bootstrap.yaml new file mode 100644 index 00000000000..5f801b8126a --- /dev/null +++ b/openstack/barbican/templates/job-tls-bootstrap.yaml @@ -0,0 +1,69 @@ +{{- if .Values.tls.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-tls-bootstrap + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: barbican + type: tls-bootstrap + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-5" + "helm.sh/hook-delete-policy": before-hook-creation +spec: + template: + spec: + restartPolicy: Never + serviceAccountName: {{ .Release.Name }}-tls-manager + containers: + - name: tls-bootstrap + image: {{ .Values.tls.lifecycle.image }}:{{ .Values.tls.lifecycle.imageTag }} + imagePullPolicy: IfNotPresent + args: + - --mode=bootstrap + - --namespace={{ .Release.Namespace }} + - --secret-name={{ .Values.tls.secretName | default "barbican-tls" }} + - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} + - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} + - --common-name={{ include "barbican_api_endpoint_host_public" . }} + - --san-dns={{ include "barbican_api_endpoint_host_public" . }} + - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + backoffLimit: 3 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-tls-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-tls-manager +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update"] + - apiGroups: ["cert-manager.io"] + resources: ["certificaterequests"] + verbs: ["create", "get", "delete", "watch"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-tls-manager +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-tls-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-tls-manager +{{- end }} From 3f83b6519347904bfe9c95c9b2fc6336c341f3b8 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Wed, 25 Mar 2026 11:16:31 +0100 Subject: [PATCH 05/54] Add Prometheus alerts for TLS certificate rotation failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fires when the weekly TLS rotation CronJob fails for either service. Severity: critical — if rotation keeps failing, the TLS certificate will expire and external HTTPS access will break. Keystone: OpenstackKeystoneTLSRotationFailed Barbican: OpenstackBarbicanTLSRotationFailed Both trigger after 1 hour of failure (one retry window). --- .../openstack/openstack-barbican.alerts | 14 ++++++++++ .../keystone/alerts/openstack/keystone.alerts | 27 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/openstack/barbican/alerts/openstack/openstack-barbican.alerts b/openstack/barbican/alerts/openstack/openstack-barbican.alerts index 368ec46d229..2710e32d31b 100644 --- a/openstack/barbican/alerts/openstack/openstack-barbican.alerts +++ b/openstack/barbican/alerts/openstack/openstack-barbican.alerts @@ -116,3 +116,17 @@ groups: annotations: description: All barbican-api server pods are down. summary: barbican is unavailable. + + - alert: OpenstackBarbicanTLSRotationFailed + expr: kube_job_status_failed{job_name=~"barbican-tls-rotation.*"} > 0 + for: 1h + labels: + context: security + dashboard: barbican + service: barbican + severity: critical + tier: os + support_group: identity + annotations: + description: The weekly TLS certificate rotation CronJob for Barbican has failed. If not resolved, the TLS certificate will expire and external HTTPS access will break. Check the CronJob logs. + summary: Barbican TLS certificate rotation failed diff --git a/openstack/keystone/alerts/openstack/keystone.alerts b/openstack/keystone/alerts/openstack/keystone.alerts index b9ed5b48d6d..2320e99e3d5 100644 --- a/openstack/keystone/alerts/openstack/keystone.alerts +++ b/openstack/keystone/alerts/openstack/keystone.alerts @@ -91,3 +91,30 @@ groups: description: Api Requests to Keystone is {{ $value }}ops/s summary: Api Requests to Keystone is {{ $value }}ops/s + - alert: OpenstackKeystoneSAMLAssertionValidationFailed + expr: increase(apache_error_log_saml_failures_total{service="keystone"}[5m]) > 0 + for: 1m + labels: + context: security + dashboard: keystone + service: keystone + severity: warning + tier: os + support_group: identity + annotations: + description: mod_shib rejected {{ $value }} SAML assertion(s) in the last 5 minutes. Check Apache error logs for signature, timing, or issuer validation failures. + summary: SAML assertion validation failures detected in Keystone + + - alert: OpenstackKeystoneTLSRotationFailed + expr: kube_job_status_failed{job_name=~"keystone-tls-rotation.*"} > 0 + for: 1h + labels: + context: security + dashboard: keystone + service: keystone + severity: critical + tier: os + support_group: identity + annotations: + description: The weekly TLS certificate rotation CronJob for Keystone has failed. If not resolved, the TLS certificate will expire and external HTTPS access will break. Check the CronJob logs. + summary: Keystone TLS certificate rotation failed From c87b48dc60da958963ff44758dfac6977d906166 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sun, 29 Mar 2026 23:18:15 +0200 Subject: [PATCH 06/54] Use Keystone LOCI image for TLS init container instead of busybox The init container only needs bash and cp, which are available in the Keystone LOCI image. This avoids pulling an external busybox image and ensures all images come from Keppel. --- openstack/keystone/templates/deployment-api.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 4cb6b2b304b..46fb9460f5b 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -112,10 +112,12 @@ spec: {{- end }} {{- if .Values.tls.enabled }} - name: tls-init - image: {{ required ".Values.global.dockerHubMirror is missing" .Values.global.dockerHubMirror }}/library/busybox - command: ["sh", "-c"] - args: + image: {{required ".Values.global.registryAlternateRegion is missing" .Values.global.registryAlternateRegion}}/loci-keystone:{{required ".Values.api.imageTag is missing" .Values.api.imageTag}} + command: + - bash + - -c - | + set -ex cp /tls-input/tls.key /mnt/secrets/tls.key cp /tls-input/tls.crt /mnt/secrets/tls.crt chmod 0440 /mnt/secrets/tls.key From 6dd908ddbb63570c113946b09c6e0016e76bc5c4 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 08:26:34 +0200 Subject: [PATCH 07/54] Lower TLS rotation alert severity from critical to warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rotation CronJob renews certificates 30 days before expiry. Even if several weekly runs fail, there are weeks of runway before impact. Warning severity is appropriate — critical should be reserved for imminent outages. --- openstack/barbican/alerts/openstack/openstack-barbican.alerts | 2 +- openstack/keystone/alerts/openstack/keystone.alerts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openstack/barbican/alerts/openstack/openstack-barbican.alerts b/openstack/barbican/alerts/openstack/openstack-barbican.alerts index 2710e32d31b..f839c6dd282 100644 --- a/openstack/barbican/alerts/openstack/openstack-barbican.alerts +++ b/openstack/barbican/alerts/openstack/openstack-barbican.alerts @@ -124,7 +124,7 @@ groups: context: security dashboard: barbican service: barbican - severity: critical + severity: warning tier: os support_group: identity annotations: diff --git a/openstack/keystone/alerts/openstack/keystone.alerts b/openstack/keystone/alerts/openstack/keystone.alerts index 2320e99e3d5..f4587ed6e8f 100644 --- a/openstack/keystone/alerts/openstack/keystone.alerts +++ b/openstack/keystone/alerts/openstack/keystone.alerts @@ -112,7 +112,7 @@ groups: context: security dashboard: keystone service: keystone - severity: critical + severity: warning tier: os support_group: identity annotations: From 0a215949eed6962c3b4eac2a4821c4df48e1d1dd Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 08:53:31 +0200 Subject: [PATCH 08/54] Use DigicertIssuer (namespace-scoped) instead of ClusterDigicertIssuer The ClusterDigicertIssuer controller is not active in any region. The namespace-scoped DigicertIssuer in kube-system is the one that processes CertificateRequests. The digicert-issuer controller falls back to kube-system when it doesn't find the issuer in the CR's namespace. --- openstack/barbican/templates/cronjob-tls-rotation.yaml | 2 +- openstack/barbican/templates/job-tls-bootstrap.yaml | 2 +- openstack/barbican/values.yaml | 2 +- openstack/keystone/templates/cronjob-tls-rotation.yaml | 2 +- openstack/keystone/templates/job-tls-bootstrap.yaml | 2 +- openstack/keystone/values.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/openstack/barbican/templates/cronjob-tls-rotation.yaml b/openstack/barbican/templates/cronjob-tls-rotation.yaml index 2b25309f0d7..a778cf1f023 100644 --- a/openstack/barbican/templates/cronjob-tls-rotation.yaml +++ b/openstack/barbican/templates/cronjob-tls-rotation.yaml @@ -30,7 +30,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "barbican-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} diff --git a/openstack/barbican/templates/job-tls-bootstrap.yaml b/openstack/barbican/templates/job-tls-bootstrap.yaml index 5f801b8126a..798dc7278fd 100644 --- a/openstack/barbican/templates/job-tls-bootstrap.yaml +++ b/openstack/barbican/templates/job-tls-bootstrap.yaml @@ -28,7 +28,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "barbican-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index bc008c51f27..71f019a6110 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -504,7 +504,7 @@ tls: imageTag: latest # cert-manager issuer configuration issuerName: digicert-issuer - issuerKind: ClusterDigicertIssuer + issuerKind: DigicertIssuer issuerGroup: certmanager.cloud.sap keyAlgorithm: ECDSAP256 diff --git a/openstack/keystone/templates/cronjob-tls-rotation.yaml b/openstack/keystone/templates/cronjob-tls-rotation.yaml index 1c71c0acbff..a2537eef4ac 100644 --- a/openstack/keystone/templates/cronjob-tls-rotation.yaml +++ b/openstack/keystone/templates/cronjob-tls-rotation.yaml @@ -30,7 +30,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "keystone-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} diff --git a/openstack/keystone/templates/job-tls-bootstrap.yaml b/openstack/keystone/templates/job-tls-bootstrap.yaml index 8d3115e038c..c84456f6d7f 100644 --- a/openstack/keystone/templates/job-tls-bootstrap.yaml +++ b/openstack/keystone/templates/job-tls-bootstrap.yaml @@ -28,7 +28,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "keystone-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterDigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 48165fab39e..7f4e11cefa3 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -67,7 +67,7 @@ tls: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager imageTag: latest issuerName: digicert-issuer - issuerKind: ClusterDigicertIssuer + issuerKind: DigicertIssuer issuerGroup: certmanager.cloud.sap keyAlgorithm: ECDSAP256 From 94024e7733b22fc53a58a92d528fc8b13ac6d253 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 09:07:52 +0200 Subject: [PATCH 09/54] Move TLS manager RBAC out of Helm hook into regular templates The ServiceAccount, Role, and RoleBinding were in the same template as the pre-install/pre-upgrade hook Job. Helm treats all resources in a hook template as hooks, causing the SA to either not exist when the Job starts or get deleted by the before-hook-creation policy. Move RBAC to separate rbac-tls-manager.yaml files (regular templates, not hooks). These are created as normal Helm resources before any hooks run, ensuring the ServiceAccount exists when the bootstrap Job starts. --- .../barbican/templates/job-tls-bootstrap.yaml | 32 --------------- .../barbican/templates/rbac-tls-manager.yaml | 39 +++++++++++++++++++ .../keystone/templates/job-tls-bootstrap.yaml | 32 --------------- .../keystone/templates/rbac-tls-manager.yaml | 39 +++++++++++++++++++ 4 files changed, 78 insertions(+), 64 deletions(-) create mode 100644 openstack/barbican/templates/rbac-tls-manager.yaml create mode 100644 openstack/keystone/templates/rbac-tls-manager.yaml diff --git a/openstack/barbican/templates/job-tls-bootstrap.yaml b/openstack/barbican/templates/job-tls-bootstrap.yaml index 798dc7278fd..f0599aada41 100644 --- a/openstack/barbican/templates/job-tls-bootstrap.yaml +++ b/openstack/barbican/templates/job-tls-bootstrap.yaml @@ -34,36 +34,4 @@ spec: - --san-dns={{ include "barbican_api_endpoint_host_public" . }} - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} backoffLimit: 3 ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Release.Name }}-tls-manager ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ .Release.Name }}-tls-manager -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "create", "update"] - - apiGroups: ["cert-manager.io"] - resources: ["certificaterequests"] - verbs: ["create", "get", "delete", "watch"] - - apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ .Release.Name }}-tls-manager -subjects: - - kind: ServiceAccount - name: {{ .Release.Name }}-tls-manager -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ .Release.Name }}-tls-manager {{- end }} diff --git a/openstack/barbican/templates/rbac-tls-manager.yaml b/openstack/barbican/templates/rbac-tls-manager.yaml new file mode 100644 index 00000000000..7ecd546ca6d --- /dev/null +++ b/openstack/barbican/templates/rbac-tls-manager.yaml @@ -0,0 +1,39 @@ +{{- if .Values.tls.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-tls-manager + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: barbican + type: tls-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-tls-manager +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update"] + - apiGroups: ["cert-manager.io"] + resources: ["certificaterequests"] + verbs: ["create", "get", "delete", "watch"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-tls-manager +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-tls-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-tls-manager +{{- end }} diff --git a/openstack/keystone/templates/job-tls-bootstrap.yaml b/openstack/keystone/templates/job-tls-bootstrap.yaml index c84456f6d7f..916c5a5a800 100644 --- a/openstack/keystone/templates/job-tls-bootstrap.yaml +++ b/openstack/keystone/templates/job-tls-bootstrap.yaml @@ -34,36 +34,4 @@ spec: - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} backoffLimit: 3 ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Release.Name }}-tls-manager ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ .Release.Name }}-tls-manager -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "create", "update"] - - apiGroups: ["cert-manager.io"] - resources: ["certificaterequests"] - verbs: ["create", "get", "delete", "watch"] - - apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ .Release.Name }}-tls-manager -subjects: - - kind: ServiceAccount - name: {{ .Release.Name }}-tls-manager -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ .Release.Name }}-tls-manager {{- end }} diff --git a/openstack/keystone/templates/rbac-tls-manager.yaml b/openstack/keystone/templates/rbac-tls-manager.yaml new file mode 100644 index 00000000000..59903402fdf --- /dev/null +++ b/openstack/keystone/templates/rbac-tls-manager.yaml @@ -0,0 +1,39 @@ +{{- if .Values.tls.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }}-tls-manager + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: keystone + type: tls-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-tls-manager +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update"] + - apiGroups: ["cert-manager.io"] + resources: ["certificaterequests"] + verbs: ["create", "get", "delete", "watch"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-tls-manager +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }}-tls-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-tls-manager +{{- end }} From 9c2f60ffc54cfd953ee3710d732028cd5b793e49 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 09:18:44 +0200 Subject: [PATCH 10/54] Make TLS manager RBAC a pre-install hook with weight -10 Regular Helm resources are applied AFTER hooks, so the ServiceAccount didn't exist when the bootstrap Job (hook weight -5) started. Making the RBAC resources hooks with weight -10 ensures they're created before the Job at weight -5. --- openstack/barbican/templates/rbac-tls-manager.yaml | 12 ++++++++++++ openstack/keystone/templates/rbac-tls-manager.yaml | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/openstack/barbican/templates/rbac-tls-manager.yaml b/openstack/barbican/templates/rbac-tls-manager.yaml index 7ecd546ca6d..ba73c685a5c 100644 --- a/openstack/barbican/templates/rbac-tls-manager.yaml +++ b/openstack/barbican/templates/rbac-tls-manager.yaml @@ -9,11 +9,19 @@ metadata: release: "{{ .Release.Name }}" component: barbican type: tls-manager + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: {{ .Release.Name }}-tls-manager + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation rules: - apiGroups: [""] resources: ["secrets"] @@ -29,6 +37,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: {{ .Release.Name }}-tls-manager + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation subjects: - kind: ServiceAccount name: {{ .Release.Name }}-tls-manager diff --git a/openstack/keystone/templates/rbac-tls-manager.yaml b/openstack/keystone/templates/rbac-tls-manager.yaml index 59903402fdf..9f02a0ba9b4 100644 --- a/openstack/keystone/templates/rbac-tls-manager.yaml +++ b/openstack/keystone/templates/rbac-tls-manager.yaml @@ -9,11 +9,19 @@ metadata: release: "{{ .Release.Name }}" component: keystone type: tls-manager + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: {{ .Release.Name }}-tls-manager + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation rules: - apiGroups: [""] resources: ["secrets"] @@ -29,6 +37,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: {{ .Release.Name }}-tls-manager + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "-10" + "helm.sh/hook-delete-policy": before-hook-creation subjects: - kind: ServiceAccount name: {{ .Release.Name }}-tls-manager From 58af1eaed0ca02c6ab917966bfe3cf73d38fd5c2 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 09:36:05 +0200 Subject: [PATCH 11/54] Enable mod_ssl for TLS termination via read-only conf for Barbican Keystone enables mod_ssl at startup with a2enmod, which writes a symlink into /etc/apache2/mods-enabled. Barbican now runs with a read-only root filesystem, so a2enmod fails there with a read-only filesystem error. Load mod_ssl (and its socache_shmcb dependency) through a LoadModule config mounted read-only into conf-enabled instead. Because conf-enabled is parsed after ports.conf, the ssl_module gate that would bind port 443 is not yet satisfied, so the TLS virtual host declares Listen 443 explicitly. The TLS hardening include is placed at server scope since SSLSessionCache is not valid inside a VirtualHost. --- openstack/barbican/templates/api-deployment.yaml | 4 ++++ openstack/barbican/templates/etc-configmap.yaml | 2 ++ openstack/barbican/templates/etc/_ssl-load.conf.tpl | 7 +++++++ openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl | 5 +++-- openstack/keystone/templates/bin/_keystone_api.sh.tpl | 4 ++++ 5 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 openstack/barbican/templates/etc/_ssl-load.conf.tpl diff --git a/openstack/barbican/templates/api-deployment.yaml b/openstack/barbican/templates/api-deployment.yaml index 1a609e1e5d6..91a30659062 100644 --- a/openstack/barbican/templates/api-deployment.yaml +++ b/openstack/barbican/templates/api-deployment.yaml @@ -169,6 +169,10 @@ spec: - name: tls-secrets mountPath: /mnt/secrets readOnly: true + - name: barbican-etc + mountPath: /etc/apache2/conf-enabled/00-ssl-load.conf + subPath: ssl-load.conf + readOnly: true - name: barbican-etc mountPath: /etc/apache2/conf-enabled/tls-hardening.conf subPath: tls-hardening.conf diff --git a/openstack/barbican/templates/etc-configmap.yaml b/openstack/barbican/templates/etc-configmap.yaml index 6d93bcce19a..2408bbf5eaf 100644 --- a/openstack/barbican/templates/etc-configmap.yaml +++ b/openstack/barbican/templates/etc-configmap.yaml @@ -12,6 +12,8 @@ data: barbican.conf: | {{ include (print .Template.BasePath "/etc/_barbican.conf.tpl") . | indent 4 }} {{- if .Values.tls.enabled }} + ssl-load.conf: | +{{ include (print .Template.BasePath "/etc/_ssl-load.conf.tpl") . | indent 4 }} tls-hardening.conf: | {{ include (print .Template.BasePath "/etc/_tls-hardening.conf.tpl") . | indent 4 }} {{- end }} diff --git a/openstack/barbican/templates/etc/_ssl-load.conf.tpl b/openstack/barbican/templates/etc/_ssl-load.conf.tpl new file mode 100644 index 00000000000..86117cbc2af --- /dev/null +++ b/openstack/barbican/templates/etc/_ssl-load.conf.tpl @@ -0,0 +1,7 @@ +{{/* +Load mod_ssl and its socache dependency via read-only conf so the module is +available under a read-only root filesystem without a2enmod. +*/}} + +LoadModule socache_shmcb_module /usr/lib/apache2/modules/mod_socache_shmcb.so +LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 8e66f25096a..2ec4afecc91 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -20,13 +20,14 @@ Listen 0.0.0.0:{{ .Values.api_port_internal }} # External HTTPS endpoint (via Ingress TLS passthrough) Listen 0.0.0.0:443 +# TLS hardening at server level (must be outside VirtualHost) +Include /etc/apache2/conf-enabled/tls-hardening.conf + SSLEngine on SSLCertificateFile /mnt/secrets/tls.crt SSLCertificateKeyFile /mnt/secrets/tls.key - Include /etc/apache2/conf-enabled/tls-hardening.conf - WSGIDaemonProcess barbican-api-tls processes={{ .Values.api.processes | default 1 }} threads=1 user=barbican group=barbican display-name=%{GROUP} WSGIProcessGroup barbican-api-tls WSGIScriptAlias / /var/www/cgi-bin/barbican/barbican-wsgi-api diff --git a/openstack/keystone/templates/bin/_keystone_api.sh.tpl b/openstack/keystone/templates/bin/_keystone_api.sh.tpl index 76f5c6dcb86..0ef2523402f 100644 --- a/openstack/keystone/templates/bin/_keystone_api.sh.tpl +++ b/openstack/keystone/templates/bin/_keystone_api.sh.tpl @@ -26,6 +26,10 @@ function start () { a2dismod status + {{- if .Values.tls.enabled }} + a2enmod ssl + {{- end }} + if [ -f /etc/apache2/envvars ]; then # Loading Apache2 ENV variables source /etc/apache2/envvars From 88b04839fa28d4aa3b69ee8538293f8ff1a872fd Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 10:15:26 +0200 Subject: [PATCH 12/54] Move TLS hardening Include outside VirtualHost block SSLSessionCache, SSLSessionCacheTimeout, and SSLStrictSNIVHostCheck are server-level directives that cannot appear inside a VirtualHost section. Moving the Include to before the VirtualHost puts all directives at server level where they apply globally. The per-VirtualHost directives (SSLProtocol, SSLCipherSuite, etc.) inherit correctly from server level. Verified locally: apache2 -t confirms Syntax OK with the Include outside VirtualHost, and confirms the error with Include inside. --- openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl | 3 +++ openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 2ec4afecc91..04c398523aa 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -17,6 +17,9 @@ WSGIDaemonProcess barbican-api processes={{ .Values.api.processes | default 1 }} Listen 0.0.0.0:{{ .Values.api_port_internal }} {{- if .Values.tls.enabled }} +# TLS hardening at server level (must be outside VirtualHost) +Include /etc/apache2/conf-enabled/tls-hardening.conf + # External HTTPS endpoint (via Ingress TLS passthrough) Listen 0.0.0.0:443 diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 6b20198807d..396ba5dfb4c 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -74,6 +74,9 @@ WSGIServerMetrics On {{- end }} {{- if .Values.tls.enabled }} +# TLS hardening at server level (must be outside VirtualHost) +Include /etc/apache2/conf-enabled/tls-hardening.conf + # External HTTPS endpoint (via Ingress TLS passthrough) Listen 0.0.0.0:443 @@ -83,7 +86,6 @@ Listen 0.0.0.0:443 SSLEngine on SSLCertificateFile /mnt/secrets/tls.crt SSLCertificateKeyFile /mnt/secrets/tls.key - Include /etc/apache2/conf-enabled/tls-hardening.conf WSGIDaemonProcess keystone-tls processes=8 threads=1 user=keystone group=keystone display-name=%{GROUP} WSGIProcessGroup keystone-tls From 4d5a411096eb2c1860dee93a874b87a7cb5f2592 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 10:25:28 +0200 Subject: [PATCH 13/54] =?UTF-8?q?Remove=20duplicate=20Listen=20443=20?= =?UTF-8?q?=E2=80=94=20ports.conf=20already=20declares=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a2enmod ssl is run, Apache's default /etc/apache2/ports.conf activates 'Listen 443' inside an block. Our WSGI templates also declared 'Listen 0.0.0.0:443', causing a duplicate bind error at startup. Remove our Listen directive and let ports.conf handle it. Verified locally: apache2 -t confirms duplicate Listen causes 'Cannot define multiple Listeners on the same IP:port', and removing ours resolves it. --- openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl | 2 +- openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 04c398523aa..cbe5bbe18b9 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -21,7 +21,7 @@ Listen 0.0.0.0:{{ .Values.api_port_internal }} Include /etc/apache2/conf-enabled/tls-hardening.conf # External HTTPS endpoint (via Ingress TLS passthrough) -Listen 0.0.0.0:443 +# Note: Listen 443 is provided by /etc/apache2/ports.conf when mod_ssl is enabled # TLS hardening at server level (must be outside VirtualHost) Include /etc/apache2/conf-enabled/tls-hardening.conf diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 396ba5dfb4c..402d4e73b52 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -78,7 +78,7 @@ WSGIServerMetrics On Include /etc/apache2/conf-enabled/tls-hardening.conf # External HTTPS endpoint (via Ingress TLS passthrough) -Listen 0.0.0.0:443 +# Note: Listen 443 is provided by /etc/apache2/ports.conf when mod_ssl is enabled ServerName {{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} From d0ef107af5dfed2d6a79d05b24ada6ecf21e5041 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 10:41:57 +0200 Subject: [PATCH 14/54] Disable Apache default site when TLS is enabled The default 000-default.conf site on port 80 interferes with VirtualHost routing when TLS passthrough is active. Apache misroutes HTTP/2 requests after a successful TLS handshake on port 443 to the default port 80 site, returning '400 Bad Request: speaking plain HTTP to an SSL-enabled server port'. Disabling the default site fixes the routing. --- openstack/keystone/templates/bin/_keystone_api.sh.tpl | 1 + 1 file changed, 1 insertion(+) diff --git a/openstack/keystone/templates/bin/_keystone_api.sh.tpl b/openstack/keystone/templates/bin/_keystone_api.sh.tpl index 0ef2523402f..189bd2565f8 100644 --- a/openstack/keystone/templates/bin/_keystone_api.sh.tpl +++ b/openstack/keystone/templates/bin/_keystone_api.sh.tpl @@ -28,6 +28,7 @@ function start () { {{- if .Values.tls.enabled }} a2enmod ssl + a2dissite 000-default {{- end }} if [ -f /etc/apache2/envvars ]; then From 2a8e378e85c3f4b122461258db21a66fffb25338 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 15:41:29 +0200 Subject: [PATCH 15/54] Move x509 client cert verification from ingress to Apache in passthrough mode The auth-tls-* ingress annotations are incompatible with ssl-passthrough because NGINX cannot verify client certificates on an encrypted stream it doesn't decrypt. When passthrough is enabled, suppress the ingress annotations and configure Apache to do the client cert verification instead: - SSLVerifyClient optional + SSLVerifyDepth 3 in the HTTPS VirtualHost - SSLCACertificateFile from the x509-ca Secret mounted into the pod - RequestHeader directives pass client cert info to the WSGI app - a2enmod headers for the RequestHeader directive The x509-ca Secret (already created by the ingress template) is mounted at /etc/apache2/x509-ca/ when both passthrough and x509 are enabled. --- .../keystone/templates/bin/_keystone_api.sh.tpl | 1 + openstack/keystone/templates/deployment-api.yaml | 11 +++++++++++ .../keystone/templates/etc/_wsgi-keystone.conf.tpl | 13 +++++++++++++ openstack/keystone/templates/ingress-api.yaml | 2 +- 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/openstack/keystone/templates/bin/_keystone_api.sh.tpl b/openstack/keystone/templates/bin/_keystone_api.sh.tpl index 189bd2565f8..381a31da107 100644 --- a/openstack/keystone/templates/bin/_keystone_api.sh.tpl +++ b/openstack/keystone/templates/bin/_keystone_api.sh.tpl @@ -28,6 +28,7 @@ function start () { {{- if .Values.tls.enabled }} a2enmod ssl + a2enmod headers a2dissite 000-default {{- end }} diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 46fb9460f5b..6aeb0e17ba2 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -279,6 +279,11 @@ spec: mountPath: /etc/apache2/conf-enabled/tls-hardening.conf subPath: tls-hardening.conf readOnly: true + {{- if .Values.services.ingress.x509.ca }} + - name: x509-ca + mountPath: /etc/apache2/x509-ca + readOnly: true + {{- end }} {{- end }} - name: keystone-etc mountPath: /etc/apache2/mods-available/mpm_event.conf @@ -412,4 +417,10 @@ spec: emptyDir: medium: Memory sizeLimit: 1Mi + {{- if .Values.services.ingress.x509.ca }} + - name: x509-ca + secret: + secretName: {{ .Release.Name }}-x509-ca + defaultMode: 0444 + {{- end }} {{- end }} diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 402d4e73b52..6ef5c7bc969 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -87,6 +87,19 @@ Include /etc/apache2/conf-enabled/tls-hardening.conf SSLCertificateFile /mnt/secrets/tls.crt SSLCertificateKeyFile /mnt/secrets/tls.key + {{- if .Values.services.ingress.x509.ca }} + # Client certificate verification (moved from ingress to Apache in passthrough mode) + SSLVerifyClient optional + SSLVerifyDepth 3 + SSLCACertificateFile /etc/apache2/x509-ca/ca.crt + + # Pass client cert info to WSGI app via request headers + # Replaces the headers that NGINX used to forward when it terminated TLS + RequestHeader set SSL_CLIENT_CERT "%{SSL_CLIENT_CERT}s" + RequestHeader set SSL_CLIENT_I_DN "%{SSL_CLIENT_I_DN}s" + RequestHeader set SSL_CLIENT_VERIFY "%{SSL_CLIENT_VERIFY}s" + {{- end }} + WSGIDaemonProcess keystone-tls processes=8 threads=1 user=keystone group=keystone display-name=%{GROUP} WSGIProcessGroup keystone-tls WSGIScriptAlias / /var/www/cgi-bin/keystone/keystone-wsgi-public diff --git a/openstack/keystone/templates/ingress-api.yaml b/openstack/keystone/templates/ingress-api.yaml index 20a05f30764..26fc907a55f 100644 --- a/openstack/keystone/templates/ingress-api.yaml +++ b/openstack/keystone/templates/ingress-api.yaml @@ -74,7 +74,7 @@ metadata: ingress.kubernetes.io/limit-connections: {{ .Values.services.ingress.limitConnections | quote }} nginx.ingress.kubernetes.io/limit-connections: {{ .Values.services.ingress.limitConnections | quote }} {{- end }} - {{- if .Values.services.ingress.x509.ca }} + {{- if and .Values.services.ingress.x509.ca (not (and .Values.tls.enabled .Values.tls.passthrough)) }} ingress.kubernetes.io/auth-tls-secret: {{ .Release.Namespace }}/{{ .Release.Name }}-x509-ca nginx.ingress.kubernetes.io/auth-tls-secret: {{ .Release.Namespace }}/{{ .Release.Name }}-x509-ca ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream: "true" From 5878e806ec9847ea0987172ba49383a5002735b9 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 16:47:38 +0200 Subject: [PATCH 16/54] Add ingress.kubernetes.io prefix for ssl-passthrough and backend-protocol The ingress controller is configured with --annotations-prefix=ingress.kubernetes.io so it only reads annotations with that prefix. The ssl-passthrough annotation was only set with the nginx.ingress.kubernetes.io prefix, causing the controller to ignore it entirely. Add both prefixes, matching the dual-prefix pattern already used by auth-tls-* and other annotations in these templates. --- openstack/barbican/templates/ingress.yaml | 2 ++ openstack/keystone/templates/ingress-api.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/openstack/barbican/templates/ingress.yaml b/openstack/barbican/templates/ingress.yaml index 59cf32b7e98..378149e2356 100644 --- a/openstack/barbican/templates/ingress.yaml +++ b/openstack/barbican/templates/ingress.yaml @@ -14,7 +14,9 @@ metadata: component: barbican annotations: {{- if and .Values.tls.enabled .Values.tls.passthrough }} + ingress.kubernetes.io/ssl-passthrough: "true" nginx.ingress.kubernetes.io/ssl-passthrough: "true" + ingress.kubernetes.io/backend-protocol: "HTTPS" nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" config.linkerd.io/skip-inbound-ports: "443" disco: "true" diff --git a/openstack/keystone/templates/ingress-api.yaml b/openstack/keystone/templates/ingress-api.yaml index 26fc907a55f..2ae8cd1a96d 100644 --- a/openstack/keystone/templates/ingress-api.yaml +++ b/openstack/keystone/templates/ingress-api.yaml @@ -47,7 +47,9 @@ metadata: type: api annotations: {{- if and .Values.tls.enabled .Values.tls.passthrough }} + ingress.kubernetes.io/ssl-passthrough: "true" nginx.ingress.kubernetes.io/ssl-passthrough: "true" + ingress.kubernetes.io/backend-protocol: "HTTPS" nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" config.linkerd.io/skip-inbound-ports: "443" nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr" From 4974d01ca9b639d48e5d0c1957282a4b373d5862 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 30 Mar 2026 17:16:43 +0200 Subject: [PATCH 17/54] Change issuerKind default from DigicertIssuer to ClusterIssuer The operations documentation specifies kind: ClusterIssuer with group: certmanager.cloud.sap for DigiCert certificates. The group field routes to the digicert-issuer controller. All working CertificateRequests in the cluster use ClusterIssuer. Using DigicertIssuer caused a nil map panic in the digicert-issuer controller. --- openstack/barbican/templates/cronjob-tls-rotation.yaml | 2 +- openstack/barbican/templates/job-tls-bootstrap.yaml | 2 +- openstack/barbican/values.yaml | 2 +- openstack/keystone/templates/cronjob-tls-rotation.yaml | 2 +- openstack/keystone/templates/job-tls-bootstrap.yaml | 2 +- openstack/keystone/values.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/openstack/barbican/templates/cronjob-tls-rotation.yaml b/openstack/barbican/templates/cronjob-tls-rotation.yaml index a778cf1f023..5e298348f4f 100644 --- a/openstack/barbican/templates/cronjob-tls-rotation.yaml +++ b/openstack/barbican/templates/cronjob-tls-rotation.yaml @@ -30,7 +30,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "barbican-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} diff --git a/openstack/barbican/templates/job-tls-bootstrap.yaml b/openstack/barbican/templates/job-tls-bootstrap.yaml index f0599aada41..e5675b4fe17 100644 --- a/openstack/barbican/templates/job-tls-bootstrap.yaml +++ b/openstack/barbican/templates/job-tls-bootstrap.yaml @@ -28,7 +28,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "barbican-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index 71f019a6110..ccc98724c69 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -504,7 +504,7 @@ tls: imageTag: latest # cert-manager issuer configuration issuerName: digicert-issuer - issuerKind: DigicertIssuer + issuerKind: ClusterIssuer issuerGroup: certmanager.cloud.sap keyAlgorithm: ECDSAP256 diff --git a/openstack/keystone/templates/cronjob-tls-rotation.yaml b/openstack/keystone/templates/cronjob-tls-rotation.yaml index a2537eef4ac..6bd457ce433 100644 --- a/openstack/keystone/templates/cronjob-tls-rotation.yaml +++ b/openstack/keystone/templates/cronjob-tls-rotation.yaml @@ -30,7 +30,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "keystone-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} diff --git a/openstack/keystone/templates/job-tls-bootstrap.yaml b/openstack/keystone/templates/job-tls-bootstrap.yaml index 916c5a5a800..03d56369249 100644 --- a/openstack/keystone/templates/job-tls-bootstrap.yaml +++ b/openstack/keystone/templates/job-tls-bootstrap.yaml @@ -28,7 +28,7 @@ spec: - --secret-name={{ .Values.tls.secretName | default "keystone-tls" }} - --key-algorithm={{ .Values.tls.keyAlgorithm | default "ECDSAP256" }} - --issuer-name={{ .Values.tls.issuerName | default "digicert-issuer" }} - - --issuer-kind={{ .Values.tls.issuerKind | default "DigicertIssuer" }} + - --issuer-kind={{ .Values.tls.issuerKind | default "ClusterIssuer" }} - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 7f4e11cefa3..060bde805e5 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -67,7 +67,7 @@ tls: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager imageTag: latest issuerName: digicert-issuer - issuerKind: DigicertIssuer + issuerKind: ClusterIssuer issuerGroup: certmanager.cloud.sap keyAlgorithm: ECDSAP256 From a9ad662e674edc47b25a326fe37042b3d99a183a Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 31 Mar 2026 17:30:46 +0200 Subject: [PATCH 18/54] Enable ssl-passthrough on the ingress-nginx controller Services might decide to use the nginx ssl-passthrough annotation to forward encrypted TLS streams directly to pods for end-to-end encryption. Without --enable-ssl-passthrough in the controller args, this annotation is silently ignored and the controller terminates TLS at the ingress instead of passing it through. Add controller.extraArgs.enable-ssl-passthrough to the ingress-nginx subchart values so passthrough works globally across all clusters. --- system/kube-system-metal/values.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/system/kube-system-metal/values.yaml b/system/kube-system-metal/values.yaml index 8dfe2357aa3..afa57e71137 100644 --- a/system/kube-system-metal/values.yaml +++ b/system/kube-system-metal/values.yaml @@ -86,6 +86,10 @@ ingress-nginx: # See https://github.com/kubernetes/ingress-nginx/issues/7837 . allowSnippetAnnotations: false + extraArgs: + # Required for ssl-passthrough ingress annotations to take effect. + enable-ssl-passthrough: true + config: custom-http-errors: "495" From 68a55540fef167580b34f1484ad69185e9bbb231 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Wed, 1 Apr 2026 11:58:43 +0200 Subject: [PATCH 19/54] Suppress tls-acme ingress annotation whenever pod-level TLS is enabled When tls.enabled is true, the bootstrap Job provisions the certificate via DigiCert. The tls-acme annotation must be suppressed regardless of whether tls.passthrough is also set, otherwise both the bootstrap Job and the ingress cert-manager flow provision certificates for the same hostname. Previously, tls-acme was only suppressed when both tls.enabled and tls.passthrough were true. With tls.enabled alone (passthrough not yet active), both certificate paths were active simultaneously. --- openstack/barbican/templates/ingress.yaml | 2 +- openstack/keystone/templates/ingress-api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openstack/barbican/templates/ingress.yaml b/openstack/barbican/templates/ingress.yaml index 378149e2356..3b6614e3b7a 100644 --- a/openstack/barbican/templates/ingress.yaml +++ b/openstack/barbican/templates/ingress.yaml @@ -20,7 +20,7 @@ metadata: nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" config.linkerd.io/skip-inbound-ports: "443" disco: "true" - {{- else if .Values.tlsacme }} + {{- else if and .Values.tlsacme (not .Values.tls.enabled) }} kubernetes.io/tls-acme: "true" disco: "true" {{- end }} diff --git a/openstack/keystone/templates/ingress-api.yaml b/openstack/keystone/templates/ingress-api.yaml index 2ae8cd1a96d..dd8ac97f1be 100644 --- a/openstack/keystone/templates/ingress-api.yaml +++ b/openstack/keystone/templates/ingress-api.yaml @@ -86,7 +86,7 @@ metadata: ingress.kubernetes.io/auth-tls-verify-depth: "3" nginx.ingress.kubernetes.io/auth-tls-verify-depth: "3" {{- end }} - {{- if and .Values.services.ingress.tlsacme (not (and .Values.tls.enabled .Values.tls.passthrough)) }} + {{- if and .Values.services.ingress.tlsacme (not .Values.tls.enabled) }} kubernetes.io/tls-acme: "true" {{- end }} {{- if .Values.global.is_global_region }} From d37516c9f99b76e20d87e3f7a5350313f47e6dfb Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Thu, 2 Apr 2026 10:03:20 +0200 Subject: [PATCH 20/54] Drop Deployment RBAC and --deployment-name from TLS rotation CronJobs Reloader watches the TLS secret and triggers rolling restarts when it changes, so the tls-lifecycle-manager no longer needs to patch Deployments directly. This removes the --deployment-name argument from both CronJob templates and the apps/deployments RBAC rule from both Role definitions, narrowing the ServiceAccount permissions to only secrets and certificaterequests. --- openstack/barbican/templates/cronjob-tls-rotation.yaml | 1 - openstack/barbican/templates/rbac-tls-manager.yaml | 3 --- openstack/keystone/templates/cronjob-tls-rotation.yaml | 1 - openstack/keystone/templates/rbac-tls-manager.yaml | 3 --- 4 files changed, 8 deletions(-) diff --git a/openstack/barbican/templates/cronjob-tls-rotation.yaml b/openstack/barbican/templates/cronjob-tls-rotation.yaml index 5e298348f4f..87991909aca 100644 --- a/openstack/barbican/templates/cronjob-tls-rotation.yaml +++ b/openstack/barbican/templates/cronjob-tls-rotation.yaml @@ -35,7 +35,6 @@ spec: - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} - --renewal-threshold-days=30 - - --deployment-name=barbican-api - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} backoffLimit: 3 {{- end }} diff --git a/openstack/barbican/templates/rbac-tls-manager.yaml b/openstack/barbican/templates/rbac-tls-manager.yaml index ba73c685a5c..7e537d0c610 100644 --- a/openstack/barbican/templates/rbac-tls-manager.yaml +++ b/openstack/barbican/templates/rbac-tls-manager.yaml @@ -29,9 +29,6 @@ rules: - apiGroups: ["cert-manager.io"] resources: ["certificaterequests"] verbs: ["create", "get", "delete", "watch"] - - apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["patch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/openstack/keystone/templates/cronjob-tls-rotation.yaml b/openstack/keystone/templates/cronjob-tls-rotation.yaml index 6bd457ce433..e06badcd416 100644 --- a/openstack/keystone/templates/cronjob-tls-rotation.yaml +++ b/openstack/keystone/templates/cronjob-tls-rotation.yaml @@ -35,7 +35,6 @@ spec: - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --renewal-threshold-days=30 - - --deployment-name={{ .Release.Name }}-api - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} backoffLimit: 3 {{- end }} diff --git a/openstack/keystone/templates/rbac-tls-manager.yaml b/openstack/keystone/templates/rbac-tls-manager.yaml index 9f02a0ba9b4..511a28d9828 100644 --- a/openstack/keystone/templates/rbac-tls-manager.yaml +++ b/openstack/keystone/templates/rbac-tls-manager.yaml @@ -29,9 +29,6 @@ rules: - apiGroups: ["cert-manager.io"] resources: ["certificaterequests"] verbs: ["create", "get", "delete", "watch"] - - apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["patch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding From f290ce966c12fbf9fa45e37f3c936a96c544b78c Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 7 Apr 2026 12:56:36 +0200 Subject: [PATCH 21/54] Add pluggable key lifecycle configuration with Helm validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace envelopeEncryption with three explicit per-region operator choices: keyGeneration, keyWrapping, keyStorage. All three are required when tls.enabled — there are no defaults. Add Helm template validation that fails the render when: - Any of the three options is missing - The forbidden combination (wrapping=none + storage=k8s-secret) is used without allowInsecureStorage: true Replace bash cp init container with tls-lifecycle-manager --mode=unwrap which reads from volume mount, unwraps via the configured wrapper, validates PEM, and writes to tmpfs. Works for all wrapping modes. Templates use required() as a second layer of enforcement alongside the validation helper. --- openstack/barbican/templates/_helpers.tpl | 17 +++++++++++++++ .../barbican/templates/api-deployment.yaml | 21 +++++++------------ .../templates/cronjob-tls-rotation.yaml | 4 +++- .../barbican/templates/job-tls-bootstrap.yaml | 5 ++++- openstack/barbican/values.yaml | 9 ++++++-- openstack/keystone/templates/_helpers.tpl | 17 +++++++++++++++ .../templates/cronjob-tls-rotation.yaml | 4 +++- .../keystone/templates/deployment-api.yaml | 18 +++++++--------- .../keystone/templates/job-tls-bootstrap.yaml | 5 ++++- openstack/keystone/values.yaml | 8 ++++++- 10 files changed, 76 insertions(+), 32 deletions(-) diff --git a/openstack/barbican/templates/_helpers.tpl b/openstack/barbican/templates/_helpers.tpl index fe785f70456..593b189cd8d 100644 --- a/openstack/barbican/templates/_helpers.tpl +++ b/openstack/barbican/templates/_helpers.tpl @@ -39,3 +39,20 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- .Release.Name }}-{{ $name }}-{{ substr 0 4 $hash }}-{{ .Values.imageVersionBarbicanApi | required "Please set barbican.imageVersionBarbicanApi" }} {{- end }} {{- end }} + +{{- define "barbican.tls.validate" -}} +{{- if .Values.tls.enabled }} + {{- if not .Values.tls.keyGeneration }} + {{- fail "tls.keyGeneration is required when tls.enabled (options: go-crypto, hsm-entropy, hsm-full, tpm-entropy)" }} + {{- end }} + {{- if not .Values.tls.keyWrapping }} + {{- fail "tls.keyWrapping is required when tls.enabled (options: none, vault-transit, hsm, tpm)" }} + {{- end }} + {{- if not .Values.tls.keyStorage }} + {{- fail "tls.keyStorage is required when tls.enabled (options: k8s-secret, vault-secret)" }} + {{- end }} + {{- if and (eq .Values.tls.keyWrapping "none") (eq .Values.tls.keyStorage "k8s-secret") (not .Values.tls.allowInsecureStorage) }} + {{- fail "tls: unwrapped keys cannot be stored as plain-text K8s Secrets. Set tls.keyWrapping or tls.keyStorage, or set tls.allowInsecureStorage: true to acknowledge." }} + {{- end }} +{{- end }} +{{- end }} diff --git a/openstack/barbican/templates/api-deployment.yaml b/openstack/barbican/templates/api-deployment.yaml index 91a30659062..38d6f99ce63 100644 --- a/openstack/barbican/templates/api-deployment.yaml +++ b/openstack/barbican/templates/api-deployment.yaml @@ -61,20 +61,13 @@ spec: {{- end }} {{- if .Values.tls.enabled }} - name: tls-init - image: {{ required ".Values.global.registry is missing" .Values.global.registry }}/loci-barbican:{{ required "Values.imageVersionBarbicanApi is missing" .Values.imageVersionBarbicanApi }} - command: - - bash - - -c - - | - set -ex - # Copy TLS key material from K8s Secret to tmpfs - # In noop envelope-encryption mode, the key is plaintext in the Secret. - # When vault-transit or TPM is enabled, this step would decrypt first. - cp /tls-input/tls.key /mnt/secrets/tls.key - cp /tls-input/tls.crt /mnt/secrets/tls.crt - chmod 0440 /mnt/secrets/tls.key - chmod 0444 /mnt/secrets/tls.crt - echo "TLS key material placed on tmpfs" + image: {{ .Values.tls.lifecycle.image }}:{{ .Values.tls.lifecycle.imageTag }} + imagePullPolicy: IfNotPresent + args: + - --mode=unwrap + - --input-dir=/tls-input + - --output-dir=/mnt/secrets + - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} volumeMounts: - name: tls-input mountPath: /tls-input diff --git a/openstack/barbican/templates/cronjob-tls-rotation.yaml b/openstack/barbican/templates/cronjob-tls-rotation.yaml index 87991909aca..e298193fa09 100644 --- a/openstack/barbican/templates/cronjob-tls-rotation.yaml +++ b/openstack/barbican/templates/cronjob-tls-rotation.yaml @@ -35,6 +35,8 @@ spec: - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} - --renewal-threshold-days=30 - - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} + - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} + - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} backoffLimit: 3 {{- end }} diff --git a/openstack/barbican/templates/job-tls-bootstrap.yaml b/openstack/barbican/templates/job-tls-bootstrap.yaml index e5675b4fe17..e7670dfecd5 100644 --- a/openstack/barbican/templates/job-tls-bootstrap.yaml +++ b/openstack/barbican/templates/job-tls-bootstrap.yaml @@ -1,4 +1,5 @@ {{- if .Values.tls.enabled }} +{{- include "barbican.tls.validate" . }} apiVersion: batch/v1 kind: Job metadata: @@ -32,6 +33,8 @@ spec: - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ include "barbican_api_endpoint_host_public" . }} - --san-dns={{ include "barbican_api_endpoint_host_public" . }} - - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} + - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} + - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} backoffLimit: 3 {{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index ccc98724c69..b91cc585e99 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -496,8 +496,13 @@ tls: secretName: barbican-tls # Enable TLS passthrough on the ingress (sends encrypted stream directly to pod) passthrough: false - # Envelope encryption mode for the private key: "none" (plaintext) or "vault-transit" - envelopeEncryption: none + # Per-region operator choices — all three are REQUIRED when tls.enabled: true. + # There are no defaults; the operator must make an explicit choice. + # keyGeneration: go-crypto | hsm-entropy | hsm-full | tpm-entropy + # keyWrapping: none | vault-transit | hsm | tpm + # keyStorage: k8s-secret | vault-secret + # Escape hatch: set allowInsecureStorage: true to allow wrapping=none + storage=k8s-secret + allowInsecureStorage: false # TLS lifecycle manager binary for bootstrap Job and rotation CronJob lifecycle: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager diff --git a/openstack/keystone/templates/_helpers.tpl b/openstack/keystone/templates/_helpers.tpl index 26b71af9dcc..83a502eb88e 100644 --- a/openstack/keystone/templates/_helpers.tpl +++ b/openstack/keystone/templates/_helpers.tpl @@ -111,3 +111,20 @@ Empty values are skipped so callers may pass "" for optional dependencies. {{- printf "http://prodel.%s.svc/check-delete_project/%%(project_id)s" $ns -}} {{- end }} {{- end }} + +{{- define "keystone.tls.validate" -}} +{{- if .Values.tls.enabled }} + {{- if not .Values.tls.keyGeneration }} + {{- fail "tls.keyGeneration is required when tls.enabled (options: go-crypto, hsm-entropy, hsm-full, tpm-entropy)" }} + {{- end }} + {{- if not .Values.tls.keyWrapping }} + {{- fail "tls.keyWrapping is required when tls.enabled (options: none, vault-transit, hsm, tpm)" }} + {{- end }} + {{- if not .Values.tls.keyStorage }} + {{- fail "tls.keyStorage is required when tls.enabled (options: k8s-secret, vault-secret)" }} + {{- end }} + {{- if and (eq .Values.tls.keyWrapping "none") (eq .Values.tls.keyStorage "k8s-secret") (not .Values.tls.allowInsecureStorage) }} + {{- fail "tls: unwrapped keys cannot be stored as plain-text K8s Secrets. Set tls.keyWrapping or tls.keyStorage, or set tls.allowInsecureStorage: true to acknowledge." }} + {{- end }} +{{- end }} +{{- end }} diff --git a/openstack/keystone/templates/cronjob-tls-rotation.yaml b/openstack/keystone/templates/cronjob-tls-rotation.yaml index e06badcd416..b1ba97303e9 100644 --- a/openstack/keystone/templates/cronjob-tls-rotation.yaml +++ b/openstack/keystone/templates/cronjob-tls-rotation.yaml @@ -35,6 +35,8 @@ spec: - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --renewal-threshold-days=30 - - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} + - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} + - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} backoffLimit: 3 {{- end }} diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 6aeb0e17ba2..282b8d430c1 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -112,17 +112,13 @@ spec: {{- end }} {{- if .Values.tls.enabled }} - name: tls-init - image: {{required ".Values.global.registryAlternateRegion is missing" .Values.global.registryAlternateRegion}}/loci-keystone:{{required ".Values.api.imageTag is missing" .Values.api.imageTag}} - command: - - bash - - -c - - | - set -ex - cp /tls-input/tls.key /mnt/secrets/tls.key - cp /tls-input/tls.crt /mnt/secrets/tls.crt - chmod 0440 /mnt/secrets/tls.key - chmod 0444 /mnt/secrets/tls.crt - echo "TLS key material placed on tmpfs" + image: {{ .Values.tls.lifecycle.image }}:{{ .Values.tls.lifecycle.imageTag }} + imagePullPolicy: IfNotPresent + args: + - --mode=unwrap + - --input-dir=/tls-input + - --output-dir=/mnt/secrets + - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} volumeMounts: - name: tls-input mountPath: /tls-input diff --git a/openstack/keystone/templates/job-tls-bootstrap.yaml b/openstack/keystone/templates/job-tls-bootstrap.yaml index 03d56369249..841d9fbeed8 100644 --- a/openstack/keystone/templates/job-tls-bootstrap.yaml +++ b/openstack/keystone/templates/job-tls-bootstrap.yaml @@ -1,4 +1,5 @@ {{- if .Values.tls.enabled }} +{{- include "keystone.tls.validate" . }} apiVersion: batch/v1 kind: Job metadata: @@ -32,6 +33,8 @@ spec: - --issuer-group={{ .Values.tls.issuerGroup | default "certmanager.cloud.sap" }} - --common-name={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - --san-dns={{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} - - --envelope-encryption={{ .Values.tls.envelopeEncryption | default "none" }} + - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} + - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} + - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} backoffLimit: 3 {{- end }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 060bde805e5..77643b6158c 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -62,7 +62,13 @@ tls: enabled: false secretName: keystone-tls passthrough: false - envelopeEncryption: none + # Per-region operator choices — all three are REQUIRED when tls.enabled: true. + # There are no defaults; the operator must make an explicit choice. + # keyGeneration: go-crypto | hsm-entropy | hsm-full | tpm-entropy + # keyWrapping: none | vault-transit | hsm | tpm + # keyStorage: k8s-secret | vault-secret + # Escape hatch: set allowInsecureStorage: true to allow wrapping=none + storage=k8s-secret + allowInsecureStorage: false lifecycle: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager imageTag: latest From 2b52103e659fc96ed4a5aa7e70ecc31376aedb20 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sat, 15 Aug 2026 16:47:35 +0200 Subject: [PATCH 22/54] Add internal-k8s-secret storage option and wire --allow-insecure-storage The tls-lifecycle-manager binary now distinguishes internal-k8s-secret (written directly to the K8s API, stays in-cluster) from k8s-secret (Secret manifest emitted for Git commit + GitOps operator review). - values.yaml (keystone, barbican): document internal-k8s-secret as the recommended keyStorage default and clarify that allowInsecureStorage now also passes --allow-insecure-storage to the binary. - _helpers.tpl tls.validate: list internal-k8s-secret in the required keyStorage options message. - bootstrap Job and rotation CronJob (keystone, barbican): pass --allow-insecure-storage when tls.allowInsecureStorage is true, so the binary permits keyWrapping=none + keyStorage=k8s-secret in dev. --- openstack/barbican/templates/_helpers.tpl | 2 +- openstack/barbican/templates/cronjob-tls-rotation.yaml | 3 +++ openstack/barbican/templates/job-tls-bootstrap.yaml | 3 +++ openstack/barbican/values.yaml | 7 +++++-- openstack/keystone/templates/_helpers.tpl | 2 +- openstack/keystone/templates/cronjob-tls-rotation.yaml | 3 +++ openstack/keystone/templates/job-tls-bootstrap.yaml | 3 +++ openstack/keystone/values.yaml | 7 +++++-- 8 files changed, 24 insertions(+), 6 deletions(-) diff --git a/openstack/barbican/templates/_helpers.tpl b/openstack/barbican/templates/_helpers.tpl index 593b189cd8d..fb4433961b1 100644 --- a/openstack/barbican/templates/_helpers.tpl +++ b/openstack/barbican/templates/_helpers.tpl @@ -49,7 +49,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- fail "tls.keyWrapping is required when tls.enabled (options: none, vault-transit, hsm, tpm)" }} {{- end }} {{- if not .Values.tls.keyStorage }} - {{- fail "tls.keyStorage is required when tls.enabled (options: k8s-secret, vault-secret)" }} + {{- fail "tls.keyStorage is required when tls.enabled (options: internal-k8s-secret, k8s-secret, vault-secret)" }} {{- end }} {{- if and (eq .Values.tls.keyWrapping "none") (eq .Values.tls.keyStorage "k8s-secret") (not .Values.tls.allowInsecureStorage) }} {{- fail "tls: unwrapped keys cannot be stored as plain-text K8s Secrets. Set tls.keyWrapping or tls.keyStorage, or set tls.allowInsecureStorage: true to acknowledge." }} diff --git a/openstack/barbican/templates/cronjob-tls-rotation.yaml b/openstack/barbican/templates/cronjob-tls-rotation.yaml index e298193fa09..b09da9ac92d 100644 --- a/openstack/barbican/templates/cronjob-tls-rotation.yaml +++ b/openstack/barbican/templates/cronjob-tls-rotation.yaml @@ -38,5 +38,8 @@ spec: - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} + {{- if .Values.tls.allowInsecureStorage }} + - --allow-insecure-storage + {{- end }} backoffLimit: 3 {{- end }} diff --git a/openstack/barbican/templates/job-tls-bootstrap.yaml b/openstack/barbican/templates/job-tls-bootstrap.yaml index e7670dfecd5..99938947c3e 100644 --- a/openstack/barbican/templates/job-tls-bootstrap.yaml +++ b/openstack/barbican/templates/job-tls-bootstrap.yaml @@ -36,5 +36,8 @@ spec: - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} + {{- if .Values.tls.allowInsecureStorage }} + - --allow-insecure-storage + {{- end }} backoffLimit: 3 {{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index b91cc585e99..f716b415894 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -500,8 +500,11 @@ tls: # There are no defaults; the operator must make an explicit choice. # keyGeneration: go-crypto | hsm-entropy | hsm-full | tpm-entropy # keyWrapping: none | vault-transit | hsm | tpm - # keyStorage: k8s-secret | vault-secret - # Escape hatch: set allowInsecureStorage: true to allow wrapping=none + storage=k8s-secret + # keyStorage: internal-k8s-secret | k8s-secret | vault-secret + # internal-k8s-secret (recommended): key written directly to the K8s API; stays in-cluster, never in Git + # k8s-secret: Secret manifest emitted for Git commit + GitOps operator review + # Escape hatch: allowInsecureStorage: true also passes --allow-insecure-storage to the binary, + # permitting wrapping=none + storage=k8s-secret (development only) allowInsecureStorage: false # TLS lifecycle manager binary for bootstrap Job and rotation CronJob lifecycle: diff --git a/openstack/keystone/templates/_helpers.tpl b/openstack/keystone/templates/_helpers.tpl index 83a502eb88e..5dcc1603622 100644 --- a/openstack/keystone/templates/_helpers.tpl +++ b/openstack/keystone/templates/_helpers.tpl @@ -121,7 +121,7 @@ Empty values are skipped so callers may pass "" for optional dependencies. {{- fail "tls.keyWrapping is required when tls.enabled (options: none, vault-transit, hsm, tpm)" }} {{- end }} {{- if not .Values.tls.keyStorage }} - {{- fail "tls.keyStorage is required when tls.enabled (options: k8s-secret, vault-secret)" }} + {{- fail "tls.keyStorage is required when tls.enabled (options: internal-k8s-secret, k8s-secret, vault-secret)" }} {{- end }} {{- if and (eq .Values.tls.keyWrapping "none") (eq .Values.tls.keyStorage "k8s-secret") (not .Values.tls.allowInsecureStorage) }} {{- fail "tls: unwrapped keys cannot be stored as plain-text K8s Secrets. Set tls.keyWrapping or tls.keyStorage, or set tls.allowInsecureStorage: true to acknowledge." }} diff --git a/openstack/keystone/templates/cronjob-tls-rotation.yaml b/openstack/keystone/templates/cronjob-tls-rotation.yaml index b1ba97303e9..7a14b5f1212 100644 --- a/openstack/keystone/templates/cronjob-tls-rotation.yaml +++ b/openstack/keystone/templates/cronjob-tls-rotation.yaml @@ -38,5 +38,8 @@ spec: - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} + {{- if .Values.tls.allowInsecureStorage }} + - --allow-insecure-storage + {{- end }} backoffLimit: 3 {{- end }} diff --git a/openstack/keystone/templates/job-tls-bootstrap.yaml b/openstack/keystone/templates/job-tls-bootstrap.yaml index 841d9fbeed8..ac6f39c36b2 100644 --- a/openstack/keystone/templates/job-tls-bootstrap.yaml +++ b/openstack/keystone/templates/job-tls-bootstrap.yaml @@ -36,5 +36,8 @@ spec: - --key-generation={{ required "tls.keyGeneration is required" .Values.tls.keyGeneration }} - --key-wrapping={{ required "tls.keyWrapping is required" .Values.tls.keyWrapping }} - --key-storage={{ required "tls.keyStorage is required" .Values.tls.keyStorage }} + {{- if .Values.tls.allowInsecureStorage }} + - --allow-insecure-storage + {{- end }} backoffLimit: 3 {{- end }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 77643b6158c..212df450975 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -66,8 +66,11 @@ tls: # There are no defaults; the operator must make an explicit choice. # keyGeneration: go-crypto | hsm-entropy | hsm-full | tpm-entropy # keyWrapping: none | vault-transit | hsm | tpm - # keyStorage: k8s-secret | vault-secret - # Escape hatch: set allowInsecureStorage: true to allow wrapping=none + storage=k8s-secret + # keyStorage: internal-k8s-secret | k8s-secret | vault-secret + # internal-k8s-secret (recommended): key written directly to the K8s API; stays in-cluster, never in Git + # k8s-secret: Secret manifest emitted for Git commit + GitOps operator review + # Escape hatch: allowInsecureStorage: true also passes --allow-insecure-storage to the binary, + # permitting wrapping=none + storage=k8s-secret (development only) allowInsecureStorage: false lifecycle: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager From 2770cb63d13cf06ccae4ff7b9e95211a07e3207d Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Sun, 16 Aug 2026 16:34:24 +0200 Subject: [PATCH 23/54] Replace ingress TLS passthrough with direct /32 exposure for Keystone and Barbican Terminate TLS at Apache in the pod and expose it directly via a dedicated BGP-announced /32, instead of nginx-ingress SSL-passthrough (ADR 0406). Each chart's own tls.enabled flag now selects one of two behaviours and the tls.passthrough flag is removed: - tls.enabled=false: nginx-ingress terminates TLS (tls-acme), backend on the internal port, x509 client-cert auth at the ingress. - tls.enabled=true: Apache terminates TLS on :443, exposed via a new dedicated :443-only Service carrying the /32 (services.externalip); no Ingress is rendered; the pod carries skip-inbound-ports: "443"; x509 is verified natively in Apache. Keystone: - Gate ingress-api.yaml on the ingress-termination path and remove the passthrough annotation and backend-port branches. - Move the x509-ca Secret into its own secret-x509-ca.yaml so it is created regardless of whether the Ingress renders; the pod mounts it for the Apache SSLCACertificateFile. Gate matches the mount (services.ingress.x509.ca). - Add service-api-tls.yaml; trim the main Service to internal :5000 only; add the pod skip-inbound-ports annotation. Barbican: - Gate ingress.yaml on the ingress-termination path and remove the passthrough branch. - Add service-tls.yaml and a services.externalip value; trim the main Service to :9311 only; add the pod skip-inbound-ports annotation. Also remove the enable-ssl-passthrough ingress-nginx controller flag in system/kube-system-metal/values.yaml, which is no longer needed. Co-authored-by: Rajiv Mucheli --- .../barbican/templates/api-deployment.yaml | 3 ++ .../templates/etc/_wsgi-barbican.conf.tpl | 9 +++-- openstack/barbican/templates/ingress.yaml | 15 ++------ openstack/barbican/templates/service-tls.yaml | 29 +++++++++++++++ openstack/barbican/templates/service.yaml | 5 --- openstack/barbican/values.yaml | 5 +-- .../keystone/templates/deployment-api.yaml | 1 + .../templates/etc/_wsgi-keystone.conf.tpl | 4 +-- openstack/keystone/templates/ingress-api.yaml | 36 +++---------------- .../keystone/templates/secret-x509-ca.yaml | 15 ++++++++ .../keystone/templates/service-api-tls.yaml | 29 +++++++++++++++ openstack/keystone/templates/service-api.yaml | 9 ----- openstack/keystone/values.yaml | 6 ++-- system/kube-system-metal/values.yaml | 4 --- 14 files changed, 97 insertions(+), 73 deletions(-) create mode 100644 openstack/barbican/templates/service-tls.yaml create mode 100644 openstack/keystone/templates/secret-x509-ca.yaml create mode 100644 openstack/keystone/templates/service-api-tls.yaml diff --git a/openstack/barbican/templates/api-deployment.yaml b/openstack/barbican/templates/api-deployment.yaml index 38d6f99ce63..81c1077a5c4 100644 --- a/openstack/barbican/templates/api-deployment.yaml +++ b/openstack/barbican/templates/api-deployment.yaml @@ -39,6 +39,9 @@ spec: annotations: configmap-etc-hash: {{ include (print $.Template.BasePath "/etc-configmap.yaml") . | sha256sum }} configmap-bin-hash: {{ include (print $.Template.BasePath "/configmap-bin.yaml") . | sha256sum }} + {{- if .Values.tls.enabled }} + config.linkerd.io/skip-inbound-ports: "443" + {{- end }} {{- if .Values.proxysql.mode }} prometheus.io/scrape: "true" prometheus.io/targets: {{ required ".Values.alerts.prometheus missing" .Values.alerts.prometheus | quote }} diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index cbe5bbe18b9..3e30c0305d7 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -17,11 +17,10 @@ WSGIDaemonProcess barbican-api processes={{ .Values.api.processes | default 1 }} Listen 0.0.0.0:{{ .Values.api_port_internal }} {{- if .Values.tls.enabled }} -# TLS hardening at server level (must be outside VirtualHost) -Include /etc/apache2/conf-enabled/tls-hardening.conf - -# External HTTPS endpoint (via Ingress TLS passthrough) -# Note: Listen 443 is provided by /etc/apache2/ports.conf when mod_ssl is enabled +# External HTTPS endpoint +# mod_ssl is loaded via a conf-enabled snippet, which is parsed after +# ports.conf, so its ssl_module-gated Listen 443 does not apply here. +Listen 0.0.0.0:443 # TLS hardening at server level (must be outside VirtualHost) Include /etc/apache2/conf-enabled/tls-hardening.conf diff --git a/openstack/barbican/templates/ingress.yaml b/openstack/barbican/templates/ingress.yaml index 3b6614e3b7a..94d26106dd2 100644 --- a/openstack/barbican/templates/ingress.yaml +++ b/openstack/barbican/templates/ingress.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.tls.enabled }} kind: Ingress apiVersion: networking.k8s.io/v1 @@ -13,14 +14,7 @@ metadata: type: api component: barbican annotations: - {{- if and .Values.tls.enabled .Values.tls.passthrough }} - ingress.kubernetes.io/ssl-passthrough: "true" - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - ingress.kubernetes.io/backend-protocol: "HTTPS" - nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - config.linkerd.io/skip-inbound-ports: "443" - disco: "true" - {{- else if and .Values.tlsacme (not .Values.tls.enabled) }} + {{- if .Values.tlsacme }} kubernetes.io/tls-acme: "true" disco: "true" {{- end }} @@ -45,8 +39,5 @@ spec: service: name: barbican-api port: - {{- if and .Values.tls.enabled .Values.tls.passthrough }} - number: 443 - {{- else }} number: {{.Values.api_port_internal}} - {{- end }} +{{- end }} diff --git a/openstack/barbican/templates/service-tls.yaml b/openstack/barbican/templates/service-tls.yaml new file mode 100644 index 00000000000..f13dc848d59 --- /dev/null +++ b/openstack/barbican/templates/service-tls.yaml @@ -0,0 +1,29 @@ +{{- if .Values.tls.enabled }} +kind: Service +apiVersion: v1 +metadata: + name: barbican-public-tls + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + system: openstack + type: api-tls + component: barbican + annotations: + {{- include "utils.linkerd.pod_and_service_annotation" . | indent 4 }} +spec: + selector: + name: barbican-api + ports: + - name: barbican-tls + protocol: "TCP" + port: 443 + targetPort: 443 + {{- if .Values.services.externalip }} + externalIPs: + - {{ .Values.services.externalip | quote }} + {{- end }} +{{- end }} diff --git a/openstack/barbican/templates/service.yaml b/openstack/barbican/templates/service.yaml index 2616d5b6e6b..9e9f6ca2899 100644 --- a/openstack/barbican/templates/service.yaml +++ b/openstack/barbican/templates/service.yaml @@ -23,8 +23,3 @@ spec: ports: - name: barbican-api port: {{.Values.api_port_internal}} - {{- if .Values.tls.enabled }} - - name: barbican-tls - port: 443 - targetPort: 443 - {{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index f716b415894..503b96c3e69 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -472,6 +472,9 @@ sentry: tlsacme: true +services: + # externalip: "192.168.1.100" + # SCIBPL-209: Ingress hardening — request size cap and connection limiting. # These annotations are applied to the nginx ingress controller. # Keystone middleware already enforces authentication; these settings add a @@ -494,8 +497,6 @@ tls: enabled: false # Name of the K8s Secret containing tls.key and tls.crt (created by TLS bootstrap Job) secretName: barbican-tls - # Enable TLS passthrough on the ingress (sends encrypted stream directly to pod) - passthrough: false # Per-region operator choices — all three are REQUIRED when tls.enabled: true. # There are no defaults; the operator must make an explicit choice. # keyGeneration: go-crypto | hsm-entropy | hsm-full | tpm-entropy diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 282b8d430c1..4b1231d3206 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -56,6 +56,7 @@ spec: {{- end }} {{- if .Values.tls.enabled }} tls-secret-hash: {{ .Values.tls.secretName | default "keystone-tls" }} + config.linkerd.io/skip-inbound-ports: "443" {{- end }} {{- if .Values.api.metrics.enabled }} prometheus.io/scrape: "true" diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 6ef5c7bc969..a558b5b6b72 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -77,7 +77,7 @@ WSGIServerMetrics On # TLS hardening at server level (must be outside VirtualHost) Include /etc/apache2/conf-enabled/tls-hardening.conf -# External HTTPS endpoint (via Ingress TLS passthrough) +# External HTTPS endpoint # Note: Listen 443 is provided by /etc/apache2/ports.conf when mod_ssl is enabled @@ -88,7 +88,7 @@ Include /etc/apache2/conf-enabled/tls-hardening.conf SSLCertificateKeyFile /mnt/secrets/tls.key {{- if .Values.services.ingress.x509.ca }} - # Client certificate verification (moved from ingress to Apache in passthrough mode) + # Client certificate verification SSLVerifyClient optional SSLVerifyDepth 3 SSLCACertificateFile /etc/apache2/x509-ca/ca.crt diff --git a/openstack/keystone/templates/ingress-api.yaml b/openstack/keystone/templates/ingress-api.yaml index dd8ac97f1be..f6fc112f710 100644 --- a/openstack/keystone/templates/ingress-api.yaml +++ b/openstack/keystone/templates/ingress-api.yaml @@ -1,21 +1,4 @@ -{{- if .Values.services.ingress.enabled }} -{{- if .Values.services.ingress.x509.ca }} -kind: Secret -apiVersion: v1 - -metadata: - name: {{ .Release.Name }}-x509-ca - labels: - app: {{ template "fullname" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - component: keystone - type: x509-ca -data: - ca.crt: {{ .Values.services.ingress.x509.ca | b64enc }} ---- -{{- end }} +{{- if and .Values.services.ingress.enabled (not .Values.tls.enabled) }} {{- if not .Values.services.ingress.tlsacme }} {{- if .Values.services.public.tlsCertificate}} kind: Secret @@ -46,14 +29,7 @@ metadata: component: keystone type: api annotations: - {{- if and .Values.tls.enabled .Values.tls.passthrough }} - ingress.kubernetes.io/ssl-passthrough: "true" - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - ingress.kubernetes.io/backend-protocol: "HTTPS" - nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - config.linkerd.io/skip-inbound-ports: "443" - nginx.ingress.kubernetes.io/upstream-hash-by: "$remote_addr" - {{- else if or .Values.federation.oidc.enabled .Values.federation.saml.enabled }} + {{- if or .Values.federation.oidc.enabled .Values.federation.saml.enabled }} # the affinity bits are required for federation; they do not affect the # cases when the API is used, but will affect browser-based workflows ingress.kubernetes.io/affinity: cookie @@ -76,7 +52,7 @@ metadata: ingress.kubernetes.io/limit-connections: {{ .Values.services.ingress.limitConnections | quote }} nginx.ingress.kubernetes.io/limit-connections: {{ .Values.services.ingress.limitConnections | quote }} {{- end }} - {{- if and .Values.services.ingress.x509.ca (not (and .Values.tls.enabled .Values.tls.passthrough)) }} + {{- if .Values.services.ingress.x509.ca }} ingress.kubernetes.io/auth-tls-secret: {{ .Release.Namespace }}/{{ .Release.Name }}-x509-ca nginx.ingress.kubernetes.io/auth-tls-secret: {{ .Release.Namespace }}/{{ .Release.Name }}-x509-ca ingress.kubernetes.io/auth-tls-pass-certificate-to-upstream: "true" @@ -86,7 +62,7 @@ metadata: ingress.kubernetes.io/auth-tls-verify-depth: "3" nginx.ingress.kubernetes.io/auth-tls-verify-depth: "3" {{- end }} - {{- if and .Values.services.ingress.tlsacme (not .Values.tls.enabled) }} + {{- if .Values.services.ingress.tlsacme }} kubernetes.io/tls-acme: "true" {{- end }} {{- if .Values.global.is_global_region }} @@ -130,9 +106,5 @@ spec: service: name: "{{if .Values.global.is_global_region }}keystone-global{{else}}keystone{{end}}" port: - {{- if and .Values.tls.enabled .Values.tls.passthrough }} - number: 443 - {{- else }} number: 5000 - {{- end }} {{- end }} diff --git a/openstack/keystone/templates/secret-x509-ca.yaml b/openstack/keystone/templates/secret-x509-ca.yaml new file mode 100644 index 00000000000..8efbad3ddc0 --- /dev/null +++ b/openstack/keystone/templates/secret-x509-ca.yaml @@ -0,0 +1,15 @@ +{{- if .Values.services.ingress.x509.ca }} +kind: Secret +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-x509-ca + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + component: keystone + type: x509-ca +data: + ca.crt: {{ .Values.services.ingress.x509.ca | b64enc }} +{{- end }} diff --git a/openstack/keystone/templates/service-api-tls.yaml b/openstack/keystone/templates/service-api-tls.yaml new file mode 100644 index 00000000000..5307883b5f4 --- /dev/null +++ b/openstack/keystone/templates/service-api-tls.yaml @@ -0,0 +1,29 @@ +{{- if .Values.tls.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-public-tls + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + system: openstack + component: keystone + type: api-tls + annotations: + {{- include "utils.linkerd.pod_and_service_annotation" . | indent 4 }} +spec: + selector: + name: {{ .Release.Name }}-api + type: {{ default "ClusterIP" .Values.services.type }} + ports: + - name: public-tls + protocol: "TCP" + port: 443 + targetPort: 443 + {{- if .Values.services.externalip }} + externalIPs: + - {{ .Values.services.externalip | quote }} + {{- end }} +{{- end }} diff --git a/openstack/keystone/templates/service-api.yaml b/openstack/keystone/templates/service-api.yaml index baf7e0d6226..d476c603368 100644 --- a/openstack/keystone/templates/service-api.yaml +++ b/openstack/keystone/templates/service-api.yaml @@ -30,16 +30,7 @@ spec: protocol: "TCP" port: 5000 targetPort: 5000 - {{- if .Values.tls.enabled }} - - name: public-tls - protocol: "TCP" - port: 443 - targetPort: 443 - {{- end }} {{- if .Values.global.keystone_internal_ip }} externalIPs: - {{ .Values.global.keystone_internal_ip | quote }} -{{- if .Values.services.externalip }} - - {{ .Values.services.externalip | quote }} -{{- end }} {{- end }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 212df450975..618e58974b6 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -61,7 +61,6 @@ osprofiler-redis: tls: enabled: false secretName: keystone-tls - passthrough: false # Per-region operator choices — all three are REQUIRED when tls.enabled: true. # There are no defaults; the operator must make an explicit choice. # keyGeneration: go-crypto | hsm-entropy | hsm-full | tpm-entropy @@ -321,7 +320,10 @@ services: # let kubernetes.io/tls-acme manage tls certificates tlsacme: true - # x509 client certificate support + # x509 client certificate support. + # NOTE: `ca` is the client-certificate trust anchor, consumed by the + # nginx-ingress (auth-tls-secret) and by Apache (SSLCACertificateFile) when + # TLS terminates in the pod. The `ingress` path in this key name is legacy. x509: # ca: issuer_attribute: HTTP_SSL_CLIENT_I_DN diff --git a/system/kube-system-metal/values.yaml b/system/kube-system-metal/values.yaml index afa57e71137..8dfe2357aa3 100644 --- a/system/kube-system-metal/values.yaml +++ b/system/kube-system-metal/values.yaml @@ -86,10 +86,6 @@ ingress-nginx: # See https://github.com/kubernetes/ingress-nginx/issues/7837 . allowSnippetAnnotations: false - extraArgs: - # Required for ssl-passthrough ingress annotations to take effect. - enable-ssl-passthrough: true - config: custom-http-errors: "495" From 6992d3223fe53fdd24f84d85e4621b945f09d594 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 17 Aug 2026 21:35:17 +0200 Subject: [PATCH 24/54] Announce Keystone TLS /32 via LoadBalancer and Calico BGP Expose the Keystone Apache TLS listener on :443 through a dedicated LoadBalancer Service whose /32 is announced over BGP via the Calico projectcalico.org/loadBalancerIPs annotation, with externalTrafficPolicy Local to preserve the client source IP. This replaces the previous ClusterIP plus bare externalIPs, which is not route-advertised, and aligns with the announce convention used by other exposed Services. Publish the public endpoint DNS through a standalone disco Record CR, rendered only when TLS is enabled and the external IP is set. Source the address from the shared global.keystone_external_ip value so the Service and Record stay consistent. --- openstack/keystone/templates/record.yaml | 18 ++++++++++++++++++ .../keystone/templates/service-api-tls.yaml | 10 ++++------ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 openstack/keystone/templates/record.yaml diff --git a/openstack/keystone/templates/record.yaml b/openstack/keystone/templates/record.yaml new file mode 100644 index 00000000000..14c3f023f00 --- /dev/null +++ b/openstack/keystone/templates/record.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.tls.enabled .Values.global.keystone_external_ip }} +apiVersion: disco.stable.sap.cc/v1 +kind: Record +metadata: + name: {{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + type: A + record: {{ .Values.global.keystone_external_ip | quote }} + hosts: + - {{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} + zoneName: {{ .Values.global.region }}.{{ .Values.global.tld }} +{{- end }} diff --git a/openstack/keystone/templates/service-api-tls.yaml b/openstack/keystone/templates/service-api-tls.yaml index 5307883b5f4..6889eb2ebe0 100644 --- a/openstack/keystone/templates/service-api-tls.yaml +++ b/openstack/keystone/templates/service-api-tls.yaml @@ -1,4 +1,4 @@ -{{- if .Values.tls.enabled }} +{{- if and .Values.tls.enabled .Values.global.keystone_external_ip }} apiVersion: v1 kind: Service metadata: @@ -12,18 +12,16 @@ metadata: component: keystone type: api-tls annotations: + projectcalico.org/loadBalancerIPs: '["{{ .Values.global.keystone_external_ip }}"]' {{- include "utils.linkerd.pod_and_service_annotation" . | indent 4 }} spec: + type: LoadBalancer + externalTrafficPolicy: Local selector: name: {{ .Release.Name }}-api - type: {{ default "ClusterIP" .Values.services.type }} ports: - name: public-tls protocol: "TCP" port: 443 targetPort: 443 - {{- if .Values.services.externalip }} - externalIPs: - - {{ .Values.services.externalip | quote }} - {{- end }} {{- end }} From 989ae9df925fc6dddef2ddd0e24af7ba317c3dd4 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 18 Aug 2026 11:40:00 +0200 Subject: [PATCH 25/54] Rebuild client-certificate headers for cc_x509 at Apache TLS When Apache terminates TLS in the pod, the cc_x509 auth method must still receive the client certificate the way it did when NGINX terminated TLS. Reconstruct the SSL-Client-Cert and SSL-Client-Verify request headers from the verified TLS session, percent-encoding the certificate to the format the plugin already parses. Strip every dash/underscore spelling of these headers before setting them so a client cannot inject a forged certificate; only the values derived from the verified handshake reach the WSGI app. Scope this to the external :443 virtual host. --- .../templates/etc/_wsgi-keystone.conf.tpl | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index a558b5b6b72..8f007db7b1c 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -88,16 +88,25 @@ Include /etc/apache2/conf-enabled/tls-hardening.conf SSLCertificateKeyFile /mnt/secrets/tls.key {{- if .Values.services.ingress.x509.ca }} - # Client certificate verification SSLVerifyClient optional SSLVerifyDepth 3 SSLCACertificateFile /etc/apache2/x509-ca/ca.crt - # Pass client cert info to WSGI app via request headers - # Replaces the headers that NGINX used to forward when it terminated TLS - RequestHeader set SSL_CLIENT_CERT "%{SSL_CLIENT_CERT}s" - RequestHeader set SSL_CLIENT_I_DN "%{SSL_CLIENT_I_DN}s" - RequestHeader set SSL_CLIENT_VERIFY "%{SSL_CLIENT_VERIFY}s" + # Reconstruct, from the verified TLS session, the client-certificate request + # headers cc_x509 consumes (same format NGINX forwarded when it terminated + # TLS). Every inbound spelling is stripped first so a client cannot inject + # them; only these Apache-set values, sourced from the verified handshake, + # reach the WSGI app. + RequestHeader unset SSL-Client-Cert + RequestHeader unset SSL_Client-Cert + RequestHeader unset SSL-Client_Cert + RequestHeader unset SSL_CLIENT_CERT + RequestHeader unset SSL-Client-Verify + RequestHeader unset SSL_Client-Verify + RequestHeader unset SSL-Client_Verify + RequestHeader unset SSL_CLIENT_VERIFY + RequestHeader set SSL-Client-Cert "expr=%{escape:%{SSL:SSL_CLIENT_CERT}}" + RequestHeader set SSL-Client-Verify "expr=%{SSL:SSL_CLIENT_VERIFY}" {{- end }} WSGIDaemonProcess keystone-tls processes=8 threads=1 user=keystone group=keystone display-name=%{GROUP} From beee4b5ac093ad1564da3944de3f011f356628e9 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 18 Aug 2026 16:58:41 +0200 Subject: [PATCH 26/54] Set pod fsGroup so the non-root tls-init can read the TLS Secret The tls-init init container runs as the distroless nonroot user (uid/gid 65532), but the tls-input Secret is mounted 0440 and, without an fsGroup, is owned root:root. The unwrap step then fails with "permission denied" on /tls-input/tls.key and the pod never starts. Add a pod-level securityContext.fsGroup: 65532 (gated on tls.enabled) to the Keystone and Barbican API deployments. The Secret files become group readable while staying non-root and non-world-readable, matching the BSI least-privilege requirement. tls-input keeps defaultMode 0440. --- openstack/barbican/templates/api-deployment.yaml | 8 ++++++++ openstack/keystone/templates/deployment-api.yaml | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/openstack/barbican/templates/api-deployment.yaml b/openstack/barbican/templates/api-deployment.yaml index 81c1077a5c4..417321e89a0 100644 --- a/openstack/barbican/templates/api-deployment.yaml +++ b/openstack/barbican/templates/api-deployment.yaml @@ -48,6 +48,14 @@ spec: {{- end }} {{- include "utils.linkerd.pod_and_service_annotation" . | indent 8 }} spec: + {{- if .Values.tls.enabled }} + securityContext: + # distroless "nonroot" gid of the tls-lifecycle-manager image; lets the + # non-root tls-init container read the 0440 tls-input Secret while keeping + # the key group-restricted (not world-readable) per the BSI least-privilege + # requirement (Sicherheitsarchitektur 4.1: non-root, restriktive Berechtigungen). + fsGroup: 65532 + {{- end }} {{- if .Values.rbac.enabled }} serviceAccountName: {{ .Release.Name }} {{- end }} diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 4b1231d3206..e3f6f91952f 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -64,6 +64,14 @@ spec: {{- end }} {{- include "utils.linkerd.pod_and_service_annotation" . | indent 8 }} spec: + {{- if .Values.tls.enabled }} + securityContext: + # distroless "nonroot" gid of the tls-lifecycle-manager image; lets the + # non-root tls-init container read the 0440 tls-input Secret while keeping + # the key group-restricted (not world-readable) per the BSI least-privilege + # requirement (Sicherheitsarchitektur 4.x: non-root, restriktive Berechtigungen). + fsGroup: 65532 + {{- end }} {{- if .Values.api.priorityClassName }} priorityClassName: {{ .Values.api.priorityClassName }} {{- end }} From 847e6b5a55512099e29b8259d550a2ee7a1a94a2 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 18 Aug 2026 17:21:10 +0200 Subject: [PATCH 27/54] Support non-Calico external IP for Keystone TLS service The public TLS service only set type: LoadBalancer with the Calico projectcalico.org/loadBalancerIPs annotation. On metal clusters (e.g. qa-de-3) there is no Calico LoadBalancer controller, so the IP is never assigned or advertised and the /32 is unreachable (No route to host). Add a calicoLoadBalancer toggle (matching designate/bind, default true to preserve existing Calico regions). When false, render spec.externalIPs so kube-parrot/externalip-operator announces the address via BGP instead. --- openstack/keystone/templates/service-api-tls.yaml | 6 ++++++ openstack/keystone/values.yaml | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/openstack/keystone/templates/service-api-tls.yaml b/openstack/keystone/templates/service-api-tls.yaml index 6889eb2ebe0..6c47be5a395 100644 --- a/openstack/keystone/templates/service-api-tls.yaml +++ b/openstack/keystone/templates/service-api-tls.yaml @@ -12,7 +12,9 @@ metadata: component: keystone type: api-tls annotations: + {{- if .Values.calicoLoadBalancer }} projectcalico.org/loadBalancerIPs: '["{{ .Values.global.keystone_external_ip }}"]' + {{- end }} {{- include "utils.linkerd.pod_and_service_annotation" . | indent 4 }} spec: type: LoadBalancer @@ -24,4 +26,8 @@ spec: protocol: "TCP" port: 443 targetPort: 443 + {{- if not .Values.calicoLoadBalancer }} + externalIPs: + - {{ .Values.global.keystone_external_ip }} + {{- end }} {{- end }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 618e58974b6..13115f4c25c 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -4,6 +4,10 @@ imageVersionKubernetesEntrypoint: latest # name: value +# Announce the public TLS /32 via Calico (type: LoadBalancer + projectcalico.org/loadBalancerIPs). +# Set to false on metal clusters (kube-parrot/externalip-operator), which announce spec.externalIPs instead. +calicoLoadBalancer: true + global: # tld: cloud.sap # region: cluster From cf3f9cadbf12ca28336ef2eb5a7b51a4189ab48c Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Thu, 20 Aug 2026 07:57:46 +0200 Subject: [PATCH 28/54] Probe served TLS certificate and alert on expiry and stale rotation Add a blackbox Probe for the public HTTPS endpoint of Keystone and Barbican that observes the certificate actually served on the wire, and alert when it expires within 7 days. Also alert when the rotation CronJob has not completed successfully in over 8 days, covering the case where it never runs at all. The Probe is disabled by default and only renders where TLS and the external endpoint are configured, so it stays inert in regions that have not switched over. The expiry and rotation alerts carry no_alert_on_absence and match on the specific endpoint and CronJob, so they never fire where the probe or rotation is absent. --- .../kubernetes/openstack-barbican.alerts | 35 +++++++++++++++++++ openstack/barbican/templates/probe-tls.yaml | 22 ++++++++++++ openstack/barbican/values.yaml | 6 ++++ .../alerts/kubernetes/keystone.alerts | 35 +++++++++++++++++++ openstack/keystone/templates/probe-tls.yaml | 22 ++++++++++++ openstack/keystone/values.yaml | 6 ++++ 6 files changed, 126 insertions(+) create mode 100644 openstack/barbican/templates/probe-tls.yaml create mode 100644 openstack/keystone/templates/probe-tls.yaml diff --git a/openstack/barbican/alerts/kubernetes/openstack-barbican.alerts b/openstack/barbican/alerts/kubernetes/openstack-barbican.alerts index 8fe19921a6a..d676d78db38 100644 --- a/openstack/barbican/alerts/kubernetes/openstack-barbican.alerts +++ b/openstack/barbican/alerts/kubernetes/openstack-barbican.alerts @@ -17,3 +17,38 @@ groups: annotations: description: 'barbican-api pod {{ $labels.pod }} not ready; recreate pod. See playbook.' summary: 'barbican-api pod not ready' + + - alert: OpenstackBarbicanTLSCertificateExpiresCritical + expr: (probe_ssl_earliest_cert_expiry{instance=~"https://keymanager-3\\..*"} - time()) / 86400 < 7 + for: 1h + labels: + no_alert_on_absence: "true" + context: security + dashboard: barbican + service: barbican + severity: critical + tier: os + support_group: identity + playbook: 'docs/support/playbook/barbican/alerts/tls-certificate-expiry/#openstackbarbicantlscertificateexpirescritical' + annotations: + description: The TLS certificate served by Barbican at {{ $labels.instance }} expires in less than 7 days. External HTTPS access will break when it expires. Check the rotation CronJob and cert-manager. + summary: Barbican served TLS certificate expires within 7 days + + - alert: OpenstackBarbicanTLSRotationStale + expr: > + time() - kube_cronjob_created{cronjob="barbican-tls-rotation"} >= 8*24*60*60 + and on(cronjob) (time() - kube_cronjob_status_last_successful_time{cronjob="barbican-tls-rotation"} + or on(cronjob) kube_cronjob_status_last_schedule_time{cronjob="barbican-tls-rotation"}) >= 8*24*60*60 + for: 1h + labels: + no_alert_on_absence: "true" + context: security + dashboard: barbican + service: barbican + severity: warning + tier: os + support_group: identity + playbook: 'docs/support/playbook/barbican/alerts/tls-certificate-expiry/#openstackbarbicantlsrotationstale' + annotations: + description: The Barbican TLS certificate rotation CronJob has not completed successfully in over 8 days. If not resolved, the certificate will expire and external HTTPS access will break. Check the CronJob and its Jobs. + summary: Barbican TLS certificate rotation is stale diff --git a/openstack/barbican/templates/probe-tls.yaml b/openstack/barbican/templates/probe-tls.yaml new file mode 100644 index 00000000000..3bef7e91923 --- /dev/null +++ b/openstack/barbican/templates/probe-tls.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.tls.enabled .Values.tls.probe.enabled .Values.services.externalip }} +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: {{ .Release.Name }}-tls + labels: + prometheus: {{ .Values.tls.probe.prometheus }} +spec: + jobName: {{ .Release.Name }}-tls + prober: + url: prober.{{ .Values.global.region }}.cloud.sap + scheme: https + module: {{ .Values.tls.probe.module }} + interval: {{ .Values.tls.probe.interval }} + scrapeTimeout: {{ .Values.tls.probe.scrapeTimeout }} + targets: + staticConfig: + static: + - https://{{ include "barbican_api_endpoint_host_public" . }} + labels: + region: {{ .Values.global.region }} +{{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index 503b96c3e69..d0da1fde3f1 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -516,6 +516,12 @@ tls: issuerKind: ClusterIssuer issuerGroup: certmanager.cloud.sap keyAlgorithm: ECDSAP256 + probe: + enabled: false + prometheus: kubernetes + module: http_200_403 + interval: 60s + scrapeTimeout: 55s # sapcc/openstack-rate-limit-middleware sapcc_rate_limit: diff --git a/openstack/keystone/alerts/kubernetes/keystone.alerts b/openstack/keystone/alerts/kubernetes/keystone.alerts index 8bb3d7cb083..bb15841690c 100644 --- a/openstack/keystone/alerts/kubernetes/keystone.alerts +++ b/openstack/keystone/alerts/kubernetes/keystone.alerts @@ -51,3 +51,38 @@ groups: description: The container {{ $labels.pod }}/{{ $labels.container }} is being CPU-throttled constantly. This is probably impacting performance, so check if we can increase the number of replicas or the resource requests/limits. + + - alert: OpenstackKeystoneTLSCertificateExpiresCritical + expr: (probe_ssl_earliest_cert_expiry{instance=~"https://identity-3\\..*"} - time()) / 86400 < 7 + for: 1h + labels: + no_alert_on_absence: "true" + context: security + dashboard: keystone + service: keystone + severity: critical + tier: os + support_group: identity + playbook: 'docs/support/playbook/keystone/alerts/tls-certificate-expiry/#openstackkeystonetlscertificateexpirescritical' + annotations: + description: The TLS certificate served by Keystone at {{ $labels.instance }} expires in less than 7 days. External HTTPS access will break when it expires. Check the rotation CronJob and cert-manager. + summary: Keystone served TLS certificate expires within 7 days + + - alert: OpenstackKeystoneTLSRotationStale + expr: > + time() - kube_cronjob_created{cronjob="keystone-tls-rotation"} >= 8*24*60*60 + and on(cronjob) (time() - kube_cronjob_status_last_successful_time{cronjob="keystone-tls-rotation"} + or on(cronjob) kube_cronjob_status_last_schedule_time{cronjob="keystone-tls-rotation"}) >= 8*24*60*60 + for: 1h + labels: + no_alert_on_absence: "true" + context: security + dashboard: keystone + service: keystone + severity: warning + tier: os + support_group: identity + playbook: 'docs/support/playbook/keystone/alerts/tls-certificate-expiry/#openstackkeystonetlsrotationstale' + annotations: + description: The Keystone TLS certificate rotation CronJob has not completed successfully in over 8 days. If not resolved, the certificate will expire and external HTTPS access will break. Check the CronJob and its Jobs. + summary: Keystone TLS certificate rotation is stale diff --git a/openstack/keystone/templates/probe-tls.yaml b/openstack/keystone/templates/probe-tls.yaml new file mode 100644 index 00000000000..f761cd13382 --- /dev/null +++ b/openstack/keystone/templates/probe-tls.yaml @@ -0,0 +1,22 @@ +{{- if and .Values.tls.enabled .Values.tls.probe.enabled .Values.global.keystone_external_ip }} +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: {{ .Release.Name }}-tls + labels: + prometheus: {{ .Values.tls.probe.prometheus }} +spec: + jobName: {{ .Release.Name }}-tls + prober: + url: prober.{{ .Values.global.region }}.cloud.sap + scheme: https + module: {{ .Values.tls.probe.module }} + interval: {{ .Values.tls.probe.interval }} + scrapeTimeout: {{ .Values.tls.probe.scrapeTimeout }} + targets: + staticConfig: + static: + - https://{{ include "keystone_api_endpoint_host_public" . }} + labels: + region: {{ .Values.global.region }} +{{- end }} diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 13115f4c25c..77e3ed85fc5 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -82,6 +82,12 @@ tls: issuerKind: ClusterIssuer issuerGroup: certmanager.cloud.sap keyAlgorithm: ECDSAP256 + probe: + enabled: false + prometheus: kubernetes + module: http_200_403 + interval: 60s + scrapeTimeout: 55s federation: oidc: From 18db635c1de9ac78d884a51b8ca39beb6d9aec05 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Fri, 21 Aug 2026 15:01:19 +0200 Subject: [PATCH 29/54] Replace runtime a2enmod/a2dissite with static Apache config for Keystone The Keystone start script mutated Apache state at boot: a2enmod ssl/headers, a2dissite 000-default, and a2dismod status. These runtime changes write into /etc/apache2 and prevent a read-only root filesystem. Replace them with static config: - Load mod_ssl, mod_socache_shmcb and mod_headers via a conf-enabled snippet (ssl-load.conf). a2enmod ssl previously pulled in socache_shmcb implicitly; static loading has no dependency resolution, so it is listed explicitly. - Because conf-enabled is parsed after ports.conf, the ssl_module-gated Listen 443 there no longer fires, so the TLS vhost declares Listen 443. - Suppress the default :80 vhost with an emptyDir over sites-enabled instead of a2dissite. - mod_status is enabled by default in the image and ships a status.conf that exposes /server-status with ExtendedStatus On. Neutralize it with a conf-enabled override (status-off.conf) rather than a2dismod, since the module cannot be unloaded statically. mod_wsgi, mod_auth_openidc and mod_shib are already enabled in the image and need no action. The root filesystem stays writable for now; this change removes the config mutations that would block making it read-only later. Verified against the deployed API image: Apache starts, ssl/socache_shmcb/ headers load, port 443 binds, TLS 1.3 negotiates, the default :80 vhost is gone, and /server-status returns 403. --- .../keystone/templates/bin/_keystone_api.sh.tpl | 8 -------- openstack/keystone/templates/configmap-etc.yaml | 4 ++++ openstack/keystone/templates/deployment-api.yaml | 12 ++++++++++++ openstack/keystone/templates/etc/_ssl-load.conf.tpl | 9 +++++++++ .../keystone/templates/etc/_status-off.conf.tpl | 13 +++++++++++++ .../keystone/templates/etc/_wsgi-keystone.conf.tpl | 4 +++- 6 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 openstack/keystone/templates/etc/_ssl-load.conf.tpl create mode 100644 openstack/keystone/templates/etc/_status-off.conf.tpl diff --git a/openstack/keystone/templates/bin/_keystone_api.sh.tpl b/openstack/keystone/templates/bin/_keystone_api.sh.tpl index 381a31da107..17e7f06c18a 100644 --- a/openstack/keystone/templates/bin/_keystone_api.sh.tpl +++ b/openstack/keystone/templates/bin/_keystone_api.sh.tpl @@ -24,14 +24,6 @@ function start () { cp -a $(type -p ${KEYSTONE_WSGI_SCRIPT}) /var/www/cgi-bin/keystone/ done - a2dismod status - - {{- if .Values.tls.enabled }} - a2enmod ssl - a2enmod headers - a2dissite 000-default - {{- end }} - if [ -f /etc/apache2/envvars ]; then # Loading Apache2 ENV variables source /etc/apache2/envvars diff --git a/openstack/keystone/templates/configmap-etc.yaml b/openstack/keystone/templates/configmap-etc.yaml index 175d0caa2bb..9c931932fde 100644 --- a/openstack/keystone/templates/configmap-etc.yaml +++ b/openstack/keystone/templates/configmap-etc.yaml @@ -21,7 +21,11 @@ data: {{ include (print .Template.BasePath "/etc/_sso_callback_template.html.tpl") . | indent 4 }} wsgi-keystone.conf: | {{ include (print .Template.BasePath "/etc/_wsgi-keystone.conf.tpl") . | indent 4 }} + status-off.conf: | +{{ include (print .Template.BasePath "/etc/_status-off.conf.tpl") . | indent 4 }} {{- if .Values.tls.enabled }} + ssl-load.conf: | +{{ include (print .Template.BasePath "/etc/_ssl-load.conf.tpl") . | indent 4 }} tls-hardening.conf: | {{ include (print .Template.BasePath "/etc/_tls-hardening.conf.tpl") . | indent 4 }} {{- end }} diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index e3f6f91952f..14e76aecf01 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -243,6 +243,12 @@ spec: mountPath: /etc/apache2/conf-enabled/wsgi-keystone.conf subPath: wsgi-keystone.conf readOnly: true + - name: keystone-etc + mountPath: /etc/apache2/conf-enabled/status-off.conf + subPath: status-off.conf + readOnly: true + - name: apache-sites + mountPath: /etc/apache2/sites-enabled {{- if .Values.federation.oidc.enabled }} - name: keystone-federation mountPath: /etc/apache2/conf-enabled/federation-oidc.conf @@ -280,6 +286,10 @@ spec: - name: tls-secrets mountPath: /mnt/secrets readOnly: true + - name: keystone-etc + mountPath: /etc/apache2/conf-enabled/00-ssl-load.conf + subPath: ssl-load.conf + readOnly: true - name: keystone-etc mountPath: /etc/apache2/conf-enabled/tls-hardening.conf subPath: tls-hardening.conf @@ -348,6 +358,8 @@ spec: emptyDir: {} - name: wsgi-keystone emptyDir: {} + - name: apache-sites + emptyDir: {} - name: keystone-etc configMap: name: keystone-etc diff --git a/openstack/keystone/templates/etc/_ssl-load.conf.tpl b/openstack/keystone/templates/etc/_ssl-load.conf.tpl new file mode 100644 index 00000000000..bd4546d2373 --- /dev/null +++ b/openstack/keystone/templates/etc/_ssl-load.conf.tpl @@ -0,0 +1,9 @@ +{{/* +Load mod_ssl and its socache dependency via read-only conf so the modules are +available without a2enmod. mod_headers is required for the x509 client-cert +header reconstruction in the TLS vhost. +*/}} + +LoadModule socache_shmcb_module /usr/lib/apache2/modules/mod_socache_shmcb.so +LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so +LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so diff --git a/openstack/keystone/templates/etc/_status-off.conf.tpl b/openstack/keystone/templates/etc/_status-off.conf.tpl new file mode 100644 index 00000000000..28ff85321d3 --- /dev/null +++ b/openstack/keystone/templates/etc/_status-off.conf.tpl @@ -0,0 +1,13 @@ +{{/* +Neutralize mod_status: it is enabled by default in the image and ships a +status.conf that exposes /server-status with ExtendedStatus On. This override +is parsed after mods-enabled, so it wins. +*/}} + + + ExtendedStatus Off + + SetHandler none + Require all denied + + diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 8f007db7b1c..5a79cdc0687 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -78,7 +78,9 @@ WSGIServerMetrics On Include /etc/apache2/conf-enabled/tls-hardening.conf # External HTTPS endpoint -# Note: Listen 443 is provided by /etc/apache2/ports.conf when mod_ssl is enabled +# mod_ssl is loaded via a conf-enabled snippet, which is parsed after +# ports.conf, so its ssl_module-gated Listen 443 does not apply here. +Listen 0.0.0.0:443 ServerName {{ .Values.services.public.host }}.{{ .Values.global.region }}.{{ .Values.global.tld }} From 6145bbbbfb3e71cbdb1f235a73d3e71c811735da Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Fri, 21 Aug 2026 18:41:34 +0200 Subject: [PATCH 30/54] Add LoadBalancer TLS service, DNS record, and Calico support for Barbican Mirror the Keystone exposure model for pod-level TLS termination: a LoadBalancer Service on port 443 that announces the public /32 via Calico loadBalancerIPs (or spec.externalIPs on metal clusters), plus a disco DNS Record for the public host. The external IP is sourced from global.barbican_external_ip with services.externalip kept as an alias. Co-authored-by: Rajiv Mucheli --- openstack/barbican/templates/_helpers.tpl | 5 +++++ openstack/barbican/templates/probe-tls.yaml | 2 +- openstack/barbican/templates/record.yaml | 19 +++++++++++++++++++ openstack/barbican/templates/service-tls.yaml | 12 +++++++++--- openstack/barbican/values.yaml | 7 +++++++ 5 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 openstack/barbican/templates/record.yaml diff --git a/openstack/barbican/templates/_helpers.tpl b/openstack/barbican/templates/_helpers.tpl index fb4433961b1..7c7f858490b 100644 --- a/openstack/barbican/templates/_helpers.tpl +++ b/openstack/barbican/templates/_helpers.tpl @@ -15,6 +15,11 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- printf "%s-%s" .Release.Name $name | trunc 63 | replace "_" "-" | trimSuffix "-" -}} {{- end -}} +{{- define "barbican.external_ip" -}} +{{- $svc := .Values.services | default dict -}} +{{- .Values.global.barbican_external_ip | default $svc.externalip -}} +{{- end -}} + {{- define "barbican.db_service" }} {{- include "utils.db_host" . }} {{- end }} diff --git a/openstack/barbican/templates/probe-tls.yaml b/openstack/barbican/templates/probe-tls.yaml index 3bef7e91923..b2ef0eb6601 100644 --- a/openstack/barbican/templates/probe-tls.yaml +++ b/openstack/barbican/templates/probe-tls.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.tls.enabled .Values.tls.probe.enabled .Values.services.externalip }} +{{- if and .Values.tls.enabled .Values.tls.probe.enabled (include "barbican.external_ip" .) }} apiVersion: monitoring.coreos.com/v1 kind: Probe metadata: diff --git a/openstack/barbican/templates/record.yaml b/openstack/barbican/templates/record.yaml new file mode 100644 index 00000000000..99c02ebb3b4 --- /dev/null +++ b/openstack/barbican/templates/record.yaml @@ -0,0 +1,19 @@ +{{- $externalIP := include "barbican.external_ip" . }} +{{- if and .Values.tls.enabled $externalIP }} +apiVersion: disco.stable.sap.cc/v1 +kind: Record +metadata: + name: {{ include "barbican_api_endpoint_host_public" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +spec: + type: A + record: {{ $externalIP | quote }} + hosts: + - {{ include "barbican_api_endpoint_host_public" . }} + zoneName: {{ .Values.global.region }}.{{ .Values.global.tld }} +{{- end }} diff --git a/openstack/barbican/templates/service-tls.yaml b/openstack/barbican/templates/service-tls.yaml index f13dc848d59..3d0b88119e4 100644 --- a/openstack/barbican/templates/service-tls.yaml +++ b/openstack/barbican/templates/service-tls.yaml @@ -1,4 +1,5 @@ -{{- if .Values.tls.enabled }} +{{- $externalIP := include "barbican.external_ip" . }} +{{- if and .Values.tls.enabled $externalIP }} kind: Service apiVersion: v1 metadata: @@ -13,8 +14,13 @@ metadata: type: api-tls component: barbican annotations: + {{- if .Values.calicoLoadBalancer }} + projectcalico.org/loadBalancerIPs: '["{{ $externalIP }}"]' + {{- end }} {{- include "utils.linkerd.pod_and_service_annotation" . | indent 4 }} spec: + type: LoadBalancer + externalTrafficPolicy: Local selector: name: barbican-api ports: @@ -22,8 +28,8 @@ spec: protocol: "TCP" port: 443 targetPort: 443 - {{- if .Values.services.externalip }} + {{- if not .Values.calicoLoadBalancer }} externalIPs: - - {{ .Values.services.externalip | quote }} + - {{ $externalIP }} {{- end }} {{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index d0da1fde3f1..173f165e285 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -11,6 +11,8 @@ global: domain_seeds: skip_hcm_domain: false linkerd_requested: false + # Public /32 for the pod-level TLS LoadBalancer service and DNS record. + # barbican_external_ip: "192.168.1.100" rbac: enabled: true @@ -472,7 +474,12 @@ sentry: tlsacme: true +# Announce the public TLS /32 via Calico LoadBalancer IPs. Set to false on +# metal clusters that announce via spec.externalIPs instead. +calicoLoadBalancer: true + services: + # Backward-compat alias for global.barbican_external_ip. # externalip: "192.168.1.100" # SCIBPL-209: Ingress hardening — request size cap and connection limiting. From 8f96fbd6dd01595d671eaa26b467c8173a083170 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 24 Aug 2026 16:09:59 +0200 Subject: [PATCH 31/54] Default the TLS service to spec.externalIPs for Keystone and Barbican Most clusters do not perform automatic IP assignment, so announce the public TLS address via Service.spec.externalIPs by default. Clusters that assign IPs via the Calico annotation set calicoLoadBalancer: true. --- openstack/barbican/values.yaml | 4 +--- openstack/keystone/values.yaml | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index 173f165e285..88637857e0b 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -474,9 +474,7 @@ sentry: tlsacme: true -# Announce the public TLS /32 via Calico LoadBalancer IPs. Set to false on -# metal clusters that announce via spec.externalIPs instead. -calicoLoadBalancer: true +calicoLoadBalancer: false services: # Backward-compat alias for global.barbican_external_ip. diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 77e3ed85fc5..638eb33b9cf 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -4,9 +4,7 @@ imageVersionKubernetesEntrypoint: latest # name: value -# Announce the public TLS /32 via Calico (type: LoadBalancer + projectcalico.org/loadBalancerIPs). -# Set to false on metal clusters (kube-parrot/externalip-operator), which announce spec.externalIPs instead. -calicoLoadBalancer: true +calicoLoadBalancer: false global: # tld: cloud.sap From b102802dca6c84bd48b3396820c63f56bbdc8b13 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Mon, 24 Aug 2026 23:37:07 +0200 Subject: [PATCH 32/54] Harden Barbican pod-level TLS and pin the TLS lifecycle image - Log external HTTPS requests on the Barbican TLS VirtualHost and cap the request body size, mirroring the internal VirtualHost. - Add a PodDisruptionBudget (minAvailable: 1) for the Barbican API so a single pod eviction cannot drop all external HTTPS traffic. - Pin the tls-lifecycle-manager image tag in the Keystone and Barbican chart defaults instead of tracking latest. --- .../templates/etc/_wsgi-barbican.conf.tpl | 10 ++++++++++ openstack/barbican/templates/pdb.yaml | 17 +++++++++++++++++ openstack/barbican/values.yaml | 4 +++- openstack/keystone/values.yaml | 2 +- 4 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 openstack/barbican/templates/pdb.yaml diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 3e30c0305d7..2039c09fb29 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -35,11 +35,21 @@ Include /etc/apache2/conf-enabled/tls-hardening.conf WSGIScriptAlias / /var/www/cgi-bin/barbican/barbican-wsgi-api WSGIApplicationGroup %{GLOBAL} WSGIPassAuthorization On + LimitRequestBody 114688 = 2.4> ErrorLogFormat "%{cu}t %M" ErrorLog /dev/stderr + {{- if .Values.use_json }} + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout json_combined env=!forwarded + CustomLog /dev/stdout json_proxy env=forwarded + {{- else }} + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout combined env=!forwarded + CustomLog /dev/stdout proxy env=forwarded + {{- end }} KeepAliveTimeout 61 diff --git a/openstack/barbican/templates/pdb.yaml b/openstack/barbican/templates/pdb.yaml new file mode 100644 index 00000000000..54931bc33db --- /dev/null +++ b/openstack/barbican/templates/pdb.yaml @@ -0,0 +1,17 @@ +{{- if .Values.api.pdb.minAvailable }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ .Release.Name }}-api + labels: + app: {{ template "fullname" . }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: "{{ .Release.Name }}" + component: barbican + type: api +spec: + minAvailable: {{ .Values.api.pdb.minAvailable }} + selector: + matchLabels: + name: barbican-api +{{- end }} diff --git a/openstack/barbican/values.yaml b/openstack/barbican/values.yaml index 88637857e0b..0b196c063da 100644 --- a/openstack/barbican/values.yaml +++ b/openstack/barbican/values.yaml @@ -44,6 +44,8 @@ api: replicas: 2 processes: 1 threads: 1 + pdb: + minAvailable: 1 metrics: enabled: true resources: @@ -515,7 +517,7 @@ tls: # TLS lifecycle manager binary for bootstrap Job and rotation CronJob lifecycle: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager - imageTag: latest + imageTag: "20260818160021" # cert-manager issuer configuration issuerName: digicert-issuer issuerKind: ClusterIssuer diff --git a/openstack/keystone/values.yaml b/openstack/keystone/values.yaml index 638eb33b9cf..d278e944184 100644 --- a/openstack/keystone/values.yaml +++ b/openstack/keystone/values.yaml @@ -75,7 +75,7 @@ tls: allowInsecureStorage: false lifecycle: image: keppel.eu-de-1.cloud.sap/ccloud/tls-lifecycle-manager - imageTag: latest + imageTag: "20260818160021" issuerName: digicert-issuer issuerKind: ClusterIssuer issuerGroup: certmanager.cloud.sap From 974cd47a09b653b79b6c05f71273ce4316978ba1 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 25 Aug 2026 10:36:26 +0200 Subject: [PATCH 33/54] Harden Apache TLS ciphers, add HTTP security headers and server hardening - Refine the TLS 1.3 cipher suites to AES-GCM and AES-CCM, and disable TLS compression on both Keystone and Barbican. - Set HSTS and standard HTTP security response headers on both TLS vhosts, loading mod_headers for Barbican. - Add a TLS-gated Apache hardening snippet (suppress the version banner, disable TRACE, and turn off directory listing) for both services. - Bump Keystone to 0.14.2 and Barbican to 0.9.3. --- openstack/barbican/Chart.yaml | 2 +- .../barbican/templates/api-deployment.yaml | 4 ++++ openstack/barbican/templates/etc-configmap.yaml | 2 ++ .../templates/etc/_apache-hardening.conf.tpl | 17 +++++++++++++++++ .../barbican/templates/etc/_ssl-load.conf.tpl | 4 +++- .../templates/etc/_tls-hardening.conf.tpl | 7 +++++-- .../templates/etc/_wsgi-barbican.conf.tpl | 7 +++++++ openstack/keystone/templates/configmap-etc.yaml | 2 ++ .../keystone/templates/deployment-api.yaml | 4 ++++ .../templates/etc/_apache-hardening.conf.tpl | 17 +++++++++++++++++ .../templates/etc/_tls-hardening.conf.tpl | 7 +++++-- .../templates/etc/_wsgi-keystone.conf.tpl | 7 +++++++ 12 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 openstack/barbican/templates/etc/_apache-hardening.conf.tpl create mode 100644 openstack/keystone/templates/etc/_apache-hardening.conf.tpl diff --git a/openstack/barbican/Chart.yaml b/openstack/barbican/Chart.yaml index 9ec9f113b0e..38f22f92a85 100644 --- a/openstack/barbican/Chart.yaml +++ b/openstack/barbican/Chart.yaml @@ -4,7 +4,7 @@ appVersion: flamingo description: A Helm chart for Openstack Barbican icon: https://www.openstack.org/themes/openstack/images/project-mascots/Barbican/OpenStack_Project_Barbican_vertical.png name: barbican -version: 0.9.2 +version: 0.9.3 dependencies: - condition: mariadb.enabled name: mariadb diff --git a/openstack/barbican/templates/api-deployment.yaml b/openstack/barbican/templates/api-deployment.yaml index 417321e89a0..c9e626e8582 100644 --- a/openstack/barbican/templates/api-deployment.yaml +++ b/openstack/barbican/templates/api-deployment.yaml @@ -181,6 +181,10 @@ spec: mountPath: /etc/apache2/conf-enabled/tls-hardening.conf subPath: tls-hardening.conf readOnly: true + - name: barbican-etc + mountPath: /etc/apache2/conf-enabled/apache-hardening.conf + subPath: apache-hardening.conf + readOnly: true {{- end }} - mountPath: /etc/barbican/barbican.conf.d name: barbican-etc-confd diff --git a/openstack/barbican/templates/etc-configmap.yaml b/openstack/barbican/templates/etc-configmap.yaml index 2408bbf5eaf..237a5eb7a96 100644 --- a/openstack/barbican/templates/etc-configmap.yaml +++ b/openstack/barbican/templates/etc-configmap.yaml @@ -16,6 +16,8 @@ data: {{ include (print .Template.BasePath "/etc/_ssl-load.conf.tpl") . | indent 4 }} tls-hardening.conf: | {{ include (print .Template.BasePath "/etc/_tls-hardening.conf.tpl") . | indent 4 }} + apache-hardening.conf: | +{{ include (print .Template.BasePath "/etc/_apache-hardening.conf.tpl") . | indent 4 }} {{- end }} policy.yaml: | {{ include (print .Template.BasePath "/etc/_barbican-policy.yaml.tpl") . | indent 4 }} diff --git a/openstack/barbican/templates/etc/_apache-hardening.conf.tpl b/openstack/barbican/templates/etc/_apache-hardening.conf.tpl new file mode 100644 index 00000000000..8ff2fa1a4db --- /dev/null +++ b/openstack/barbican/templates/etc/_apache-hardening.conf.tpl @@ -0,0 +1,17 @@ +{{/* +Server-level Apache hardening: suppress the version banner, disable the TRACE +method, and turn off directory listing for the document root. Parsed after the +packaged apache2.conf so these directives take precedence. +*/}} + +# Suppress server version banner and disable TRACE +ServerTokens Prod +ServerSignature Off +TraceEnable Off + +# Disable directory listing for the document root + + Options FollowSymLinks + AllowOverride None + Require all granted + diff --git a/openstack/barbican/templates/etc/_ssl-load.conf.tpl b/openstack/barbican/templates/etc/_ssl-load.conf.tpl index 86117cbc2af..ed96306c1db 100644 --- a/openstack/barbican/templates/etc/_ssl-load.conf.tpl +++ b/openstack/barbican/templates/etc/_ssl-load.conf.tpl @@ -1,7 +1,9 @@ {{/* Load mod_ssl and its socache dependency via read-only conf so the module is -available under a read-only root filesystem without a2enmod. +available under a read-only root filesystem without a2enmod. mod_headers is +required for the HTTP security response headers set in the TLS vhost. */}} LoadModule socache_shmcb_module /usr/lib/apache2/modules/mod_socache_shmcb.so LoadModule ssl_module /usr/lib/apache2/modules/mod_ssl.so +LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so diff --git a/openstack/barbican/templates/etc/_tls-hardening.conf.tpl b/openstack/barbican/templates/etc/_tls-hardening.conf.tpl index 5598d9dd0ce..e85cd57fd4d 100644 --- a/openstack/barbican/templates/etc/_tls-hardening.conf.tpl +++ b/openstack/barbican/templates/etc/_tls-hardening.conf.tpl @@ -10,8 +10,8 @@ SSLProtocol -all +TLSv1.3 +TLSv1.2 # TLS 1.2 Cipher Suites (ECDHE + AEAD only, PFS required) SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256 -# TLS 1.3 Cipher Suites -SSLCipherSuite TLSv1.3 TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 +# TLS 1.3 Cipher Suites (AES-GCM and AES-CCM) +SSLCipherSuite TLSv1.3 TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_AES_128_CCM_SHA256 # Server chooses cipher order SSLHonorCipherOrder on @@ -31,3 +31,6 @@ SSLSessionTickets off # Strict SNI SSLStrictSNIVHostCheck on + +# Disable TLS compression (CRIME mitigation) +SSLCompression off diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 2039c09fb29..36344bde727 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -30,6 +30,13 @@ Include /etc/apache2/conf-enabled/tls-hardening.conf SSLCertificateFile /mnt/secrets/tls.crt SSLCertificateKeyFile /mnt/secrets/tls.key + # HTTP security response headers + Header always set Strict-Transport-Security "max-age=63072000; includeSubDomains" + Header always set X-Content-Type-Options "nosniff" + Header always set X-Frame-Options "DENY" + Header always set Content-Security-Policy "default-src 'self'" + Header always set Referrer-Policy "strict-origin-when-cross-origin" + WSGIDaemonProcess barbican-api-tls processes={{ .Values.api.processes | default 1 }} threads=1 user=barbican group=barbican display-name=%{GROUP} WSGIProcessGroup barbican-api-tls WSGIScriptAlias / /var/www/cgi-bin/barbican/barbican-wsgi-api diff --git a/openstack/keystone/templates/configmap-etc.yaml b/openstack/keystone/templates/configmap-etc.yaml index 9c931932fde..18d7f4a0d5e 100644 --- a/openstack/keystone/templates/configmap-etc.yaml +++ b/openstack/keystone/templates/configmap-etc.yaml @@ -28,6 +28,8 @@ data: {{ include (print .Template.BasePath "/etc/_ssl-load.conf.tpl") . | indent 4 }} tls-hardening.conf: | {{ include (print .Template.BasePath "/etc/_tls-hardening.conf.tpl") . | indent 4 }} + apache-hardening.conf: | +{{ include (print .Template.BasePath "/etc/_apache-hardening.conf.tpl") . | indent 4 }} {{- end }} {{- if .Values.watcher.enabled }} watcher.yaml: | diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index 14e76aecf01..d89f015ca96 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -294,6 +294,10 @@ spec: mountPath: /etc/apache2/conf-enabled/tls-hardening.conf subPath: tls-hardening.conf readOnly: true + - name: keystone-etc + mountPath: /etc/apache2/conf-enabled/apache-hardening.conf + subPath: apache-hardening.conf + readOnly: true {{- if .Values.services.ingress.x509.ca }} - name: x509-ca mountPath: /etc/apache2/x509-ca diff --git a/openstack/keystone/templates/etc/_apache-hardening.conf.tpl b/openstack/keystone/templates/etc/_apache-hardening.conf.tpl new file mode 100644 index 00000000000..8ff2fa1a4db --- /dev/null +++ b/openstack/keystone/templates/etc/_apache-hardening.conf.tpl @@ -0,0 +1,17 @@ +{{/* +Server-level Apache hardening: suppress the version banner, disable the TRACE +method, and turn off directory listing for the document root. Parsed after the +packaged apache2.conf so these directives take precedence. +*/}} + +# Suppress server version banner and disable TRACE +ServerTokens Prod +ServerSignature Off +TraceEnable Off + +# Disable directory listing for the document root + + Options FollowSymLinks + AllowOverride None + Require all granted + diff --git a/openstack/keystone/templates/etc/_tls-hardening.conf.tpl b/openstack/keystone/templates/etc/_tls-hardening.conf.tpl index 5598d9dd0ce..e85cd57fd4d 100644 --- a/openstack/keystone/templates/etc/_tls-hardening.conf.tpl +++ b/openstack/keystone/templates/etc/_tls-hardening.conf.tpl @@ -10,8 +10,8 @@ SSLProtocol -all +TLSv1.3 +TLSv1.2 # TLS 1.2 Cipher Suites (ECDHE + AEAD only, PFS required) SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256 -# TLS 1.3 Cipher Suites -SSLCipherSuite TLSv1.3 TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 +# TLS 1.3 Cipher Suites (AES-GCM and AES-CCM) +SSLCipherSuite TLSv1.3 TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256:TLS_AES_128_CCM_SHA256 # Server chooses cipher order SSLHonorCipherOrder on @@ -31,3 +31,6 @@ SSLSessionTickets off # Strict SNI SSLStrictSNIVHostCheck on + +# Disable TLS compression (CRIME mitigation) +SSLCompression off diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 5a79cdc0687..7d45809cd5a 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -89,6 +89,13 @@ Listen 0.0.0.0:443 SSLCertificateFile /mnt/secrets/tls.crt SSLCertificateKeyFile /mnt/secrets/tls.key + # HTTP security response headers + Header always set Strict-Transport-Security "max-age=63072000; includeSubDomains" + Header always set X-Content-Type-Options "nosniff" + Header always set X-Frame-Options "DENY" + Header always set Content-Security-Policy "default-src 'self'" + Header always set Referrer-Policy "strict-origin-when-cross-origin" + {{- if .Values.services.ingress.x509.ca }} SSLVerifyClient optional SSLVerifyDepth 3 From 4ad1bca7c2d2cf1add9654b3d5aee4ab45322401 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Tue, 25 Aug 2026 11:09:28 +0200 Subject: [PATCH 34/54] Suppress absence alerting on the TLS rotation failure alerts The TLS rotation failure alerts for Keystone and Barbican lacked no_alert_on_absence, unlike the other TLS alerts. When TLS is disabled the rotation jobs do not exist and the metric is absent, so add the label to avoid spurious absence alerts. --- openstack/barbican/alerts/openstack/openstack-barbican.alerts | 1 + openstack/keystone/alerts/openstack/keystone.alerts | 1 + 2 files changed, 2 insertions(+) diff --git a/openstack/barbican/alerts/openstack/openstack-barbican.alerts b/openstack/barbican/alerts/openstack/openstack-barbican.alerts index f839c6dd282..49263ccd982 100644 --- a/openstack/barbican/alerts/openstack/openstack-barbican.alerts +++ b/openstack/barbican/alerts/openstack/openstack-barbican.alerts @@ -127,6 +127,7 @@ groups: severity: warning tier: os support_group: identity + no_alert_on_absence: "true" annotations: description: The weekly TLS certificate rotation CronJob for Barbican has failed. If not resolved, the TLS certificate will expire and external HTTPS access will break. Check the CronJob logs. summary: Barbican TLS certificate rotation failed diff --git a/openstack/keystone/alerts/openstack/keystone.alerts b/openstack/keystone/alerts/openstack/keystone.alerts index f4587ed6e8f..9dd279cf977 100644 --- a/openstack/keystone/alerts/openstack/keystone.alerts +++ b/openstack/keystone/alerts/openstack/keystone.alerts @@ -115,6 +115,7 @@ groups: severity: warning tier: os support_group: identity + no_alert_on_absence: "true" annotations: description: The weekly TLS certificate rotation CronJob for Keystone has failed. If not resolved, the TLS certificate will expire and external HTTPS access will break. Check the CronJob logs. summary: Keystone TLS certificate rotation failed From 821fd307063cdeef33fbc65678e979584371f928 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Wed, 26 Aug 2026 08:33:50 +0200 Subject: [PATCH 35/54] Remove non-functional tls-secret-hash pod annotation --- openstack/keystone/templates/deployment-api.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/openstack/keystone/templates/deployment-api.yaml b/openstack/keystone/templates/deployment-api.yaml index d89f015ca96..eeeb4f5c674 100644 --- a/openstack/keystone/templates/deployment-api.yaml +++ b/openstack/keystone/templates/deployment-api.yaml @@ -55,7 +55,6 @@ spec: federation-saml-hash: {{ include (print $.Template.BasePath "/federation-saml.yaml") . | sha256sum }} {{- end }} {{- if .Values.tls.enabled }} - tls-secret-hash: {{ .Values.tls.secretName | default "keystone-tls" }} config.linkerd.io/skip-inbound-ports: "443" {{- end }} {{- if .Values.api.metrics.enabled }} From 223a5005faa87de26f4b01d9828b95fdce9a9a4f Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Wed, 26 Aug 2026 22:44:11 +0200 Subject: [PATCH 36/54] Include per-tenant SAML config in the TLS vhost --- openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 7d45809cd5a..8ecc093d95f 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -140,7 +140,7 @@ Listen 0.0.0.0:443 SetHandler shib - IncludeOptional /etc/apache2/conf-enabled/federation-saml.conf + IncludeOptional /etc/apache2/conf-enabled/federation-saml.d/*.conf {{- end }} {{- end }} From d1f52b686405c48a49d2a8daf8f871a985e2662e Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Thu, 27 Aug 2026 06:12:34 +0200 Subject: [PATCH 37/54] Require an external IP when pod-level TLS is enabled --- openstack/barbican/templates/_helpers.tpl | 4 ++++ openstack/keystone/templates/_helpers.tpl | 3 +++ 2 files changed, 7 insertions(+) diff --git a/openstack/barbican/templates/_helpers.tpl b/openstack/barbican/templates/_helpers.tpl index 7c7f858490b..b6b858cd389 100644 --- a/openstack/barbican/templates/_helpers.tpl +++ b/openstack/barbican/templates/_helpers.tpl @@ -59,5 +59,9 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- if and (eq .Values.tls.keyWrapping "none") (eq .Values.tls.keyStorage "k8s-secret") (not .Values.tls.allowInsecureStorage) }} {{- fail "tls: unwrapped keys cannot be stored as plain-text K8s Secrets. Set tls.keyWrapping or tls.keyStorage, or set tls.allowInsecureStorage: true to acknowledge." }} {{- end }} + {{- $svc := .Values.services | default dict }} + {{- if not (.Values.global.barbican_external_ip | default $svc.externalip) }} + {{- fail "tls.enabled requires an external IP (global.barbican_external_ip or services.externalip): enabling TLS removes the ingress and the public TLS Service only renders once the external IP is set." }} + {{- end }} {{- end }} {{- end }} diff --git a/openstack/keystone/templates/_helpers.tpl b/openstack/keystone/templates/_helpers.tpl index 5dcc1603622..aa9ea61c99d 100644 --- a/openstack/keystone/templates/_helpers.tpl +++ b/openstack/keystone/templates/_helpers.tpl @@ -126,5 +126,8 @@ Empty values are skipped so callers may pass "" for optional dependencies. {{- if and (eq .Values.tls.keyWrapping "none") (eq .Values.tls.keyStorage "k8s-secret") (not .Values.tls.allowInsecureStorage) }} {{- fail "tls: unwrapped keys cannot be stored as plain-text K8s Secrets. Set tls.keyWrapping or tls.keyStorage, or set tls.allowInsecureStorage: true to acknowledge." }} {{- end }} + {{- if not .Values.global.keystone_external_ip }} + {{- fail "tls.enabled requires global.keystone_external_ip: enabling TLS removes the ingress and the public TLS Service only renders once the external IP is set." }} + {{- end }} {{- end }} {{- end }} From feeb03855ac2dfa3f622159e95035d920fd49e42 Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Thu, 27 Aug 2026 22:37:41 +0200 Subject: [PATCH 38/54] Harden x509 header handling, unify Barbican WSGI group, restore keystone external IP Rebuild and sanitize the client-certificate issuer and subject DN request headers on the Keystone TLS vhost so tokenless x509 auth reads verified values and clients cannot inject them. Collapse the Barbican TLS vhost onto the single barbican-api WSGI daemon group to keep one PKCS#11 session per pod and honor api.threads. Restore the services.externalip entry on the plain Keystone API Service. --- .../barbican/templates/etc/_wsgi-barbican.conf.tpl | 3 +-- .../keystone/templates/etc/_wsgi-keystone.conf.tpl | 11 +++++++++++ openstack/keystone/templates/service-api.yaml | 3 +++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl index 36344bde727..e8031d378aa 100644 --- a/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl +++ b/openstack/barbican/templates/etc/_wsgi-barbican.conf.tpl @@ -37,8 +37,7 @@ Include /etc/apache2/conf-enabled/tls-hardening.conf Header always set Content-Security-Policy "default-src 'self'" Header always set Referrer-Policy "strict-origin-when-cross-origin" - WSGIDaemonProcess barbican-api-tls processes={{ .Values.api.processes | default 1 }} threads=1 user=barbican group=barbican display-name=%{GROUP} - WSGIProcessGroup barbican-api-tls + WSGIProcessGroup barbican-api WSGIScriptAlias / /var/www/cgi-bin/barbican/barbican-wsgi-api WSGIApplicationGroup %{GLOBAL} WSGIPassAuthorization On diff --git a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl index 8ecc093d95f..b31ad1c6d0f 100644 --- a/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl +++ b/openstack/keystone/templates/etc/_wsgi-keystone.conf.tpl @@ -114,8 +114,19 @@ Listen 0.0.0.0:443 RequestHeader unset SSL_Client-Verify RequestHeader unset SSL-Client_Verify RequestHeader unset SSL_CLIENT_VERIFY + {{- /* The issuer/subject DN env vars have four underscore-separated tokens, + so a client can spell the injected header with '-' or '_' at each of + the three separators; strip every combination. */}} + {{- range $dn := list "I" "S" }} + {{- range $s1 := list "-" "_" }} + {{- range $s2 := list "-" "_" }} + {{- range $s3 := list "-" "_" }} + RequestHeader unset SSL{{ $s1 }}Client{{ $s2 }}{{ $dn }}{{ $s3 }}DN + {{- end }}{{- end }}{{- end }}{{- end }} RequestHeader set SSL-Client-Cert "expr=%{escape:%{SSL:SSL_CLIENT_CERT}}" RequestHeader set SSL-Client-Verify "expr=%{SSL:SSL_CLIENT_VERIFY}" + RequestHeader set SSL-Client-I-DN "expr=%{SSL:SSL_CLIENT_I_DN}" + RequestHeader set SSL-Client-S-DN "expr=%{SSL:SSL_CLIENT_S_DN}" {{- end }} WSGIDaemonProcess keystone-tls processes=8 threads=1 user=keystone group=keystone display-name=%{GROUP} diff --git a/openstack/keystone/templates/service-api.yaml b/openstack/keystone/templates/service-api.yaml index d476c603368..15f7d815eb0 100644 --- a/openstack/keystone/templates/service-api.yaml +++ b/openstack/keystone/templates/service-api.yaml @@ -33,4 +33,7 @@ spec: {{- if .Values.global.keystone_internal_ip }} externalIPs: - {{ .Values.global.keystone_internal_ip | quote }} +{{- if .Values.services.externalip }} + - {{ .Values.services.externalip | quote }} +{{- end }} {{- end }} From c638e005e0d90df713f00058960e237b9c40a02d Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Fri, 28 Aug 2026 11:05:19 +0200 Subject: [PATCH 39/54] Allow the WebSSO auto-submit script under the Keystone CSP The TLS vhost CSP default-src 'self' blocked the inline auto-submit script in the WebSSO callback, leaving browser federation logins stuck. Allow just that script via its sha256 hash, and guard the pinned hash at render time so a change to the callback template fails loudly instead of silently breaking WebSSO. --- openstack/keystone/templates/_helpers.tpl | 13 +++++++++++++ .../keystone/templates/etc/_wsgi-keystone.conf.tpl | 5 ++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/openstack/keystone/templates/_helpers.tpl b/openstack/keystone/templates/_helpers.tpl index aa9ea61c99d..74a350470ed 100644 --- a/openstack/keystone/templates/_helpers.tpl +++ b/openstack/keystone/templates/_helpers.tpl @@ -129,5 +129,18 @@ Empty values are skipped so callers may pass "" for optional dependencies. {{- if not .Values.global.keystone_external_ip }} {{- fail "tls.enabled requires global.keystone_external_ip: enabling TLS removes the ingress and the public TLS Service only renders once the external IP is set." }} {{- end }} + {{- /* Guard the CSP script-src hash against drift: the TLS vhost pins a + sha256 of the WebSSO callback's inline auto-submit script, and Helm + cannot recompute a base64 digest, so fail loudly if the script changes + and both pins must be refreshed together. */}} + {{- $ssoTpl := include (print $.Template.BasePath "/etc/_sso_callback_template.html.tpl") . }} + {{- $ssoParts := splitList "" (index $ssoParts 1)) 0 }} + {{- if ne (sha256sum $ssoScript) "a016a19411507a6fa732cd49c20701074dc7cbc9d18797bca8419339cc6602e3" }} + {{- fail "SSO callback inline script changed; recompute the CSP script-src hash (sha256/base64 of the