diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index cbff462d45..8b0ef7242f 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -268,6 +268,51 @@ If the gateway exits with `failed to read sandbox JWT signing key from `sandbox-jwt` secret at `/etc/openshell-jwt`. The sandbox JWT mount is required even when local Helm values disable TLS. +If `certManager.serverIssuerRef` points the server certificate at an external +Issuer or ClusterIssuer (for example an ACME issuer, for a publicly-trusted +cert on an OpenShift `Route` with TLS passthrough — see +`openshiftRoute.enabled`), the chart creates **two** server certificates: an +internal one (chart CA, internal SANs) and an external one (from the configured +issuer, external SANs only). The gateway uses SNI to present the right cert. + +Check the external `Certificate`/`CertificateRequest`/`Challenge` resources +directly when the external secret never becomes Ready: + +```bash +kubectl -n openshell get certificate,certificaterequest,challenge +kubectl -n openshell describe certificate openshell-server-external +oc -n openshell get route +``` + +ACME issuers reject certificate requests that include internal-only names +(`*.svc.cluster.local`, `localhost`, loopback IPs) and require the +`commonName` to also be a SAN — the external `Certificate` only requests the +hostnames in `certManager.serverDnsNames`, for exactly this reason. + +If sandbox supervisors fail their TLS handshake to the gateway with +`UnknownCA` after configuring `serverIssuerRef`, the most likely cause is +`server.grpcEndpoint` set to the external hostname. This forces supervisors +to connect via the external hostname, receiving the ACME cert (via SNI) which +they cannot verify against the chart CA. Remove `server.grpcEndpoint` or set +it to the internal service name so supervisors receive the internal cert: + +```bash +helm -n openshell get values openshell | grep -E 'grpcEndpoint|clientCaFromServerTlsSecret|clientCaSecretName|serverIssuerRef|caSecretName' +# server.grpcEndpoint should be unset or point to internal service name +``` + +Less commonly, `UnknownCA` can occur if the gateway's client-verification CA +is misconfigured. The chart fails at render time if +`clientCaFromServerTlsSecret` is still true when `serverIssuerRef` is set, but +this can still occur with pre-existing releases or manual overrides. Set +`certManager.clientCaFromServerTlsSecret=false` and +`server.tls.clientCaSecretName` to a secret that actually contains the CA that +signs the client certificate (`certManager.caSecretName`'s value by default): + +```bash +kubectl -n openshell get secret openshell-ca-tls -o jsonpath='{.data.ca\.crt}' | base64 -d | openssl x509 -noout -subject +``` + If `server.providerTokenGrants.spiffe.enabled=true`, the gateway should still render `[openshell.gateway.gateway_jwt]` and mount the `sandbox-jwt` Secret. SPIRE is used only by sandbox pods for dynamic provider token grants. Verify @@ -452,6 +497,8 @@ openshell logs | `K8s namespace not ready` with `envoy-gateway-openshell.yaml: the server could not find the requested resource` | Optional Gateway API manifest was applied without Envoy Gateway CRDs, or k3s Helm controller startup exceeded the namespace wait | Apply `deploy/kube/manifests/envoy-gateway-openshell.yaml` manually only after Envoy Gateway is installed and `grpcRoute` is enabled | | HTTPS ingress (`grpcRoute.gateway.listener.protocol=HTTPS`) connection resets or TLS handshake hangs | Envoy terminates TLS but the gateway pod still expects TLS, so the plaintext backend hop fails | Set `server.disableTls=true` so Envoy forwards plaintext to the pod; verify the listener `certificateRefs` Secret exists in the release namespace and `openshell status` over `https://` | | HTTPS ingress returns `Unauthenticated` after connecting | TLS terminates at Envoy, so the gateway never sees a client cert; no OIDC issuer is configured for identity | Configure `server.oidc.issuer` and register with `openshell gateway add https:// --oidc-issuer `, or set `server.auth.allowUnauthenticatedUsers=true` for a trusted-proxy/dev cluster | +| External server `Certificate` never becomes Ready with `certManager.serverIssuerRef` set | ACME issuer rejected internal-only SANs, a loopback IP, or a `commonName` absent from the SANs | `kubectl -n openshell describe certificate openshell-server-external`; confirm `certManager.serverDnsNames` lists only real, externally-resolvable hostnames | +| Sandbox supervisors fail TLS handshake with `UnknownCA` after configuring `certManager.serverIssuerRef` | `server.grpcEndpoint` is set to the external hostname, forcing supervisors to receive the ACME cert (via SNI) which they can't verify against chart CA | Remove `server.grpcEndpoint` or set it to the internal service name; supervisors should connect via internal service name to receive the internal cert | ## Reporting diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index e138e1eee1..380732ce7b 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -14,7 +14,7 @@ repository.workspace = true glob = { workspace = true } prost = { workspace = true } prost-types = { workspace = true } -tonic = { workspace = true, features = ["channel", "tls-native-roots"] } +tonic = { workspace = true, features = ["channel", "tls-ring"] } tonic-prost = { workspace = true } tokio = { workspace = true } thiserror = { workspace = true } diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 5d4cceeab3..f012cdd25f 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -546,6 +546,24 @@ pub struct TlsConfig { /// When `false`, client certificates are accepted but not required. #[serde(default)] pub require_client_auth: bool, + + /// Path to an external TLS certificate file (e.g. ACME/publicly-trusted). + /// When set, the server uses SNI-based certificate selection: connections + /// whose SNI hostname matches `external_server_names` receive this cert, + /// all others receive the primary (internal) cert. + #[serde(default)] + pub external_cert_path: Option, + + /// Path to the private key for the external TLS certificate. + #[serde(default)] + pub external_key_path: Option, + + /// Hostnames that should be served with the external certificate. + /// Connections whose SNI matches one of these names receive the external + /// cert; all other connections (including those with no SNI) receive the + /// primary (internal) cert. + #[serde(default)] + pub external_server_names: Vec, } /// OIDC (`OpenID` Connect) configuration for JWT-based authentication. diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 579ee4a5b3..aa2344d7f7 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -167,6 +167,17 @@ async fn build_plain_channel(endpoint: &str) -> Result { .into_diagnostic() .wrap_err_with(|| format!("failed to read client key from {key_path}"))?; + // Trust only the configured CA — this is the chart's internal CA + // that signs both the gateway's internal server certificate and + // this client's identity certificate. The gateway uses SNI-based + // certificate selection to present this internal cert to supervisor + // connections, so no public root trust is needed here. + // + // Do NOT add `.with_native_roots()` or `.with_webpki_roots()` here: + // the supervisor runs inside the user-selected sandbox image + // (Docker/Podman drivers), and broadening the trust store would let + // an attacker who controls the image + DNS present a publicly valid + // certificate and intercept the supervisor→gateway TLS connection. let mut tls_config = ClientTlsConfig::new() .ca_certificate(Certificate::from_pem(ca_pem)) .identity(Identity::from_pem(cert_pem, key_pem)); diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index dd4d9ef0f0..bc5e9adca1 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -2250,6 +2250,11 @@ fn build_environment_for_oci_user( environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + // Prevent user-supplied environment from overriding the TLS server name + // the supervisor verifies — a sandbox user who can redirect the gateway + // hostname could otherwise present a certificate for a name they control + // and intercept the sandbox JWT. + environment.remove(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); environment.insert( openshell_core::sandbox_env::OCI_IMAGE_USER.to_string(), oci_user.to_string(), diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index ac525c705c..786cb0ac63 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -594,6 +594,26 @@ fn build_environment_protects_oci_identity_metadata() { assert!(!env.iter().any(|entry| entry.ends_with("=9999"))); } +#[test] +fn build_environment_strips_gateway_tls_server_name() { + let mut sandbox = test_sandbox(); + let spec = sandbox.spec.as_mut().unwrap(); + spec.environment.insert( + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME.to_string(), + "evil.attacker.example.com".to_string(), + ); + + let env = build_environment(&sandbox, &runtime_config()); + + assert!( + !env.iter().any(|entry| entry.starts_with(&format!( + "{}=", + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME + ))), + "GATEWAY_TLS_SERVER_NAME must be stripped from the supervisor environment" + ); +} + #[test] fn container_creation_uses_inspected_immutable_image() { let sandbox = test_sandbox(); diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 005f688a19..df61a13e2d 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -483,6 +483,11 @@ fn build_env( env.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); env.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + // Prevent user-supplied environment from overriding the TLS server name + // the supervisor verifies — a sandbox user who can redirect the gateway + // hostname could otherwise present a certificate for a name they control + // and intercept the sandbox JWT. + env.remove(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); env.insert( openshell_core::sandbox_env::OCI_IMAGE_USER.into(), oci_user.to_string(), @@ -1413,6 +1418,24 @@ mod tests { ); } + #[test] + fn build_env_strips_gateway_tls_server_name() { + let mut sandbox = test_sandbox("test-id", "test-name"); + let spec = sandbox.spec.get_or_insert_default(); + spec.environment.insert( + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME.to_string(), + "evil.attacker.example.com".to_string(), + ); + + let container = build_container_spec(&sandbox, &test_config()); + + assert_eq!( + container["env"].get(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME), + None, + "GATEWAY_TLS_SERVER_NAME must be stripped from the supervisor environment" + ); + } + #[test] fn volume_name_uses_id() { assert_eq!( diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 9b6c0dd6ce..fd7b2c8ae5 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -4268,6 +4268,11 @@ fn build_guest_environment( ); environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + // Prevent user-supplied environment from overriding the TLS server name + // the supervisor verifies — a sandbox user who can redirect the gateway + // hostname could otherwise present a certificate for a name they control + // and intercept the sandbox JWT. + environment.remove(openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); if sandbox .spec .as_ref() @@ -6716,6 +6721,36 @@ mod tests { ))); } + #[test] + fn build_guest_environment_strips_gateway_tls_server_name() { + let config = VmDriverConfig { + openshell_endpoint: "http://127.0.0.1:8080".to_string(), + ..Default::default() + }; + let sandbox = Sandbox { + id: "sandbox-123".to_string(), + name: "sandbox-123".to_string(), + spec: Some(SandboxSpec { + environment: HashMap::from([( + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME.to_string(), + "evil.attacker.example.com".to_string(), + )]), + ..Default::default() + }), + ..Default::default() + }; + + let env = build_guest_environment(&sandbox, &config, None); + + assert!( + !env.iter().any(|v| v.starts_with(&format!( + "{}=", + openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME + ))), + "GATEWAY_TLS_SERVER_NAME must be stripped from the guest environment" + ); + } + #[test] fn build_guest_environment_uses_deployment_telemetry_toggle() { let _guard = ENV_LOCK.lock().unwrap(); diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 8b18034947..1ac77367ee 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -294,11 +294,27 @@ fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result, require_client_auth: bool, + external_cert_path: Option, + external_key_path: Option, + external_server_names: Vec, reload_spawned: Arc, } @@ -64,14 +68,28 @@ impl TlsAcceptor { key_path: &Path, client_ca_path: Option<&Path>, require_client_auth: bool, + external_cert_path: Option<&Path>, + external_key_path: Option<&Path>, + external_server_names: Vec, ) -> Result { - let config = build_server_config(cert_path, key_path, client_ca_path, require_client_auth)?; + let config = build_server_config( + cert_path, + key_path, + client_ca_path, + require_client_auth, + external_cert_path, + external_key_path, + &external_server_names, + )?; Ok(Self { config: Arc::new(ArcSwap::from(config)), cert_path: cert_path.to_path_buf(), key_path: key_path.to_path_buf(), client_ca_path: client_ca_path.map(Path::to_path_buf), require_client_auth, + external_cert_path: external_cert_path.map(Path::to_path_buf), + external_key_path: external_key_path.map(Path::to_path_buf), + external_server_names, reload_spawned: Arc::new(AtomicBool::new(false)), }) } @@ -87,6 +105,9 @@ impl TlsAcceptor { &self.key_path, self.client_ca_path.as_deref(), self.require_client_auth, + self.external_cert_path.as_deref(), + self.external_key_path.as_deref(), + &self.external_server_names, )?; self.config.store(new_config); @@ -144,10 +165,22 @@ impl TlsAcceptor { } if let Some(ref ca) = self.client_ca_path { let ca_dir = ca.parent().unwrap_or_else(|| Path::new(".")); - if ca_dir != cert_dir && ca_dir != key_dir { + if !dirs.contains(&ca_dir.to_path_buf()) { dirs.push(ca_dir.to_path_buf()); } } + if let Some(ref ext_cert) = self.external_cert_path { + let ext_dir = ext_cert.parent().unwrap_or_else(|| Path::new(".")); + if !dirs.contains(&ext_dir.to_path_buf()) { + dirs.push(ext_dir.to_path_buf()); + } + } + if let Some(ref ext_key) = self.external_key_path { + let ext_dir = ext_key.parent().unwrap_or_else(|| Path::new(".")); + if !dirs.contains(&ext_dir.to_path_buf()) { + dirs.push(ext_dir.to_path_buf()); + } + } let debounce = Duration::from_secs(1); @@ -244,12 +277,102 @@ impl TlsAcceptor { } } +/// SNI-based certificate resolver that presents an external (e.g. ACME) +/// certificate for configured hostnames and the internal (chart CA) certificate +/// for everything else, including connections with no SNI. +struct DualCertResolver { + internal: Arc, + external: Arc, + external_names: Vec, +} + +/// Check whether `sni` matches a configured external name. +/// +/// Supports exact matches and single-level wildcard matches per RFC 6125: +/// `*.example.com` matches `foo.example.com` but not `bar.foo.example.com` +/// or `example.com` itself. +fn sni_matches(pattern: &str, sni: &str) -> bool { + pattern.strip_prefix("*.").map_or(pattern == sni, |suffix| { + // Wildcard: SNI must have exactly one label before the suffix. + // e.g. "foo." for "foo.example.com" against "*.example.com" + sni.strip_suffix(suffix).is_some_and(|prefix| { + prefix.ends_with('.') && !prefix[..prefix.len() - 1].contains('.') + }) + }) +} + +impl ResolvesServerCert for DualCertResolver { + fn resolve(&self, client_hello: ClientHello<'_>) -> Option> { + if let Some(name) = client_hello.server_name() + && self.external_names.iter().any(|n| sni_matches(n, name)) + { + return Some(self.external.clone()); + } + Some(self.internal.clone()) + } +} + +impl std::fmt::Debug for DualCertResolver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DualCertResolver") + .field("external_names", &self.external_names) + .finish() + } +} + +/// Build a `CertifiedKey` from certificate and key file paths. +fn load_certified_key(cert_path: &Path, key_path: &Path) -> Result> { + let certs = load_certs(cert_path)?; + let key = load_key(key_path)?; + let signing_key = sign::any_supported_type(&key) + .map_err(|e| Error::tls(format!("unsupported private key type: {e}")))?; + Ok(Arc::new(CertifiedKey::new(certs, signing_key))) +} + +/// Build an SNI-based cert resolver when an external certificate is configured. +/// Returns `None` when no external cert is configured (single-cert mode). +fn build_cert_resolver( + cert_path: &Path, + key_path: &Path, + external_cert_path: Option<&Path>, + external_key_path: Option<&Path>, + external_server_names: &[String], +) -> Result>> { + match (external_cert_path, external_key_path) { + (None, None) => Ok(None), + (Some(_), None) => Err(Error::tls( + "external_cert_path is set but external_key_path is missing", + )), + (None, Some(_)) => Err(Error::tls( + "external_key_path is set but external_cert_path is missing", + )), + (Some(ext_cert_path), Some(ext_key_path)) => { + if external_server_names.is_empty() { + return Err(Error::tls( + "external certificate is configured but external_server_names is empty — \ + the external cert would never be served", + )); + } + let internal = load_certified_key(cert_path, key_path)?; + let external = load_certified_key(ext_cert_path, ext_key_path)?; + Ok(Some(Arc::new(DualCertResolver { + internal, + external, + external_names: external_server_names.to_vec(), + }))) + } + } +} + /// Build a `ServerConfig` from certificate, key, and optional client CA files. fn build_server_config( cert_path: &Path, key_path: &Path, client_ca_path: Option<&Path>, require_client_auth: bool, + external_cert_path: Option<&Path>, + external_key_path: Option<&Path>, + external_server_names: &[String], ) -> Result> { let certs = load_certs(cert_path)?; let key = load_key(key_path)?; @@ -259,6 +382,14 @@ fn build_server_config( sign::any_supported_type(&key) .map_err(|e| Error::tls(format!("unsupported private key type: {e}")))?; + let resolver = build_cert_resolver( + cert_path, + key_path, + external_cert_path, + external_key_path, + external_server_names, + )?; + let mut config = if let Some(ca_path) = client_ca_path { let ca_certs = load_certs(ca_path)?; let mut root_store = rustls::RootCertStore::empty(); @@ -277,15 +408,23 @@ fn build_server_config( .build() .map_err(|e| Error::tls(format!("failed to build client verifier: {e}")))?; - ServerConfig::builder() - .with_client_cert_verifier(verifier) - .with_single_cert(certs, key) - .map_err(|e| Error::tls(format!("failed to create TLS config: {e}")))? + let builder = ServerConfig::builder().with_client_cert_verifier(verifier); + if let Some(resolver) = resolver { + builder.with_cert_resolver(resolver) + } else { + builder + .with_single_cert(certs, key) + .map_err(|e| Error::tls(format!("failed to create TLS config: {e}")))? + } } else { - ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(certs, key) - .map_err(|e| Error::tls(format!("failed to create TLS config: {e}")))? + let builder = ServerConfig::builder().with_no_client_auth(); + if let Some(resolver) = resolver { + builder.with_cert_resolver(resolver) + } else { + builder + .with_single_cert(certs, key) + .map_err(|e| Error::tls(format!("failed to create TLS config: {e}")))? + } }; config @@ -402,6 +541,9 @@ mod tests { &dir.path().join("server-key.pem"), Some(&dir.path().join("ca.pem")), false, + None, + None, + &[], ) .expect("failed to build server config"); @@ -420,6 +562,9 @@ mod tests { &dir.path().join("server-key.pem"), Some(&dir.path().join("ca.pem")), false, + None, + None, + Vec::new(), ) .expect("failed to build acceptor"); @@ -438,6 +583,9 @@ mod tests { &dir.path().join("server-key.pem"), Some(&dir.path().join("ca.pem")), false, + None, + None, + Vec::new(), ) .expect("failed to build acceptor"); @@ -468,6 +616,9 @@ mod tests { &dir.path().join("server-key.pem"), Some(&dir.path().join("ca.pem")), false, + None, + None, + Vec::new(), ) .expect("failed to build acceptor"); @@ -560,6 +711,9 @@ mod tests { &dir.path().join("server-key.pem"), Some(&dir.path().join("ca.pem")), false, + None, + None, + Vec::new(), ) .expect("failed to build acceptor"); @@ -633,6 +787,9 @@ mod tests { &dir.path().join("server-key.pem"), Some(&dir.path().join("ca.pem")), false, + None, + None, + Vec::new(), ) .expect("failed to build acceptor"); @@ -664,6 +821,9 @@ mod tests { &dir.path().join("server-key.pem"), Some(&dir.path().join("ca.pem")), false, + None, + None, + Vec::new(), ) .expect("failed to build acceptor"); @@ -752,6 +912,9 @@ mod tests { &dir.path().join("server-key.pem"), Some(&dir.path().join("ca.pem")), true, // require mTLS + None, + None, + Vec::new(), ) .expect("failed to build acceptor with mTLS"); @@ -905,4 +1068,275 @@ mod tests { server_task.await.expect("server task failed"); } + + /// Generate a cert+key pair with given SANs, signed by the provided CA, + /// and write them to the specified files in `dir`. + fn generate_named_cert( + ca_cert: &rcgen::Certificate, + ca_key: &KeyPair, + dir: &Path, + cert_file: &str, + key_file: &str, + san: &str, + ) { + let params = + CertificateParams::new(vec![san.to_string()]).expect("failed to create cert params"); + let key = KeyPair::generate().expect("failed to generate key"); + let cert = params + .signed_by(&key, ca_cert, ca_key) + .expect("failed to sign cert"); + write_test_file(dir, cert_file, cert.pem().as_bytes()); + write_test_file(dir, key_file, key.serialize_pem().as_bytes()); + } + + #[test] + fn test_sni_matches_exact() { + assert!(sni_matches("example.com", "example.com")); + assert!(!sni_matches("example.com", "other.com")); + assert!(!sni_matches("example.com", "sub.example.com")); + } + + #[test] + fn test_sni_matches_wildcard() { + assert!(sni_matches("*.example.com", "foo.example.com")); + assert!(sni_matches("*.example.com", "bar.example.com")); + // Must not match bare domain. + assert!(!sni_matches("*.example.com", "example.com")); + // Must not match nested subdomains (RFC 6125). + assert!(!sni_matches("*.example.com", "sub.foo.example.com")); + // Must not match unrelated domain with same suffix. + assert!(!sni_matches("*.example.com", "notexample.com")); + } + + #[test] + fn test_build_cert_resolver_returns_none_when_no_external() { + install_rustls_provider(); + let dir = tempfile::tempdir().expect("failed to create tempdir"); + generate_test_certs_with_ca(dir.path()); + + let result = build_cert_resolver( + &dir.path().join("server-cert.pem"), + &dir.path().join("server-key.pem"), + None, + None, + &[], + ) + .expect("build_cert_resolver should succeed"); + assert!(result.is_none(), "should return None when no external cert"); + } + + #[test] + fn test_build_cert_resolver_errors_on_cert_without_key() { + install_rustls_provider(); + let dir = tempfile::tempdir().expect("failed to create tempdir"); + generate_test_certs_with_ca(dir.path()); + + let result = build_cert_resolver( + &dir.path().join("server-cert.pem"), + &dir.path().join("server-key.pem"), + Some(&dir.path().join("server-cert.pem")), + None, + &["example.com".to_string()], + ); + let err = result.expect_err("should error when key is missing"); + assert!( + err.to_string().contains("external_key_path is missing"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_build_cert_resolver_errors_on_key_without_cert() { + install_rustls_provider(); + let dir = tempfile::tempdir().expect("failed to create tempdir"); + generate_test_certs_with_ca(dir.path()); + + let result = build_cert_resolver( + &dir.path().join("server-cert.pem"), + &dir.path().join("server-key.pem"), + None, + Some(&dir.path().join("server-key.pem")), + &["example.com".to_string()], + ); + let err = result.expect_err("should error when cert is missing"); + assert!( + err.to_string().contains("external_cert_path is missing"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_build_cert_resolver_errors_on_empty_server_names() { + install_rustls_provider(); + let dir = tempfile::tempdir().expect("failed to create tempdir"); + let (ca_cert, ca_key) = generate_test_certs_with_ca(dir.path()); + generate_named_cert( + &ca_cert, + &ca_key, + dir.path(), + "ext-cert.pem", + "ext-key.pem", + "external.example.com", + ); + + let result = build_cert_resolver( + &dir.path().join("server-cert.pem"), + &dir.path().join("server-key.pem"), + Some(&dir.path().join("ext-cert.pem")), + Some(&dir.path().join("ext-key.pem")), + &[], + ); + let err = result.expect_err("should error when server names are empty"); + assert!( + err.to_string().contains("external_server_names is empty"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_dual_cert_resolver_returns_external_on_sni_match() { + install_rustls_provider(); + let dir = tempfile::tempdir().expect("failed to create tempdir"); + let (ca_cert, ca_key) = generate_test_certs_with_ca(dir.path()); + generate_named_cert( + &ca_cert, + &ca_key, + dir.path(), + "ext-cert.pem", + "ext-key.pem", + "external.example.com", + ); + + let internal = load_certified_key( + &dir.path().join("server-cert.pem"), + &dir.path().join("server-key.pem"), + ) + .expect("load internal"); + let external = load_certified_key( + &dir.path().join("ext-cert.pem"), + &dir.path().join("ext-key.pem"), + ) + .expect("load external"); + + let internal_der = internal.cert[0].as_ref().to_vec(); + let external_der = external.cert[0].as_ref().to_vec(); + + // `ClientHello` cannot be constructed directly in tests, so + // SNI-based selection is exercised in the async integration test + // below. Here we verify the certs are distinct so the integration + // test's DER comparisons are meaningful. + assert_ne!( + internal_der, external_der, + "internal and external certs should be distinct" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_dual_cert_resolver_sni_selects_correct_cert() { + install_rustls_provider(); + + let dir = tempfile::tempdir().expect("failed to create tempdir"); + let (ca_cert, ca_key) = generate_test_certs_with_ca(dir.path()); + generate_named_cert( + &ca_cert, + &ca_key, + dir.path(), + "ext-cert.pem", + "ext-key.pem", + "external.example.com", + ); + + // Snapshot the DER of the internal and external leaf certs. + let internal_der = load_certs(&dir.path().join("server-cert.pem")) + .expect("load internal certs")[0] + .as_ref() + .to_vec(); + let external_der = load_certs(&dir.path().join("ext-cert.pem")) + .expect("load external certs")[0] + .as_ref() + .to_vec(); + + let acceptor = TlsAcceptor::from_files( + &dir.path().join("server-cert.pem"), + &dir.path().join("server-key.pem"), + Some(&dir.path().join("ca.pem")), + false, + Some(&dir.path().join("ext-cert.pem")), + Some(&dir.path().join("ext-key.pem")), + vec!["external.example.com".to_string()], + ) + .expect("failed to build acceptor"); + + let client_config = build_test_client_config(&dir.path().join("ca.pem")); + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("failed to bind"); + let listen_addr = listener.local_addr().expect("failed to get local addr"); + + // --- Connection 1: SNI matches external name → external cert --- + let acceptor_srv = acceptor.clone(); + let server_task = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept failed"); + acceptor_srv + .acceptor() + .accept(stream) + .await + .expect("TLS accept failed") + }); + + let connector = tokio_rustls::TlsConnector::from(client_config.clone()); + let tcp = TcpStream::connect(listen_addr) + .await + .expect("connect failed"); + let server_name = "external.example.com" + .try_into() + .expect("invalid server name"); + let tls = connector + .connect(server_name, tcp) + .await + .expect("TLS connect failed"); + + assert_eq!( + peer_cert_der(&tls), + external_der, + "SNI matching external name should serve external cert" + ); + drop(tls); + let _ = server_task.await; + + // --- Connection 2: SNI = "localhost" → internal cert --- + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("failed to bind"); + let listen_addr = listener.local_addr().expect("failed to get local addr"); + + let acceptor_srv = acceptor.clone(); + let server_task = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept failed"); + acceptor_srv + .acceptor() + .accept(stream) + .await + .expect("TLS accept failed") + }); + + let connector = tokio_rustls::TlsConnector::from(client_config); + let tcp = TcpStream::connect(listen_addr) + .await + .expect("connect failed"); + let server_name = "localhost".try_into().expect("invalid server name"); + let tls = connector + .connect(server_name, tcp) + .await + .expect("TLS connect failed"); + + assert_eq!( + peer_cert_der(&tls), + internal_der, + "SNI not matching external name should serve internal cert" + ); + drop(tls); + let _ = server_task.await; + } } diff --git a/crates/openshell-server/tests/edge_tunnel_auth.rs b/crates/openshell-server/tests/edge_tunnel_auth.rs index 4df221f117..be70e45675 100644 --- a/crates/openshell-server/tests/edge_tunnel_auth.rs +++ b/crates/openshell-server/tests/edge_tunnel_auth.rs @@ -168,6 +168,9 @@ async fn mtls_valid_client_cert_accepted() { &temp.path().join("server-key.pem"), Some(temp.path().join("ca.pem").as_path()), false, + None, + None, + Vec::new(), ) .unwrap(); @@ -209,6 +212,9 @@ async fn no_client_cert_accepted_with_ca_configured() { &temp.path().join("server-key.pem"), Some(temp.path().join("ca.pem").as_path()), false, + None, + None, + Vec::new(), ) .unwrap(); @@ -252,6 +258,9 @@ async fn bearer_header_reaches_server_without_client_cert() { &temp.path().join("server-key.pem"), Some(temp.path().join("ca.pem").as_path()), false, + None, + None, + Vec::new(), ) .unwrap(); @@ -283,6 +292,9 @@ async fn rogue_cert_rejected() { &temp.path().join("server-key.pem"), Some(temp.path().join("ca.pem").as_path()), false, + None, + None, + Vec::new(), ) .unwrap(); @@ -329,6 +341,9 @@ async fn https_only_no_client_cert_required() { &temp.path().join("server-key.pem"), None, false, + None, + None, + Vec::new(), ) .unwrap(); diff --git a/crates/openshell-server/tests/multiplex_tls_integration.rs b/crates/openshell-server/tests/multiplex_tls_integration.rs index 4e17fdef97..3447aad517 100644 --- a/crates/openshell-server/tests/multiplex_tls_integration.rs +++ b/crates/openshell-server/tests/multiplex_tls_integration.rs @@ -62,6 +62,9 @@ async fn serves_grpc_and_http_over_tls_on_same_port() { &temp.path().join("server-key.pem"), Some(temp.path().join("ca.pem").as_path()), false, + None, + None, + Vec::new(), ) .unwrap(); @@ -101,6 +104,9 @@ async fn mtls_valid_client_cert_accepted() { &temp.path().join("server-key.pem"), Some(temp.path().join("ca.pem").as_path()), false, + None, + None, + Vec::new(), ) .unwrap(); @@ -129,6 +135,9 @@ async fn no_client_cert_accepted_with_ca() { &temp.path().join("server-key.pem"), Some(temp.path().join("ca.pem").as_path()), false, + None, + None, + Vec::new(), ) .unwrap(); @@ -165,6 +174,9 @@ async fn no_client_cert_rejected_when_required() { &temp.path().join("server-key.pem"), Some(temp.path().join("ca.pem").as_path()), true, + None, + None, + Vec::new(), ) .unwrap(); @@ -202,6 +214,9 @@ async fn mtls_wrong_ca_client_cert_rejected() { &temp.path().join("server-key.pem"), Some(temp.path().join("ca.pem").as_path()), false, + None, + None, + Vec::new(), ) .unwrap(); diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index d4310cb9a7..e8b660df25 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -144,10 +144,11 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | certManager.caSecretName | string | `"openshell-ca-tls"` | Secret created for the intermediate CA (Certificate with isCA: true). | | certManager.certificateDuration | string | `"8760h"` | Duration for cert-manager-issued certificates. | | certManager.certificateRenewBefore | string | `"720h"` | Renewal window for cert-manager-issued certificates. | -| certManager.clientCaFromServerTlsSecret | bool | `true` | Mount gateway client CA from the server TLS secret's ca.crt (populated by cert-manager for certs issued by a CA Issuer). Avoids a separate openshell-server-client-ca Secret. | +| certManager.clientCaFromServerTlsSecret | bool | `true` | Mount gateway client CA from the server TLS secret's ca.crt (populated by cert-manager for certs issued by a CA Issuer). Set to false when serverIssuerRef points at an external issuer (its secret's ca.crt would be that issuer's chain, not the CA that signs the client cert). When false, also set server.tls.clientCaSecretName to a secret containing the actual client CA — for example caSecretName's value, since that's the CA the client certificate above is issued from by default. | | certManager.enabled | bool | `false` | Create cert-manager Issuer and Certificate resources. When enabled, cert-manager owns TLS and the chart runs a JWT-only certgen hook to create the sandbox JWT signing Secret that cert-manager does not manage. | | certManager.serverDnsNames | list | `["openshell","openshell.openshell.svc","openshell.openshell.svc.cluster.local","localhost","openshell.localhost","*.openshell.localhost","host.docker.internal"]` | DNS SANs on the cert-manager-issued server certificate. | | certManager.serverIpAddresses | list | `["127.0.0.1"]` | IP SANs on the cert-manager-issued server certificate. | +| certManager.serverIssuerRef | object | `{"group":"","kind":"","name":""}` | Override the issuerRef for the server Certificate (e.g. a real ACME ClusterIssuer for a publicly-trusted cert on an external hostname). Leave name empty to use the chart's own self-signed CA issuer (default). | | fullnameOverride | string | `""` | Override the full generated resource name. | | grpcRoute.enabled | bool | `false` | Create a Gateway API GRPCRoute for the gateway service. | | grpcRoute.gateway.className | string | `"eg"` | GatewayClass to reference. Envoy Gateway installs one named "eg". | @@ -166,6 +167,9 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | nameOverride | string | `"openshell"` | Override the chart name used in generated resource names. | | networkPolicy.enabled | bool | `true` | Create a NetworkPolicy restricting SSH ingress on sandbox pods to the gateway. | | nodeSelector | object | `{}` | Node selector for the gateway pod. | +| openshiftRoute.annotations | object | `{}` | Extra annotations on the Route (e.g. haproxy.router.openshift.io/*). | +| openshiftRoute.enabled | bool | `false` | Create an OpenShift Route with TLS passthrough. | +| openshiftRoute.host | string | `""` | Hostname for the Route. Must match a SAN on the gateway's server cert. | | pkiInitJob.enabled | bool | `true` | Run a pre-install/pre-upgrade Job that creates gateway and client mTLS Secrets. When certManager.enabled=true, cert-manager owns TLS and this same hook runs in JWT-only mode even if pkiInitJob.enabled remains true. | | pkiInitJob.serverDnsNames | list | `[]` | Extra DNS SANs to append to the server certificate. | | pkiInitJob.serverIpAddresses | list | `[]` | Extra IP SANs to append to the server certificate. | diff --git a/deploy/helm/openshell/ci/values-openshift-route-cert-manager.yaml b/deploy/helm/openshell/ci/values-openshift-route-cert-manager.yaml new file mode 100644 index 0000000000..f1e8b4f7ef --- /dev/null +++ b/deploy/helm/openshell/ci/values-openshift-route-cert-manager.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Render-coverage overlay for cert-manager issuing the server certificate from +# an external Issuer/ClusterIssuer (e.g. a real ACME issuer), plus an +# OpenShift Route with TLS passthrough. Merge after values.yaml: +# helm lint deploy/helm/openshell -f ci/values-openshift-route-cert-manager.yaml +# +# The ClusterIssuer name below is a placeholder for render coverage; a real +# deployment must reference an Issuer/ClusterIssuer that's actually installed +# and Ready in the target cluster. See docs/kubernetes/managing-certificates.mdx. + +server: + disableTls: false + tls: + # Must name a secret containing the actual client CA when + # clientCaFromServerTlsSecret is false — here, the same CA secret + # cert-manager creates for signing the client (mTLS) certificate. + clientCaSecretName: openshell-ca-tls + +certManager: + enabled: true + clientCaFromServerTlsSecret: false + serverIssuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + serverDnsNames: + - openshell.example.com + +openshiftRoute: + enabled: true + host: openshell.example.com diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 5931047e5f..d455cefab3 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -84,6 +84,11 @@ spec: - name: tls-cert mountPath: /etc/openshell-tls/server readOnly: true + {{- if .Values.certManager.serverIssuerRef.name }} + - name: tls-external-cert + mountPath: /etc/openshell-tls/server-external + readOnly: true + {{- end }} {{- if or .Values.server.tls.clientCaSecretName (and .Values.pkiInitJob.enabled (not .Values.certManager.enabled)) (and .Values.certManager.enabled .Values.certManager.clientCaFromServerTlsSecret) }} - name: tls-client-ca mountPath: /etc/openshell-tls/client-ca @@ -144,6 +149,11 @@ spec: - name: tls-cert secret: secretName: {{ .Values.server.tls.certSecretName }} + {{- if .Values.certManager.serverIssuerRef.name }} + - name: tls-external-cert + secret: + secretName: {{ include "openshell.fullname" . }}-server-external-tls + {{- end }} {{- if or .Values.server.tls.clientCaSecretName (and .Values.pkiInitJob.enabled (not .Values.certManager.enabled)) (and .Values.certManager.enabled .Values.certManager.clientCaFromServerTlsSecret) }} - name: tls-client-ca secret: diff --git a/deploy/helm/openshell/templates/cert-manager-pki.yaml b/deploy/helm/openshell/templates/cert-manager-pki.yaml index fdd702a305..cd9da7ebab 100644 --- a/deploy/helm/openshell/templates/cert-manager-pki.yaml +++ b/deploy/helm/openshell/templates/cert-manager-pki.yaml @@ -42,6 +42,27 @@ spec: ca: secretName: {{ .Values.certManager.caSecretName | quote }} --- +{{- $externalServerIssuer := .Values.certManager.serverIssuerRef.name }} +{{- if and (not .Values.certManager.clientCaFromServerTlsSecret) (eq .Values.server.tls.clientCaSecretName "openshell-server-client-ca") }} +{{- fail "certManager.clientCaFromServerTlsSecret is false but server.tls.clientCaSecretName is still the default (openshell-server-client-ca), which nothing creates when cert-manager owns TLS. Set server.tls.clientCaSecretName to the secret containing the client CA (e.g. the value of certManager.caSecretName, which defaults to openshell-ca-tls), or set it to empty to disable mTLS client verification." }} +{{- end }} +{{- if and $externalServerIssuer .Values.certManager.clientCaFromServerTlsSecret }} +{{- fail "certManager.serverIssuerRef.name is set but certManager.clientCaFromServerTlsSecret is still true \u2014 the server cert Secret's ca.crt comes from the external issuer, not the CA that signs the client (mTLS) certificate. Set certManager.clientCaFromServerTlsSecret=false and set server.tls.clientCaSecretName to the secret containing the client CA (e.g. the value of certManager.caSecretName, which defaults to openshell-ca-tls)." }} +{{- end }} +{{- if $externalServerIssuer }} +{{- if not .Values.certManager.serverDnsNames }} +{{- fail "certManager.serverIssuerRef.name is set but certManager.serverDnsNames is empty — the external certificate requires at least one externally-resolvable DNS name." }} +{{- end }} +{{- range .Values.certManager.serverDnsNames }} +{{- /* Single-label names (e.g. "openshell") are also rejected by ACME CAs but are intentionally not checked here — the guard targets recognisable internal-network patterns. */ -}} +{{- if or (eq . "localhost") (hasSuffix ".localhost" .) (hasSuffix ".svc.cluster.local" .) (hasSuffix ".svc" .) (eq . "host.docker.internal") (eq . "host.containers.internal") }} +{{- fail (printf "certManager.serverIssuerRef.name is set (external issuer) but certManager.serverDnsNames contains %q — external CAs (e.g. ACME / Let's Encrypt) reject internal-only names per CA/Browser Forum baseline requirements. Override certManager.serverDnsNames with your externally-resolvable hostname(s)." .) }} +{{- end }} +{{- end }} +{{- end }} +# Internal server certificate — always issued by the chart’s own CA with +# internal SANs. Supervisors connect via internal hostnames and verify +# this cert against the chart CA they already trust. apiVersion: cert-manager.io/v1 kind: Certificate metadata: @@ -53,14 +74,16 @@ spec: secretName: {{ .Values.server.tls.certSecretName | quote }} duration: {{ .Values.certManager.certificateDuration | quote }} renewBefore: {{ .Values.certManager.certificateRenewBefore | quote }} - commonName: openshell-server + commonName: {{ include "openshell.fullname" . }} dnsNames: {{- range (include "openshell.defaultServerDnsNames" . | fromYamlArray) }} - {{ . | quote }} {{- end }} + {{- if not $externalServerIssuer }} {{- range .Values.certManager.serverDnsNames }} - {{ . | quote }} {{- end }} + {{- end }} {{- if .Values.certManager.serverIpAddresses }} ipAddresses: {{- toYaml .Values.certManager.serverIpAddresses | nindent 4 }} @@ -76,6 +99,41 @@ spec: name: {{ include "openshell.fullname" . }}-ca-issuer kind: Issuer group: cert-manager.io +{{- if $externalServerIssuer }} +--- +# External server certificate — issued by the operator-configured issuer +# (e.g. ACME/Let’s Encrypt) with only externally-resolvable SANs. +# The gateway uses SNI to present this cert for external hostnames. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "openshell.fullname" . }}-server-external + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell.labels" . | nindent 4 }} +spec: + secretName: {{ include "openshell.fullname" . }}-server-external-tls + duration: {{ .Values.certManager.certificateDuration | quote }} + renewBefore: {{ .Values.certManager.certificateRenewBefore | quote }} + {{- if .Values.certManager.serverDnsNames }} + commonName: {{ first .Values.certManager.serverDnsNames | quote }} + {{- end }} + dnsNames: + {{- range .Values.certManager.serverDnsNames }} + - {{ . | quote }} + {{- end }} + privateKey: + algorithm: ECDSA + size: 256 + usages: + - server auth + - digital signature + - key encipherment + issuerRef: + name: {{ .Values.certManager.serverIssuerRef.name }} + kind: {{ .Values.certManager.serverIssuerRef.kind | default "Issuer" }} + group: {{ .Values.certManager.serverIssuerRef.group | default "cert-manager.io" }} +{{- end }} --- apiVersion: cert-manager.io/v1 kind: Certificate diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 0c2fc3bbd4..1fcff7996b 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -81,6 +81,11 @@ data: cert_path = "/etc/openshell-tls/server/tls.crt" key_path = "/etc/openshell-tls/server/tls.key" client_ca_path = "/etc/openshell-tls/client-ca/ca.crt" + {{- if .Values.certManager.serverIssuerRef.name }} + external_cert_path = "/etc/openshell-tls/server-external/tls.crt" + external_key_path = "/etc/openshell-tls/server-external/tls.key" + external_server_names = [{{- range $i, $name := .Values.certManager.serverDnsNames }}{{ if $i }}, {{ end }}{{ $name | quote }}{{- end }}] + {{- end }} {{- end }} {{- if .Values.server.auth.allowUnauthenticatedUsers }} diff --git a/deploy/helm/openshell/templates/route.yaml b/deploy/helm/openshell/templates/route.yaml new file mode 100644 index 0000000000..292b71b67a --- /dev/null +++ b/deploy/helm/openshell/templates/route.yaml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.openshiftRoute.enabled }} +{{- if .Values.server.disableTls }} +{{- fail "openshiftRoute.enabled=true requires TLS (server.disableTls must be false) \u2014 a passthrough Route forwards encrypted traffic by SNI, so the gateway must terminate its own TLS." }} +{{- end }} +{{- if and .Values.openshiftRoute.host .Values.certManager.serverIssuerRef.name .Values.certManager.serverDnsNames (not (has .Values.openshiftRoute.host .Values.certManager.serverDnsNames)) }} +{{- fail (printf "openshiftRoute.host %q is not listed in certManager.serverDnsNames %v — the Route will forward SNI for a hostname the external certificate does not cover, causing TLS verification failures for CLI clients." .Values.openshiftRoute.host .Values.certManager.serverDnsNames) }} +{{- end }} +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ include "openshell.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "openshell.labels" . | nindent 4 }} + {{- with .Values.openshiftRoute.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.openshiftRoute.host }} + host: {{ .Values.openshiftRoute.host | quote }} + {{- end }} + to: + kind: Service + name: {{ include "openshell.fullname" . }} + port: + targetPort: grpc + tls: + termination: passthrough + wildcardPolicy: None +{{- end }} diff --git a/deploy/helm/openshell/tests/cert_manager_pki_test.yaml b/deploy/helm/openshell/tests/cert_manager_pki_test.yaml new file mode 100644 index 0000000000..089b3b5037 --- /dev/null +++ b/deploy/helm/openshell/tests/cert_manager_pki_test.yaml @@ -0,0 +1,192 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: cert-manager PKI issuerRef overrides +templates: + - templates/cert-manager-pki.yaml +release: + name: openshell + namespace: my-namespace + +tests: + - it: defaults both server and client Certificates to the chart's own CA issuer + template: templates/cert-manager-pki.yaml + set: + certManager.enabled: true + asserts: + - equal: + path: spec.issuerRef.name + value: openshell-ca-issuer + documentIndex: 3 + - equal: + path: spec.issuerRef.kind + value: Issuer + documentIndex: 3 + - equal: + path: spec.issuerRef.name + value: openshell-ca-issuer + documentIndex: 4 + - equal: + path: spec.issuerRef.kind + value: Issuer + documentIndex: 4 + + - it: default server Certificate includes internal SANs, IPs, and a fixed commonName + template: templates/cert-manager-pki.yaml + set: + certManager.enabled: true + asserts: + - equal: + path: spec.commonName + value: openshell + documentIndex: 3 + - contains: + path: spec.dnsNames + content: openshell.my-namespace.svc.cluster.local + documentIndex: 3 + - contains: + path: spec.dnsNames + content: localhost + documentIndex: 3 + - equal: + path: spec.ipAddresses[0] + value: 127.0.0.1 + documentIndex: 3 + + - it: internal server cert keeps chart CA issuer and internal SANs when serverIssuerRef is set + template: templates/cert-manager-pki.yaml + set: + certManager.enabled: true + certManager.serverIssuerRef.name: letsencrypt-prod + certManager.serverIssuerRef.kind: ClusterIssuer + certManager.clientCaFromServerTlsSecret: false + server.tls.clientCaSecretName: openshell-ca-tls + certManager.serverDnsNames: + - openshell.example.com + asserts: + # Internal cert (doc 3) stays on chart CA + - equal: + path: spec.issuerRef.name + value: openshell-ca-issuer + documentIndex: 3 + - equal: + path: spec.issuerRef.kind + value: Issuer + documentIndex: 3 + # Internal cert has internal SANs + - equal: + path: spec.commonName + value: openshell + documentIndex: 3 + - contains: + path: spec.dnsNames + content: openshell.my-namespace.svc.cluster.local + documentIndex: 3 + - contains: + path: spec.dnsNames + content: localhost + documentIndex: 3 + # Internal cert does NOT include external-only hostnames + - notContains: + path: spec.dnsNames + content: openshell.example.com + documentIndex: 3 + + - it: creates external server Certificate from serverIssuerRef with external SANs only + template: templates/cert-manager-pki.yaml + set: + certManager.enabled: true + certManager.serverIssuerRef.name: letsencrypt-prod + certManager.serverIssuerRef.kind: ClusterIssuer + certManager.clientCaFromServerTlsSecret: false + server.tls.clientCaSecretName: openshell-ca-tls + certManager.serverDnsNames: + - openshell.example.com + asserts: + # External cert (doc 4) uses the external issuer + - equal: + path: spec.issuerRef.name + value: letsencrypt-prod + documentIndex: 4 + - equal: + path: spec.issuerRef.kind + value: ClusterIssuer + documentIndex: 4 + - equal: + path: spec.issuerRef.group + value: cert-manager.io + documentIndex: 4 + # External cert has only external SANs + - equal: + path: spec.commonName + value: openshell.example.com + documentIndex: 4 + - equal: + path: spec.dnsNames + value: + - openshell.example.com + documentIndex: 4 + # External cert has no IP addresses + - notExists: + path: spec.ipAddresses + documentIndex: 4 + # External cert has no internal names + - notContains: + path: spec.dnsNames + content: localhost + documentIndex: 4 + + - it: fails when serverIssuerRef is set but clientCaFromServerTlsSecret is true + template: templates/cert-manager-pki.yaml + set: + certManager.enabled: true + certManager.serverIssuerRef.name: letsencrypt-prod + certManager.serverIssuerRef.kind: ClusterIssuer + certManager.clientCaFromServerTlsSecret: true + certManager.serverDnsNames: + - openshell.example.com + asserts: + - failedTemplate: + errorMessage: "certManager.serverIssuerRef.name is set but certManager.clientCaFromServerTlsSecret is still true \u2014 the server cert Secret's ca.crt comes from the external issuer, not the CA that signs the client (mTLS) certificate. Set certManager.clientCaFromServerTlsSecret=false and set server.tls.clientCaSecretName to the secret containing the client CA (e.g. the value of certManager.caSecretName, which defaults to openshell-ca-tls)." + + - it: fails when serverIssuerRef is set but serverDnsNames contains internal-only names + template: templates/cert-manager-pki.yaml + set: + certManager.enabled: true + certManager.serverIssuerRef.name: letsencrypt-prod + certManager.clientCaFromServerTlsSecret: false + server.tls.clientCaSecretName: openshell-ca-tls + # serverDnsNames is left at the default which contains "openshell.openshell.svc", "localhost", etc. + asserts: + - failedTemplate: + errorMessage: "certManager.serverIssuerRef.name is set (external issuer) but certManager.serverDnsNames contains \"openshell.openshell.svc\" \u2014 external CAs (e.g. ACME / Let's Encrypt) reject internal-only names per CA/Browser Forum baseline requirements. Override certManager.serverDnsNames with your externally-resolvable hostname(s)." + + - it: fails when clientCaFromServerTlsSecret is false but clientCaSecretName is the default + template: templates/cert-manager-pki.yaml + set: + certManager.enabled: true + certManager.clientCaFromServerTlsSecret: false + asserts: + - failedTemplate: + errorMessage: "certManager.clientCaFromServerTlsSecret is false but server.tls.clientCaSecretName is still the default (openshell-server-client-ca), which nothing creates when cert-manager owns TLS. Set server.tls.clientCaSecretName to the secret containing the client CA (e.g. the value of certManager.caSecretName, which defaults to openshell-ca-tls), or set it to empty to disable mTLS client verification." + + - it: client Certificate issuerRef is unaffected by serverIssuerRef + template: templates/cert-manager-pki.yaml + set: + certManager.enabled: true + certManager.serverIssuerRef.name: letsencrypt-prod + certManager.serverIssuerRef.kind: ClusterIssuer + certManager.clientCaFromServerTlsSecret: false + server.tls.clientCaSecretName: openshell-ca-tls + certManager.serverDnsNames: + - openshell.example.com + asserts: + # Client cert is now doc 5 (after internal + external server certs) + - equal: + path: spec.issuerRef.name + value: openshell-ca-issuer + documentIndex: 5 + - equal: + path: spec.issuerRef.kind + value: Issuer + documentIndex: 5 diff --git a/deploy/helm/openshell/tests/route_test.yaml b/deploy/helm/openshell/tests/route_test.yaml new file mode 100644 index 0000000000..e12b7ce2b7 --- /dev/null +++ b/deploy/helm/openshell/tests/route_test.yaml @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: OpenShift Route +templates: + - templates/route.yaml +release: + name: openshell + namespace: my-namespace + +tests: + - it: renders nothing by default + asserts: + - hasDocuments: + count: 0 + + - it: renders a passthrough Route when enabled + set: + openshiftRoute.enabled: true + openshiftRoute.host: openshell.apps.example.com + asserts: + - isKind: + of: Route + - equal: + path: apiVersion + value: route.openshift.io/v1 + - equal: + path: spec.host + value: openshell.apps.example.com + - equal: + path: spec.to.kind + value: Service + - equal: + path: spec.to.name + value: openshell + - equal: + path: spec.port.targetPort + value: grpc + - equal: + path: spec.tls.termination + value: passthrough + - equal: + path: spec.wildcardPolicy + value: None + + - it: omits host when not set + set: + openshiftRoute.enabled: true + asserts: + - notExists: + path: spec.host + + - it: fails when passthrough Route is enabled with TLS disabled + set: + openshiftRoute.enabled: true + server.disableTls: true + asserts: + - failedTemplate: + errorMessage: "openshiftRoute.enabled=true requires TLS (server.disableTls must be false) \u2014 a passthrough Route forwards encrypted traffic by SNI, so the gateway must terminate its own TLS." + + - it: renders custom annotations + set: + openshiftRoute.enabled: true + openshiftRoute.host: openshell.apps.example.com + openshiftRoute.annotations: + haproxy.router.openshift.io/balance: roundrobin + asserts: + - equal: + path: metadata.annotations["haproxy.router.openshift.io/balance"] + value: roundrobin diff --git a/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml b/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml index a7b02310cf..1d744b35aa 100644 --- a/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml +++ b/deploy/helm/openshell/tests/statefulset_client_ca_test.yaml @@ -53,13 +53,14 @@ tests: certManager.enabled: true certManager.clientCaFromServerTlsSecret: false pkiInitJob.enabled: true + server.tls.clientCaSecretName: openshell-ca-tls asserts: - equal: path: spec.template.spec.volumes[3].name value: tls-client-ca - equal: path: spec.template.spec.volumes[3].secret.secretName - value: openshell-server-client-ca + value: openshell-ca-tls - notExists: path: spec.template.spec.volumes[3].secret.items diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 0525ed475d..c12d093d0b 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -345,7 +345,7 @@ pkiInitJob: serverIpAddresses: [] # cert-manager Certificate/Issuer resources (requires cert-manager CRDs in-cluster). -# Uses namespaced Issuers only (no ClusterIssuer). Does not install cert-manager itself. +# Does not install cert-manager itself. certManager: # -- Create cert-manager Issuer and Certificate resources. When enabled, # cert-manager owns TLS and the chart runs a JWT-only certgen hook to create @@ -353,9 +353,20 @@ certManager: enabled: false # -- Secret created for the intermediate CA (Certificate with isCA: true). caSecretName: openshell-ca-tls + # -- Override the issuerRef for the server Certificate (e.g. a real ACME + # ClusterIssuer for a publicly-trusted cert on an external hostname). Leave + # name empty to use the chart's own self-signed CA issuer (default). + serverIssuerRef: + name: "" + kind: "" + group: "" # -- Mount gateway client CA from the server TLS secret's ca.crt (populated by - # cert-manager for certs issued by a CA Issuer). Avoids a separate - # openshell-server-client-ca Secret. + # cert-manager for certs issued by a CA Issuer). Set to false when + # serverIssuerRef points at an external issuer (its secret's ca.crt would be + # that issuer's chain, not the CA that signs the client cert). When false, + # also set server.tls.clientCaSecretName to a secret containing the actual + # client CA — for example caSecretName's value, since that's the CA the + # client certificate above is issued from by default. clientCaFromServerTlsSecret: true # -- Duration for cert-manager-issued certificates. certificateDuration: 8760h @@ -415,3 +426,15 @@ grpcRoute: # or the existing openshell-server-tls Secret (its SANs must include the # external hostname). certificateRefs: [] + +# OpenShift Route with TLS passthrough. The gateway terminates its own +# TLS/mTLS; the router only forwards based on SNI, so it never sees plaintext +# or the client certificate. Requires server.disableTls=false and a server +# cert whose SANs include the Route host (see certManager.serverIssuerRef). +openshiftRoute: + # -- Create an OpenShift Route with TLS passthrough. + enabled: false + # -- Hostname for the Route. Must match a SAN on the gateway's server cert. + host: "" + # -- Extra annotations on the Route (e.g. haproxy.router.openshift.io/*). + annotations: {} diff --git a/docs/kubernetes/managing-certificates.mdx b/docs/kubernetes/managing-certificates.mdx index b66419b505..39de163cbe 100644 --- a/docs/kubernetes/managing-certificates.mdx +++ b/docs/kubernetes/managing-certificates.mdx @@ -60,6 +60,65 @@ The chart also runs a pre-install hook in JWT-only mode to create the gateway's sandbox JWT signing Secret. That Secret is separate from the cert-manager TLS certificate Secrets and is mounted at `/etc/openshell-jwt`. +## Using a real Issuer for the server certificate + +By default, cert-manager issues both the server and client certificates from +a self-signed CA the chart creates — this rotates automatically, but the +server certificate is still not publicly trusted. `certManager.serverIssuerRef` +overrides the `issuerRef` on the server `Certificate` resource to point at a +real `Issuer` or `ClusterIssuer` instead, for example an ACME issuer: + +```shell +helm upgrade --install openshell \ + oci://ghcr.io/nvidia/openshell/helm-chart \ + --version \ + --namespace openshell \ + --set certManager.enabled=true \ + --set certManager.clientCaFromServerTlsSecret=false \ + --set server.tls.clientCaSecretName=openshell-ca-tls \ + --set certManager.serverIssuerRef.name=letsencrypt-prod \ + --set certManager.serverIssuerRef.kind=ClusterIssuer \ + --set certManager.serverDnsNames[0]=openshell.example.com +``` + +### Dual certificate architecture + +When `serverIssuerRef` is set, the chart creates **two** server certificates: + +1. **Internal certificate** (`openshell-server-tls`): signed by the chart CA + with internal SANs (`*.svc.cluster.local`, `localhost`, etc.). +2. **External certificate** (`openshell-server-external-tls`): signed by the + configured issuer (e.g. ACME) with only the hostnames from + `certManager.serverDnsNames`. + +The gateway uses **SNI** to select which certificate to present: +supervisors connect via internal service names and receive the internal +certificate (verified against the chart CA they already trust), while CLI +users connecting through a Route or ingress use the external hostname and +receive the ACME certificate. This keeps supervisor trust pinned to only +the operator's chart CA — no WebPKI root trust is needed. + + +Public CAs such as Let's Encrypt reject certificate requests that include +internal-only names per CA/Browser Forum baseline requirements. The chart +validates this at install time and fails with an actionable error if +`certManager.serverDnsNames` contains internal-only entries while +`serverIssuerRef` is set. + +You do **not** need to set `server.grpcEndpoint` to the external hostname. +Supervisors connect via the internal service name automatically. Setting +`server.grpcEndpoint` to an external hostname would cause supervisors to +receive the ACME certificate (via SNI) which they cannot verify against the +chart CA. + + +Set `certManager.clientCaFromServerTlsSecret=false` whenever `serverIssuerRef` +is set, and set `server.tls.clientCaSecretName` to a secret containing the +client CA — by default that's `certManager.caSecretName` (`openshell-ca-tls`), +the CA the chart issues the client certificate from. + ## Next Steps -Return to [Setup](/kubernetes/setup) to complete the installation. +Return to [Setup](/kubernetes/setup) to complete the installation. For +exposing the gateway externally on OpenShift with a real certificate, see +[OpenShift](/kubernetes/openshift). diff --git a/docs/kubernetes/openshift.mdx b/docs/kubernetes/openshift.mdx index 7512eaa65e..43666b9410 100644 --- a/docs/kubernetes/openshift.mdx +++ b/docs/kubernetes/openshift.mdx @@ -87,8 +87,59 @@ openshell gateway add http://127.0.0.1:8080 --local --name openshift openshell status ``` +## Production: expose externally with a real certificate + +The steps above run the gateway over plaintext HTTP for quick evaluation. For +a real deployment, cert-manager can issue the gateway's server certificate +from a real Issuer or ClusterIssuer (for example, an ACME issuer), and an +OpenShift Route with TLS passthrough exposes it externally while the gateway +keeps terminating its own TLS and mTLS. + +Install cert-manager and configure a working `ClusterIssuer` first — see +[Managing Certificates](/kubernetes/managing-certificates) for the +`certManager.serverIssuerRef` details. Configure an OIDC provider as described +in [Access Control](/kubernetes/access-control) — remote gateways authenticate +CLI users via OIDC, not mTLS, so the gateway must know the OIDC issuer URL. +Install the chart with: + +```shell +helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ + --version \ + --namespace openshell \ + --set podSecurityContext.fsGroup=null \ + --set securityContext.runAsUser=null \ + --set server.disableTls=false \ + --set certManager.enabled=true \ + --set certManager.clientCaFromServerTlsSecret=false \ + --set server.tls.clientCaSecretName=openshell-ca-tls \ + --set certManager.serverIssuerRef.name= \ + --set certManager.serverIssuerRef.kind=ClusterIssuer \ + --set certManager.serverDnsNames[0]= \ + --set openshiftRoute.enabled=true \ + --set openshiftRoute.host= \ + --set server.oidc.issuer= \ + --set server.oidc.audience= +``` + +| Override | Reason | +|---|---| +| `certManager.clientCaFromServerTlsSecret=false` + `server.tls.clientCaSecretName` | Points the gateway's client-verification CA at `certManager.caSecretName` (the chart's own CA secret, `openshell-ca-tls` by default). Supervisors connect via internal service names and receive the internal certificate signed by this CA. | +| `certManager.serverIssuerRef` | Creates a second server certificate from your Issuer or ClusterIssuer for external clients. The gateway uses SNI to present this cert for the external hostname while continuing to present the internal (chart CA) cert to supervisors. | +| `openshiftRoute.enabled` / `openshiftRoute.host` | Creates an OpenShift Route with TLS passthrough — the router forwards the encrypted connection by SNI without decrypting, so the gateway uses the SNI hostname to select the external certificate. | +| `server.oidc.issuer` / `server.oidc.audience` | Configures server-side OIDC validation. Without these, the gateway expects mTLS client certificates and rejects OIDC-only CLI connections. See [Access Control](/kubernetes/access-control). | + +Register the gateway with the CLI over OIDC. Remote gateways authenticate CLI +users via OIDC, not mTLS — see [Access Control](/kubernetes/access-control): + +```shell +openshell gateway add https:// \ + --name openshift \ + --oidc-issuer +openshell gateway login openshift +``` + ## Next Steps -- For TLS-enabled deployments, refer to [Managing Certificates](/kubernetes/managing-certificates). -- To expose the gateway externally, refer to [Ingress](/kubernetes/ingress). +- For more on certificate provisioning modes, refer to [Managing Certificates](/kubernetes/managing-certificates). +- To expose the gateway externally through the Kubernetes Gateway API instead of a Route, refer to [Ingress](/kubernetes/ingress). - To configure OIDC authentication, refer to [Access Control](/kubernetes/access-control). diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 4db3c9c472..05735a8377 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -133,6 +133,10 @@ cert_path = "/etc/openshell/certs/gateway.pem" key_path = "/etc/openshell/certs/gateway-key.pem" client_ca_path = "/etc/openshell/certs/client-ca.pem" require_client_auth = false +# Optional: SNI-based dual certificate for external (e.g. ACME) TLS. +# external_cert_path = "/etc/openshell/certs/external.pem" +# external_key_path = "/etc/openshell/certs/external-key.pem" +# external_server_names = ["gateway.example.com"] [openshell.gateway.gateway_jwt] signing_key_path = "/etc/openshell/jwt/signing.pem" @@ -184,6 +188,8 @@ phases = ["validate"] Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth] enabled = true` to authenticate CLI callers from verified client certificates. Kubernetes deployments must leave this unset and use OIDC or a trusted access proxy; the Helm chart does not render this table. +`[openshell.gateway.tls]` supports optional SNI-based dual-certificate mode for deployments that need separate internal and external server certificates. Set `external_cert_path` and `external_key_path` to point at the external (e.g. ACME/publicly-trusted) certificate and key. List the hostnames that should be served with the external certificate in `external_server_names`. Connections whose TLS SNI hostname matches one of those names receive the external certificate; all other connections (including those with no SNI) receive the primary internal certificate from `cert_path`/`key_path`. Both fields must be set together — providing only one is a configuration error. On Kubernetes with the Helm chart, the external certificate is managed automatically when `certManager.serverIssuerRef.name` is set; the chart populates these fields from the cert-manager-issued external server certificate. + `[openshell.gateway] policy_validation_failure_mode` controls what sandbox supervisors do when a complete candidate policy fails runtime validation. The default, `fail_closed`, deactivates the previous network policy, closes relays pinned to it, and denies new egress until a valid generation loads. `retain_last_valid` leaves the previous valid generation active. Both modes reject the candidate atomically; startup always fails closed when no previous valid generation exists. Gateway mutation paths that can preflight a known effective scope reject invalid candidates before persistence and leave the active policy unchanged regardless of this setting. Changing the value requires restarting the gateway so it can reload `gateway.toml` and distribute the new posture to sandbox supervisors. `[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. When omitted, it defaults to `0`: the token `exp` claim and `expires_at_ms` response field become `0`, and the sandbox JWT does not expire. Use that default only for local single-player Docker, Podman, or VM gateways. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway uses `0`. @@ -315,6 +321,10 @@ compute_drivers = ["kubernetes"] cert_path = "/etc/openshell-tls/server/tls.crt" key_path = "/etc/openshell-tls/server/tls.key" client_ca_path = "/etc/openshell-tls/client-ca/ca.crt" +# When cert-manager serverIssuerRef is configured, these are populated by Helm: +# external_cert_path = "/etc/openshell-tls/server-external/tls.crt" +# external_key_path = "/etc/openshell-tls/server-external/tls.key" +# external_server_names = ["gateway.example.com"] [openshell.drivers.kubernetes] namespace = "agents"