feat: TLS and env-based endpoint for jumpstarter-telemetry

controller/cmd/telemetry/main.go

@@ -18,6 +18,18 @@ limitations under the License.
 // via the PushLogs gRPC RPC and writes them to structured stdout for downstream
 // log shippers (Promtail, Grafana Alloy, Vector) to forward to Loki.
 //
+// TLS: always enabled. Set EXTERNAL_CERT_PEM and EXTERNAL_KEY_PEM to file paths of
+// operator-mounted cert/key (e.g. from a cert-manager Secret); when absent a
+// self-signed certificate is generated. The self-signed cert PEM is logged at
+// startup — copy it into the controller ConfigMap's telemetry.certificate field
+// so exporters can verify the TLS connection.
+//
+// Endpoint: GRPC_TELEMETRY_ENDPOINT must be set on BOTH this pod and the controller
+// pod to the same value (e.g. "jumpstarter-telemetry.jumpstarter.svc:9093").
+// The telemetry service uses it to generate the correct SAN in the self-signed
+// certificate; the controller uses it to advertise the address to exporters via
+// GetServiceEndpoints. A mismatch causes TLS hostname verification failures.
+//
 // Future phases will add direct Loki push and MetricsStream for reverse-scrape
 // of exporter prometheus_client registries.
 package main
@@ -79,14 +91,16 @@ func main() {
 		Signer:   signer,
 	}
 
+	// Register signal handler before starting the service so no signal
+	// is missed in the window between goroutine start and Notify.
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+
 	errCh := make(chan error, 1)
 	go func() {
 		errCh <- svc.Start(ctx)
 	}()
 
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
-
 	select {
 	case sig := <-sigs:
 		logger.Info("received signal, shutting down", "signal", sig)

controller/internal/config/config.go

@@ -1,8 +1,11 @@
 package config
 
 import (
+	"cmp"
 	"context"
 	"fmt"
+	"net"
+	"os"
 	"time"
 
 	"github.com/jumpstarter-dev/jumpstarter/controller/internal/oidc"
@@ -43,6 +46,34 @@ func LoadRouterConfiguration(
 	return serverOptions, nil
 }
 
+// resolveTelemetryConfig validates and resolves the telemetry endpoint for a
+// Telemetry config block. The GRPC_TELEMETRY_ENDPOINT env var takes priority
+// over the ConfigMap value, allowing operators to override at the pod level.
+// Returns nil when t is nil or disabled.
+func resolveTelemetryConfig(t *Telemetry) (*Telemetry, error) {
+	if t == nil || !t.Enabled {
+		return nil, nil
+	}
+	if err := t.Validate(); err != nil {
+		return nil, err
+	}
+	// Env var takes priority over ConfigMap, allowing operators to override
+	// at the pod level without modifying the ConfigMap. Resolving here ensures
+	// LoadedConfig.Telemetry.Endpoint is always the complete value — callers
+	// don't need to re-check the env var.
+	t.Endpoint = cmp.Or(os.Getenv("GRPC_TELEMETRY_ENDPOINT"), t.Endpoint)
+	if ep := t.Endpoint; ep != "" {
+		host, _, err := net.SplitHostPort(ep)
+		if err != nil {
+			return nil, fmt.Errorf("telemetry endpoint %q is not a valid host:port: %w", ep, err)
+		}
+		if host == "" {
+			return nil, fmt.Errorf("telemetry endpoint %q has no host", ep)
+		}
+	}
+	return t, nil
+}
+
 func LoadConfiguration(
 	ctx context.Context,
 	client client.Reader,
@@ -122,18 +153,9 @@ func LoadConfiguration(
 		return nil, err
 	}
 
-	var telemetry *Telemetry
-	if config.Telemetry != nil && config.Telemetry.Enabled {
-		if err := config.Telemetry.Validate(); err != nil {
-			return nil, err
-		}
-		// Auto-derive the gRPC address when the operator has not overridden it.
-		// The well-known service name follows the same pattern as the controller
-		// and router: <service>.<namespace>.svc (in-cluster DNS).
-		if config.Telemetry.Endpoint == "" {
-			config.Telemetry.Endpoint = "jumpstarter-telemetry." + key.Namespace + ":9093"
-		}
-		telemetry = config.Telemetry
+	telemetry, err := resolveTelemetryConfig(config.Telemetry)
+	if err != nil {
+		return nil, err
 	}
 
 	return &LoadedConfig{

controller/internal/config/types.go

@@ -30,15 +30,22 @@ type Telemetry struct {
 	// When true the controller advertises the endpoint returned by GetServiceEndpoints.
 	Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty"`
 
-	// Endpoint is an optional override for the telemetry gRPC address.
-	// When empty and Enabled is true, defaults to
-	// "jumpstarter-telemetry.<namespace>:9093" derived from the controller namespace.
+	// Endpoint is an optional override for the telemetry gRPC address advertised
+	// to exporters. When empty the controller reads GRPC_TELEMETRY_ENDPOINT from its
+	// own environment (set by the operator on the controller Deployment).
 	Endpoint string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"`
 
-	// Certificate is reserved for a future phase where the telemetry service manages
-	// its own TLS credentials. Leave empty for Phase 1 deployments — the telemetry
-	// server listens on plaintext gRPC and exporters that receive a certificate here
-	// will fail to connect.
+	// Certificate is the PEM-encoded CA certificate that exporters use to verify
+	// the telemetry server's TLS certificate.
+	//
+	// When the operator provisions the telemetry service with a cert-manager-issued
+	// certificate, set this to the issuer's CA certificate.
+	//
+	// When the telemetry service runs in self-signed mode (no EXTERNAL_CERT_PEM /
+	// EXTERNAL_KEY_PEM set), it logs the generated certificate PEM at startup under
+	// the key "certPEM". Copy that value here so exporters can pin and verify it.
+	// A self-signed certificate is not trusted by the system CA pool, so leaving
+	// this field empty means exporters cannot establish a verified TLS connection.
 	Certificate string `json:"certificate,omitempty" yaml:"certificate,omitempty"`
 
 	// Logging configures the log ingestion path to the telemetry service.

controller/internal/config/types_test.go

@@ -276,6 +276,97 @@ func TestDeprecatedLabelsOmitEmpty(t *testing.T) {
 	}
 }
 
+func TestTelemetryEndpointResolution(t *testing.T) {
+	tests := []struct {
+		name         string
+		cfg          *Telemetry
+		envValue     string
+		wantNil      bool
+		wantEndpoint string
+		wantErr      bool
+	}{
+		{
+			name:    "nil config returns nil",
+			cfg:     nil,
+			wantNil: true,
+		},
+		{
+			name:    "disabled config returns nil",
+			cfg:     &Telemetry{Enabled: false, Endpoint: "telemetry:9093"},
+			wantNil: true,
+		},
+		{
+			name:         "env var takes precedence over ConfigMap",
+			cfg:          &Telemetry{Enabled: true, Endpoint: "telemetry.ns.svc:9093"},
+			envValue:     "env-telemetry:9093",
+			wantEndpoint: "env-telemetry:9093",
+		},
+		{
+			name:         "ConfigMap fallback when env var is empty",
+			cfg:          &Telemetry{Enabled: true, Endpoint: "telemetry.ns.svc:9093"},
+			wantEndpoint: "telemetry.ns.svc:9093",
+		},
+		{
+			name:         "both empty yields empty endpoint (no error)",
+			cfg:          &Telemetry{Enabled: true},
+			wantEndpoint: "",
+		},
+		{
+			name:    "malformed ConfigMap value is rejected",
+			cfg:     &Telemetry{Enabled: true, Endpoint: "no-port"},
+			wantErr: true,
+		},
+		{
+			name:     "malformed env var is rejected",
+			cfg:      &Telemetry{Enabled: true},
+			envValue: "garbage-no-port",
+			wantErr:  true,
+		},
+		{
+			name:    "port-only ConfigMap value is rejected",
+			cfg:     &Telemetry{Enabled: true, Endpoint: ":9093"},
+			wantErr: true,
+		},
+		{
+			name:     "port-only env var is rejected",
+			cfg:      &Telemetry{Enabled: true},
+			envValue: ":9093",
+			wantErr:  true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Setenv("GRPC_TELEMETRY_ENDPOINT", tt.envValue)
+
+			resolved, err := resolveTelemetryConfig(tt.cfg)
+
+			if tt.wantErr {
+				if err == nil {
+					t.Fatalf("expected validation error, got nil (resolved=%+v)", resolved)
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if tt.wantNil {
+				if resolved != nil {
+					t.Fatalf("expected nil, got %+v", resolved)
+				}
+				return
+			}
+			var gotEndpoint string
+			if resolved != nil {
+				gotEndpoint = resolved.Endpoint
+			}
+			if gotEndpoint != tt.wantEndpoint {
+				t.Errorf("resolved.Endpoint = %q, want %q", gotEndpoint, tt.wantEndpoint)
+			}
+		})
+	}
+}
+
 func TestParseDuration(t *testing.T) {
 	tests := []struct {
 		input    string

controller/internal/service/controller_service.go

@@ -327,6 +327,11 @@ func (s *ControllerService) GetServiceEndpoints(
 	resp := &pb.GetServiceEndpointsResponse{}
 
 	if s.TelemetryConfig != nil && s.TelemetryConfig.Enabled {
+		// Endpoint is resolved at config-load time (ConfigMap value or GRPC_TELEMETRY_ENDPOINT
+		// env var fallback), so TelemetryConfig.Endpoint is always the complete value here.
+		if s.TelemetryConfig.Endpoint == "" {
+			return nil, status.Error(codes.FailedPrecondition, "telemetry is enabled but no endpoint is configured; set telemetry.endpoint in the ConfigMap or GRPC_TELEMETRY_ENDPOINT on the controller pod")
+		}
 		resp.TelemetryEndpoints = append(resp.TelemetryEndpoints, &pb.TelemetryEndpoint{
 			Endpoint:    s.TelemetryConfig.Endpoint,
 			Certificate: s.TelemetryConfig.Certificate,
@@ -1196,32 +1201,9 @@ func (s *ControllerService) Start(ctx context.Context) error {
 		return err
 	}
 
-	// Load external certificate if provided via environment variables.
-	// Environment variables EXTERNAL_CERT_PEM and EXTERNAL_KEY_PEM should contain the PEM-encoded
-	// certificate and private key respectively. If both are set, they are used; otherwise
-	// a self-signed certificate is generated.
-	var cert *tls.Certificate
-	certPEMPath := os.Getenv("EXTERNAL_CERT_PEM")
-	keyPEMPath := os.Getenv("EXTERNAL_KEY_PEM")
-	if certPEMPath != "" && keyPEMPath != "" {
-		certPEMBytes, err := os.ReadFile(certPEMPath)
-		if err != nil {
-			return fmt.Errorf("failed to read external certificate file: %w", err)
-		}
-		keyPEMBytes, err := os.ReadFile(keyPEMPath)
-		if err != nil {
-			return fmt.Errorf("failed to read external key file: %w", err)
-		}
-		parsedCert, err := tls.X509KeyPair(certPEMBytes, keyPEMBytes)
-		if err != nil {
-			return fmt.Errorf("failed to parse external certificate: %w", err)
-		}
-		cert = &parsedCert
-	} else {
-		cert, err = NewSelfSignedCertificate("jumpstarter controller", dnsnames, ipaddresses)
-		if err != nil {
-			return err
-		}
+	cert, _, err := LoadTLSCertificate("jumpstarter controller", dnsnames, ipaddresses)
+	if err != nil {
+		return err
 	}
 
 	opts := append(s.ServerOptions,
@@ -1264,8 +1246,11 @@ func (s *ControllerService) Start(ctx context.Context) error {
 	// Register gRPC gateway
 	gwmux := gwruntime.NewServeMux()
 
+	// The controller multiplexes gRPC (h2) and REST (http/1.1) on a single port,
+	// so it needs NextProtos — which LoadTLSCredentials doesn't expose.
 	listener, err := tls.Listen("tcp", ":8082", &tls.Config{
 		Certificates: []tls.Certificate{*cert},
+		MinVersion:   tls.VersionTLS12,
 		NextProtos:   []string{"http/1.1", "h2"},
 	})
 	if err != nil {

controller/internal/service/controller_service_test.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/golang-jwt/jwt/v5"
 	jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1"
+	"github.com/jumpstarter-dev/jumpstarter/controller/internal/config"
 	jlog "github.com/jumpstarter-dev/jumpstarter/controller/internal/log"
 	pb "github.com/jumpstarter-dev/jumpstarter/controller/internal/protocol/jumpstarter/v1"
 	"google.golang.org/grpc"
@@ -39,9 +40,11 @@ import (
 	"google.golang.org/grpc/status"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8sruntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	ctrlzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
@@ -2061,6 +2064,57 @@ func (noopAuthorizer) Authorize(_ context.Context, _ authorizer.Attributes) (aut
 	return authorizer.DecisionNoOpinion, "", nil
 }
 
+// passingAuthenticator always authenticates successfully with a fixed user name.
+type passingAuthenticator struct{ userName string }
+
+func (p *passingAuthenticator) AuthenticateContext(_ context.Context) (*authenticator.Response, bool, error) {
+	return &authenticator.Response{User: &user.DefaultInfo{Name: p.userName}}, true, nil
+}
+
+// exporterAttributesGetter returns attributes that identify a fixed Exporter object.
+type exporterAttributesGetter struct{ namespace, name string }
+
+func (e *exporterAttributesGetter) ContextAttributes(_ context.Context, u user.Info) (authorizer.Attributes, error) {
+	return authorizer.AttributesRecord{
+		User:      u,
+		Namespace: e.namespace,
+		Resource:  "Exporter",
+		Name:      e.name,
+	}, nil
+}
+
+// passingAuthorizer always allows.
+type passingAuthorizer struct{}
+
+func (passingAuthorizer) Authorize(_ context.Context, _ authorizer.Attributes) (authorizer.Decision, string, error) {
+	return authorizer.DecisionAllow, "", nil
+}
+
+// authSuccessServiceCtx builds a ControllerService whose authentication always
+// succeeds. A pre-populated Exporter object is stored in the fake client so
+// that VerifyExporterObjectToken can fetch it.
+func authSuccessServiceCtx(t *testing.T, cfg *config.Telemetry) (*ControllerService, context.Context) {
+	t.Helper()
+
+	scheme := k8sruntime.NewScheme()
+	if err := jumpstarterdevv1alpha1.AddToScheme(scheme); err != nil {
+		t.Fatalf("failed to add scheme: %v", err)
+	}
+	exporter := &jumpstarterdevv1alpha1.Exporter{
+		ObjectMeta: metav1.ObjectMeta{Name: "test-exporter", Namespace: "default"},
+	}
+	fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(exporter).Build()
+
+	svc := &ControllerService{
+		Client:          fakeClient,
+		Authn:           &passingAuthenticator{userName: "test-user"},
+		Authz:           passingAuthorizer{},
+		Attr:            &exporterAttributesGetter{namespace: "default", name: "test-exporter"},
+		TelemetryConfig: cfg,
+	}
+	return svc, context.Background()
+}
+
 // authFailureServiceCtx builds a ControllerService whose authentication always
 // fails, plus a context carrying a peer address, a captured logger, and the
 // jlog.LogContext enrichment applied by the gRPC interceptors in production.