diff --git a/controller/Makefile b/controller/Makefile index 1fbd0ef9a..f7f5a20c5 100644 --- a/controller/Makefile +++ b/controller/Makefile @@ -33,7 +33,8 @@ endif # tools. (i.e. podman) CONTAINER_TOOL ?= podman -# Cluster type: kind (default) or k3s +# Cluster type: kind (default), k3s, or openshift (bring your own logged-in +# OpenShift/CRC cluster) CLUSTER_TYPE ?= kind export CLUSTER_TYPE @@ -308,9 +309,12 @@ cluster: else ifeq ($(CLUSTER_TYPE),kind) cluster: $(KIND) $(KIND) get clusters | grep jumpstarter || $(KIND) create cluster --name jumpstarter --config hack/kind_cluster.yaml +else ifeq ($(CLUSTER_TYPE),openshift) +cluster: + @source hack/utils && create_cluster else cluster: - $(error Unknown CLUSTER_TYPE=$(CLUSTER_TYPE). Use 'kind' or 'k3s') + $(error Unknown CLUSTER_TYPE=$(CLUSTER_TYPE). Use 'kind', 'k3s', or 'openshift') endif ifeq ($(CLUSTER_TYPE),k3s) @@ -319,9 +323,12 @@ clean: else ifeq ($(CLUSTER_TYPE),kind) clean: $(KIND) $(KIND) delete cluster --name jumpstarter +else ifeq ($(CLUSTER_TYPE),openshift) +clean: + @source hack/utils && delete_cluster else clean: - $(error Unknown CLUSTER_TYPE=$(CLUSTER_TYPE). Use 'kind' or 'k3s') + $(error Unknown CLUSTER_TYPE=$(CLUSTER_TYPE). Use 'kind', 'k3s', or 'openshift') endif diff --git a/controller/hack/deploy_vars b/controller/hack/deploy_vars index a021ca5b0..503209b4f 100755 --- a/controller/hack/deploy_vars +++ b/controller/hack/deploy_vars @@ -2,9 +2,22 @@ # Common deployment variables for Jumpstarter hack scripts # This file should be sourced after utils -# Calculate external IP and networking configuration -IP=${IP:-$(get_external_ip)} -BASEDOMAIN=${BASEDOMAIN:-"jumpstarter.${IP}.nip.io"} +# Calculate BASEDOMAIN. OpenShift clusters have their own routable wildcard +# domain (e.g. apps-crc.testing), so we detect it instead of using a nip.io +# domain derived from the local outbound IP. +if [ -z "${BASEDOMAIN:-}" ]; then + if [ "${CLUSTER_TYPE}" == "openshift" ]; then + OPENSHIFT_CLUSTER_DOMAIN=$(kubectl get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}' 2>/dev/null || true) + if [ -z "${OPENSHIFT_CLUSTER_DOMAIN}" ]; then + echo "Could not auto-detect the OpenShift cluster domain (ingresses.config.openshift.io/cluster). Set BASEDOMAIN explicitly." + exit 1 + fi + BASEDOMAIN="jumpstarter.${OPENSHIFT_CLUSTER_DOMAIN}" + else + IP=${IP:-$(get_external_ip)} + BASEDOMAIN="jumpstarter.${IP}.nip.io" + fi +fi IMG=${IMG:-quay.io/jumpstarter-dev/jumpstarter-controller:latest} OPERATOR_IMG=${OPERATOR_IMG:-$(make -C deploy/operator --no-print-directory -s print-img 2>/dev/null || echo "quay.io/jumpstarter-dev/jumpstarter-operator:latest")} EXPORTER_SET_CONTROLLER_IMG=${EXPORTER_SET_CONTROLLER_IMG:-quay.io/jumpstarter-dev/jumpstarter-exporterset-controller:latest} @@ -14,6 +27,11 @@ if [ "${NETWORKING_MODE}" == "ingress" ]; then GRPC_ENDPOINT="grpc.${BASEDOMAIN}:5443" GRPC_ROUTER_ENDPOINT="router.${BASEDOMAIN}:5443" LOGIN_ENDPOINT="login.${BASEDOMAIN}" +elif [ "${NETWORKING_MODE}" == "route" ]; then + # OpenShift Routes always terminate on the router's standard HTTPS port (443) + GRPC_ENDPOINT="grpc.${BASEDOMAIN}:443" + GRPC_ROUTER_ENDPOINT="router.${BASEDOMAIN}:443" + LOGIN_ENDPOINT="login.${BASEDOMAIN}" elif [ "${CLUSTER_TYPE}" == "k3s" ]; then # k3s exposes NodePorts directly on the host, no port mapping needed GRPC_ENDPOINT="grpc.${BASEDOMAIN}:30010" diff --git a/controller/hack/deploy_with_operator.sh b/controller/hack/deploy_with_operator.sh index ff9f0b2a6..453daf4d0 100755 --- a/controller/hack/deploy_with_operator.sh +++ b/controller/hack/deploy_with_operator.sh @@ -8,14 +8,17 @@ USE_CERTMANAGER=${USE_CERTMANAGER:-true} # Source common utilities source "${SCRIPT_DIR}/utils" -# Source common deployment variables -source "${SCRIPT_DIR}/deploy_vars" - set_kubectl_context +# Source common deployment variables (depends on kubectl being ready, e.g. to +# auto-detect the OpenShift cluster domain) +source "${SCRIPT_DIR}/deploy_vars" + # Install nginx ingress if in ingress mode if [ "${NETWORKING_MODE}" = "ingress" ]; then install_nginx_ingress +elif [ "${NETWORKING_MODE}" = "route" ]; then + echo -e "${GREEN}Deploying with OpenShift Routes ...${NC}" else echo -e "${GREEN}Deploying with nodeport ...${NC}" fi @@ -29,15 +32,31 @@ if [ "${USE_CERTMANAGER}" = "true" ]; then fi fi -# load the container images into the cluster +# Load the container images into the cluster. For CLUSTER_TYPE=openshift this +# also rewrites IMG/OPERATOR_IMG/EXPORTER_SET_CONTROLLER_IMG to the pushed +# internal-registry pull spec, since the originals aren't reachable from the cluster. load_image "${IMG}" +IMG="${LOADED_IMAGE}" load_image "${OPERATOR_IMG}" +OPERATOR_IMG="${LOADED_IMAGE}" load_image "${EXPORTER_SET_CONTROLLER_IMG}" +EXPORTER_SET_CONTROLLER_IMG="${LOADED_IMAGE}" + +# Recompute the repo/tag split now that IMG may have been rewritten above +IMAGE_TAG="${IMG##*:}" +IMAGE_REPO="${IMG%:*}" # Deploy the operator echo -e "${GREEN}Deploying Jumpstarter operator ...${NC}" kubectl apply -f deploy/operator/dist/install.yaml +# On OpenShift, install.yaml bakes in the original (unreachable) operator image +# reference, so point the deployment at the one we actually pushed. +if [ "${CLUSTER_TYPE}" = "openshift" ]; then + kubectl set image deployment/jumpstarter-operator-controller-manager \ + manager="${OPERATOR_IMG}" -n jumpstarter-operator-system +fi + # If operator deployment already exists, restart it to pick up the new image if kubectl get deployment jumpstarter-operator-controller-manager -n jumpstarter-operator-system > /dev/null 2>&1; then echo -e "${GREEN}Restarting operator deployment to pick up new image ...${NC}" @@ -92,6 +111,29 @@ END class: "nginx" END ) +elif [ "${NETWORKING_MODE}" == "route" ]; then + # OpenShift Routes always listen on the router's standard HTTPS port (443), + # so no custom port can be specified in the address. + CONTROLLER_ENDPOINT_CONFIG=$(cat <<-END + - address: grpc.${BASEDOMAIN} + route: + enabled: true +END +) + ROUTER_ENDPOINT_CONFIG=$(cat <<-END + - address: router.${BASEDOMAIN} + route: + enabled: true +END +) + LOGIN_ENDPOINT_CONFIG=$(cat <<-END + login: + endpoints: + - address: login.${BASEDOMAIN} + route: + enabled: true +END +) else # For kind, NodePorts are mapped to host ports via extraPortMappings (30010->8082, etc.) # For k3s, NodePorts are directly accessible on the host diff --git a/controller/hack/utils b/controller/hack/utils index 0b142398c..2f911875c 100755 --- a/controller/hack/utils +++ b/controller/hack/utils @@ -10,14 +10,29 @@ get_script_dir() { } # Environment variable defaults +export CLUSTER_TYPE=${CLUSTER_TYPE:-kind} export KIND=${KIND:-bin/kind} export GRPCURL=${GRPCURL:-bin/grpcurl} -export NETWORKING_MODE=${NETWORKING_MODE:-nodeport} +# OpenShift has no NodePort-friendly local port mapping, so Routes are the natural default there. +if [ "${CLUSTER_TYPE}" = "openshift" ]; then + export NETWORKING_MODE=${NETWORKING_MODE:-route} +else + export NETWORKING_MODE=${NETWORKING_MODE:-nodeport} +fi export CERTMANAGER_VERSION=${CERTMANAGER_VERSION:-v1.19.2} -export CLUSTER_TYPE=${CLUSTER_TYPE:-kind} export K3S_KUBECONFIG=${K3S_KUBECONFIG:-/etc/rancher/k3s/k3s.yaml} +# Namespace used to host locally-built images pushed to the OpenShift internal +# registry when CLUSTER_TYPE=openshift (see openshift_load_image). +export OPENSHIFT_IMAGE_NAMESPACE=${OPENSHIFT_IMAGE_NAMESPACE:-jumpstarter-lab} +# TLS verification flag for the OpenShift internal registry. Defaults to +# --tls-verify=false for CRC/local clusters whose registry certificate is +# self-signed. Override with e.g. OPENSHIFT_TLS_VERIFY="--tls-verify=true" for +# production clusters that have a trusted CA. +export OPENSHIFT_TLS_VERIFY=${OPENSHIFT_TLS_VERIFY:-"--tls-verify=false"} # Color codes for terminal output +export RED='\033[0;31m' +export YELLOW='\033[0;33m' export GREEN='\033[0;32m' export NC='\033[0m' # No Color @@ -36,14 +51,23 @@ get_external_ip() { # Validate CLUSTER_TYPE and exit if unknown _require_valid_cluster_type() { case "${CLUSTER_TYPE}" in - kind|k3s) ;; + kind|k3s|openshift) ;; *) - echo "Unknown CLUSTER_TYPE=${CLUSTER_TYPE}. Use 'kind' or 'k3s'" + echo "Unknown CLUSTER_TYPE=${CLUSTER_TYPE}. Use 'kind', 'k3s', or 'openshift'" exit 1 ;; esac } +# Print instructions for logging into an existing OpenShift/CRC cluster. +_print_openshift_login_help() { + echo -e "${RED}No active OpenShift context found.${NC}" + echo "For CLUSTER_TYPE=openshift, log in to your cluster first, e.g. for CRC:" + echo " eval \$(crc oc-env)" + echo " oc login -u kubeadmin -p \$(crc console --credentials -o json | jq -r .clusterConfig.password) \\" + echo " \$(crc console --credentials -o json | jq -r .clusterConfig.url)" +} + set_kubectl_context() { _require_valid_cluster_type case "${CLUSTER_TYPE}" in @@ -60,12 +84,26 @@ set_kubectl_context() { export KUBECONFIG="${user_kubeconfig}" echo -e "${GREEN}Using k3s kubeconfig (copied to ${user_kubeconfig})${NC}" ;; + openshift) + # CLUSTER_TYPE=openshift does not manage login/context (context names vary + # per cluster/login flow); it just uses whatever context is already active. + if ! kubectl cluster-info > /dev/null 2>&1; then + _print_openshift_login_help + exit 1 + fi + echo -e "${GREEN}Using existing OpenShift context: $(kubectl config current-context)${NC}" + ;; esac } +# Loads a locally built image into the target cluster and updates LOADED_IMAGE +# with the image reference that should actually be used to deploy it (this +# only differs from the input for CLUSTER_TYPE=openshift, see +# openshift_load_image). load_image() { _require_valid_cluster_type local image=$1 + LOADED_IMAGE="${image}" case "${CLUSTER_TYPE}" in kind) kind_load_image "${image}" @@ -73,6 +111,9 @@ load_image() { k3s) echo -e "${GREEN}k3s pulls images directly from registries, skipping load for ${image}${NC}" ;; + openshift) + LOADED_IMAGE=$(openshift_load_image "${image}") + ;; esac } @@ -96,6 +137,15 @@ create_cluster() { echo -e "${GREEN}Waiting for k3s node to be ready...${NC}" kubectl wait --for=condition=ready node --all --timeout=120s ;; + openshift) + # CLUSTER_TYPE=openshift does not manage cluster lifecycle (bring your own + # OpenShift/CRC cluster and log in before running); just verify we're connected. + if ! kubectl cluster-info > /dev/null 2>&1; then + _print_openshift_login_help + exit 1 + fi + echo -e "${GREEN}Using existing OpenShift cluster: $(kubectl config current-context)${NC}" + ;; esac } @@ -113,6 +163,9 @@ delete_cluster() { echo -e "${GREEN}k3s uninstall script not found, skipping${NC}" fi ;; + openshift) + echo -e "${GREEN}CLUSTER_TYPE=openshift does not manage cluster lifecycle, skipping delete (use 'crc stop'/'crc delete' if using CRC)${NC}" + ;; esac } @@ -146,6 +199,94 @@ kind_load_image() { fi } +# Get the hostname of the OpenShift internal image registry's default route, +# exposing the route if it isn't already. The result is cached in +# _OPENSHIFT_REGISTRY_HOST for the rest of the script's run. +openshift_registry_host() { + if [ -z "${_OPENSHIFT_REGISTRY_HOST:-}" ]; then + echo -e "${YELLOW}WARNING:${NC} Enabling the default route on the OpenShift image registry" >&2 + echo -e " (configs.imageregistry.operator.openshift.io/cluster .spec.defaultRoute=true)." >&2 + echo -e " This exposes the internal registry externally. Safe for CRC/dev clusters," >&2 + echo -e " but review your security posture on shared/production clusters." >&2 + kubectl patch configs.imageregistry.operator.openshift.io/cluster \ + --type=merge -p '{"spec":{"defaultRoute":true}}' > /dev/null + + echo -e "${GREEN} * Waiting for the OpenShift image registry route ...${NC}" >&2 + local timeout=60 + _OPENSHIFT_REGISTRY_HOST="" + while [ -z "${_OPENSHIFT_REGISTRY_HOST}" ]; do + _OPENSHIFT_REGISTRY_HOST=$(kubectl get route default-route -n openshift-image-registry -o jsonpath='{.spec.host}' 2>/dev/null || true) + if [ -n "${_OPENSHIFT_REGISTRY_HOST}" ]; then + break + fi + sleep 2 + timeout=$((timeout - 2)) + if [ ${timeout} -le 0 ]; then + echo "Timed out waiting for the OpenShift image registry default route" >&2 + exit 1 + fi + done + fi + echo "${_OPENSHIFT_REGISTRY_HOST}" +} + +# Load a locally built image into an OpenShift cluster by pushing it to the +# cluster's internal image registry (there is no local-load mechanism like +# kind's, and CRC's podman socket does not share storage with CRI-O). Prints +# the in-cluster pull spec (image-registry.openshift-image-registry.svc:5000/...) +# that manifests should use instead of the original registry/tag. +# Requires the `oc` CLI and a container tool (${CONTAINER_TOOL:-podman}) logged +# in locally with the image already present. +openshift_load_image() { + local image=$1 + local name tag registry_host external_ref internal_ref + + if ! command -v oc &> /dev/null; then + echo -e "${RED}The 'oc' CLI is required for CLUSTER_TYPE=openshift image loading${NC}" >&2 + exit 1 + fi + + kubectl create namespace "${OPENSHIFT_IMAGE_NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - > /dev/null + # Allow every service account in the cluster to pull from this namespace, + # since the operator and Jumpstarter components run in different namespaces. + oc policy add-role-to-group system:image-puller system:serviceaccounts \ + --namespace="${OPENSHIFT_IMAGE_NAMESPACE}" > /dev/null 2>&1 || true + + registry_host=$(openshift_registry_host) + + # Parse the image name and tag from the final path component so that + # registry ports (e.g. localhost:5000/team/controller:latest) don't + # confuse the split. + local leaf + leaf="${image##*/}" # e.g. "controller:latest" + if [[ "${leaf}" == *:* ]]; then + name="${leaf%:*}" # e.g. "controller" + tag="${leaf##*:}" # e.g. "latest" + else + name="${leaf}" + tag="latest" + fi + + external_ref="${registry_host}/${OPENSHIFT_IMAGE_NAMESPACE}/${name}:${tag}" + internal_ref="image-registry.openshift-image-registry.svc:5000/${OPENSHIFT_IMAGE_NAMESPACE}/${name}:${tag}" + + echo -e "${GREEN}Pushing ${image} to OpenShift internal registry as ${internal_ref} ...${NC}" >&2 + + local _ct="${CONTAINER_TOOL:-podman}" + local xtrace_on=0 + [[ $- == *x* ]] && xtrace_on=1 + set +x 2>/dev/null + oc whoami -t | "${_ct}" login -u kubeadmin --password-stdin ${OPENSHIFT_TLS_VERIFY} "${registry_host}" > /dev/null + [[ $xtrace_on -eq 1 ]] && set -x + "${_ct}" tag "${image}" "${external_ref}" + if ! "${_ct}" push ${OPENSHIFT_TLS_VERIFY} "${external_ref}" > /dev/null; then + echo "Error pushing ${image} to OpenShift internal registry." >&2 + exit 1 + fi + + echo "${internal_ref}" +} + # Install nginx ingress in kind cluster # This function deploys nginx ingress and waits for it to be ready NGINX_INGRESS_VERSION=${NGINX_INGRESS_VERSION:-controller-v1.12.1}