From 2137281b1e5481f658bc22df3e92ec9302b6e3f8 Mon Sep 17 00:00:00 2001 From: Artem Goncharov Date: Mon, 14 Sep 2026 09:52:55 +0200 Subject: [PATCH 1/2] feat(tools): Add local devstack-full VM verification script Local, throwaway-VM equivalent of the devstack-full CI job so SPIRE integration / service-auth changes can be iterated on without waiting on CI. Verified end-to-end in a 40G-disk VM: stack.sh brings up mysql/rabbit/nova/cinder/glance/neutron-OVN/SPIRE successfully. cargo build -p uses the Cargo package names (openstack-keystone, openstack-keystone-cli-manage), not the binary names, and the script chowns its DEST dir before devstack's own unprivileged git_clone runs against it. Signed-off-by: Artem Goncharov --- tools/devstack-full/README.md | 158 ++++++++++++++++++++++++ tools/devstack-full/run.sh | 223 ++++++++++++++++++++++++++++++++++ 2 files changed, 381 insertions(+) create mode 100644 tools/devstack-full/README.md create mode 100755 tools/devstack-full/run.sh diff --git a/tools/devstack-full/README.md b/tools/devstack-full/README.md new file mode 100644 index 000000000..8ed5b1c6c --- /dev/null +++ b/tools/devstack-full/README.md @@ -0,0 +1,158 @@ +# Local devstack-full replica + +Local, throwaway-VM equivalent of the `devstack-full` CI job +(`.github/workflows/devstack.yml`). Runs the full single-node stack - +Nova/Cinder/Glance/Neutron(OVN)/SPIRE on top of rust Keystone - so you can +iterate on SPIRE integration / service-auth changes without waiting on CI. + +`run.sh` is the extracted job body (steps 337-488 of the workflow). Both CI +and this script should stay thin wrappers around the same shape so fixes +made locally land in CI too - if you change one, check the other still +matches. + +## What it needs + +- Ubuntu 24.04 (noble), systemd. devstack drives services via + `devstack@*` systemd units. +- ~4 vCPU / 8-16 GB RAM / 60-100 GB disk. mysql + rabbit + OVN + + nova/cinder/glance + 4 spiffe-helpers + tempest is heavier than a + keystone-only devstack. +- **Nested KVM**, for `n-cpu` and the cirros boot/delete smoke test: + nova-compute runs libvirt inside what is itself a VM. GitHub's + `ubuntu-24.04` runners are themselves VMs with nested KVM enabled and + this passes there, so it's proven to work in this exact shape. +- A real branch checked out (not detached HEAD) is *not* required locally - + `run.sh` bypasses devstack's git clone for the key-rs/SPIRE plugins + entirely (see "How local state gets in" below). + +Run `run.sh` only inside a disposable VM/host you don't care about - it +installs system packages/services and is destructive to `/opt/stack`. + +## One-time VM setup (Option A: virt-install + host-passthrough) + +On the host (needs libvirt/qemu-kvm): + +```sh +virt-install \ + --name devstack-full \ + --vcpus 4 --memory 16384 \ + --disk size=100 \ + --cpu host-passthrough \ + --network network=default \ + --os-variant ubuntu24.04 \ + --cloud-init user-data=cloud-init.yaml \ + --location +``` + +Seed `cloud-init.yaml` (or equivalent NoCloud config) to install +prerequisites and enable nesting: + +```yaml +#cloud-config +package_update: true +packages: + - git + - curl + - rsync + - qemu-kvm + - libvirt-daemon-system +users: + - default + - name: ubuntu + groups: [kvm, libvirt] + sudo: ALL=(ALL) NOPASSWD:ALL +``` + +Verify nesting actually works *inside* the guest before doing anything +else - if this fails, `n-cpu` will fail too, much later and much more +confusingly: + +```sh +test -e /dev/kvm && egrep -c '(vmx|svm)' /proc/cpuinfo +``` + +Put the VM on NAT or a dedicated libvirt network, not your workstation's +LAN bridge - OVN/flat bridges inside the guest will otherwise fight host +addressing. + +## Getting the repo in + +You don't need to push anywhere. Either: + +- **virtiofs/9p mount** the keystone-rs worktree into the guest (fastest + iteration - edits on the host show up immediately), or +- **`git clone` the worktree itself** as a local-path origin inside the + guest (`git clone /path/to/worktree /home/ubuntu/keystone-rs` from a host + mount, or `rsync` it in once and `git pull` from a mount for updates). + +Either way, `run.sh --workspace ` just needs a working tree it can +read and rsync from - it doesn't care how that tree got onto the VM. + +## How local state gets in (no push, no commit) + +The CI job points devstack's `enable_plugin key-rs file:// ` +at the checked-out branch, which only sees *committed* state. `run.sh` +does something different: it `rsync`s your live working tree (staged, +unstaged, untracked - everything except `.git`/`target`) straight into +`/opt/stack/key-rs` before `stack.sh` runs. + +devstack's `git_clone()` (functions-common) is a no-op whenever its +destination directory already exists and `RECLONE` isn't set - so with +`/opt/stack/key-rs` pre-populated, devstack never clones or fetches +anything for that plugin; it just uses what's there. Same for the SPIRE +plugin at `/opt/stack/spire`. Net effect: no commit and no push required +to test uncommitted local changes, and re-running the script re-syncs + +re-stacks with whatever the working tree looks like *right now*. + +## Running it + +Inside the guest, with a prebuilt release binary (mirrors the CI +artifact - skips a redundant `cargo build --release` inside the run): + +```sh +cargo build --release -p openstack-keystone -p openstack-keystone-cli-manage # on the host, or in the guest +tools/devstack-full/run.sh --workspace /path/to/keystone-rs --bin-dir /path/to/target/release +``` + +Or let it build for you: + +```sh +tools/devstack-full/run.sh --workspace /path/to/keystone-rs +``` + +Useful flags: + +- `--skip-build` - reuse an existing `target/release` under `--workspace` + without rebuilding. +- `--skip-smoke` - skip the cirros boot/delete smoke test (still runs + everything else - service health, SPIRE checks). +- `--skip-verify` - stop right after `stack.sh`, skip all verification. +- `--restack` - run devstack's own `unstack.sh` + `clean.sh` before + re-stacking, for when you want a truly clean run rather than devstack's + normal incremental re-stack. + +See `run.sh --help` for the full list. + +## Iterating + +Keep the VM around between runs. A typical loop: + +1. Edit code on the host (or in the guest, if you cloned there directly). +2. `cargo build --release` if binaries changed. +3. Re-run `tools/devstack-full/run.sh --workspace ... --bin-dir ...` - + re-syncs the plugin dirs and re-stacks. + +Cold run (fresh VM): ~40-60 min. Warm re-runs: ~20-30 min. + +## Caveats + +- Nested virtualization makes the cirros smoke test noticeably slower than + bare metal; use `--skip-smoke` for faster inner-loop iteration and drop + it for the occasional full validation pass. +- GitHub-runner-specific package purges (stock MySQL, `esl-erlang`, the + `/etc/hosts` sed, stray postgres) are deliberately **not** in `run.sh` - + a clean noble image doesn't have any of those conflicts to begin with. +- Day-to-day keystone-rs/SPIRE-plugin changes are already covered by + `cargo test` and the lighter `devstack` CI job (keystone-only, no KVM + needed). Reserve this full nested-VM run for pre-PR validation, not + every edit. diff --git a/tools/devstack-full/run.sh b/tools/devstack-full/run.sh new file mode 100755 index 000000000..8f7073048 --- /dev/null +++ b/tools/devstack-full/run.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +# Local-runnable replica of the devstack-full CI job +# (.github/workflows/devstack.yml, devstack-full: steps 337-488). +# +# Meant to run *inside* a throwaway VM/host with nested KVM (see +# tools/devstack-full/README.md for the recommended virt-install setup) - +# it is destructive to /opt/stack and installs system packages/services. +# Never run it against a machine you care about. +# +# Unlike the CI job, this does NOT go through devstack's file:// git clone +# of the key-rs plugin (which only sees committed refs). Instead it rsyncs +# the *current working tree* - staged, unstaged, untracked included - into +# $DEST/key-rs before stack.sh runs. devstack's git_clone() is a no-op +# whenever the destination directory already exists (functions-common, +# RECLONE defaults to False), so pre-populating it this way makes devstack +# skip cloning entirely and just use what's there. No commit, no push, no +# branch ref required - re-running this script re-syncs and re-stacks. +set -euo pipefail + +WORKSPACE="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +DEST="/opt/stack" +BIN_DIR="" +SKIP_BUILD=false +SKIP_SMOKE=false +SKIP_VERIFY=false +RESTACK=false + +usage() { + cat <&2; usage; exit 1 ;; + esac +done + +if [[ -z "$BIN_DIR" ]] && [[ "$SKIP_BUILD" == false ]]; then + echo "==> Building keystone-rs release binaries" + (cd "$WORKSPACE" && cargo build --release -p openstack-keystone -p openstack-keystone-cli-manage) + BIN_DIR="$WORKSPACE/target/release" +elif [[ -z "$BIN_DIR" ]]; then + BIN_DIR="$WORKSPACE/target/release" +fi +for bin in keystone keystone-manage; do + test -x "$BIN_DIR/$bin" || { echo "missing $BIN_DIR/$bin (build first or pass --bin-dir)" >&2; exit 1; } +done + +if [[ "$RESTACK" == true ]] && [[ -x "$DEST/devstack/unstack.sh" ]]; then + echo "==> Tearing down previous stack" + (cd "$DEST/devstack" && ./unstack.sh || true) + sudo "$DEST/devstack/clean.sh" || true +fi + +# On a fresh disposable VM, $DEST pre-exists root-owned (or doesn't exist +# yet); devstack's own git_clone calls (e.g. for openstack/requirements) +# run unprivileged as $USER and fail with "Permission denied" creating +# their destination dir under $DEST unless $DEST itself is writable first. +sudo mkdir -p "$DEST" +sudo chown "$USER" "$DEST" + +if [[ ! -d "$DEST/devstack/.git" ]]; then + echo "==> Cloning devstack" + sudo git clone https://opendev.org/openstack/devstack "$DEST/devstack" + sudo chown -R "$USER" "$DEST/devstack" +fi + +echo "==> Syncing local keystone-rs worktree into $DEST/key-rs" +sudo mkdir -p "$DEST/key-rs" +sudo rsync -a --delete \ + --exclude .git \ + --exclude target \ + --exclude tests/loadtest/target \ + "$WORKSPACE"/ "$DEST/key-rs"/ +sudo chown -R "$USER" "$DEST/key-rs" + +echo "==> Syncing SPIRE devstack plugin into $DEST/spire" +sudo mkdir -p "$DEST/spire/devstack" +sudo cp -rT "$WORKSPACE/tools/devstack-plugin-spire" "$DEST/spire/devstack" +sudo chown -R "$USER" "$DEST/spire" + +echo "==> Writing local.conf" +cat </dev/null +[[local|localrc]] +ADMIN_PASSWORD=password +DATABASE_PASSWORD=password +RABBIT_PASSWORD=password +SERVICE_PASSWORD=password +SERVICE_TOKEN=service-token +GIT_BASE=https://github.com + +disable_all_services +enable_service mysql +enable_service rabbit +enable_service key +enable_service key-rs +enable_service tempest +enable_service spire + +# Nova (+ placement, a hard Nova dependency in modern devstack) +enable_service n-api n-cpu n-sch n-cond n-novnc n-crt n-api-meta +enable_service placement-api placement-client +# Cinder (c-vol defaults to an LVM-over-loopback backend when no +# CINDER_DRIVER is set - no real block device needed) +enable_service c-api c-sch c-vol +# Glance +enable_service g-api +# Neutron with OVN (devstack's default ML2 mechanism driver since 2023). +# The legacy agents (q-agt/q-dhcp/q-l3/q-meta) are mutually exclusive with +# OVN - devstack's ovn_sanity_check aborts the run if any of them is +# enabled. OVN's own services must be listed explicitly because +# disable_all_services drops stackrc's defaults. +enable_service q-svc q-ovn-agent +enable_service ovn-controller ovn-northd ovs-vswitchd ovsdb-server + +# Both plugin dirs already exist (rsynced above), so devstack's git_clone +# skips cloning and uses the local working tree as-is - the "local" ref +# below is unused in that case, just a placeholder enable_plugin requires. +enable_plugin spire file://$DEST/spire local +enable_plugin key-rs file://$DEST/key-rs local + +SPIRE_TRUST_DOMAIN=cloud.trust.domain + +KEYSTONE_RS_BIN_DIR=$BIN_DIR + +LOGFILE=\$HOME/devstack.log +LOG_COLOR=False +EOF +cat "$DEST/devstack/local.conf" + +echo "==> Running stack.sh" +(cd "$DEST/devstack" && FORCE=yes ./stack.sh) + +if [[ "$SKIP_VERIFY" == true ]]; then + echo "==> Stack up, skipping verification (--skip-verify)" + exit 0 +fi + +echo "==> Verifying rust Keystone is serving directly" +curl -sf http://127.0.0.1:8080/v3 >/dev/null + +echo "==> Issuing a token via the openstack CLI" +source "$DEST/devstack/openrc" admin admin +openstack token issue + +echo "==> Verifying OpenStack service health" +openstack compute service list +openstack volume service list +openstack image list +openstack network agent list +# devstack no longer binds per-service ports (8774/8776/9292/9696): all +# APIs are proxied by a single Apache on port 80 via URL path prefixes. +# -f is omitted on purpose: unauthenticated requests get 3xx/401 back, +# which still proves the API is up and answering. +curl -s -o /dev/null http://127.0.0.1/compute/ +curl -s -o /dev/null http://127.0.0.1/volume/v3/ +curl -s -o /dev/null http://127.0.0.1/image/ +curl -s -o /dev/null http://127.0.0.1/networking/ + +if [[ "$SKIP_SMOKE" == false ]]; then + echo "==> Smoke test - boot and delete a cirros instance" + image_id=$(openstack image list -f value -c ID | head -n1) + network_id=$(openstack network list -f value -c ID --long -c "Name" | awk '$0 !~ /public/ {print $1; exit}') + openstack server create --image "$image_id" \ + --flavor cirros256 --network "$network_id" smoke-test --wait + openstack server delete smoke-test --wait +fi + +echo "==> Verifying SPIRE server and agent are healthy" +/usr/local/bin/spire-server healthcheck -socketPath "$DEST/data/spire/server.sock" +/usr/local/bin/spire-agent healthcheck -socketPath "$DEST/data/spire/agent.sock" + +echo "==> Verifying pre-registered SPIRE entries exist" +/usr/local/bin/spire-server entry show -socketPath "$DEST/data/spire/server.sock" | tee /tmp/spire-entries.txt +grep -q "service/keystone" /tmp/spire-entries.txt +grep -q "service/nova-api" /tmp/spire-entries.txt +grep -q "service/neutron" /tmp/spire-entries.txt +grep -q "service/cinder" /tmp/spire-entries.txt +grep -q "service/glance" /tmp/spire-entries.txt +grep -q "service/nova-compute/host/" /tmp/spire-entries.txt + +echo "==> Verifying SPIRE CA bundle was exported" +test -s /etc/keystone/spiffe/ca.crt + +echo "==> Verifying spiffe-helper processes are running" +for svc in nova cinder glance neutron; do + sudo systemctl is-active "devstack@spiffe-helper-$svc" +done + +echo "==> Verifying spiffe-helper certs were issued and are valid" +# SPIRE does not put the SPIFFE ID in the cert's Subject (it is C=US, +# O=SPIRE) - the identity lives in the URI SAN. So assert the exact +# per-service identity each helper is pinned to, and verify the SVID +# chains to the trust bundle the helper stored alongside it. +for pair in nova:nova-api cinder:cinder glance:glance neutron:neutron; do + svc=${pair%%:*} + dir="$DEST/data/spire/certs/$svc" + openssl x509 -in "$dir/tls.crt" -noout -checkend 0 + openssl verify -CAfile "$dir/ca.crt" "$dir/tls.crt" + openssl x509 -in "$dir/tls.crt" -noout -text | grep -q "URI:spiffe://cloud.trust.domain/service/${pair#*:}" +done + +echo "==> devstack-full OK" From 4ea4de48c25b5bb380fbfe5f8fe73b157dfc02a1 Mon Sep 17 00:00:00 2001 From: Artem Goncharov Date: Mon, 14 Sep 2026 10:26:43 +0200 Subject: [PATCH 2/2] fix(api-types): Allow empty domain description description is Option; validator's length(min=1) fired on Some("") the same as an actual too-short value, rejecting the empty string devstack sends on its idempotent domain-create/update calls with a 400. Drop the min bound, keep max=255. Signed-off-by: Artem Goncharov --- crates/api-types/src/v3/domain.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/api-types/src/v3/domain.rs b/crates/api-types/src/v3/domain.rs index 69fcfa279..e5ee9d34c 100644 --- a/crates/api-types/src/v3/domain.rs +++ b/crates/api-types/src/v3/domain.rs @@ -61,7 +61,7 @@ pub struct Domain { /// The description of the domain. #[cfg_attr(feature = "builder", builder(default))] #[serde(skip_serializing_if = "Option::is_none")] - #[cfg_attr(feature = "validate", validate(length(min = 1, max = 255)))] + #[cfg_attr(feature = "validate", validate(length(max = 255)))] pub description: Option, /// If set to true, domain is enabled. If set to false, domain is disabled. @@ -105,7 +105,7 @@ pub struct Domain { pub struct DomainCreate { /// The description of the domain. #[cfg_attr(feature = "builder", builder(default))] - #[cfg_attr(feature = "validate", validate(length(min = 1, max = 255)))] + #[cfg_attr(feature = "validate", validate(length(max = 255)))] #[serde(skip_serializing_if = "Option::is_none")] pub description: Option, @@ -174,7 +174,7 @@ pub struct DomainCreateRequest { pub struct DomainUpdate { /// The description of the domain. #[cfg_attr(feature = "builder", builder(default))] - #[cfg_attr(feature = "validate", validate(length(min = 1, max = 255)))] + #[cfg_attr(feature = "validate", validate(length(max = 255)))] pub description: Option, /// If set to true, domain is enabled. If set to false, domain is disabled.