diff --git a/.ansible-lint b/.ansible-lint index d6ddfd2a6b..d566b5dc16 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -14,6 +14,7 @@ exclude_paths: - zuul.d mock_roles: + - ensure-go - opendev.container_registry skip_list: diff --git a/.gitignore b/.gitignore index 3c967aed17..6b4c1f2c92 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ atmosphere/_version.py [._]*.un~ Session.vim Sessionx.vim +bin/ diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000000..60b20cd0a3 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,31 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# SPDX-License-Identifier: Apache-2.0 + +version: 2 + +builds: + - id: atmosphere + main: ./cmd/atmosphere + binary: atmosphere + env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + +archives: + - id: atmosphere + builds: + - atmosphere + format: tar.gz + name_template: "atmosphere_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + +checksum: + name_template: "checksums.txt" + +changelog: + sort: asc diff --git a/.zuul.yaml b/.zuul.yaml index d736ba5f07..829aec9df2 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,3 +1,95 @@ +- job: + name: atmosphere-molecule-aio-openvswitch-selective + parent: atmosphere-molecule-aio-openvswitch + pre-run: + - test-playbooks/molecule/prepare.yml + run: + - test-playbooks/molecule/selective-run.yml + vars: + atmosphere_ci_job: aio-openvswitch + +- job: + name: atmosphere-molecule-aio-ovn-selective + parent: atmosphere-molecule-aio-ovn + pre-run: + - test-playbooks/molecule/prepare.yml + run: + - test-playbooks/molecule/selective-run.yml + vars: + atmosphere_ci_job: aio-ovn + +- job: + name: atmosphere-molecule-csi-local-path-provisioner-selective + parent: atmosphere-molecule-csi-local-path-provisioner + pre-run: + - test-playbooks/molecule/prepare.yml + run: + - test-playbooks/molecule/selective-run.yml + vars: + atmosphere_ci_job: csi-local-path-provisioner + atmosphere_deploy_tags: + - kubernetes + - csi + +- job: + name: atmosphere-molecule-csi-rbd-selective + parent: atmosphere-molecule-csi-rbd + pre-run: + - test-playbooks/molecule/prepare.yml + run: + - test-playbooks/molecule/selective-run.yml + vars: + atmosphere_ci_job: csi-rbd + atmosphere_deploy_tags: + - ceph + - kubernetes + - csi + +- job: + name: atmosphere-molecule-keycloak-selective + parent: atmosphere-molecule-keycloak + pre-run: + - test-playbooks/molecule/prepare.yml + run: + - test-playbooks/molecule/selective-run.yml + vars: + atmosphere_ci_job: keycloak + atmosphere_deploy_tags: + - kubernetes + - csi + - cert-manager + - cluster-issuer + - ingress-nginx + - rabbitmq-cluster-operator + - percona-xtradb-cluster-operator + - percona-xtradb-cluster + - valkey + - memcached + - keycloak + - kube-prometheus-stack + - keystone + - horizon + - project: - templates: - - atmosphere-molecule-jobs + vars: + atmosphere_ci_plan_base: "origin/{{ zuul.branch }}" + ceph_public_network: >- + {{ + ansible_facts['default_ipv4']['network'] + + '/' + + (ansible_facts['default_ipv4']['prefix'] | string) + }} + check: + jobs: + - atmosphere-molecule-aio-openvswitch-selective + - atmosphere-molecule-aio-ovn-selective + - atmosphere-molecule-csi-local-path-provisioner-selective + - atmosphere-molecule-csi-rbd-selective + - atmosphere-molecule-keycloak-selective + gate: + jobs: + - atmosphere-molecule-aio-openvswitch-selective + - atmosphere-molecule-aio-ovn-selective + - atmosphere-molecule-csi-local-path-provisioner-selective + - atmosphere-molecule-csi-rbd-selective + - atmosphere-molecule-keycloak-selective diff --git a/ci/molecule-plan.yaml b/ci/molecule-plan.yaml new file mode 100644 index 0000000000..6130c4f536 --- /dev/null +++ b/ci/molecule-plan.yaml @@ -0,0 +1,233 @@ +--- +version: 1 + +# Ordinary service changes use the current Atmosphere default backend. Direct +# networking changes can opt into both variants below. +canonical_network_backend: openvswitch +full_network_backends: + - openvswitch + - ovn + +# The AIO Molecule inventory uses local-path storage for Kubernetes workloads. +# OpenStack services which require Ceph still pull it into their closure through +# ceph-provisioners and their service dependencies. +dependency_options: + csi_driver: local-path-provisioner + +full_verification_profiles: + - full + +# Zuul jobs are static, but each job consumes an explicit decision from the +# plan. Backend jobs run the matching AIO variant. Dedicated scenario jobs run +# only when one of their verification profiles is requested. +jobs: + aio-openvswitch: + scenario: aio + network_backend: openvswitch + skip_if_only_verification_profiles: + - csi + - keycloak-federation + + aio-ovn: + scenario: aio + network_backend: ovn + skip_if_only_verification_profiles: + - csi + - keycloak-federation + + csi-local-path-provisioner: + scenario: csi + verification_profiles: + - csi + + csi-rbd: + scenario: csi + verification_profiles: + - csi + + keycloak: + scenario: keycloak + verification_profiles: + - keycloak-federation + +rules: + - name: planner-policy + action: full + reason: CI selection policy changed and must validate its conservative fallback + paths: + - ci/molecule-plan.yaml + - cmd/atmosphere/ci.go + - cmd/atmosphere/deploy.go + - cmd/atmosphere/main.go + - internal/ciplan/** + - internal/deploy/component.go + - internal/deploy/orchestrator.go + - pkg/dag/** + - molecule/aio/** + - test-playbooks/molecule/** + - .zuul.yaml + - tox.ini + + - name: shared-deployment-code + action: full + reason: shared deployment code can affect multiple unrelated components + paths: + - .charts.yml + - galaxy.yml + - go.mod + - go.sum + - pyproject.toml + - requirements.txt + - uv.lock + - playbooks/generate_workspace.yml + - playbooks/infrastructure.yml + - playbooks/monitoring.yml + - playbooks/openstack.yml + - playbooks/site.yml + - molecule/shared/** + - plugins/** + - roles/defaults/** + - roles/ingress/** + - roles/openstack_helm_endpoints/** + - roles/openstack_helm_ingress/** + - roles/openstacksdk/** + - roles/rabbitmq/** + - roles/tempest/** + - charts/tempest/** + - charts/patches/tempest/** + + - name: openvswitch-runtime + action: targets + targets: + - openvswitch + verification_profiles: + - openvswitch + network_backends: + - openvswitch + paths: + - Cargo.lock + - Cargo.toml + - crates/ovsinit/** + + - name: non-runtime + action: ignore + paths: + - .agents/** + - .github/** + - .gitignore + - .gitreview + - .pre-commit-config.yaml + - .vale.ini + - AGENTS.md + - CLAUDE.md + - CODEOWNERS + - LICENSE + - README.md + - doc/** + - releasenotes/** + - reno.yaml + - renovate.json + +components: + cert-manager: + paths: + - charts/cert-manager-webhook-infoblox-wapi/** + - charts/patches/cert-manager-webhook-infoblox-wapi/** + - charts/godaddy-webhook/** + - charts/patches/godaddy-webhook/** + verification_profiles: + - cert-manager + + csi: + paths: + - roles/ceph_csi_rbd/** + - roles/local_path_provisioner/** + - roles/powerstore_csi/** + - roles/portworx/** + - roles/storpool_csi/** + - roles/ibm_block_csi_driver/** + - charts/ceph-csi-rbd/** + - charts/local-path-provisioner/** + verification_profiles: + - csi + + percona-xtradb-cluster-operator: + paths: + - charts/pxc-operator/** + - charts/patches/pxc-operator/** + verification_profiles: + - percona-xtradb-cluster + + keycloak: + test_requires: + - keystone + verification_profiles: + - identity + - keycloak-federation + + keystone: + verification_profiles: + - identity + - keycloak-federation + + glance: + verification_profiles: + - image + + cinder: + verification_profiles: + - volume + + placement: + verification_profiles: + - placement + + nova: + test_requires: + - glance-images + - neutron + verification_profiles: + - compute + network_backends: + - canonical + + neutron: + verification_profiles: + - network + network_backends: + - openvswitch + - ovn + + heat: + test_requires: + - glance-images + - neutron + verification_profiles: + - orchestration + network_backends: + - canonical + + octavia: + verification_profiles: + - load-balancer + network_backends: + - canonical + + manila: + verification_profiles: + - shared-file-system + network_backends: + - canonical + + magnum: + test_requires: + - barbican + - cinder + - glance-images + - heat + - neutron + - octavia + verification_profiles: + - container-infrastructure + network_backends: + - canonical diff --git a/cmd/atmosphere/ci.go b/cmd/atmosphere/ci.go new file mode 100644 index 0000000000..35ad7b2d1f --- /dev/null +++ b/cmd/atmosphere/ci.go @@ -0,0 +1,201 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "io" + "os" + "strings" + + "github.com/spf13/cobra" + "github.com/vexxhost/atmosphere/internal/ciplan" +) + +func newCICmd() *cobra.Command { + command := &cobra.Command{ + Use: "ci", + Short: "Plan selective CI deployments and verification", + } + command.AddCommand(newCIPlanCmd()) + command.AddCommand(newCIValidateCmd()) + return command +} + +func newCIValidateCmd() *cobra.Command { + var configPath string + command := &cobra.Command{ + Use: "validate", + Short: "Validate the selective CI policy and dependency graph", + RunE: func(command *cobra.Command, args []string) error { + config, err := ciplan.LoadConfig(configPath) + if err != nil { + return err + } + if _, err := ciplan.New(config); err != nil { + return err + } + _, err = fmt.Fprintf(command.OutOrStdout(), "CI plan policy %s is valid\n", configPath) + return err + }, + } + command.Flags().StringVar( + &configPath, + "config", + "ci/molecule-plan.yaml", + "Path to the CI planning policy", + ) + return command +} + +func newCIPlanCmd() *cobra.Command { + var ( + configPath string + changedFiles []string + filesFrom string + base string + head string + format string + outputPath string + ) + + command := &cobra.Command{ + Use: "plan", + Short: "Map changed files to Molecule deployment variants", + RunE: func(command *cobra.Command, args []string) error { + changes, err := collectChanges( + command, + changedFiles, + filesFrom, + base, + head, + ) + if err != nil { + return err + } + + config, err := ciplan.LoadConfig(configPath) + if err != nil { + return err + } + planner, err := ciplan.New(config) + if err != nil { + return err + } + plan, err := planner.Plan(changes) + if err != nil { + return err + } + + output, closeOutput, err := planOutput(command.OutOrStdout(), outputPath) + if err != nil { + return err + } + defer closeOutput() + + switch format { + case "json": + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(plan) + case "text": + return ciplan.WriteText(output, plan) + default: + return fmt.Errorf("unsupported output format %q", format) + } + }, + } + + command.Flags().StringVar( + &configPath, + "config", + "ci/molecule-plan.yaml", + "Path to the CI planning policy", + ) + command.Flags().StringSliceVar( + &changedFiles, + "changed-file", + nil, + "Changed repository path; may be provided more than once", + ) + command.Flags().StringVar( + &filesFrom, + "files-from", + "", + "Read paths or git name-status records from a file, or - for stdin", + ) + command.Flags().StringVar(&base, "base", "", "Base Git revision for a three-dot diff") + command.Flags().StringVar(&head, "head", "", "Head Git revision for a three-dot diff") + command.Flags().StringVar(&format, "format", "text", "Output format: text or json") + command.Flags().StringVar(&outputPath, "output", "-", "Write the plan to this file, or - for stdout") + return command +} + +func collectChanges( + command *cobra.Command, + changedFiles []string, + filesFrom string, + base string, + head string, +) ([]ciplan.Change, error) { + usingChangedFiles := len(changedFiles) > 0 + usingFilesFrom := strings.TrimSpace(filesFrom) != "" + usingGit := strings.TrimSpace(base) != "" || strings.TrimSpace(head) != "" + + sources := 0 + for _, used := range []bool{usingChangedFiles, usingFilesFrom, usingGit} { + if used { + sources++ + } + } + if sources != 1 { + return nil, fmt.Errorf( + "provide exactly one change source: --changed-file, --files-from, or --base with --head", + ) + } + + if usingChangedFiles { + changes := make([]ciplan.Change, 0, len(changedFiles)) + for _, changedFile := range changedFiles { + changes = append(changes, ciplan.Change{Status: "M", Path: changedFile}) + } + return changes, nil + } + + if usingFilesFrom { + var ( + reader io.Reader + file *os.File + err error + ) + if filesFrom == "-" { + reader = command.InOrStdin() + } else { + file, err = os.Open(filesFrom) + if err != nil { + return nil, fmt.Errorf("opening changed-file input: %w", err) + } + defer file.Close() + reader = file + } + return ciplan.ParseChanges(reader) + } + + if strings.TrimSpace(base) == "" || strings.TrimSpace(head) == "" { + return nil, fmt.Errorf("--base and --head must be provided together") + } + return ciplan.GitChanges(command.Context(), base, head) +} + +func planOutput(stdout io.Writer, outputPath string) (io.Writer, func(), error) { + if outputPath == "" || outputPath == "-" { + return stdout, func() {}, nil + } + file, err := os.Create(outputPath) + if err != nil { + return nil, nil, fmt.Errorf("creating plan output: %w", err) + } + return file, func() { _ = file.Close() }, nil +} diff --git a/cmd/atmosphere/deploy.go b/cmd/atmosphere/deploy.go new file mode 100644 index 0000000000..f57abeab31 --- /dev/null +++ b/cmd/atmosphere/deploy.go @@ -0,0 +1,119 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "os/signal" + "strings" + "syscall" + + "github.com/spf13/cobra" + "github.com/vexxhost/atmosphere/internal/deploy" +) + +func newDeployCmd() *cobra.Command { + var ( + inventory string + tags string + concurrency int + withDependencies bool + dependencyValues []string + ) + + cmd := &cobra.Command{ + Use: "deploy", + Short: "Deploy Atmosphere components", + Long: `Deploy Atmosphere components using parallel execution. + +Without --tags, deploys all components in parallel waves based on the +dependency graph. With a single --tags value, passes through directly +to ansible-playbook for backwards compatibility. With multiple +comma-separated --tags, resolves ordering from the dependency graph +and runs them in parallel where possible. + +Use --with-dependencies with one or more tags when bootstrapping a fresh +environment. It expands the selected tags to their complete transitive +dependency closure before deploying them.`, + RunE: func(cmd *cobra.Command, args []string) error { + if inventory == "" { + return fmt.Errorf("--inventory is required") + } + if withDependencies && strings.TrimSpace(tags) == "" { + return fmt.Errorf("--with-dependencies requires --tags") + } + + dependencyOptions, err := parseDependencyOptions(dependencyValues) + if err != nil { + return err + } + + deployer := &deploy.AnsibleDeployer{ + Inventory: inventory, + Output: os.Stdout, + } + + orchestrator := &deploy.Orchestrator{ + Deployer: deployer, + Inventory: inventory, + Output: os.Stdout, + Concurrency: concurrency, + WithDependencies: withDependencies, + DependencyOptions: dependencyOptions, + } + + // Parse tags + var tagList []string + if tags != "" { + tagList = strings.Split(tags, ",") + for i, t := range tagList { + tagList[i] = strings.TrimSpace(t) + } + } + + // Cancel the context on SIGINT/SIGTERM so in-flight + // ansible-playbook/helm/ssh subprocesses (started with + // exec.CommandContext) are torn down instead of being + // orphaned when the user hits Ctrl-C. + ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) + defer stop() + + return orchestrator.Deploy(ctx, tagList) + }, + } + + cmd.Flags().StringVarP(&inventory, "inventory", "i", "", "Path to Ansible inventory file (required)") + cmd.Flags().StringVarP(&tags, "tags", "t", "", "Comma-separated list of component tags to deploy") + cmd.Flags().IntVar(&concurrency, "concurrency", 0, "Max concurrent deployments per wave (0 = unlimited)") + cmd.Flags().BoolVar(&withDependencies, "with-dependencies", false, "Include transitive dependencies needed by a fresh environment") + cmd.Flags().StringSliceVar( + &dependencyValues, + "dependency-option", + nil, + "Dependency option in key=value form (for example csi_driver=local-path-provisioner)", + ) + + return cmd +} + +func parseDependencyOptions(values []string) (deploy.DependencyOptions, error) { + options := deploy.DependencyOptions{} + for _, value := range values { + key, optionValue, found := strings.Cut(value, "=") + key = strings.TrimSpace(key) + optionValue = strings.TrimSpace(optionValue) + if !found || key == "" || optionValue == "" { + return nil, fmt.Errorf("invalid dependency option %q: expected key=value", value) + } + if _, exists := options[key]; exists { + return nil, fmt.Errorf("dependency option %q was provided more than once", key) + } + options[key] = optionValue + } + if err := deploy.ValidateDependencyOptions(options); err != nil { + return nil, err + } + return options, nil +} diff --git a/cmd/atmosphere/deploy_test.go b/cmd/atmosphere/deploy_test.go new file mode 100644 index 0000000000..ec754dad57 --- /dev/null +++ b/cmd/atmosphere/deploy_test.go @@ -0,0 +1,44 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import "testing" + +func TestParseDependencyOptions(t *testing.T) { + options, err := parseDependencyOptions([]string{ + "csi_driver=local-path-provisioner", + "network_backend=ovn", + }) + if err != nil { + t.Fatalf("parseDependencyOptions returned an error: %v", err) + } + if options["csi_driver"] != "local-path-provisioner" { + t.Errorf("unexpected CSI driver: %q", options["csi_driver"]) + } + if options["network_backend"] != "ovn" { + t.Errorf("unexpected network backend: %q", options["network_backend"]) + } +} + +func TestParseDependencyOptionsRejectsInvalidValues(t *testing.T) { + tests := []struct { + name string + values []string + }{ + {name: "missing separator", values: []string{"csi_driver"}}, + {name: "missing key", values: []string{"=rbd"}}, + {name: "missing value", values: []string{"csi_driver="}}, + {name: "duplicate key", values: []string{"csi_driver=rbd", "csi_driver=local-path-provisioner"}}, + {name: "unknown key", values: []string{"csi_drivr=rbd"}}, + {name: "unknown network backend", values: []string{"network_backend=linuxbridge"}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := parseDependencyOptions(test.values); err == nil { + t.Fatalf("expected an error for %v", test.values) + } + }) + } +} diff --git a/cmd/atmosphere/main.go b/cmd/atmosphere/main.go new file mode 100644 index 0000000000..4aa7486596 --- /dev/null +++ b/cmd/atmosphere/main.go @@ -0,0 +1,27 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" +) + +func main() { + rootCmd := &cobra.Command{ + Use: "atmosphere", + Short: "Atmosphere deployment orchestrator", + Long: "Parallel deployment orchestrator for the Atmosphere cloud platform.", + } + + rootCmd.AddCommand(newDeployCmd()) + rootCmd.AddCommand(newCICmd()) + + if err := rootCmd.Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/doc/source/admin/upgrading.rst b/doc/source/admin/upgrading.rst index b3c3fcf891..75cf4b0180 100644 --- a/doc/source/admin/upgrading.rst +++ b/doc/source/admin/upgrading.rst @@ -7,14 +7,14 @@ This document shows the most common way of upgrading your Atmosphere deployment. .. admonition:: Avoid jumping Atmosphere major releases :class: warning - It is important to avoid jumping major versions in Atmosphere, which is the + It's important to avoid jumping major versions in Atmosphere, which is the same advice in OpenStack. For example, If you are running Atmosphere Zed release (version 1) and want to move to Bobcat (version 3) you should perform 2 upgrades: version 1 to version 2 and then version 3. - If you dont do this, you may face database inconsistencies and failures on + If you don't do this, you may face database inconsistencies and failures on services like Nova or Neutron, or failed upgrades of components such as RabbitMQ. @@ -32,7 +32,7 @@ Atmosphere release you want to upgrade to. - name: vexxhost.atmosphere version: X.Y.Z -Once that is done, you should update your collections by running: +Once that's done, you should update your collections by running: .. code-block:: console @@ -40,7 +40,7 @@ Once that is done, you should update your collections by running: It's important to review your inventory, specifically image overrides to make sure that the image overrides are still necessary, otherwise you may end up -with a broken deployment since the images will not be the ones the Atmosphere +with a broken deployment since the images won't be the ones the Atmosphere collection expects. ******************* @@ -56,6 +56,15 @@ which gives you the most granular control. $ ansible-playbook -i hosts.ini playbooks/site.yml +Alternatively, run the parallel orchestrator, which deploys independent +components concurrently: + +.. code-block:: console + + $ atmosphere deploy --inventory hosts.ini + +For details, see :doc:`/deploy/parallel`. + You can also run the Atmosphere provided playbooks by pointing to a specific playbook of the Ansible collection, in this case, the Ceph playbook: @@ -72,7 +81,7 @@ following command: $ ansible-playbook -i hosts.ini vexxhost.atmosphere.openstack --tags keystone During the upgrade, you may find it useful to have a monitor on all of the pods -in the cluster to ensure that they are becoming ready and not failing. You can +in the cluster to make sure they're becoming READY and not failing. You can do this by running the following command: .. code-block:: console diff --git a/doc/source/deploy/deployment.rst b/doc/source/deploy/deployment.rst index 08ebea4681..051a4d587d 100644 --- a/doc/source/deploy/deployment.rst +++ b/doc/source/deploy/deployment.rst @@ -20,3 +20,7 @@ has ssh access to all the hosts in the inventory file. You can edit the site.yml to include only the roles that you need to deploy if you want to deploy only a subset of the platform at a time. + +For faster deployments, Atmosphere also ships a parallel orchestrator +that runs independent components concurrently. See +:doc:`parallel` for details. diff --git a/doc/source/deploy/index.rst b/doc/source/deploy/index.rst index 63101bdac6..af6e8d4edd 100644 --- a/doc/source/deploy/index.rst +++ b/doc/source/deploy/index.rst @@ -6,9 +6,8 @@ Welcome to the Deployment Guide for the Atmosphere project. This guide provides detailed instructions and resources for setting up and configuring the Atmosphere deployment on a Kubernetes cluster. -It is designed to assist you through the entire process, from initial system -requirements to post-deployment verification, ensuring a successful installation -and setup. +It helps you through the entire process, from initial system requirements to +post-deployment verification, ensuring a successful installation and setup. .. toctree:: :maxdepth: 2 @@ -20,3 +19,5 @@ and setup. glance neutron deployment + parallel + selective-ci diff --git a/doc/source/deploy/parallel.rst b/doc/source/deploy/parallel.rst new file mode 100644 index 0000000000..5e315f5cb9 --- /dev/null +++ b/doc/source/deploy/parallel.rst @@ -0,0 +1,104 @@ +================================ +Parallel deployment orchestrator +================================ + +Atmosphere ships a Go binary, ``atmosphere``, that deploys components in +parallel waves based on a directed acyclic graph (DAG) of dependencies. +Components without a direct dependency on each other run concurrently, +which shortens full deployment time compared to the sequential +role-by-role flow driven by ``site.yml``. + +The orchestrator is additive. It doesn't change Ansible roles, +variables, or playbooks. Existing ``ansible-playbook site.yml`` flows +continue to work unchanged. + +Building the binary +=================== + +Build the binary from the repository root: + +.. code-block:: console + + $ CGO_ENABLED=0 go build -o ./bin/atmosphere ./cmd/atmosphere + +The resulting binary lives at ``./bin/atmosphere``. Copy it onto +``PATH`` if you'd like to run it from any directory. + +Running a deployment +==================== + +The ``deploy`` command takes an inventory path and an optional list of +component tags: + +.. code-block:: console + + $ atmosphere deploy --inventory ./inventory.yaml + +The command requires the same Ansible runtime that ``site.yml`` needs. +Activate the project virtual environment, or otherwise make +``ansible-playbook`` and the ``vexxhost.atmosphere`` collection +available on ``PATH``, before invoking ``atmosphere deploy``. + +Operating modes +=============== + +The orchestrator supports three modes, chosen by how you pass +``--tags``: + +Full deployment + Without ``--tags``, the orchestrator runs every component across + parallel waves derived from the dependency graph. Components in + the same wave run concurrently; later waves start once their + prerequisites finish. + +Single tag + With one tag, for example ``--tags keystone``, the orchestrator + delegates to ``ansible-playbook site.yml --tags keystone``. The + behavior matches the existing tag-based flow. + +Multiple tags in an existing environment + With several comma-separated tags, for example + ``--tags nova,keystone``, the orchestrator builds a subgraph that + contains only the requested components, then runs them in parallel + waves. This mode assumes that dependencies outside the selected + subgraph are already deployed. + +Fresh environment + Add ``--with-dependencies`` when selected components must be + bootstrapped in a fresh environment: + + .. code-block:: console + + $ atmosphere deploy --inventory ./inventory.yaml \ + --tags manila \ + --with-dependencies \ + --dependency-option csi_driver=local-path-provisioner \ + --dependency-option network_backend=openvswitch + + This mode includes all transitive main-role and pre-role dependencies. + Dependency options select conditional relationships. For example, the + ``rbd`` CSI driver requires Ceph, while ``local-path-provisioner`` does + not. Neutron selects CoreDNS for Open vSwitch and OVN for the OVN + backend. + +Concurrency control +=================== + +Use ``--concurrency`` to cap the number of components that run at the +same time inside a wave: + +.. code-block:: console + + $ atmosphere deploy --inventory ./inventory.yaml --concurrency 4 + +The default, ``0``, lets every component in a wave run concurrently. +Lower values reduce peak load on the deploy host and target nodes at +the cost of longer wave durations. + +Output +====== + +Output from each component streams to standard output with a +``[component-name]`` prefix so that you can tell concurrent log lines +apart. A non-zero exit from any component cancels the remaining work +in the current wave and surfaces the failure. diff --git a/doc/source/deploy/selective-ci.rst b/doc/source/deploy/selective-ci.rst new file mode 100644 index 0000000000..e57eac128b --- /dev/null +++ b/doc/source/deploy/selective-ci.rst @@ -0,0 +1,179 @@ +=============================== +Selective Molecule CI planning +=============================== + +Atmosphere can map pull request changes to a smaller Molecule deployment. +The planner keeps change impact separate from deployment dependencies: + +* Changed paths select direct test targets. +* Functional test requirements add any extra test targets. +* The deployment graph expands those roots into the components required by a + fresh environment. +* Verification profiles describe the focused checks that should run. + +The policy lives in ``ci/molecule-plan.yaml``. The Go deployment registry +supplies component role and chart paths. The policy contains additional paths, +shared-code rules, functional test requirements, backend selection, and +verification profiles. It also maps the static Zuul jobs to the variants or +verification profiles they can execute. + +Creating a plan +=============== + +Build the ``atmosphere`` binary, then pass changed paths directly: + +.. code-block:: console + + $ atmosphere ci plan \ + --changed-file roles/manila/tasks/main.yml + +The planner also accepts ``git diff --name-status`` records: + +.. code-block:: console + + $ git diff --name-status --find-renames origin/main...HEAD | + atmosphere ci plan --files-from - + +Or let the planner collect the Git diff: + +.. code-block:: console + + $ atmosphere ci plan --base origin/main --head HEAD + +Use JSON when another CI step will consume the plan: + +.. code-block:: console + + $ atmosphere ci plan \ + --base origin/main \ + --head HEAD \ + --format json \ + --output ci-plan.json + +Plan modes +========== + +``noop`` + Every changed path matches an explicit Molecule ignore rule. + +``selective`` + The plan contains direct targets, deployment roots, verification profiles, + and one component closure for each required network backend. + +``full`` + The plan requires both complete network-backend variants. + +The policy fails safely. Unknown runtime paths, shared deployment code, CI +policy changes, and planner changes select ``full`` rather than silently +omitting tests. Renames and copies evaluate both the old and new paths. +Failure to generate or read a plan in Zuul also runs every Molecule job with +its complete deployment. + +Job selection +============= + +Zuul still declares every Molecule job statically so that its configuration is +easy to inspect. At runtime each job reads its decision from ``ci-plan.json``: + +* An AIO job runs only when the plan contains its network backend. +* Dedicated CSI and Keycloak jobs run only when their verification profile is + requested. +* A documentation-only plan skips every Molecule scenario. +* A complete fallback runs every job. + +The ``jobs`` mapping controls this behavior: + +.. code-block:: yaml + + jobs: + aio-openvswitch: + scenario: aio + network_backend: openvswitch + skip_if_only_verification_profiles: + - csi + - keycloak-federation + + keycloak: + scenario: keycloak + verification_profiles: + - keycloak-federation + +For a selective AIO job, the deployment variant becomes the comma-separated +``atmosphere deploy --tags`` list. Tempest receives the same component list and +marks services that were not deployed as unavailable. This keeps focused +verification from querying unrelated services. + +The deployment runs directly as a Zuul ``command`` task rather than as a +nested Molecule Ansible process. This makes the deployment's Ansible output +visible in the live Zuul console. The same deployment is run a second time to +retain the idempotence check before Molecule verification. + +Selection examples +================== + +The current policy produces these representative job decisions: + +``roles/keystone/**`` + Run the Open vSwitch AIO closure and the dedicated Keycloak federation job. + Skip OVN and both CSI jobs. + +``roles/manila/**`` + Run only the Open vSwitch AIO closure, including Manila's storage, image, + compute, placement, and network test requirements. + +``roles/magnum/**`` + Run only the Open vSwitch AIO job with Magnum's broader orchestration, + secrets, storage, compute, network, and load-balancing environment. + +``roles/neutron/**`` + Run both AIO network variants. + +``roles/ceph_csi_rbd/**`` + Run the two dedicated CSI scenarios and skip both AIO jobs. + +Extending the policy +==================== + +Add service-specific behavior under ``components``. For example: + +.. code-block:: yaml + + components: + manila: + verification_profiles: + - shared-file-system + network_backends: + - canonical + +``test_requires`` contains services needed by functional verification but not +by normal deployment ordering. Magnum uses this to request the broader compute, +network, orchestration, secrets, storage, and load-balancing environment. + +Use a rule for paths which affect several components or which must force the +full fallback: + +.. code-block:: yaml + + rules: + - name: shared-deployment-code + action: full + paths: + - roles/openstack_helm_endpoints/** + +Run the Go tests after every policy change. Configuration validation rejects +unknown components, invalid actions, unsupported globs, and invalid backend +references. + +Validate the policy directly with: + +.. code-block:: console + + $ atmosphere ci validate + +Zuul execution +============== + +The pre-run playbook writes ``ci-plan.json`` into the build logs and prints the +human-readable explanation. Every job then reports its run or skip decision, +reason, component closure, and verification profiles before doing any +deployment work. diff --git a/internal/ciplan/changes.go b/internal/ciplan/changes.go new file mode 100644 index 0000000000..b3d3220818 --- /dev/null +++ b/internal/ciplan/changes.go @@ -0,0 +1,79 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package ciplan + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "os/exec" + "strings" +) + +// ParseChanges accepts either one path per line or git diff --name-status +// output. Rename and copy records preserve both the old and new paths. +func ParseChanges(reader io.Reader) ([]Change, error) { + var changes []Change + scanner := bufio.NewScanner(reader) + lineNumber := 0 + for scanner.Scan() { + lineNumber++ + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + fields := strings.Split(line, "\t") + switch len(fields) { + case 1: + changes = append(changes, Change{Status: "M", Path: fields[0]}) + case 2: + if fields[0] == "" || fields[1] == "" { + return nil, fmt.Errorf("invalid changed-file record on line %d", lineNumber) + } + changes = append(changes, Change{Status: fields[0], Path: fields[1]}) + case 3: + if !strings.HasPrefix(fields[0], "R") && !strings.HasPrefix(fields[0], "C") { + return nil, fmt.Errorf( + "three-column changed-file record on line %d must be a rename or copy", + lineNumber, + ) + } + changes = append(changes, Change{ + Status: fields[0], + PreviousPath: fields[1], + Path: fields[2], + }) + default: + return nil, fmt.Errorf("invalid changed-file record on line %d", lineNumber) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("reading changed files: %w", err) + } + return changes, nil +} + +// GitChanges obtains name-status records for a three-dot Git diff. +func GitChanges(ctx context.Context, base, head string) ([]Change, error) { + if strings.TrimSpace(base) == "" || strings.TrimSpace(head) == "" { + return nil, fmt.Errorf("both base and head revisions are required") + } + + command := exec.CommandContext( + ctx, + "git", + "diff", + "--name-status", + "--find-renames", + base+"..."+head, + ) + output, err := command.Output() + if err != nil { + return nil, fmt.Errorf("collecting changed files from Git: %w", err) + } + return ParseChanges(bytes.NewReader(output)) +} diff --git a/internal/ciplan/config.go b/internal/ciplan/config.go new file mode 100644 index 0000000000..9c14448959 --- /dev/null +++ b/internal/ciplan/config.go @@ -0,0 +1,238 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package ciplan + +import ( + "fmt" + "os" + "slices" + + "github.com/goccy/go-yaml" + "github.com/vexxhost/atmosphere/internal/deploy" +) + +// LoadConfig reads and validates a planner configuration. +func LoadConfig(path string) (Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return Config{}, fmt.Errorf("reading CI plan configuration: %w", err) + } + return ParseConfig(data) +} + +// ParseConfig parses and validates planner configuration data. +func ParseConfig(data []byte) (Config, error) { + var config Config + if err := yaml.Unmarshal(data, &config); err != nil { + return Config{}, fmt.Errorf("parsing CI plan configuration: %w", err) + } + if err := config.Validate(); err != nil { + return Config{}, err + } + return config, nil +} + +// Validate checks references and patterns before any change is planned. +func (c Config) Validate() error { + if c.Version != 1 { + return fmt.Errorf("unsupported CI plan configuration version %d", c.Version) + } + if c.CanonicalNetworkBackend == "" { + return fmt.Errorf("canonical_network_backend is required") + } + if len(c.FullNetworkBackends) == 0 { + return fmt.Errorf("full_network_backends must not be empty") + } + if !slices.Contains(c.FullNetworkBackends, c.CanonicalNetworkBackend) { + return fmt.Errorf( + "canonical network backend %q is not in full_network_backends", + c.CanonicalNetworkBackend, + ) + } + if len(c.FullVerificationProfiles) == 0 { + return fmt.Errorf("full_verification_profiles must not be empty") + } + + componentNames := make(map[string]bool, len(deploy.Components)) + for _, component := range deploy.Components { + componentNames[component.Name] = true + } + + ruleNames := make(map[string]bool, len(c.Rules)) + for _, rule := range c.Rules { + if rule.Name == "" { + return fmt.Errorf("rule name must not be empty") + } + if ruleNames[rule.Name] { + return fmt.Errorf("rule %q is defined more than once", rule.Name) + } + ruleNames[rule.Name] = true + if len(rule.Paths) == 0 { + return fmt.Errorf("rule %q has no paths", rule.Name) + } + switch rule.Action { + case ActionIgnore, ActionFull: + if len(rule.Targets) > 0 { + return fmt.Errorf("rule %q action %q must not define targets", rule.Name, rule.Action) + } + case ActionTargets: + if len(rule.Targets) == 0 { + return fmt.Errorf("rule %q action targets requires at least one target", rule.Name) + } + default: + return fmt.Errorf("rule %q has unsupported action %q", rule.Name, rule.Action) + } + for _, target := range rule.Targets { + if !componentNames[target] { + return fmt.Errorf("rule %q references unknown target %q", rule.Name, target) + } + } + for _, pattern := range rule.Paths { + if _, err := compileGlob(pattern); err != nil { + return fmt.Errorf("rule %q path %q: %w", rule.Name, pattern, err) + } + } + } + + for name, policy := range c.Components { + if !componentNames[name] { + return fmt.Errorf("policy references unknown component %q", name) + } + for _, requirement := range policy.TestRequires { + if !componentNames[requirement] { + return fmt.Errorf( + "component %q has unknown test requirement %q", + name, + requirement, + ) + } + } + for _, pattern := range policy.Paths { + if _, err := compileGlob(pattern); err != nil { + return fmt.Errorf("component %q path %q: %w", name, pattern, err) + } + } + } + + if len(c.Jobs) == 0 { + return fmt.Errorf("jobs must not be empty") + } + verificationProfiles := configuredVerificationProfiles(c) + for name, policy := range c.Jobs { + if name == "" { + return fmt.Errorf("job name must not be empty") + } + if policy.Scenario == "" { + return fmt.Errorf("job %q scenario must not be empty", name) + } + if policy.NetworkBackend == "" && len(policy.VerificationProfiles) == 0 { + return fmt.Errorf( + "job %q must define network_backend or verification_profiles", + name, + ) + } + if policy.NetworkBackend != "" && len(policy.VerificationProfiles) > 0 { + return fmt.Errorf( + "job %q cannot define both network_backend and verification_profiles", + name, + ) + } + if policy.NetworkBackend == "" && len(policy.SkipIfOnlyVerificationProfiles) > 0 { + return fmt.Errorf( + "job %q cannot skip profiles without a network_backend", + name, + ) + } + for _, profile := range append( + slices.Clone(policy.VerificationProfiles), + policy.SkipIfOnlyVerificationProfiles..., + ) { + if !verificationProfiles[profile] { + return fmt.Errorf( + "job %q references unknown verification profile %q", + name, + profile, + ) + } + } + } + + for _, backend := range append( + slices.Clone(c.FullNetworkBackends), + configuredBackends(c.Rules, c.Components, c.Jobs)..., + ) { + if backend == "canonical" { + continue + } + if !slices.Contains(c.FullNetworkBackends, backend) { + return fmt.Errorf("unknown network backend %q", backend) + } + } + + options := deploy.DependencyOptions(c.DependencyOptions) + for _, backend := range c.FullNetworkBackends { + optionsForBackend := cloneOptions(options) + optionsForBackend[deploy.DependencyOptionNetworkBackend] = backend + if graph, err := deploy.BuildGraphWithOptions(optionsForBackend); err != nil { + return fmt.Errorf("building deployment graph for backend %q: %w", backend, err) + } else if _, err := graph.Waves(); err != nil { + return fmt.Errorf("validating deployment graph for backend %q: %w", backend, err) + } + for _, component := range deploy.Components { + if _, err := deploy.BootstrapComponentNames( + []string{component.Name}, + optionsForBackend, + ); err != nil { + return fmt.Errorf( + "validating bootstrap closure for component %q on backend %q: %w", + component.Name, + backend, + err, + ) + } + } + } + + return nil +} + +func configuredBackends( + rules []Rule, + policies map[string]ComponentPolicy, + jobs map[string]JobPolicy, +) []string { + var backends []string + for _, rule := range rules { + backends = append(backends, rule.NetworkBackends...) + } + for _, policy := range policies { + backends = append(backends, policy.NetworkBackends...) + } + for _, job := range jobs { + if job.NetworkBackend != "" { + backends = append(backends, job.NetworkBackend) + } + } + return backends +} + +func configuredVerificationProfiles(config Config) map[string]bool { + profiles := make(map[string]bool) + addAll(profiles, config.FullVerificationProfiles) + for _, rule := range config.Rules { + addAll(profiles, rule.VerificationProfiles) + } + for _, policy := range config.Components { + addAll(profiles, policy.VerificationProfiles) + } + return profiles +} + +func cloneOptions(options map[string]string) map[string]string { + cloned := make(map[string]string, len(options)+1) + for key, value := range options { + cloned[key] = value + } + return cloned +} diff --git a/internal/ciplan/glob.go b/internal/ciplan/glob.go new file mode 100644 index 0000000000..bfded4c203 --- /dev/null +++ b/internal/ciplan/glob.go @@ -0,0 +1,51 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package ciplan + +import ( + "fmt" + "regexp" + "strings" +) + +// compileGlob supports repository-oriented globs. A single star does not +// cross a path separator; a double star does. +func compileGlob(pattern string) (*regexp.Regexp, error) { + if pattern == "" { + return nil, fmt.Errorf("glob must not be empty") + } + if strings.HasPrefix(pattern, "/") { + return nil, fmt.Errorf("glob must be repository-relative") + } + + var expression strings.Builder + expression.WriteString("^") + for index := 0; index < len(pattern); { + switch pattern[index] { + case '*': + if index+1 < len(pattern) && pattern[index+1] == '*' { + expression.WriteString(".*") + index += 2 + } else { + expression.WriteString("[^/]*") + index++ + } + case '?': + expression.WriteString("[^/]") + index++ + case '[', ']': + return nil, fmt.Errorf("character classes are not supported") + default: + expression.WriteString(regexp.QuoteMeta(pattern[index : index+1])) + index++ + } + } + expression.WriteString("$") + + compiled, err := regexp.Compile(expression.String()) + if err != nil { + return nil, fmt.Errorf("compiling glob: %w", err) + } + return compiled, nil +} diff --git a/internal/ciplan/output.go b/internal/ciplan/output.go new file mode 100644 index 0000000000..2ac261eb96 --- /dev/null +++ b/internal/ciplan/output.go @@ -0,0 +1,134 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package ciplan + +import ( + "fmt" + "io" + "strings" +) + +// WriteText writes an explainable summary suitable for a Zuul console log. +func WriteText(writer io.Writer, plan Plan) error { + if _, err := fmt.Fprintf(writer, "CI plan mode: %s\n", plan.Mode); err != nil { + return err + } + + if len(plan.Changes) > 0 { + if _, err := fmt.Fprintln(writer, "Changed files:"); err != nil { + return err + } + for _, change := range plan.Changes { + if change.PreviousPath == "" { + if _, err := fmt.Fprintf(writer, " %s\t%s\n", change.Status, change.Path); err != nil { + return err + } + } else if _, err := fmt.Fprintf( + writer, + " %s\t%s -> %s\n", + change.Status, + change.PreviousPath, + change.Path, + ); err != nil { + return err + } + } + } + + if len(plan.Matches) > 0 { + if _, err := fmt.Fprintln(writer, "Matches:"); err != nil { + return err + } + for _, match := range plan.Matches { + targets := "" + if len(match.Targets) > 0 { + targets = " -> " + strings.Join(match.Targets, ",") + } + if _, err := fmt.Fprintf( + writer, + " %s: %s (%s)%s\n", + match.Path, + match.Rule, + match.Action, + targets, + ); err != nil { + return err + } + } + } + + if len(plan.Targets) > 0 { + if _, err := fmt.Fprintf(writer, "Direct targets: %s\n", strings.Join(plan.Targets, ", ")); err != nil { + return err + } + } + if len(plan.DeploymentRoots) > 0 { + if _, err := fmt.Fprintf( + writer, + "Deployment roots: %s\n", + strings.Join(plan.DeploymentRoots, ", "), + ); err != nil { + return err + } + } + if len(plan.VerificationProfiles) > 0 { + if _, err := fmt.Fprintf( + writer, + "Verification profiles: %s\n", + strings.Join(plan.VerificationProfiles, ", "), + ); err != nil { + return err + } + } + + for _, variant := range plan.Variants { + if _, err := fmt.Fprintf( + writer, + "Variant %s (%d components): %s\n", + variant.NetworkBackend, + len(variant.Components), + strings.Join(variant.Components, ", "), + ); err != nil { + return err + } + } + + if len(plan.Jobs) > 0 { + if _, err := fmt.Fprintln(writer, "Molecule jobs:"); err != nil { + return err + } + for _, job := range plan.Jobs { + action := "skip" + if job.Run { + action = "run" + } + components := "" + if len(job.Components) > 0 { + components = fmt.Sprintf(" (%d components)", len(job.Components)) + } + if _, err := fmt.Fprintf( + writer, + " %s: %s%s - %s\n", + job.Name, + action, + components, + job.Reason, + ); err != nil { + return err + } + } + } + + if len(plan.Reasons) > 0 { + if _, err := fmt.Fprintln(writer, "Reasons:"); err != nil { + return err + } + for _, reason := range plan.Reasons { + if _, err := fmt.Fprintf(writer, " - %s\n", reason); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/ciplan/planner.go b/internal/ciplan/planner.go new file mode 100644 index 0000000000..20aa727328 --- /dev/null +++ b/internal/ciplan/planner.go @@ -0,0 +1,502 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package ciplan + +import ( + "fmt" + "path" + "regexp" + "slices" + "sort" + "strings" + + "github.com/vexxhost/atmosphere/internal/deploy" +) + +type compiledRule struct { + rule Rule + patterns []*regexp.Regexp +} + +type compiledPolicy struct { + policy ComponentPolicy + patterns []*regexp.Regexp +} + +// Planner evaluates changed paths using a validated Config and the deployment +// component registry. +type Planner struct { + config Config + rules []compiledRule + policies map[string]compiledPolicy + components map[string]deploy.Component + rolesToComponents map[string]string + chartsToComponents map[string]string + playbookComponents map[string]string +} + +// New creates a planner and precompiles all configured path patterns. +func New(config Config) (*Planner, error) { + if err := config.Validate(); err != nil { + return nil, err + } + + planner := &Planner{ + config: config, + policies: make(map[string]compiledPolicy, len(config.Components)), + components: make(map[string]deploy.Component, len(deploy.Components)), + rolesToComponents: make(map[string]string, len(deploy.Components)), + chartsToComponents: make(map[string]string, len(deploy.Components)), + playbookComponents: make(map[string]string, len(deploy.Components)), + } + + for _, component := range deploy.Components { + planner.components[component.Name] = component + planner.chartsToComponents[component.Name] = component.Name + if component.RoleName != "" { + planner.rolesToComponents[component.RoleName] = component.Name + } + if component.PreRoleName != "" { + planner.rolesToComponents[component.PreRoleName] = component.Name + } + if component.Playbook != "" { + planner.playbookComponents[component.Playbook+".yml"] = component.Name + } + } + + for _, rule := range config.Rules { + compiled := compiledRule{rule: rule} + for _, pattern := range rule.Paths { + matcher, err := compileGlob(pattern) + if err != nil { + return nil, fmt.Errorf("compiling rule %q: %w", rule.Name, err) + } + compiled.patterns = append(compiled.patterns, matcher) + } + planner.rules = append(planner.rules, compiled) + } + + for name, policy := range config.Components { + compiled := compiledPolicy{policy: policy} + for _, pattern := range policy.Paths { + matcher, err := compileGlob(pattern) + if err != nil { + return nil, fmt.Errorf("compiling component %q: %w", name, err) + } + compiled.patterns = append(compiled.patterns, matcher) + } + planner.policies[name] = compiled + } + + return planner, nil +} + +// Plan maps changed files to deployment and verification variants. +func (p *Planner) Plan(changes []Change) (Plan, error) { + if len(changes) == 0 { + return p.fullPlan(Plan{ + Mode: ModeFull, + Reasons: []string{"no changed files were provided, so the planner cannot select a safe subset"}, + }), nil + } + + normalizedChanges := make([]Change, 0, len(changes)) + targets := map[string]bool{} + verificationProfiles := map[string]bool{} + requestedBackends := map[string]bool{} + var matches []Match + var reasons []string + full := false + + for _, change := range changes { + normalized, err := normalizeChange(change) + if err != nil { + return Plan{}, err + } + normalizedChanges = append(normalizedChanges, normalized) + + for _, changedPath := range changePaths(normalized) { + pathMatched := false + + for _, rule := range p.rules { + if !matchesAny(rule.patterns, changedPath) { + continue + } + pathMatched = true + matches = append(matches, Match{ + Path: changedPath, + Rule: rule.rule.Name, + Action: rule.rule.Action, + Targets: slices.Clone(rule.rule.Targets), + }) + switch rule.rule.Action { + case ActionFull: + full = true + reasons = appendUnique(reasons, ruleReason(rule.rule, changedPath)) + case ActionTargets: + addAll(targets, rule.rule.Targets) + addAll(verificationProfiles, rule.rule.VerificationProfiles) + addAll(requestedBackends, rule.rule.NetworkBackends) + } + } + + for _, componentName := range p.matchComponents(changedPath) { + pathMatched = true + targets[componentName] = true + matches = append(matches, Match{ + Path: changedPath, + Rule: "component:" + componentName, + Action: ActionTargets, + Targets: []string{componentName}, + }) + } + + if !pathMatched { + full = true + reasons = appendUnique( + reasons, + fmt.Sprintf("unclassified runtime path %q requires the full fallback", changedPath), + ) + matches = append(matches, Match{ + Path: changedPath, + Rule: "unclassified", + Action: ActionFull, + }) + } + } + } + + plan := Plan{ + Changes: normalizedChanges, + Matches: matches, + Reasons: reasons, + } + + if full { + return p.fullPlan(plan), nil + } + if len(targets) == 0 { + plan.Mode = ModeNoop + plan.Reasons = appendUnique(plan.Reasons, "all changed paths are ignored by the Molecule policy") + plan.Jobs = p.planJobs(plan) + return plan, nil + } + + directTargets := sortedKeys(targets) + deploymentRoots := map[string]bool{} + queue := slices.Clone(directTargets) + for len(queue) > 0 { + name := queue[0] + queue = queue[1:] + if deploymentRoots[name] { + continue + } + deploymentRoots[name] = true + + policy := p.policies[name].policy + for _, requirement := range policy.TestRequires { + if !deploymentRoots[requirement] { + queue = append(queue, requirement) + } + } + } + + for _, target := range directTargets { + policy := p.policies[target].policy + if len(policy.VerificationProfiles) == 0 { + verificationProfiles[target] = true + } else { + addAll(verificationProfiles, policy.VerificationProfiles) + } + addAll(requestedBackends, policy.NetworkBackends) + } + if len(requestedBackends) == 0 { + requestedBackends["canonical"] = true + } + + backends, err := p.resolveBackends(sortedKeys(requestedBackends)) + if err != nil { + return Plan{}, err + } + roots := sortedKeys(deploymentRoots) + + plan.Mode = ModeSelective + plan.Targets = directTargets + plan.DeploymentRoots = roots + plan.VerificationProfiles = sortedKeys(verificationProfiles) + + for _, backend := range backends { + options := cloneOptions(p.config.DependencyOptions) + options[deploy.DependencyOptionNetworkBackend] = backend + components, err := deploy.BootstrapComponentNames( + roots, + deploy.DependencyOptions(options), + ) + if err != nil { + return Plan{}, fmt.Errorf("planning backend %q: %w", backend, err) + } + plan.Variants = append(plan.Variants, Variant{ + NetworkBackend: backend, + DependencyOptions: options, + Components: components, + }) + } + + plan.Jobs = p.planJobs(plan) + return plan, nil +} + +func (p *Planner) fullPlan(plan Plan) Plan { + plan.Mode = ModeFull + plan.VerificationProfiles = slices.Clone(p.config.FullVerificationProfiles) + + components := make([]string, 0, len(p.components)) + for name := range p.components { + components = append(components, name) + } + sort.Strings(components) + + for _, backend := range p.config.FullNetworkBackends { + options := cloneOptions(p.config.DependencyOptions) + options[deploy.DependencyOptionNetworkBackend] = backend + plan.Variants = append(plan.Variants, Variant{ + NetworkBackend: backend, + DependencyOptions: options, + Components: slices.Clone(components), + }) + } + plan.Jobs = p.planJobs(plan) + return plan +} + +func (p *Planner) planJobs(plan Plan) []JobPlan { + names := make([]string, 0, len(p.config.Jobs)) + for name := range p.config.Jobs { + names = append(names, name) + } + sort.Strings(names) + + jobs := make([]JobPlan, 0, len(names)) + for _, name := range names { + policy := p.config.Jobs[name] + job := JobPlan{ + Name: name, + Scenario: policy.Scenario, + NetworkBackend: policy.NetworkBackend, + VerificationProfiles: slices.Clone(plan.VerificationProfiles), + } + + switch plan.Mode { + case ModeFull: + job.Run = true + job.Reason = "full fallback runs every Molecule job" + case ModeNoop: + job.Reason = "no runtime changes require Molecule" + case ModeSelective: + if policy.NetworkBackend != "" { + variant, found := findVariant(plan.Variants, policy.NetworkBackend) + if !found { + job.Reason = fmt.Sprintf( + "plan has no %s deployment variant", + policy.NetworkBackend, + ) + break + } + job.Components = slices.Clone(variant.Components) + if profilesOnly( + plan.VerificationProfiles, + policy.SkipIfOnlyVerificationProfiles, + ) { + job.Reason = fmt.Sprintf( + "verification is handled by dedicated profiles: %s", + strings.Join(plan.VerificationProfiles, ", "), + ) + break + } + job.Run = true + job.Reason = fmt.Sprintf( + "executes the %s deployment variant", + policy.NetworkBackend, + ) + break + } + + matchedProfiles := intersect( + plan.VerificationProfiles, + policy.VerificationProfiles, + ) + if len(matchedProfiles) == 0 { + job.Reason = "none of the job's verification profiles were requested" + break + } + job.Run = true + job.Reason = fmt.Sprintf( + "requested verification profiles: %s", + strings.Join(matchedProfiles, ", "), + ) + } + + if job.Run && policy.NetworkBackend != "" && len(job.Components) == 0 { + if variant, found := findVariant(plan.Variants, policy.NetworkBackend); found { + job.Components = slices.Clone(variant.Components) + } + } + jobs = append(jobs, job) + } + return jobs +} + +func findVariant(variants []Variant, backend string) (Variant, bool) { + for _, variant := range variants { + if variant.NetworkBackend == backend { + return variant, true + } + } + return Variant{}, false +} + +func profilesOnly(requested, dedicated []string) bool { + if len(requested) == 0 || len(dedicated) == 0 { + return false + } + for _, profile := range requested { + if !slices.Contains(dedicated, profile) { + return false + } + } + return true +} + +func intersect(left, right []string) []string { + var values []string + for _, value := range left { + if slices.Contains(right, value) { + values = append(values, value) + } + } + return values +} + +func (p *Planner) matchComponents(changedPath string) []string { + matched := map[string]bool{} + segments := strings.Split(changedPath, "/") + + if len(segments) >= 3 && segments[0] == "roles" { + if component, ok := p.rolesToComponents[segments[1]]; ok { + matched[component] = true + } + } + if len(segments) >= 3 && segments[0] == "charts" && segments[1] != "patches" { + if component, ok := p.chartsToComponents[segments[1]]; ok { + matched[component] = true + } + } + if len(segments) >= 4 && segments[0] == "charts" && segments[1] == "patches" { + if component, ok := p.chartsToComponents[segments[2]]; ok { + matched[component] = true + } + } + if len(segments) == 2 && segments[0] == "playbooks" { + if component, ok := p.playbookComponents[segments[1]]; ok { + matched[component] = true + } + } + + for name, policy := range p.policies { + if matchesAny(policy.patterns, changedPath) { + matched[name] = true + } + } + return sortedKeys(matched) +} + +func (p *Planner) resolveBackends(requested []string) ([]string, error) { + resolved := map[string]bool{} + for _, backend := range requested { + if backend == "canonical" { + backend = p.config.CanonicalNetworkBackend + } + if !slices.Contains(p.config.FullNetworkBackends, backend) { + return nil, fmt.Errorf("network backend %q is not configured", backend) + } + resolved[backend] = true + } + return sortedKeys(resolved), nil +} + +func normalizeChange(change Change) (Change, error) { + var err error + change.Path, err = normalizeRepoPath(change.Path) + if err != nil { + return Change{}, fmt.Errorf("normalizing changed path: %w", err) + } + if change.PreviousPath != "" { + change.PreviousPath, err = normalizeRepoPath(change.PreviousPath) + if err != nil { + return Change{}, fmt.Errorf("normalizing previous path: %w", err) + } + } + if change.Status == "" { + change.Status = "M" + } + return change, nil +} + +func normalizeRepoPath(value string) (string, error) { + value = strings.TrimPrefix(strings.ReplaceAll(value, "\\", "/"), "./") + cleaned := path.Clean(value) + if value == "" || cleaned == "." { + return "", fmt.Errorf("path must not be empty") + } + if strings.HasPrefix(cleaned, "../") || strings.HasPrefix(cleaned, "/") { + return "", fmt.Errorf("path %q is outside the repository", value) + } + return cleaned, nil +} + +func changePaths(change Change) []string { + paths := []string{change.Path} + if change.PreviousPath != "" && change.PreviousPath != change.Path { + paths = append(paths, change.PreviousPath) + } + return paths +} + +func matchesAny(patterns []*regexp.Regexp, value string) bool { + for _, pattern := range patterns { + if pattern.MatchString(value) { + return true + } + } + return false +} + +func addAll(set map[string]bool, values []string) { + for _, value := range values { + set[value] = true + } +} + +func appendUnique(values []string, value string) []string { + if slices.Contains(values, value) { + return values + } + return append(values, value) +} + +func sortedKeys(values map[string]bool) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func ruleReason(rule Rule, changedPath string) string { + if rule.Reason != "" { + return fmt.Sprintf("%s: %s", changedPath, rule.Reason) + } + return fmt.Sprintf("%s matched full-fallback rule %q", changedPath, rule.Name) +} diff --git a/internal/ciplan/planner_test.go b/internal/ciplan/planner_test.go new file mode 100644 index 0000000000..b40c6b72c1 --- /dev/null +++ b/internal/ciplan/planner_test.go @@ -0,0 +1,373 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package ciplan + +import ( + "os" + "slices" + "strings" + "testing" + + "github.com/goccy/go-yaml" + "github.com/vexxhost/atmosphere/internal/deploy" +) + +func testPlanner(t *testing.T) *Planner { + t.Helper() + + data, err := os.ReadFile("../../ci/molecule-plan.yaml") + if err != nil { + t.Fatalf("reading planner configuration: %v", err) + } + config, err := ParseConfig(data) + if err != nil { + t.Fatalf("parsing planner configuration: %v", err) + } + planner, err := New(config) + if err != nil { + t.Fatalf("creating planner: %v", err) + } + return planner +} + +func planForPath(t *testing.T, changedPath string) Plan { + t.Helper() + plan, err := testPlanner(t).Plan([]Change{{Status: "M", Path: changedPath}}) + if err != nil { + t.Fatalf("planning %q: %v", changedPath, err) + } + return plan +} + +func jobForName(t *testing.T, plan Plan, name string) JobPlan { + t.Helper() + for _, job := range plan.Jobs { + if job.Name == name { + return job + } + } + t.Fatalf("plan does not contain job %q: %#v", name, plan.Jobs) + return JobPlan{} +} + +func TestZuulSelectiveJobsMatchPolicy(t *testing.T) { + configData, err := os.ReadFile("../../ci/molecule-plan.yaml") + if err != nil { + t.Fatalf("reading planner configuration: %v", err) + } + config, err := ParseConfig(configData) + if err != nil { + t.Fatalf("parsing planner configuration: %v", err) + } + + zuulData, err := os.ReadFile("../../.zuul.yaml") + if err != nil { + t.Fatalf("reading Zuul configuration: %v", err) + } + var entries []struct { + Job struct { + Name string `yaml:"name"` + Vars map[string]any `yaml:"vars"` + } `yaml:"job"` + } + if err := yaml.Unmarshal(zuulData, &entries); err != nil { + t.Fatalf("parsing Zuul configuration: %v", err) + } + + configuredJobs := make([]string, 0, len(config.Jobs)) + for name := range config.Jobs { + configuredJobs = append(configuredJobs, name) + } + slices.Sort(configuredJobs) + + var zuulJobs []string + for _, entry := range entries { + if entry.Job.Vars == nil { + continue + } + name, ok := entry.Job.Vars["atmosphere_ci_job"].(string) + if !ok { + continue + } + if !strings.HasSuffix(entry.Job.Name, "-selective") { + t.Errorf("selective policy job %q uses unexpected Zuul job %q", name, entry.Job.Name) + } + zuulJobs = append(zuulJobs, name) + } + slices.Sort(zuulJobs) + + if !slices.Equal(zuulJobs, configuredJobs) { + t.Fatalf("Zuul policy jobs = %v, want %v", zuulJobs, configuredJobs) + } +} + +func TestPlanKeystoneUsesSmallLocalPathClosure(t *testing.T) { + plan := planForPath(t, "roles/keystone/tasks/main.yml") + + if plan.Mode != ModeSelective { + t.Fatalf("mode = %q, want selective", plan.Mode) + } + if !slices.Equal(plan.Targets, []string{"keystone"}) { + t.Fatalf("targets = %v, want keystone", plan.Targets) + } + if len(plan.Variants) != 1 { + t.Fatalf("variants = %d, want 1", len(plan.Variants)) + } + + components := plan.Variants[0].Components + for _, required := range []string{ + "cluster-issuer", + "csi", + "keycloak", + "keystone", + "kubernetes", + "percona-xtradb-cluster", + } { + if !slices.Contains(components, required) { + t.Errorf("Keystone closure is missing %q: %v", required, components) + } + } + for _, excluded := range []string{"ceph", "glance", "nova", "neutron", "manila", "magnum"} { + if slices.Contains(components, excluded) { + t.Errorf("Keystone closure unexpectedly contains %q: %v", excluded, components) + } + } + + for _, name := range []string{"aio-openvswitch", "keycloak"} { + if job := jobForName(t, plan, name); !job.Run { + t.Errorf("Keystone plan skips required job %q: %#v", name, job) + } + } + for _, name := range []string{"aio-ovn", "csi-local-path-provisioner", "csi-rbd"} { + if job := jobForName(t, plan, name); job.Run { + t.Errorf("Keystone plan unexpectedly runs job %q: %#v", name, job) + } + } +} + +func TestPlanManilaIncludesFunctionalDependencies(t *testing.T) { + plan := planForPath(t, "charts/patches/manila/0001-example.patch") + components := plan.Variants[0].Components + + for _, required := range []string{ + "ceph", + "cinder", + "glance", + "keystone", + "manila", + "neutron", + "nova", + "placement", + } { + if !slices.Contains(components, required) { + t.Errorf("Manila closure is missing %q: %v", required, components) + } + } + for _, excluded := range []string{ + "barbican", + "heat", + "horizon", + "magnum", + "octavia", + "openstack-exporter", + } { + if slices.Contains(components, excluded) { + t.Errorf("Manila closure unexpectedly contains %q: %v", excluded, components) + } + } + + if job := jobForName(t, plan, "aio-openvswitch"); !job.Run { + t.Errorf("Manila plan skips canonical AIO job: %#v", job) + } + if job := jobForName(t, plan, "aio-ovn"); job.Run { + t.Errorf("Manila plan unexpectedly runs OVN AIO job: %#v", job) + } +} + +func TestPlanMagnumUsesBroadTestEnvironment(t *testing.T) { + plan := planForPath(t, "roles/magnum/tasks/main.yml") + components := plan.Variants[0].Components + + for _, required := range []string{ + "barbican", + "cinder", + "glance-images", + "heat", + "magnum", + "neutron", + "nova", + "octavia", + } { + if !slices.Contains(components, required) { + t.Errorf("Magnum closure is missing %q: %v", required, components) + } + } + for _, excluded := range []string{"horizon", "manila", "openstack-exporter"} { + if slices.Contains(components, excluded) { + t.Errorf("Magnum closure unexpectedly contains %q: %v", excluded, components) + } + } + + if job := jobForName(t, plan, "aio-openvswitch"); !job.Run { + t.Errorf("Magnum plan skips canonical AIO job: %#v", job) + } + for _, name := range []string{ + "aio-ovn", + "csi-local-path-provisioner", + "csi-rbd", + "keycloak", + } { + if job := jobForName(t, plan, name); job.Run { + t.Errorf("Magnum plan unexpectedly runs job %q: %#v", name, job) + } + } +} + +func TestPlanNeutronCreatesBackendSpecificVariants(t *testing.T) { + plan := planForPath(t, "roles/neutron/tasks/main.yml") + if len(plan.Variants) != 2 { + t.Fatalf("variants = %d, want 2: %#v", len(plan.Variants), plan.Variants) + } + + var openvswitch, ovn *Variant + for index := range plan.Variants { + switch plan.Variants[index].NetworkBackend { + case "openvswitch": + openvswitch = &plan.Variants[index] + case "ovn": + ovn = &plan.Variants[index] + } + } + if openvswitch == nil || ovn == nil { + t.Fatalf("expected Open vSwitch and OVN variants: %#v", plan.Variants) + } + if !slices.Contains(openvswitch.Components, "coredns") { + t.Errorf("Open vSwitch variant is missing CoreDNS: %v", openvswitch.Components) + } + if slices.Contains(openvswitch.Components, "ovn") { + t.Errorf("Open vSwitch variant unexpectedly contains OVN: %v", openvswitch.Components) + } + if !slices.Contains(ovn.Components, "ovn") { + t.Errorf("OVN variant is missing OVN: %v", ovn.Components) + } + if slices.Contains(ovn.Components, "coredns") { + t.Errorf("OVN variant unexpectedly contains CoreDNS: %v", ovn.Components) + } + for _, name := range []string{"aio-openvswitch", "aio-ovn"} { + if job := jobForName(t, plan, name); !job.Run { + t.Errorf("Neutron plan skips required backend job %q: %#v", name, job) + } + } +} + +func TestPlanCSIRunsOnlyDedicatedScenarios(t *testing.T) { + plan := planForPath(t, "roles/ceph_csi_rbd/tasks/main.yml") + + for _, name := range []string{"csi-local-path-provisioner", "csi-rbd"} { + if job := jobForName(t, plan, name); !job.Run { + t.Errorf("CSI plan skips dedicated job %q: %#v", name, job) + } + } + for _, name := range []string{"aio-openvswitch", "aio-ovn", "keycloak"} { + if job := jobForName(t, plan, name); job.Run { + t.Errorf("CSI plan unexpectedly runs job %q: %#v", name, job) + } + } +} + +func TestPlanIgnoredFilesIsNoop(t *testing.T) { + plan := planForPath(t, "doc/source/deploy/parallel.rst") + if plan.Mode != ModeNoop { + t.Fatalf("mode = %q, want noop: %#v", plan.Mode, plan) + } + for _, job := range plan.Jobs { + if job.Run { + t.Errorf("noop plan unexpectedly runs job %q: %#v", job.Name, job) + } + } +} + +func TestPlanUnknownPathFallsBackToFull(t *testing.T) { + plan := planForPath(t, "new-runtime-area/config.yaml") + if plan.Mode != ModeFull { + t.Fatalf("mode = %q, want full", plan.Mode) + } + if len(plan.Variants) != 2 { + t.Fatalf("full fallback variants = %d, want 2", len(plan.Variants)) + } + for _, variant := range plan.Variants { + if len(variant.Components) != len(deploy.Components) { + t.Errorf( + "full fallback has %d components, want %d", + len(variant.Components), + len(deploy.Components), + ) + } + } + for _, job := range plan.Jobs { + if !job.Run { + t.Errorf("full fallback skips job %q: %#v", job.Name, job) + } + } +} + +func TestPlanEmptyChangeListFallsBackToFull(t *testing.T) { + plan, err := testPlanner(t).Plan(nil) + if err != nil { + t.Fatalf("planning an empty change list: %v", err) + } + if plan.Mode != ModeFull { + t.Fatalf("mode = %q, want full", plan.Mode) + } +} + +func TestPlanPolicyChangeFallsBackToFull(t *testing.T) { + plan := planForPath(t, "ci/molecule-plan.yaml") + if plan.Mode != ModeFull { + t.Fatalf("mode = %q, want full", plan.Mode) + } +} + +func TestPlanRenameEvaluatesOldAndNewPaths(t *testing.T) { + plan, err := testPlanner(t).Plan([]Change{{ + Status: "R100", + PreviousPath: "roles/keystone/tasks/old.yml", + Path: "roles/manila/tasks/new.yml", + }}) + if err != nil { + t.Fatalf("planning rename: %v", err) + } + if !slices.Equal(plan.Targets, []string{"keystone", "manila"}) { + t.Fatalf("rename targets = %v, want keystone and manila", plan.Targets) + } +} + +func TestParseChangesNameStatus(t *testing.T) { + changes, err := ParseChanges(strings.NewReader( + "M\troles/keystone/tasks/main.yml\n" + + "R100\troles/manila/tasks/old.yml\troles/manila/tasks/new.yml\n", + )) + if err != nil { + t.Fatalf("ParseChanges returned an error: %v", err) + } + if len(changes) != 2 { + t.Fatalf("changes = %d, want 2", len(changes)) + } + if changes[1].PreviousPath != "roles/manila/tasks/old.yml" { + t.Errorf("unexpected previous path: %#v", changes[1]) + } +} + +func TestCompileGlobDoubleStar(t *testing.T) { + matcher, err := compileGlob("roles/manila/**") + if err != nil { + t.Fatalf("compileGlob returned an error: %v", err) + } + if !matcher.MatchString("roles/manila/tasks/main.yml") { + t.Error("double-star glob did not match a nested path") + } + if matcher.MatchString("roles/magnum/tasks/main.yml") { + t.Error("double-star glob matched another role") + } +} diff --git a/internal/ciplan/types.go b/internal/ciplan/types.go new file mode 100644 index 0000000000..8597f436d2 --- /dev/null +++ b/internal/ciplan/types.go @@ -0,0 +1,105 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package ciplan maps changed repository files to the smallest safe Molecule +// deployment and verification plan. +package ciplan + +// Change describes a changed repository path. PreviousPath is populated for +// renames and copies so both sides of the change are evaluated. +type Change struct { + Status string `json:"status"` + Path string `json:"path"` + PreviousPath string `json:"previous_path,omitempty"` +} + +// Config is the declarative CI impact and verification policy. +type Config struct { + Version int `yaml:"version"` + CanonicalNetworkBackend string `yaml:"canonical_network_backend"` + FullNetworkBackends []string `yaml:"full_network_backends"` + DependencyOptions map[string]string `yaml:"dependency_options"` + Rules []Rule `yaml:"rules"` + Components map[string]ComponentPolicy `yaml:"components"` + Jobs map[string]JobPolicy `yaml:"jobs"` + FullVerificationProfiles []string `yaml:"full_verification_profiles"` +} + +// Rule handles shared, ignored, or otherwise non-component-specific paths. +type Rule struct { + Name string `yaml:"name"` + Paths []string `yaml:"paths"` + Action string `yaml:"action"` + Targets []string `yaml:"targets"` + VerificationProfiles []string `yaml:"verification_profiles"` + NetworkBackends []string `yaml:"network_backends"` + Reason string `yaml:"reason"` +} + +// ComponentPolicy adds functional-test requirements and verification metadata +// to a deployable component from the deployment registry. +type ComponentPolicy struct { + Paths []string `yaml:"paths"` + TestRequires []string `yaml:"test_requires"` + VerificationProfiles []string `yaml:"verification_profiles"` + NetworkBackends []string `yaml:"network_backends"` +} + +// JobPolicy maps a static Zuul job to the part of a CI plan it can execute. +// Backend jobs consume a matching deployment variant. Profile jobs run when +// any listed verification profile is requested. +type JobPolicy struct { + Scenario string `yaml:"scenario"` + NetworkBackend string `yaml:"network_backend"` + VerificationProfiles []string `yaml:"verification_profiles"` + SkipIfOnlyVerificationProfiles []string `yaml:"skip_if_only_verification_profiles"` +} + +// Match explains why one changed path affected the plan. +type Match struct { + Path string `json:"path"` + Rule string `json:"rule"` + Action string `json:"action"` + Targets []string `json:"targets,omitempty"` +} + +// Variant is the deployment closure for one network backend. +type Variant struct { + NetworkBackend string `json:"network_backend"` + DependencyOptions map[string]string `json:"dependency_options"` + Components []string `json:"components"` +} + +// JobPlan is the executable decision for one static Zuul Molecule job. +type JobPlan struct { + Name string `json:"name"` + Scenario string `json:"scenario"` + Run bool `json:"run"` + NetworkBackend string `json:"network_backend,omitempty"` + Components []string `json:"components,omitempty"` + VerificationProfiles []string `json:"verification_profiles,omitempty"` + Reason string `json:"reason"` +} + +// Plan is both machine-readable CI input and an explainable build artifact. +type Plan struct { + Mode string `json:"mode"` + Changes []Change `json:"changes"` + Matches []Match `json:"matches"` + Targets []string `json:"targets,omitempty"` + DeploymentRoots []string `json:"deployment_roots,omitempty"` + VerificationProfiles []string `json:"verification_profiles,omitempty"` + Variants []Variant `json:"variants,omitempty"` + Jobs []JobPlan `json:"jobs"` + Reasons []string `json:"reasons,omitempty"` +} + +const ( + ActionIgnore = "ignore" + ActionFull = "full" + ActionTargets = "targets" + + ModeNoop = "noop" + ModeSelective = "selective" + ModeFull = "full" +) diff --git a/internal/deploy/bootstrap_test.go b/internal/deploy/bootstrap_test.go new file mode 100644 index 0000000000..8e3d9acbba --- /dev/null +++ b/internal/deploy/bootstrap_test.go @@ -0,0 +1,143 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package deploy + +import ( + "context" + "io" + "slices" + "testing" +) + +func TestBootstrapComponentNamesKeystoneLocalPath(t *testing.T) { + components, err := BootstrapComponentNames( + []string{"keystone"}, + DependencyOptions{DependencyOptionCSIDriver: "local-path-provisioner"}, + ) + if err != nil { + t.Fatalf("BootstrapComponentNames returned an error: %v", err) + } + + for _, expected := range []string{ + "csi", + "ingress-nginx", + "keycloak", + "keystone", + "kubernetes", + "memcached", + "percona-xtradb-cluster", + "percona-xtradb-cluster-operator", + "rabbitmq-cluster-operator", + } { + if !slices.Contains(components, expected) { + t.Errorf("expected bootstrap closure to contain %q: %v", expected, components) + } + } + if slices.Contains(components, "ceph") { + t.Errorf("local-path Keystone closure must not contain Ceph: %v", components) + } +} + +func TestBootstrapComponentNamesCSIDriverDependencies(t *testing.T) { + tests := []struct { + name string + driver string + wantCeph bool + }{ + {name: "rbd", driver: "rbd", wantCeph: true}, + {name: "local path", driver: "local-path-provisioner", wantCeph: false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + components, err := BootstrapComponentNames( + []string{"csi"}, + DependencyOptions{DependencyOptionCSIDriver: test.driver}, + ) + if err != nil { + t.Fatalf("BootstrapComponentNames returned an error: %v", err) + } + if got := slices.Contains(components, "ceph"); got != test.wantCeph { + t.Errorf("Ceph presence = %v, want %v: %v", got, test.wantCeph, components) + } + }) + } +} + +func TestBootstrapComponentNamesNeutronBackendDependencies(t *testing.T) { + tests := []struct { + name string + backend string + want string + doesNotWant string + }{ + { + name: "Open vSwitch", + backend: "openvswitch", + want: "coredns", + doesNotWant: "ovn", + }, + { + name: "OVN", + backend: "ovn", + want: "ovn", + doesNotWant: "coredns", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + components, err := BootstrapComponentNames( + []string{"neutron"}, + DependencyOptions{ + DependencyOptionCSIDriver: "local-path-provisioner", + DependencyOptionNetworkBackend: test.backend, + }, + ) + if err != nil { + t.Fatalf("BootstrapComponentNames returned an error: %v", err) + } + if !slices.Contains(components, "openvswitch") { + t.Errorf("Neutron closure must contain Open vSwitch: %v", components) + } + if !slices.Contains(components, test.want) { + t.Errorf("expected closure to contain %q: %v", test.want, components) + } + if slices.Contains(components, test.doesNotWant) { + t.Errorf("closure must not contain %q: %v", test.doesNotWant, components) + } + }) + } +} + +func TestBootstrapComponentNamesUnknownTarget(t *testing.T) { + if _, err := BootstrapComponentNames([]string{"does-not-exist"}, nil); err == nil { + t.Fatal("expected an error for an unknown component") + } +} + +func TestOrchestratorWithDependencies(t *testing.T) { + mock := &mockDeployer{} + orchestrator := &Orchestrator{ + Deployer: mock, + Concurrency: 1, + Output: io.Discard, + Preflight: noopPreflight, + WithDependencies: true, + DependencyOptions: DependencyOptions{ + DependencyOptionCSIDriver: "local-path-provisioner", + }, + } + + if err := orchestrator.Deploy(context.Background(), []string{"keystone"}); err != nil { + t.Fatalf("Deploy returned an error: %v", err) + } + + if !slices.Contains(mock.deployed, "keycloak") { + t.Errorf("fresh Keystone deployment must include its pre-role dependency: %v", mock.deployed) + } + if slices.Contains(mock.deployed, "ceph") { + t.Errorf("fresh local-path Keystone deployment must not include Ceph: %v", mock.deployed) + } +} diff --git a/internal/deploy/completion.go b/internal/deploy/completion.go new file mode 100644 index 0000000000..3e733ba8bf --- /dev/null +++ b/internal/deploy/completion.go @@ -0,0 +1,65 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package deploy + +import ( + "context" + "sync" +) + +// completionTracker signals when named components finish deploying. Components +// whose pre-role has an asymmetric dependency (see Component.PreRoleDependsOn) +// use it to gate the pre-role while the main role runs eagerly. +// +// Signals are one-shot: MarkDone is idempotent; Wait returns immediately for +// already-completed names. +type completionTracker struct { + mu sync.Mutex + done map[string]chan struct{} +} + +// newCompletionTracker preallocates channels for every component name so that +// Wait can be called before MarkDone without a race. +func newCompletionTracker(names []string) *completionTracker { + m := make(map[string]chan struct{}, len(names)) + for _, n := range names { + m[n] = make(chan struct{}) + } + return &completionTracker{done: m} +} + +// MarkDone signals that the component with the given name has finished. +// Safe to call multiple times; only the first call closes the channel. +func (t *completionTracker) MarkDone(name string) { + t.mu.Lock() + ch, ok := t.done[name] + t.mu.Unlock() + if !ok { + return + } + select { + case <-ch: + default: + close(ch) + } +} + +// Wait blocks until every named component is done or the context is cancelled. +// Unknown names are ignored so the caller does not need to pre-validate them. +func (t *completionTracker) Wait(ctx context.Context, names []string) error { + for _, n := range names { + t.mu.Lock() + ch, ok := t.done[n] + t.mu.Unlock() + if !ok { + continue + } + select { + case <-ch: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil +} diff --git a/internal/deploy/completion_test.go b/internal/deploy/completion_test.go new file mode 100644 index 0000000000..473484d387 --- /dev/null +++ b/internal/deploy/completion_test.go @@ -0,0 +1,66 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package deploy + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" +) + +func TestCompletionTracker_WaitBlocksUntilDone(t *testing.T) { + tr := newCompletionTracker([]string{"a", "b"}) + + var waited atomic.Bool + done := make(chan struct{}) + go func() { + defer close(done) + if err := tr.Wait(context.Background(), []string{"a"}); err != nil { + t.Errorf("unexpected error: %v", err) + return + } + waited.Store(true) + }() + + // Give the goroutine a chance to block on the channel. + time.Sleep(20 * time.Millisecond) + if waited.Load() { + t.Fatal("Wait returned before MarkDone was called") + } + + tr.MarkDone("a") + <-done + if !waited.Load() { + t.Fatal("Wait did not return after MarkDone") + } +} + +func TestCompletionTracker_MarkDoneIdempotent(t *testing.T) { + tr := newCompletionTracker([]string{"a"}) + tr.MarkDone("a") + tr.MarkDone("a") // must not panic on double close + if err := tr.Wait(context.Background(), []string{"a"}); err != nil { + t.Fatalf("Wait on already-done name returned error: %v", err) + } +} + +func TestCompletionTracker_UnknownNameIgnored(t *testing.T) { + tr := newCompletionTracker([]string{"a"}) + if err := tr.Wait(context.Background(), []string{"unknown"}); err != nil { + t.Fatalf("Wait on unknown name should be a no-op, got: %v", err) + } +} + +func TestCompletionTracker_ContextCancellation(t *testing.T) { + tr := newCompletionTracker([]string{"a"}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := tr.Wait(ctx, []string{"a"}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled, got: %v", err) + } +} diff --git a/internal/deploy/component.go b/internal/deploy/component.go new file mode 100644 index 0000000000..313c16e46a --- /dev/null +++ b/internal/deploy/component.go @@ -0,0 +1,733 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package deploy + +import ( + "fmt" + "slices" + "sort" + "strings" + + "github.com/vexxhost/atmosphere/pkg/dag" +) + +// ComponentType distinguishes between full-playbook and single-role components. +type ComponentType int + +const ( + // RoleType: single Ansible role → generates a minimal playbook at runtime + RoleType ComponentType = iota + // PlaybookType: full multi-play playbook → runs the file directly + PlaybookType +) + +// DependencyOptions contains runtime values which affect the component graph. +// Defaults preserve the behavior of a normal production deployment. +type DependencyOptions map[string]string + +const ( + DependencyOptionCSIDriver = "csi_driver" + DependencyOptionNetworkBackend = "network_backend" +) + +// DefaultDependencyOptions returns dependency option defaults matching the +// defaults used by the Ansible roles. +func DefaultDependencyOptions() DependencyOptions { + return DependencyOptions{ + DependencyOptionCSIDriver: "rbd", + DependencyOptionNetworkBackend: "openvswitch", + } +} + +// ValidateDependencyOptions rejects misspelled or unsupported graph options. +func ValidateDependencyOptions(options DependencyOptions) error { + for key, value := range options { + if value == "" { + return fmt.Errorf("dependency option %q must not be empty", key) + } + switch key { + case DependencyOptionCSIDriver: + case DependencyOptionNetworkBackend: + if value != "openvswitch" && value != "ovn" { + return fmt.Errorf("unsupported network backend %q", value) + } + default: + return fmt.Errorf("unknown dependency option %q", key) + } + } + return nil +} + +// ConditionalDependency is enabled only when a dependency option has one of +// the configured values. +type ConditionalDependency struct { + Name string + Option string + Values []string +} + +func (d ConditionalDependency) enabled(options DependencyOptions) bool { + return slices.Contains(d.Values, options[d.Option]) +} + +// Component represents a deployable unit in the Atmosphere platform. +type Component struct { + // Name is the unique identifier (kebab-case), e.g., "cert-manager" + Name string + // Type is RoleType or PlaybookType + Type ComponentType + // Tag is the Ansible tag (if different from Name). Empty means use Name. + Tag string + // Playbook is the playbook filename (without .yml) for PlaybookType, e.g., "ceph" + Playbook string + // RoleName is the Ansible role name for RoleType, e.g., "cert_manager" + RoleName string + // Hosts is the Ansible host pattern, e.g., "controllers[0]", "controllers:computes" + Hosts string + // DependsOn lists component Names that must complete before this one + DependsOn []string + // ConditionalDependsOn lists main-role dependencies which vary with + // deployment configuration, such as the selected CSI driver. + ConditionalDependsOn []ConditionalDependency + // When is an optional Ansible conditional expression + When string + // Environment contains optional play-level environment variables + Environment map[string]string + // GatherFacts controls fact gathering. nil = default (true), ptr to false = skip + GatherFacts *bool + // Resources lists named resources this component requires exclusive access to. + // Components sharing a resource are serialized even if the DAG allows parallelism. + // Example: []string{"apt"} serializes all components that use package management. + Resources []string + // PreRoleName is an optional Ansible role to run concurrently with the main + // role. This enables intra-component parallelism: pre-work (e.g., creating + // OpenStack resources, downloading images) overlaps with the Helm install. + // Both must complete before the component is considered done. + PreRoleName string + // PreRoleDependsOn lists component Names that must complete before the + // pre-role may start. When empty, the pre-role starts as soon as the + // component's own DependsOn are satisfied (the current behaviour). When + // set, the main role still starts eagerly while the pre-role waits. + // This enables asymmetric gating — e.g. keystone's Keycloak realm + // creation (pre-role) waits for keycloak while the keystone Helm install + // (main role) runs in parallel with keycloak startup. + PreRoleDependsOn []string + // ConditionalPreRoleDependsOn lists configuration-dependent dependencies + // which must complete before the pre-role starts. + ConditionalPreRoleDependsOn []ConditionalDependency +} + +// EffectiveTag returns the Ansible tag for this component. +func (c Component) EffectiveTag() string { + if c.Tag != "" { + return c.Tag + } + return c.Name +} + +// boolPtr returns a pointer to a bool value. +func boolPtr(b bool) *bool { + return &b +} + +var cephEnvironment = map[string]string{ + "CEPH_CONTAINER_IMAGE": `{{ cephadm_image | default('quay.io/ceph/ceph:v' + (ceph_version | default('18.2.7'))) }}`, + "CEPH_CONTAINER_BINARY": "docker", +} + +// Components is the full registry of all deployment components. +var Components = []Component{ + // Foundation (PlaybookType) + { + Name: "ceph", + Type: PlaybookType, + Playbook: "ceph", + // "apt": ceph installs packages; serialize against other apt + // consumers to avoid dpkg lock contention. + // "containerd": both the ceph and kubernetes playbooks include + // the vexxhost.containers.containerd role, which calls + // systemctl daemon-reload + enable on containerd.service. + // When run in parallel, the two enable calls race on dbus and + // intermittently fail with: + // "Unable to enable service containerd: Failed to enable + // unit: Message recipient disconnected from message bus + // without replying" + Resources: []string{"apt", "containerd", "ca-certificates"}, + }, + { + Name: "kubernetes", + Type: PlaybookType, + Playbook: "kubernetes", + // "apt": kubernetes installs apt packages; serialize against + // other apt consumers to avoid dpkg lock contention. + // "containerd": shares vexxhost.containers.containerd with the + // ceph playbook — see ceph above for the dbus race details. + Resources: []string{"apt", "containerd"}, + }, + { + Name: "csi", + Type: PlaybookType, + Playbook: "csi", + DependsOn: []string{"kubernetes"}, + ConditionalDependsOn: []ConditionalDependency{ + { + Name: "ceph", + Option: DependencyOptionCSIDriver, + Values: []string{"rbd"}, + }, + }, + }, + + // Infrastructure (RoleType, Hosts: "controllers") + { + Name: "cert-manager", + Type: RoleType, + RoleName: "cert_manager", + Hosts: "controllers", + DependsOn: []string{"kubernetes"}, + }, + { + Name: "cluster-issuer", + Type: RoleType, + RoleName: "cluster_issuer", + Hosts: "controllers", + DependsOn: []string{"cert-manager"}, + // Updating the host CA trust while another component is opening a TLS + // connection can produce transient certificate verification failures. + Resources: []string{"ca-certificates"}, + }, + { + Name: "ingress-nginx", + Type: RoleType, + RoleName: "ingress_nginx", + Hosts: "controllers", + DependsOn: []string{"kubernetes"}, + // Helm install talks to apiserver; counts toward k8s-api cap to + // prevent Wave 2 fan-out from overloading a single-node apiserver. + Resources: []string{"k8s-api"}, + }, + { + Name: "rabbitmq-cluster-operator", + Type: RoleType, + RoleName: "rabbitmq_cluster_operator", + Hosts: "controllers", + DependsOn: []string{"cert-manager"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "percona-xtradb-cluster-operator", + Type: RoleType, + RoleName: "percona_xtradb_cluster_operator", + Hosts: "controllers", + DependsOn: []string{"cert-manager"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "percona-xtradb-cluster", + Type: RoleType, + RoleName: "percona_xtradb_cluster", + Hosts: "controllers", + DependsOn: []string{"percona-xtradb-cluster-operator", "csi"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "valkey", + Type: RoleType, + RoleName: "valkey", + Hosts: "controllers", + DependsOn: []string{"kubernetes", "csi", "cluster-issuer"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "keycloak", + Type: RoleType, + RoleName: "keycloak", + Hosts: "controllers", + DependsOn: []string{"percona-xtradb-cluster", "ingress-nginx", "cluster-issuer"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "keepalived", + Type: RoleType, + RoleName: "keepalived", + Hosts: "controllers", + DependsOn: []string{"kubernetes"}, + }, + + // Monitoring (RoleType, Hosts: "controllers[0]") + { + Name: "node-feature-discovery", + Type: RoleType, + RoleName: "node_feature_discovery", + Hosts: "controllers[0]", + DependsOn: []string{"kubernetes"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "kube-prometheus-stack", + Type: RoleType, + RoleName: "kube_prometheus_stack", + Hosts: "controllers[0]", + DependsOn: []string{"kubernetes", "csi", "cluster-issuer", "keycloak"}, + Resources: []string{"k8s-api", "keycloak-admin"}, + }, + { + Name: "loki", + Type: RoleType, + RoleName: "loki", + Hosts: "controllers[0]", + DependsOn: []string{"kubernetes", "csi"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "vector", + Type: RoleType, + RoleName: "vector", + Hosts: "controllers[0]", + DependsOn: []string{"loki"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "goldpinger", + Type: RoleType, + RoleName: "goldpinger", + Hosts: "controllers[0]", + DependsOn: []string{"kubernetes"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "ipmi-exporter", + Type: RoleType, + RoleName: "ipmi_exporter", + Hosts: "controllers[0]", + DependsOn: []string{"kube-prometheus-stack"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "prometheus-pushgateway", + Type: RoleType, + RoleName: "prometheus_pushgateway", + Hosts: "controllers[0]", + DependsOn: []string{"kube-prometheus-stack"}, + Resources: []string{"k8s-api"}, + }, + + // OS Configuration (RoleType, Hosts: "controllers:computes") + // These are pure host config — no K8s dependency. + { + Name: "lpfc", + Type: RoleType, + RoleName: "lpfc", + Hosts: "controllers:computes", + }, + { + Name: "multipathd", + Type: RoleType, + RoleName: "multipathd", + Hosts: "controllers:computes", + Resources: []string{"apt"}, + }, + { + Name: "iscsi", + Type: RoleType, + RoleName: "iscsi", + Hosts: "controllers:computes", + Resources: []string{"apt"}, + }, + { + Name: "udev", + Type: RoleType, + RoleName: "udev", + Hosts: "controllers:computes", + }, + + // OpenStack (RoleType, Hosts: "controllers[0]") + { + // Pre-pull all known Atmosphere container images on every node + // so later component Helm installs do not stall in + // ImagePulling. Best-effort: failures are non-fatal because + // the kubelet falls back to on-demand pulls. + Name: "image-warmup", + Type: RoleType, + RoleName: "image_warmup", + Hosts: "controllers:computes", + DependsOn: []string{"kubernetes"}, + GatherFacts: boolPtr(false), + }, + { + Name: "memcached", + Type: RoleType, + RoleName: "memcached", + Hosts: "controllers[0]", + DependsOn: []string{"kubernetes"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "keystone", + Type: RoleType, + RoleName: "keystone", + PreRoleName: "keystone_pre", + Hosts: "controllers[0]", + DependsOn: []string{"ingress-nginx", "rabbitmq-cluster-operator", "percona-xtradb-cluster", "memcached"}, + PreRoleDependsOn: []string{"keycloak"}, + Resources: []string{"k8s-api", "keycloak-admin"}, + }, + { + Name: "barbican", + Type: RoleType, + RoleName: "barbican", + Hosts: "controllers[0]", + DependsOn: []string{"keystone"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "rook-ceph", + Type: RoleType, + RoleName: "rook_ceph", + Hosts: "controllers[0]", + DependsOn: []string{"kubernetes"}, + Environment: cephEnvironment, + Resources: []string{"k8s-api"}, + }, + { + Name: "rook-ceph-cluster", + Type: RoleType, + RoleName: "rook_ceph_cluster", + Hosts: "controllers[0]", + DependsOn: []string{"rook-ceph", "ceph", "keystone"}, + Environment: cephEnvironment, + Resources: []string{"k8s-api"}, + }, + { + Name: "ceph-provisioners", + Type: RoleType, + RoleName: "ceph_provisioners", + Hosts: "controllers[0]", + DependsOn: []string{"ceph", "kubernetes"}, + Environment: cephEnvironment, + Resources: []string{"k8s-api"}, + }, + { + Name: "glance", + Type: RoleType, + RoleName: "glance", + Hosts: "controllers[0]", + DependsOn: []string{"keystone", "ceph-provisioners"}, + Resources: []string{"k8s-api"}, + }, + { + // Image uploads run as a separate component so they do not block + // downstream services (Nova, Magnum, etc.) which only need the + // Glance API to be deployed. + Name: "glance-images", + Type: RoleType, + RoleName: "glance_images", + Hosts: "controllers[0]", + DependsOn: []string{"glance"}, + }, + { + Name: "staffeln", + Type: RoleType, + RoleName: "staffeln", + Hosts: "controllers[0]", + DependsOn: []string{"cinder"}, + When: `atmosphere_staffeln_enabled | default(true)`, + }, + { + Name: "cinder", + Type: RoleType, + RoleName: "cinder", + Hosts: "controllers[0]", + DependsOn: []string{"keystone", "ceph-provisioners"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "placement", + Type: RoleType, + RoleName: "placement", + Hosts: "controllers[0]", + DependsOn: []string{"keystone"}, + Resources: []string{"k8s-api"}, + }, + + // SDN (RoleType, Hosts: "controllers:computes", GatherFacts: false) + { + Name: "openvswitch", + Type: RoleType, + RoleName: "openvswitch", + Hosts: "controllers:computes", + DependsOn: []string{"kubernetes"}, + GatherFacts: boolPtr(false), + }, + { + Name: "frr-k8s", + Type: RoleType, + RoleName: "frr_k8s", + Tag: "frr_k8s", + Hosts: "controllers:computes", + DependsOn: []string{"kubernetes"}, + When: `ovn_bgp_agent_enabled | default(false)`, + GatherFacts: boolPtr(false), + }, + { + Name: "ovn", + Type: RoleType, + RoleName: "ovn", + Hosts: "controllers:computes", + DependsOn: []string{"openvswitch"}, + GatherFacts: boolPtr(false), + }, + + // Remaining OpenStack (RoleType, Hosts: "controllers[0]") + { + Name: "libvirt", + Type: RoleType, + RoleName: "libvirt", + Hosts: "controllers[0]", + DependsOn: []string{"kubernetes", "cluster-issuer"}, + }, + { + Name: "coredns", + Type: RoleType, + RoleName: "coredns", + Hosts: "controllers[0]", + DependsOn: []string{"kubernetes"}, + }, + { + Name: "nova", + Type: RoleType, + RoleName: "nova", + Hosts: "controllers[0]", + DependsOn: []string{"placement", "glance", "libvirt"}, + Resources: []string{"k8s-api"}, + }, + { + // The pre-role does the Helm install (heavy, ~5 min) and only + // requires Keystone and the configured network backend. The main + // role's only remaining + // work is the post-install "Create networks" task, which hits + // neutron-server's AZ check that needs Nova compute to have + // registered the default availability zone "nova". Splitting + // the role lets the install overlap with Nova while the cheap + // network creation continues to wait on Nova. + Name: "neutron", + Type: RoleType, + RoleName: "neutron", + PreRoleName: "neutron_pre", + Hosts: "controllers[0]", + DependsOn: []string{"nova"}, + PreRoleDependsOn: []string{"keystone", "openvswitch"}, + ConditionalPreRoleDependsOn: []ConditionalDependency{ + { + Name: "coredns", + Option: DependencyOptionNetworkBackend, + Values: []string{"openvswitch"}, + }, + { + Name: "ovn", + Option: DependencyOptionNetworkBackend, + Values: []string{"ovn"}, + }, + }, + Resources: []string{"k8s-api"}, + }, + { + Name: "heat", + Type: RoleType, + RoleName: "heat", + Hosts: "controllers[0]", + DependsOn: []string{"keystone"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "octavia", + Type: RoleType, + RoleName: "octavia", + PreRoleName: "octavia_pre", + Hosts: "controllers[0]", + DependsOn: []string{"keystone", "nova", "neutron"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "magnum", + Type: RoleType, + RoleName: "magnum", + PreRoleName: "magnum_pre", + Hosts: "controllers[0]", + DependsOn: []string{"keystone", "glance"}, + PreRoleDependsOn: []string{"glance-images"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "manila", + Type: RoleType, + RoleName: "manila", + Hosts: "controllers[0]", + DependsOn: []string{"keystone", "nova", "neutron", "cinder"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "horizon", + Type: RoleType, + RoleName: "horizon", + Hosts: "controllers[0]", + DependsOn: []string{"keystone"}, + Resources: []string{"k8s-api"}, + }, + { + Name: "openstack-exporter", + Type: RoleType, + RoleName: "openstack_exporter", + Hosts: "controllers[0]", + DependsOn: []string{"cinder", "neutron"}, + }, + { + Name: "openstack-cli", + Type: RoleType, + RoleName: "openstack_cli", + Hosts: "controllers", + DependsOn: []string{"keystone"}, + }, +} + +// BuildGraph constructs a DAG from the component registry. +func BuildGraph() (*dag.Graph[Component], error) { + return BuildGraphWithOptions(nil) +} + +// BuildGraphWithOptions constructs a DAG using configuration-dependent +// component relationships. +func BuildGraphWithOptions(options DependencyOptions) (*dag.Graph[Component], error) { + if err := ValidateDependencyOptions(options); err != nil { + return nil, err + } + options = dependencyOptionsWithDefaults(options) + + g := dag.NewGraph[Component]() + for _, c := range Components { + if err := g.AddNode(c.Name, c); err != nil { + return nil, fmt.Errorf("adding node %s: %w", c.Name, err) + } + } + for _, c := range Components { + for _, dep := range mainDependencies(c, options) { + if err := g.AddEdge(c.Name, dep); err != nil { + return nil, fmt.Errorf("adding edge %s -> %s: %w", c.Name, dep, err) + } + } + } + return g, nil +} + +// BootstrapComponentNames returns the requested components and every +// transitive dependency required to deploy them in a fresh environment. +// Pre-role dependencies are included because, unlike normal --tags behavior, +// a fresh environment cannot assume that they are already available. +func BootstrapComponentNames(targets []string, options DependencyOptions) ([]string, error) { + if err := ValidateDependencyOptions(options); err != nil { + return nil, err + } + options = dependencyOptionsWithDefaults(options) + + registry := make(map[string]Component, len(Components)) + for _, component := range Components { + registry[component.Name] = component + } + + const ( + unvisited = iota + visiting + visited + ) + state := make(map[string]int, len(Components)) + selected := make(map[string]bool, len(Components)) + var stack []string + + var visit func(string) error + visit = func(name string) error { + component, ok := registry[name] + if !ok { + return fmt.Errorf("component %q not found", name) + } + + switch state[name] { + case visiting: + cycleStart := slices.Index(stack, name) + cycle := append(slices.Clone(stack[cycleStart:]), name) + return fmt.Errorf("dependency cycle detected: %s", strings.Join(cycle, " -> ")) + case visited: + return nil + } + + state[name] = visiting + stack = append(stack, name) + for _, dependency := range bootstrapDependencies(component, options) { + if err := visit(dependency); err != nil { + return err + } + } + stack = stack[:len(stack)-1] + state[name] = visited + selected[name] = true + return nil + } + + for _, target := range targets { + if err := visit(target); err != nil { + return nil, err + } + } + + names := make([]string, 0, len(selected)) + for name := range selected { + names = append(names, name) + } + sort.Strings(names) + return names, nil +} + +func dependencyOptionsWithDefaults(options DependencyOptions) DependencyOptions { + resolved := DefaultDependencyOptions() + for key, value := range options { + resolved[key] = value + } + return resolved +} + +func mainDependencies(component Component, options DependencyOptions) []string { + dependencies := slices.Clone(component.DependsOn) + for _, dependency := range component.ConditionalDependsOn { + if dependency.enabled(options) { + dependencies = append(dependencies, dependency.Name) + } + } + return dependencies +} + +func preRoleDependencies(component Component, options DependencyOptions) []string { + dependencies := slices.Clone(component.PreRoleDependsOn) + for _, dependency := range component.ConditionalPreRoleDependsOn { + if dependency.enabled(options) { + dependencies = append(dependencies, dependency.Name) + } + } + return dependencies +} + +func bootstrapDependencies(component Component, options DependencyOptions) []string { + return append( + mainDependencies(component, options), + preRoleDependencies(component, options)..., + ) +} + +// FindComponent looks up a component by name or tag. +func FindComponent(nameOrTag string) (Component, bool) { + for _, c := range Components { + if c.Name == nameOrTag || c.EffectiveTag() == nameOrTag { + return c, true + } + } + return Component{}, false +} diff --git a/internal/deploy/component_test.go b/internal/deploy/component_test.go new file mode 100644 index 0000000000..6cc49219ea --- /dev/null +++ b/internal/deploy/component_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package deploy + +import ( + "reflect" + "slices" + "testing" +) + +func TestNeutronInstallDoesNotDependOnNova(t *testing.T) { + neutron, ok := FindComponent("neutron") + if !ok { + t.Fatal("neutron component not found") + } + + value := reflect.ValueOf(neutron) + preRole := value.FieldByName("PreRoleName") + if preRole.IsValid() && preRole.Kind() == reflect.String && preRole.String() != "" { + preRoleDependsOn := value.FieldByName("PreRoleDependsOn") + if !preRoleDependsOn.IsValid() { + t.Fatal("neutron has a pre-role but no pre-role dependencies") + } + for i := 0; i < preRoleDependsOn.Len(); i++ { + if preRoleDependsOn.Index(i).String() == "nova" { + t.Fatal("neutron pre-role must not depend on nova") + } + } + return + } + + for _, dep := range neutron.DependsOn { + if dep == "nova" { + t.Fatal("neutron must not depend on nova") + } + } +} + +func TestMagnumImageUploadWaitsForGlanceImages(t *testing.T) { + magnum, ok := FindComponent("magnum") + if !ok { + t.Fatal("magnum component not found") + } + + if !slices.Contains(magnum.PreRoleDependsOn, "glance-images") { + t.Fatal("magnum pre-role must wait for glance-images to avoid concurrent image preparation") + } + if slices.Contains(magnum.DependsOn, "glance-images") { + t.Fatal("magnum main role must remain independent from glance-images") + } +} + +func TestCephAndClusterIssuerSerializeCAStoreAccess(t *testing.T) { + for _, componentName := range []string{"ceph", "cluster-issuer"} { + component, ok := FindComponent(componentName) + if !ok { + t.Fatalf("%s component not found", componentName) + } + + if !slices.Contains(component.Resources, "ca-certificates") { + t.Fatalf("%s must serialize access to the host CA store", componentName) + } + } +} diff --git a/internal/deploy/deployer.go b/internal/deploy/deployer.go new file mode 100644 index 0000000000..8233579e3e --- /dev/null +++ b/internal/deploy/deployer.go @@ -0,0 +1,209 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package deploy + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "os/exec" + "sort" + "strings" + "sync" + "syscall" + "time" + + "golang.org/x/sync/errgroup" +) + +// subprocessCancelGracePeriod is the time the runtime gives a cancelled +// subprocess to exit and release its stdout/stderr pipes before forcibly +// closing them and reaping the process. It exists because ansible-playbook +// frequently spawns long-lived helpers (ssh, python) that inherit our pipes; +// SIGKILL on the direct child does not always reach them, so without a +// fallback the parent's pipe-reading goroutines block forever. +const subprocessCancelGracePeriod = 10 * time.Second + +// configureSubprocess prepares cmd so that context cancellation reliably tears +// down ansible-playbook *and* every descendant it spawned. Without this, +// long-running grandchild processes keep the pipes open and prevent +// prefixOutput's scanners (and therefore cmd.Wait) from ever returning. +func configureSubprocess(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + if cmd.Process == nil { + return os.ErrProcessDone + } + // Negative PID targets the entire process group created via Setpgid, + // so descendants of ansible-playbook are signalled too. + return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } + cmd.WaitDelay = subprocessCancelGracePeriod +} + +// Deployer deploys a single component. When preGate is non-nil, it gates the +// pre-role: the pre-role goroutine waits for preGate to return nil (typically +// after PreRoleDependsOn components complete) before running. The main role +// always starts immediately. For components without a pre-role, preGate is +// ignored. +type Deployer interface { + Deploy(ctx context.Context, component Component, preGate func(context.Context) error) error +} + +// AnsibleDeployer deploys components by spawning ansible-playbook subprocesses. +type AnsibleDeployer struct { + // Inventory is the path to the Ansible inventory file. + Inventory string + // Output is the writer for prefixed output (defaults to os.Stdout). + Output io.Writer +} + +func (a *AnsibleDeployer) Deploy(ctx context.Context, component Component, preGate func(context.Context) error) error { + if component.PreRoleName == "" { + return a.runRole(ctx, component, component.RoleName) + } + + g, ctx := errgroup.WithContext(ctx) + + g.Go(func() error { + if preGate != nil { + if err := preGate(ctx); err != nil { + return err + } + } + return a.runRole(ctx, component, component.PreRoleName) + }) + + g.Go(func() error { + return a.runRole(ctx, component, component.RoleName) + }) + + return g.Wait() +} + +func (a *AnsibleDeployer) runRole(ctx context.Context, component Component, roleName string) error { + output := a.Output + if output == nil { + output = os.Stdout + } + + var cmd *exec.Cmd + + switch component.Type { + case PlaybookType: + playbookRef := "vexxhost.atmosphere." + component.Playbook + cmd = exec.CommandContext(ctx, "ansible-playbook", playbookRef, + "--inventory", a.Inventory) + case RoleType: + playbook := renderPlaybook(component, roleName) + cmd = exec.CommandContext(ctx, "ansible-playbook", "/dev/stdin", + "--inventory", a.Inventory) + cmd.Stdin = strings.NewReader(playbook) + default: + return fmt.Errorf("unknown component type: %d", component.Type) + } + + configureSubprocess(cmd) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("stdout pipe: %w", err) + } + stderr, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("stderr pipe: %w", err) + } + + if err := cmd.Start(); err != nil { + return fmt.Errorf("starting ansible-playbook for %s/%s: %w", component.Name, executionLabel(component, roleName), err) + } + + prefix := component.Name + "/" + executionLabel(component, roleName) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + prefixOutput(prefix, stdout, output) + }() + go func() { + defer wg.Done() + prefixOutput(prefix, stderr, output) + }() + + wg.Wait() + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("ansible-playbook failed for %s/%s: %w", component.Name, executionLabel(component, roleName), err) + } + + return nil +} + +func executionLabel(component Component, roleName string) string { + if roleName != "" { + return roleName + } + + if component.Type == PlaybookType { + if component.Playbook != "" { + return component.Playbook + } + return component.Name + } + + return component.Name +} + +func renderPlaybook(c Component, roleName string) string { + isMainWithPreRole := c.PreRoleName != "" && roleName == c.RoleName + + var b strings.Builder + b.WriteString("---\n") + b.WriteString(fmt.Sprintf("- hosts: %s\n", c.Hosts)) + b.WriteString(" become: true\n") + + if c.GatherFacts != nil && !*c.GatherFacts { + b.WriteString(" gather_facts: false\n") + } + + if len(c.Environment) > 0 { + b.WriteString(" environment:\n") + keys := make([]string, 0, len(c.Environment)) + for k := range c.Environment { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + b.WriteString(fmt.Sprintf(" %s: %q\n", k, c.Environment[k])) + } + } + + b.WriteString(" roles:\n") + fqRoleName := roleName + if !strings.Contains(fqRoleName, ".") { + fqRoleName = "vexxhost.atmosphere." + fqRoleName + } + b.WriteString(fmt.Sprintf(" - role: %s\n", fqRoleName)) + if c.When != "" { + b.WriteString(fmt.Sprintf(" when: %q\n", c.When)) + } + if isMainWithPreRole { + b.WriteString(" vars:\n") + b.WriteString(" _pre_role_active: true\n") + } + + return b.String() +} + +func prefixOutput(component string, r io.Reader, w io.Writer) { + scanner := bufio.NewScanner(r) + buf := make([]byte, 0, 64*1024) + scanner.Buffer(buf, 1024*1024) + for scanner.Scan() { + fmt.Fprintf(w, "[%s] %s\n", component, scanner.Text()) + } +} diff --git a/internal/deploy/deployer_test.go b/internal/deploy/deployer_test.go new file mode 100644 index 0000000000..aa0a2802ef --- /dev/null +++ b/internal/deploy/deployer_test.go @@ -0,0 +1,240 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package deploy + +import ( + "bytes" + "context" + "fmt" + "strings" + "sync" + "testing" + "time" +) + +// trackingDeployer records role names deployed and their timing to verify parallelism. +type trackingDeployer struct { + mu sync.Mutex + events []deployEvent + roleTime time.Duration +} + +type deployEvent struct { + component string + roleName string + startedAt time.Time + endedAt time.Time +} + +func (d *trackingDeployer) Deploy(ctx context.Context, component Component, preGate func(context.Context) error) error { + // Mirror only the PreRoleName branching needed by this test helper; + // this doesn't exercise AnsibleDeployer internals. + if component.PreRoleName == "" { + d.recordRole(component.Name, component.RoleName) + return nil + } + + // Simulate parallel pre-role + main role + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + if preGate != nil { + if err := preGate(ctx); err != nil { + return + } + } + d.recordRole(component.Name, component.PreRoleName) + }() + + go func() { + defer wg.Done() + d.recordRole(component.Name, component.RoleName) + }() + + wg.Wait() + return nil +} + +func (d *trackingDeployer) recordRole(component, roleName string) { + start := time.Now() + time.Sleep(d.roleTime) + end := time.Now() + + d.mu.Lock() + defer d.mu.Unlock() + d.events = append(d.events, deployEvent{ + component: component, + roleName: roleName, + startedAt: start, + endedAt: end, + }) +} + +func TestRenderPlaybook_WithPreRole(t *testing.T) { + c := Component{ + Name: "octavia", + Type: RoleType, + RoleName: "octavia", + PreRoleName: "octavia_pre", + Hosts: "controllers[0]", + } + + // Main role should include _pre_role_active var + mainPlaybook := renderPlaybook(c, c.RoleName) + if !strings.Contains(mainPlaybook, "_pre_role_active: true") { + t.Errorf("main role playbook should contain _pre_role_active var, got:\n%s", mainPlaybook) + } + + // Pre-role should NOT include _pre_role_active var + prePlaybook := renderPlaybook(c, c.PreRoleName) + if strings.Contains(prePlaybook, "_pre_role_active") { + t.Errorf("pre-role playbook should not contain _pre_role_active var, got:\n%s", prePlaybook) + } +} + +func TestRenderPlaybook_WithoutPreRole(t *testing.T) { + c := Component{ + Name: "keystone", + Type: RoleType, + RoleName: "keystone", + Hosts: "controllers[0]", + } + + playbook := renderPlaybook(c, c.RoleName) + if strings.Contains(playbook, "_pre_role_active") { + t.Errorf("playbook without pre-role should not contain _pre_role_active, got:\n%s", playbook) + } +} + +func TestRenderPlaybook_PreRoleUsesCorrectRoleName(t *testing.T) { + c := Component{ + Name: "magnum", + Type: RoleType, + RoleName: "magnum", + PreRoleName: "magnum_pre", + Hosts: "controllers[0]", + } + + prePlaybook := renderPlaybook(c, c.PreRoleName) + if !strings.Contains(prePlaybook, "vexxhost.atmosphere.magnum_pre") { + t.Errorf("pre-role playbook should reference magnum_pre, got:\n%s", prePlaybook) + } + + mainPlaybook := renderPlaybook(c, c.RoleName) + if !strings.Contains(mainPlaybook, "vexxhost.atmosphere.magnum") { + t.Errorf("main role playbook should reference magnum, got:\n%s", mainPlaybook) + } +} + +func TestDeploy_PreRoleRunsInParallel(t *testing.T) { + deployer := &trackingDeployer{roleTime: 50 * time.Millisecond} + + component := Component{ + Name: "magnum", + Type: RoleType, + RoleName: "magnum", + PreRoleName: "magnum_pre", + Hosts: "controllers[0]", + } + + deployer.Deploy(context.Background(), component, nil) + + if len(deployer.events) != 2 { + t.Fatalf("expected 2 deploy events, got %d", len(deployer.events)) + } + + var preEvent, mainEvent *deployEvent + for i := range deployer.events { + if deployer.events[i].roleName == "magnum_pre" { + preEvent = &deployer.events[i] + } else if deployer.events[i].roleName == "magnum" { + mainEvent = &deployer.events[i] + } + } + + if preEvent == nil || mainEvent == nil { + t.Fatal("expected both magnum_pre and magnum events") + } + + // Verify the two deployments overlapped in time. + if preEvent.endedAt.Before(mainEvent.startedAt) || mainEvent.endedAt.Before(preEvent.startedAt) { + t.Errorf( + "pre-role and main role should overlap: pre=[%v,%v] main=[%v,%v]", + preEvent.startedAt, + preEvent.endedAt, + mainEvent.startedAt, + mainEvent.endedAt, + ) + } +} + +func TestDeploy_NoPreRoleRunsSingle(t *testing.T) { + deployer := &trackingDeployer{roleTime: 10 * time.Millisecond} + + component := Component{ + Name: "keystone", + Type: RoleType, + RoleName: "keystone", + Hosts: "controllers[0]", + } + + deployer.Deploy(context.Background(), component, nil) + + if len(deployer.events) != 1 { + t.Fatalf("expected 1 deploy event, got %d", len(deployer.events)) + } + if deployer.events[0].roleName != "keystone" { + t.Errorf("expected keystone role, got %s", deployer.events[0].roleName) + } +} + +func TestRenderPlaybook_WithEnvironmentAndPreRole(t *testing.T) { + c := Component{ + Name: "test", + Type: RoleType, + RoleName: "test_main", + PreRoleName: "test_pre", + Hosts: "controllers[0]", + Environment: map[string]string{"FOO": "bar"}, + } + + out := &bytes.Buffer{} + _ = out + + mainPlaybook := renderPlaybook(c, c.RoleName) + if !strings.Contains(mainPlaybook, "FOO:") { + t.Error("main playbook should include environment") + } + if !strings.Contains(mainPlaybook, "_pre_role_active: true") { + t.Error("main playbook should include _pre_role_active") + } +} + +func TestComponentRegistry_PreRoleComponents(t *testing.T) { +preRoleComponents := []string{} +for _, c := range Components { +if c.PreRoleName != "" { +preRoleComponents = append(preRoleComponents, fmt.Sprintf("%s (pre: %s)", c.Name, c.PreRoleName)) +} +} + +if len(preRoleComponents) == 0 { +t.Fatal("expected at least one component with PreRoleName") +} + +// Verify known components have pre-roles +found := map[string]bool{} +for _, c := range Components { +found[c.Name] = c.PreRoleName != "" +} + +if !found["octavia"] { +t.Error("octavia should have a pre-role") +} +if !found["magnum"] { +t.Error("magnum should have a pre-role") +} +} diff --git a/internal/deploy/orchestrator.go b/internal/deploy/orchestrator.go new file mode 100644 index 0000000000..22fb8f59c3 --- /dev/null +++ b/internal/deploy/orchestrator.go @@ -0,0 +1,351 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package deploy + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "strings" + "sync" +) + +// preflightPlaybook contains validation checks that mirror pre_tasks from the +// original sequential playbooks (e.g., playbooks/openstack.yml). The parallel +// orchestrator generates minimal single-role playbooks for RoleType components, +// which bypasses pre_tasks defined in the original playbook files. Running these +// checks before the DAG ensures configuration errors are caught early. +const preflightPlaybook = `--- +- name: Preflight checks + hosts: controllers[0] + become: true + gather_facts: false + tasks: + - name: Fail if atmosphere_ceph_enabled is set + ansible.builtin.fail: + msg: >- + The "atmosphere_ceph_enabled" variable is no longer supported. + Please use the "atmosphere_storage" variable to configure storage + backends instead. Refer to the storage configuration documentation + for migration instructions. + when: atmosphere_ceph_enabled is defined +` + +// Orchestrator coordinates the parallel deployment of Atmosphere components. +type Orchestrator struct { + // Deployer is the deployment backend (e.g., AnsibleDeployer). + Deployer Deployer + // Inventory is the path to the Ansible inventory file. + Inventory string + // Output is the writer for status messages (defaults to os.Stdout). + Output io.Writer + // Concurrency limits parallel deployments per wave (0 = unlimited). + Concurrency int + // WithDependencies expands selected tags to the transitive dependency + // closure required for bootstrapping a fresh environment. + WithDependencies bool + // DependencyOptions controls configuration-dependent graph edges. + DependencyOptions DependencyOptions + // Preflight is invoked before any component deployment and is expected + // to return an error if the environment is misconfigured. When nil, the + // default implementation shells out to ansible-playbook with the + // preflightPlaybook defined in this file. Tests inject a no-op to + // keep the unit tests hermetic. + Preflight func(ctx context.Context, output io.Writer) error +} + +// preflight dispatches to a caller-supplied Preflight hook when set, falling +// back to the ansible-playbook-based implementation otherwise. +func (o *Orchestrator) preflight(ctx context.Context, output io.Writer) error { + if o.Preflight != nil { + return o.Preflight(ctx, output) + } + return o.runPreflightChecks(ctx, output) +} + +// Deploy runs the deployment based on the provided tags. +// - No tags: full DAG, all components, parallel waves +// - Single tag: pass-through to ansible-playbook site.yml --tags +// - Multiple tags: extract subgraph, resolve DAG ordering, parallel waves +func (o *Orchestrator) Deploy(ctx context.Context, tags []string) error { + output := o.Output + if output == nil { + output = os.Stdout + } + + if len(tags) > 0 && o.WithDependencies { + return o.deployTagsWithDependencies(ctx, tags, output) + } + + switch len(tags) { + case 0: + return o.deployFullDAG(ctx, output) + case 1: + return o.deploySingleTag(ctx, tags[0], output) + default: + return o.deployMultipleTags(ctx, tags, output) + } +} + +// runPreflightChecks runs validation checks before any component deployment. +// This ensures that deprecated or invalid configuration is caught early, mirroring +// the pre_tasks from the original sequential playbooks (e.g., playbooks/openstack.yml). +func (o *Orchestrator) runPreflightChecks(ctx context.Context, output io.Writer) error { + fmt.Fprintln(output, "==> Running preflight checks") + + cmd := exec.CommandContext(ctx, "ansible-playbook", "/dev/stdin", + "--inventory", o.Inventory) + cmd.Stdin = strings.NewReader(preflightPlaybook) + configureSubprocess(cmd) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("stdout pipe: %w", err) + } + stderr, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("stderr pipe: %w", err) + } + + if err := cmd.Start(); err != nil { + return fmt.Errorf("starting preflight checks: %w", err) + } + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + prefixOutput("preflight", stdout, output) + }() + go func() { + defer wg.Done() + prefixOutput("preflight", stderr, output) + }() + wg.Wait() + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("preflight checks failed: %w", err) + } + + fmt.Fprintln(output, "==> Preflight checks passed") + return nil +} + +// deployFullDAG runs all components in parallel waves. +func (o *Orchestrator) deployFullDAG(ctx context.Context, output io.Writer) error { + if err := o.preflight(ctx, output); err != nil { + return err + } + + g, err := BuildGraphWithOptions(o.DependencyOptions) + if err != nil { + return fmt.Errorf("building dependency graph: %w", err) + } + + rc := NewResourceCoordinator(Components, nil) + tracker := newCompletionTracker(allComponentNames(Components)) + + fmt.Fprintln(output, "==> Starting parallel deployment") + return g.Run(ctx, o.Concurrency, func(ctx context.Context, id string, comp Component) error { + fmt.Fprintf(output, "==> [%s] Starting deployment\n", id) + + release, err := rc.Acquire(ctx, comp) + if err != nil { + return fmt.Errorf("component %s: %w", id, err) + } + defer release() + + preGate := buildPreGate(comp, tracker, o.DependencyOptions) + if err := o.Deployer.Deploy(ctx, comp, preGate); err != nil { + return fmt.Errorf("component %s failed: %w", id, err) + } + // Only mark done on success; on failure, downstream pre-roles + // must not be unblocked because the dependency is not actually + // ready. errgroup cancellation will tear them down promptly. + tracker.MarkDone(id) + fmt.Fprintf(output, "==> [%s] Deployment complete\n", id) + return nil + }) +} + +// deploySingleTag passes through to ansible-playbook with the tag. +// This is identical to running: ansible-playbook vexxhost.atmosphere.site --tags +func (o *Orchestrator) deploySingleTag(ctx context.Context, tag string, output io.Writer) error { + fmt.Fprintf(output, "==> Single tag mode: %s\n", tag) + + cmd := exec.CommandContext(ctx, "ansible-playbook", "vexxhost.atmosphere.site", + "--inventory", o.Inventory, + "--tags", tag) + configureSubprocess(cmd) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("stdout pipe: %w", err) + } + stderr, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("stderr pipe: %w", err) + } + + if err := cmd.Start(); err != nil { + return fmt.Errorf("starting ansible-playbook: %w", err) + } + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + prefixOutput(tag, stdout, output) + }() + go func() { + defer wg.Done() + prefixOutput(tag, stderr, output) + }() + wg.Wait() + + if err := cmd.Wait(); err != nil { + return fmt.Errorf("ansible-playbook --tags %s failed: %w", tag, err) + } + + return nil +} + +// deployMultipleTags extracts a subgraph for the specified tags and runs them +// in DAG order with parallel waves. +func (o *Orchestrator) deployMultipleTags(ctx context.Context, tags []string, output io.Writer) error { + fmt.Fprintf(output, "==> Multi-tag mode: %s\n", strings.Join(tags, ", ")) + + if err := o.preflight(ctx, output); err != nil { + return err + } + + // Resolve tag names to component names + componentNames := make([]string, 0, len(tags)) + for _, tag := range tags { + comp, ok := FindComponent(tag) + if !ok { + return fmt.Errorf("unknown component or tag: %q", tag) + } + componentNames = append(componentNames, comp.Name) + } + + // Build full graph, then extract subgraph + fullGraph, err := BuildGraphWithOptions(o.DependencyOptions) + if err != nil { + return fmt.Errorf("building dependency graph: %w", err) + } + + subGraph, err := fullGraph.Subgraph(componentNames) + if err != nil { + return fmt.Errorf("extracting subgraph: %w", err) + } + + rc := NewResourceCoordinator(Components, nil) + // Only allocate tracker channels for components actually in the + // subgraph. Components outside the subgraph are treated as + // already-complete by completionTracker.Wait (it skips unknown + // names), matching the semantic that --tags assumes everything + // not selected is already deployed. Without this, a pre-role + // gated on an out-of-subgraph component would block forever. + tracker := newCompletionTracker(componentNames) + + fmt.Fprintln(output, "==> Starting parallel deployment (subgraph)") + return subGraph.Run(ctx, o.Concurrency, func(ctx context.Context, id string, comp Component) error { + fmt.Fprintf(output, "==> [%s] Starting deployment\n", id) + + release, err := rc.Acquire(ctx, comp) + if err != nil { + return fmt.Errorf("component %s: %w", id, err) + } + defer release() + + preGate := buildPreGate(comp, tracker, o.DependencyOptions) + if err := o.Deployer.Deploy(ctx, comp, preGate); err != nil { + return fmt.Errorf("component %s failed: %w", id, err) + } + // Only mark done on success; see comment in deployFullGraph. + tracker.MarkDone(id) + fmt.Fprintf(output, "==> [%s] Deployment complete\n", id) + return nil + }) +} + +// deployTagsWithDependencies expands selected components to everything needed +// to bootstrap them in a fresh environment, then deploys that subgraph. +func (o *Orchestrator) deployTagsWithDependencies(ctx context.Context, tags []string, output io.Writer) error { + fmt.Fprintf(output, "==> Fresh-environment mode: %s\n", strings.Join(tags, ", ")) + + if err := o.preflight(ctx, output); err != nil { + return err + } + + targets := make([]string, 0, len(tags)) + for _, tag := range tags { + component, ok := FindComponent(tag) + if !ok { + return fmt.Errorf("unknown component or tag: %q", tag) + } + targets = append(targets, component.Name) + } + + componentNames, err := BootstrapComponentNames(targets, o.DependencyOptions) + if err != nil { + return fmt.Errorf("resolving bootstrap dependencies: %w", err) + } + + fullGraph, err := BuildGraphWithOptions(o.DependencyOptions) + if err != nil { + return fmt.Errorf("building dependency graph: %w", err) + } + subGraph, err := fullGraph.Subgraph(componentNames) + if err != nil { + return fmt.Errorf("extracting bootstrap subgraph: %w", err) + } + + rc := NewResourceCoordinator(Components, nil) + tracker := newCompletionTracker(componentNames) + + fmt.Fprintln(output, "==> Starting parallel deployment (fresh dependency closure)") + return subGraph.Run(ctx, o.Concurrency, func(ctx context.Context, id string, comp Component) error { + fmt.Fprintf(output, "==> [%s] Starting deployment\n", id) + + release, err := rc.Acquire(ctx, comp) + if err != nil { + return fmt.Errorf("component %s: %w", id, err) + } + defer release() + + preGate := buildPreGate(comp, tracker, o.DependencyOptions) + if err := o.Deployer.Deploy(ctx, comp, preGate); err != nil { + return fmt.Errorf("component %s failed: %w", id, err) + } + tracker.MarkDone(id) + fmt.Fprintf(output, "==> [%s] Deployment complete\n", id) + return nil + }) +} + +// allComponentNames extracts the Name field from each component for +// preallocating the completion tracker. +func allComponentNames(comps []Component) []string { + names := make([]string, 0, len(comps)) + for _, c := range comps { + names = append(names, c.Name) + } + return names +} + +// buildPreGate returns a preGate closure for the deployer, or nil when the +// component has no asymmetric pre-role dependency. +func buildPreGate(comp Component, tracker *completionTracker, options DependencyOptions) func(context.Context) error { + dependencies := preRoleDependencies(comp, dependencyOptionsWithDefaults(options)) + if comp.PreRoleName == "" || len(dependencies) == 0 { + return nil + } + return func(ctx context.Context) error { + return tracker.Wait(ctx, dependencies) + } +} diff --git a/internal/deploy/orchestrator_subgraph_pregate_test.go b/internal/deploy/orchestrator_subgraph_pregate_test.go new file mode 100644 index 0000000000..bca163a771 --- /dev/null +++ b/internal/deploy/orchestrator_subgraph_pregate_test.go @@ -0,0 +1,132 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package deploy + +import ( + "context" + "errors" + "io" + "testing" + "time" +) + +// gateAwareDeployer invokes preGate (mirroring AnsibleDeployer behavior), +// unlike the simpler mocks in orchestrator_test.go that drop it. This is +// what surfaces the multi-tag pre-role-dependency hang. +type gateAwareDeployer struct{} + +func (g *gateAwareDeployer) Deploy(ctx context.Context, c Component, preGate func(context.Context) error) error { + if preGate != nil { + if err := preGate(ctx); err != nil { + return err + } + } + return nil +} + +// TestDeployMultipleTags_PreRoleDependencyOutsideSubgraph guards against the +// regression where a selected component's PreRoleDependsOn pointed at a +// component outside the user-selected --tags subgraph and caused the pre-role +// goroutine to block forever. With the fix, out-of-subgraph names are +// treated as already-complete, matching --tags semantics. +func TestDeployMultipleTags_PreRoleDependencyOutsideSubgraph(t *testing.T) { + o := &Orchestrator{ + Deployer: &gateAwareDeployer{}, + Inventory: "/dev/null", + Output: io.Discard, + Preflight: func(_ context.Context, _ io.Writer) error { return nil }, + } + + // neutron has PreRoleDependsOn=[keystone, ovn, coredns]; none of + // these are in --tags neutron,nova. Before the fix, neutron's + // pre-role goroutine waited on those channels forever. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := o.Deploy(ctx, []string{"neutron", "nova"}) + if err != nil { + t.Fatalf("Deploy returned error (likely a hang regression): %v", err) + } + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatal("context deadline exceeded - subgraph deployment hung") + } +} + +// TestDeployMultipleTags_KeystoneSubgraph covers the keystone variant of +// the C1 hang (PreRoleDependsOn=[keycloak]). +// +// The orchestrator only routes through deployMultipleTags — and therefore +// only exercises the buggy pre-role gating path — when **two or more** +// distinct tags are selected. A single tag falls into the Mode 2 +// pass-through path, which shells out to a real ansible-playbook and +// ignores o.Deployer entirely, making it unusable for unit testing. +// +// We pair keystone with barbican (which DependsOn=[keystone]) so the +// subgraph contains keystone+barbican but NOT keycloak, reproducing +// the original hang condition. +func TestDeployMultipleTags_KeystoneSubgraph(t *testing.T) { + o := &Orchestrator{ + Deployer: &gateAwareDeployer{}, + Inventory: "/dev/null", + Output: io.Discard, + Preflight: func(_ context.Context, _ io.Writer) error { return nil }, + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := o.Deploy(ctx, []string{"keystone", "barbican"}); err != nil { + t.Fatalf("Deploy returned error: %v", err) + } + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatal("context deadline exceeded - subgraph deployment hung") + } +} + +// failingDeployer fails for a named target, succeeds otherwise. It also +// invokes preGate so we can observe whether MarkDone was triggered after +// a failure (which would unblock the gated downstream pre-role). +type failingDeployer struct { + failOn string + preGated chan string +} + +func (f *failingDeployer) Deploy(ctx context.Context, c Component, preGate func(context.Context) error) error { + if preGate != nil { + if err := preGate(ctx); err != nil { + return err + } + select { + case f.preGated <- c.Name: + default: + } + } + if c.Name == f.failOn { + return errors.New("synthetic failure") + } + return nil +} + +// TestDeployFailure_DoesNotMarkDone confirms that a failed component does +// not mark itself done and therefore does not unblock downstream pre-role +// gates. The test runs the keystone -> neutron pre-role chain (well: only +// the part observable in the in-subgraph case) and asserts that when +// keystone fails, the run aborts with the failure rather than silently +// proceeding. +func TestDeployFailure_DoesNotMarkDone(t *testing.T) { + o := &Orchestrator{ + Deployer: &failingDeployer{ + failOn: "keycloak", + preGated: make(chan string, 8), + }, + Inventory: "/dev/null", + Output: io.Discard, + Preflight: func(_ context.Context, _ io.Writer) error { return nil }, + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := o.Deploy(ctx, []string{"keycloak", "keystone"}) + if err == nil { + t.Fatal("expected failure from keycloak to propagate, got nil") + } +} diff --git a/internal/deploy/orchestrator_test.go b/internal/deploy/orchestrator_test.go new file mode 100644 index 0000000000..1642c41e66 --- /dev/null +++ b/internal/deploy/orchestrator_test.go @@ -0,0 +1,191 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package deploy + +import ( + "bytes" + "context" + "io" + "sync" + "testing" +) + +// noopPreflight is injected into Orchestrator under test so unit tests do not +// shell out to ansible-playbook for the preflight checks. +func noopPreflight(_ context.Context, _ io.Writer) error { return nil } + +// mockDeployer records which components were deployed and in what order. +type mockDeployer struct { + mu sync.Mutex + deployed []string +} + +func (m *mockDeployer) Deploy(_ context.Context, component Component, _ func(context.Context) error) error { + m.mu.Lock() + defer m.mu.Unlock() + m.deployed = append(m.deployed, component.Name) + return nil +} + +func TestOrchestrator_FullDAG(t *testing.T) { + mock := &mockDeployer{} + out := &bytes.Buffer{} + + orch := &Orchestrator{ + Deployer: mock, + Concurrency: 1, // serial execution for deterministic ordering + Output: out, + Preflight: noopPreflight, + } + + if err := orch.Deploy(context.Background(), nil); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + deployed := mock.deployed + + if len(deployed) == 0 { + t.Fatal("no components deployed") + } + + // Total: len(Components) components + if len(deployed) != len(Components) { + t.Errorf("expected %d deployed components, got %d", len(Components), len(deployed)) + } + + // keystone must appear before nova (nova depends on keystone transitively) + keystoneIdx := -1 + novaIdx := -1 + for i, name := range deployed { + if name == "keystone" { + keystoneIdx = i + } + if name == "nova" { + novaIdx = i + } + } + if keystoneIdx == -1 { + t.Fatal("keystone not found in deployed list") + } + if novaIdx == -1 { + t.Fatal("nova not found in deployed list") + } + if keystoneIdx >= novaIdx { + t.Errorf("keystone (index %d) must deploy before nova (index %d)", keystoneIdx, novaIdx) + } +} + +func TestOrchestrator_MultipleTags(t *testing.T) { + mock := &mockDeployer{} + out := &bytes.Buffer{} + + orch := &Orchestrator{ + Deployer: mock, + Concurrency: 1, + Output: out, + Preflight: noopPreflight, + } + + if err := orch.Deploy(context.Background(), []string{"nova", "keystone"}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + deployed := mock.deployed + + // Only nova and keystone should be deployed + if len(deployed) != 2 { + t.Fatalf("expected 2 deployed components, got %d: %v", len(deployed), deployed) + } + + // keystone must come before nova (nova depends on keystone via placement/glance) + // In the subgraph with only these two, the edge nova->keystone is NOT preserved + // because nova doesn't directly depend on keystone. Let's verify both are present. + found := map[string]bool{} + for _, name := range deployed { + found[name] = true + } + if !found["keystone"] { + t.Error("keystone not in deployed list") + } + if !found["nova"] { + t.Error("nova not in deployed list") + } +} + +func TestOrchestrator_MultipleTags_UnknownTag(t *testing.T) { + mock := &mockDeployer{} + out := &bytes.Buffer{} + + orch := &Orchestrator{ + Deployer: mock, + Concurrency: 1, + Output: out, + Preflight: noopPreflight, + } + + err := orch.Deploy(context.Background(), []string{"keystone", "nonexistent-component"}) + if err == nil { + t.Fatal("expected error for unknown tag, got nil") + } + + if len(mock.deployed) != 0 { + t.Errorf("expected no deployments on error, got %d", len(mock.deployed)) + } +} + +// preGateRecorder captures whether preGate was invoked and what it returned +// for each component deployed, allowing tests to assert pre-role gating. +type preGateRecorder struct { +mu sync.Mutex +gates map[string]bool // component name -> gate was invoked +gateErr map[string]error +} + +func newPreGateRecorder() *preGateRecorder { +return &preGateRecorder{ +gates: map[string]bool{}, +gateErr: map[string]error{}, +} +} + +func (p *preGateRecorder) Deploy(ctx context.Context, component Component, preGate func(context.Context) error) error { +if preGate != nil { +err := preGate(ctx) +p.mu.Lock() +p.gates[component.Name] = true +p.gateErr[component.Name] = err +p.mu.Unlock() +if err != nil { +return err +} +} else { +p.mu.Lock() +p.gates[component.Name] = false +p.mu.Unlock() +} +return nil +} + +func TestOrchestrator_BuildPreGate(t *testing.T) { +tracker := newCompletionTracker([]string{"x", "y"}) + +withPre := Component{ +Name: "withpre", +PreRoleName: "withpre_pre", +PreRoleDependsOn: []string{"x"}, +} +if gate := buildPreGate(withPre, tracker, nil); gate == nil { +t.Fatal("expected non-nil gate for component with PreRoleDependsOn") +} + +noPreDeps := Component{Name: "n1", PreRoleName: "n1_pre"} +if gate := buildPreGate(noPreDeps, tracker, nil); gate != nil { +t.Errorf("expected nil gate when PreRoleDependsOn is empty") +} + +noPreRole := Component{Name: "n2", PreRoleDependsOn: []string{"x"}} +if gate := buildPreGate(noPreRole, tracker, nil); gate != nil { +t.Errorf("expected nil gate when PreRoleName is empty") +} +} diff --git a/internal/deploy/resource.go b/internal/deploy/resource.go new file mode 100644 index 0000000000..2182605f4e --- /dev/null +++ b/internal/deploy/resource.go @@ -0,0 +1,112 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package deploy + +import ( + "context" + "fmt" +) + +// ResourceCoordinator manages named semaphores to serialize components that +// share a resource (e.g., "apt" for package management) or cap the parallel +// fan-out to a shared backend (e.g., "k8s-api"). Components declaring the +// same resource respect its concurrency cap; components without shared +// resources remain fully parallel. +type ResourceCoordinator struct { + semas map[string]chan struct{} +} + +// defaultResourceConcurrency returns the default cap for a resource when the +// caller does not override it. Historically every resource was a mutex +// (cap=1); newer resources may be backends we merely want to rate-limit. +var defaultResourceConcurrency = map[string]int{ + // apt: package manager. Must be serialized per host, but we also + // serialize globally today since components touching apt span all hosts. + "apt": 1, + // k8s-api: the Kubernetes apiserver. Cap limits how many Helm/Kubernetes + // operations run concurrently across heavy OpenStack components, smoothing + // apiserver load during wave fan-out without serializing fully. + "k8s-api": 6, + // keycloak-admin: the Keycloak admin HTTP endpoint. The community.general + // keycloak_* modules rebuild auth state per call; concurrent realm/client + // creates race and fail. Serialize globally (cap=1). + "keycloak-admin": 1, + // containerd: the vexxhost.containers.containerd role calls + // systemctl daemon-reload + enable on containerd.service. Two + // concurrent enable calls race on dbus and intermittently fail + // with "Message recipient disconnected from message bus without + // replying". Serialize globally (cap=1) so any pair of + // components touching the role can't collide on systemd. + "containerd": 1, + // ca-certificates: the cluster issuer updates and rehashes the host CA + // store. Serialize that mutation with components which download over TLS + // during deployment, otherwise an in-flight certificate verification can + // observe the trust store between updates and fail intermittently. + "ca-certificates": 1, +} + +// NewResourceCoordinator builds semaphores for every resource declared across +// the given components. Each resource's capacity is looked up in overrides, +// falling back to defaultResourceConcurrency, and finally to 1 (mutex). +func NewResourceCoordinator(components []Component, overrides map[string]int) *ResourceCoordinator { + resources := make(map[string]bool) + for _, c := range components { + for _, r := range c.Resources { + resources[r] = true + } + } + + semas := make(map[string]chan struct{}, len(resources)) + for r := range resources { + cap := resourceCapacity(r, overrides) + semas[r] = make(chan struct{}, cap) + } + + return &ResourceCoordinator{semas: semas} +} + +func resourceCapacity(name string, overrides map[string]int) int { + if v, ok := overrides[name]; ok && v > 0 { + return v + } + if v, ok := defaultResourceConcurrency[name]; ok && v > 0 { + return v + } + return 1 +} + +// Acquire blocks until all resources required by the component are available +// or the context is cancelled. Returns a release function and nil on success, +// or a nil function and an error if the context was cancelled while waiting. +func (rc *ResourceCoordinator) Acquire(ctx context.Context, comp Component) (func(), error) { + if len(comp.Resources) == 0 { + return func() {}, nil + } + + acquired := make([]string, 0, len(comp.Resources)) + for _, r := range comp.Resources { + if ch, ok := rc.semas[r]; ok { + select { + case ch <- struct{}{}: + acquired = append(acquired, r) + case <-ctx.Done(): + // Release any resources already acquired + for _, ar := range acquired { + if ach, ok := rc.semas[ar]; ok { + <-ach + } + } + return nil, fmt.Errorf("context cancelled waiting for resource %q: %w", r, ctx.Err()) + } + } + } + + return func() { + for _, r := range comp.Resources { + if ch, ok := rc.semas[r]; ok { + <-ch + } + } + }, nil +} diff --git a/internal/deploy/resource_concurrency_test.go b/internal/deploy/resource_concurrency_test.go new file mode 100644 index 0000000000..14392a6928 --- /dev/null +++ b/internal/deploy/resource_concurrency_test.go @@ -0,0 +1,68 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package deploy + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestResourceCoordinator_CustomConcurrency(t *testing.T) { + components := []Component{ + {Name: "a", Resources: []string{"k8s-api"}}, + {Name: "b", Resources: []string{"k8s-api"}}, + {Name: "c", Resources: []string{"k8s-api"}}, + {Name: "d", Resources: []string{"k8s-api"}}, + } + + rc := NewResourceCoordinator(components, map[string]int{"k8s-api": 2}) + ctx := context.Background() + + var concurrent int64 + var maxConcurrent int64 + var wg sync.WaitGroup + + for _, comp := range components { + wg.Add(1) + go func(c Component) { + defer wg.Done() + release, err := rc.Acquire(ctx, c) + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + defer release() + + cur := atomic.AddInt64(&concurrent, 1) + for { + old := atomic.LoadInt64(&maxConcurrent) + if cur <= old || atomic.CompareAndSwapInt64(&maxConcurrent, old, cur) { + break + } + } + time.Sleep(20 * time.Millisecond) + atomic.AddInt64(&concurrent, -1) + }(comp) + } + + wg.Wait() + + if maxConcurrent != 2 { + t.Errorf("expected max concurrency of 2 for k8s-api override, got %d", maxConcurrent) + } +} + +func TestResourceCoordinator_DefaultK8sApiCap(t *testing.T) { + // Without an override, k8s-api should use defaultResourceConcurrency (6). + // Verify the semaphore's buffered capacity matches. + components := []Component{{Name: "a", Resources: []string{"k8s-api"}}} + rc := NewResourceCoordinator(components, nil) + + if got := cap(rc.semas["k8s-api"]); got != 6 { + t.Errorf("expected default k8s-api cap of 6, got %d", got) + } +} diff --git a/internal/deploy/resource_test.go b/internal/deploy/resource_test.go new file mode 100644 index 0000000000..cb0223b74b --- /dev/null +++ b/internal/deploy/resource_test.go @@ -0,0 +1,149 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package deploy + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestResourceCoordinator_Serializes(t *testing.T) { + components := []Component{ + {Name: "a", Resources: []string{"apt"}}, + {Name: "b", Resources: []string{"apt"}}, + {Name: "c"}, + } + + rc := NewResourceCoordinator(components, nil) + ctx := context.Background() + + var concurrent int64 + var maxConcurrent int64 + var wg sync.WaitGroup + + for _, comp := range components[:2] { + wg.Add(1) + go func(c Component) { + defer wg.Done() + release, err := rc.Acquire(ctx, c) + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + defer release() + + cur := atomic.AddInt64(&concurrent, 1) + for { + old := atomic.LoadInt64(&maxConcurrent) + if cur <= old || atomic.CompareAndSwapInt64(&maxConcurrent, old, cur) { + break + } + } + + time.Sleep(10 * time.Millisecond) + atomic.AddInt64(&concurrent, -1) + }(comp) + } + + wg.Wait() + + if maxConcurrent != 1 { + t.Errorf("expected max concurrency of 1 for shared resource, got %d", maxConcurrent) + } +} + +func TestResourceCoordinator_NoResource(t *testing.T) { + components := []Component{ + {Name: "a"}, + {Name: "b"}, + } + + rc := NewResourceCoordinator(components, nil) + ctx := context.Background() + + release, err := rc.Acquire(ctx, components[0]) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + release() + release, err = rc.Acquire(ctx, components[1]) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + release() +} + +func TestResourceCoordinator_DifferentResources(t *testing.T) { + components := []Component{ + {Name: "a", Resources: []string{"apt"}}, + {Name: "b", Resources: []string{"helm"}}, + } + + rc := NewResourceCoordinator(components, nil) + ctx := context.Background() + + var concurrent int64 + var maxConcurrent int64 + var wg sync.WaitGroup + + for _, comp := range components { + wg.Add(1) + go func(c Component) { + defer wg.Done() + release, err := rc.Acquire(ctx, c) + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + defer release() + + cur := atomic.AddInt64(&concurrent, 1) + for { + old := atomic.LoadInt64(&maxConcurrent) + if cur <= old || atomic.CompareAndSwapInt64(&maxConcurrent, old, cur) { + break + } + } + + time.Sleep(10 * time.Millisecond) + atomic.AddInt64(&concurrent, -1) + }(comp) + } + + wg.Wait() + + if maxConcurrent != 2 { + t.Errorf("expected max concurrency of 2 for different resources, got %d", maxConcurrent) + } +} + +func TestResourceCoordinator_ContextCancellation(t *testing.T) { + components := []Component{ + {Name: "a", Resources: []string{"apt"}}, + {Name: "b", Resources: []string{"apt"}}, + } + + rc := NewResourceCoordinator(components, nil) + ctx := context.Background() + + // Acquire the resource with component a + release, err := rc.Acquire(ctx, components[0]) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Try to acquire with a cancelled context — should fail + cancelCtx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err = rc.Acquire(cancelCtx, components[1]) + if err == nil { + t.Fatal("expected error from cancelled context, got nil") + } + + release() +} diff --git a/internal/deploy/subprocess_linux_test.go b/internal/deploy/subprocess_linux_test.go new file mode 100644 index 0000000000..e702754346 --- /dev/null +++ b/internal/deploy/subprocess_linux_test.go @@ -0,0 +1,87 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux + +package deploy + +import ( + "bufio" + "context" + "io" + "os/exec" + "sync" + "testing" + "time" +) + +// TestConfigureSubprocess_ReapsGrandchildrenOnCancel verifies that when the +// context is cancelled, descendants of the direct subprocess are killed and +// the parent's pipe-reading goroutines unblock — even if those grandchildren +// inherit stdout/stderr and would otherwise hold the pipes open. This is the +// exact failure mode that caused the molecule-aio-ovn job to hang for over an +// hour after magnum's main role failed mid-deploy. +func TestConfigureSubprocess_ReapsGrandchildrenOnCancel(t *testing.T) { + if _, err := exec.LookPath("sh"); err != nil { + t.Skipf("sh not available: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Spawn a shell that detaches a child with `setsid sleep 60` inheriting + // our stdout. When the parent shell exits, the grandchild stays alive + // holding the pipe — mirroring how ansible-playbook leaves ssh helpers + // behind. Without configureSubprocess, scanner.Scan() below would block + // for the full 60 s. + cmd := exec.CommandContext(ctx, "sh", "-c", "sleep 60 & wait") + configureSubprocess(cmd) + + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatalf("stdout pipe: %v", err) + } + + if err := cmd.Start(); err != nil { + t.Fatalf("start: %v", err) + } + + scannerDone := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + } + close(scannerDone) + }() + + // Give the subprocess a moment to start, then cancel. + time.Sleep(100 * time.Millisecond) + cancel() + + waitDone := make(chan error, 1) + go func() { waitDone <- cmd.Wait() }() + + // configureSubprocess sets WaitDelay = subprocessCancelGracePeriod. + // Allow a comfortable margin on top of that for goroutine scheduling. + deadline := subprocessCancelGracePeriod + 5*time.Second + + select { + case <-waitDone: + case <-time.After(deadline): + t.Fatalf("cmd.Wait did not return within %v after ctx cancel", deadline) + } + + select { + case <-scannerDone: + case <-time.After(deadline): + t.Fatalf("stdout scanner did not unblock within %v after ctx cancel", deadline) + } + + wg.Wait() +} + +// readAllUntilEOF is a tiny helper kept around for clarity in error messages. +var _ = io.Discard diff --git a/molecule/aio/converge.yml b/molecule/aio/converge.yml index 7fc4a91f19..de5e9808a1 100644 --- a/molecule/aio/converge.yml +++ b/molecule/aio/converge.yml @@ -1,34 +1,33 @@ -# Copyright (c) 2024 VEXXHOST, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -- name: Install Ceph - ansible.builtin.import_playbook: vexxhost.atmosphere.ceph - -- name: Install Kubernetes - ansible.builtin.import_playbook: vexxhost.atmosphere.kubernetes - -- name: Install CSI - ansible.builtin.import_playbook: vexxhost.atmosphere.csi - -- name: Deploy Infrastructure - ansible.builtin.import_playbook: vexxhost.atmosphere.infrastructure - -- name: Deploy Monitoring stack - ansible.builtin.import_playbook: vexxhost.atmosphere.monitoring +--- +- hosts: all + gather_facts: false + tasks: + - name: Build atmosphere binary + ansible.builtin.command: + cmd: go build -o ./bin/atmosphere ./cmd/atmosphere + args: + chdir: "{{ zuul.project.src_dir | default(playbook_dir + '/../..') }}" + run_once: true + changed_when: false + environment: + CGO_ENABLED: "0" + PATH: "/usr/local/go/bin:{{ ansible_env.PATH | default(lookup('env', 'PATH')) }}" -- name: Deploy OpenStack - ansible.builtin.import_playbook: vexxhost.atmosphere.openstack + - name: Deploy with parallel orchestrator + ansible.builtin.shell: + cmd: > + . .venv/bin/activate && + ./bin/atmosphere deploy + --inventory ./inventory.yaml + --concurrency {{ atmosphere_deploy_concurrency | default(3) | int }} + {{ '--tags ' + atmosphere_deploy_tags if atmosphere_deploy_tags is defined else '' }} + --dependency-option csi_driver={{ csi_driver | default('rbd') | quote }} + --dependency-option network_backend={{ atmosphere_network_backend | default('openvswitch') | quote }} + executable: /bin/bash + args: + chdir: "{{ zuul.project.src_dir | default(playbook_dir + '/../..') }}" + run_once: true + changed_when: false - name: Configure networking hosts: all diff --git a/molecule/csi/converge.yml b/molecule/csi/converge.yml index 704a763a13..7084509beb 100644 --- a/molecule/csi/converge.yml +++ b/molecule/csi/converge.yml @@ -12,21 +12,29 @@ # License for the specific language governing permissions and limitations # under the License. -# NOTE(mnaser): This play is required for csi_driver to be evaluated properly -# so the when condition on import_playbook works correctly -- name: Debug CSI driver value - hosts: all +- hosts: all + gather_facts: false tasks: - - name: Display CSI driver value and environment variable - ansible.builtin.debug: - msg: "csi_driver={{ csi_driver }}, MOLECULE_CSI_DRIVER={{ lookup('env', 'MOLECULE_CSI_DRIVER') }}" + - name: Build atmosphere binary + ansible.builtin.command: + cmd: go build -o ./bin/atmosphere ./cmd/atmosphere + args: + chdir: "{{ zuul.project.src_dir | default(playbook_dir + '/../..') }}" + run_once: true + changed_when: false + environment: + CGO_ENABLED: "0" + PATH: "/usr/local/go/bin:{{ ansible_env.PATH | default(lookup('env', 'PATH')) }}" -- name: Install Ceph - ansible.builtin.import_playbook: vexxhost.atmosphere.ceph - when: csi_driver == 'rbd' - -- name: Install Kubernetes - ansible.builtin.import_playbook: vexxhost.atmosphere.kubernetes - -- name: Install CSI - ansible.builtin.import_playbook: vexxhost.atmosphere.csi + - name: Deploy with parallel orchestrator + ansible.builtin.shell: + cmd: > + . .venv/bin/activate && + ./bin/atmosphere deploy + --inventory ./inventory.yaml + {{ '--tags ' + atmosphere_deploy_tags if atmosphere_deploy_tags is defined else '' }} + executable: /bin/bash + args: + chdir: "{{ zuul.project.src_dir | default(playbook_dir + '/../..') }}" + run_once: true + changed_when: false diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml index 24b07ffbee..9724cebaea 100644 --- a/molecule/default/converge.yml +++ b/molecule/default/converge.yml @@ -12,4 +12,29 @@ # License for the specific language governing permissions and limitations # under the License. -- import_playbook: vexxhost.atmosphere.site +- hosts: all + gather_facts: false + tasks: + - name: Build atmosphere binary + ansible.builtin.command: + cmd: go build -o ./bin/atmosphere ./cmd/atmosphere + args: + chdir: "{{ zuul.project.src_dir | default(playbook_dir + '/../..') }}" + run_once: true + changed_when: false + environment: + CGO_ENABLED: "0" + PATH: "/usr/local/go/bin:{{ ansible_env.PATH | default(lookup('env', 'PATH')) }}" + + - name: Deploy with parallel orchestrator + ansible.builtin.shell: + cmd: > + . .venv/bin/activate && + ./bin/atmosphere deploy + --inventory {{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/workspace + {{ '--tags ' + atmosphere_deploy_tags if atmosphere_deploy_tags is defined else '' }} + executable: /bin/bash + args: + chdir: "{{ zuul.project.src_dir | default(playbook_dir + '/../..') }}" + run_once: true + changed_when: false diff --git a/molecule/keycloak/converge.yml b/molecule/keycloak/converge.yml index dd9097df2a..70fc02a1a7 100644 --- a/molecule/keycloak/converge.yml +++ b/molecule/keycloak/converge.yml @@ -12,21 +12,29 @@ # License for the specific language governing permissions and limitations # under the License. -- hosts: controllers - become: true - roles: - - vexxhost.atmosphere.cert_manager - - vexxhost.atmosphere.cluster_issuer - - vexxhost.atmosphere.ingress_nginx - - vexxhost.atmosphere.rabbitmq_cluster_operator - - vexxhost.atmosphere.percona_xtradb_cluster_operator - - vexxhost.atmosphere.percona_xtradb_cluster - - vexxhost.atmosphere.memcached +- hosts: all + gather_facts: false + tasks: + - name: Build atmosphere binary + ansible.builtin.command: + cmd: go build -o ./bin/atmosphere ./cmd/atmosphere + args: + chdir: "{{ zuul.project.src_dir | default(playbook_dir + '/../..') }}" + run_once: true + changed_when: false + environment: + CGO_ENABLED: "0" + PATH: "/usr/local/go/bin:{{ ansible_env.PATH | default(lookup('env', 'PATH')) }}" -- hosts: controllers - become: true - roles: - - vexxhost.atmosphere.keycloak - - vexxhost.atmosphere.kube_prometheus_stack - - vexxhost.atmosphere.keystone - - vexxhost.atmosphere.horizon + - name: Deploy with parallel orchestrator + ansible.builtin.shell: + cmd: > + . .venv/bin/activate && + ./bin/atmosphere deploy + --inventory ./inventory.yaml + {{ '--tags ' + atmosphere_deploy_tags if atmosphere_deploy_tags is defined else '' }} + executable: /bin/bash + args: + chdir: "{{ zuul.project.src_dir | default(playbook_dir + '/../..') }}" + run_once: true + changed_when: false diff --git a/molecule/pxc/converge.yml b/molecule/pxc/converge.yml index 5a69a7795b..b73bcf15a8 100644 --- a/molecule/pxc/converge.yml +++ b/molecule/pxc/converge.yml @@ -12,7 +12,29 @@ # License for the specific language governing permissions and limitations # under the License. -- hosts: controllers - become: true - roles: - - vexxhost.atmosphere.percona_xtradb_cluster +- hosts: all + gather_facts: false + tasks: + - name: Build atmosphere binary + ansible.builtin.command: + cmd: go build -o ./bin/atmosphere ./cmd/atmosphere + args: + chdir: "{{ zuul.project.src_dir | default(playbook_dir + '/../..') }}" + run_once: true + changed_when: false + environment: + CGO_ENABLED: "0" + PATH: "/usr/local/go/bin:{{ ansible_env.PATH | default(lookup('env', 'PATH')) }}" + + - name: Deploy with parallel orchestrator + ansible.builtin.shell: + cmd: > + . .venv/bin/activate && + ./bin/atmosphere deploy + --inventory {{ lookup('env', 'MOLECULE_INVENTORY_FILE') }} + {{ '--tags ' + atmosphere_deploy_tags if atmosphere_deploy_tags is defined else '' }} + executable: /bin/bash + args: + chdir: "{{ zuul.project.src_dir | default(playbook_dir + '/../..') }}" + run_once: true + changed_when: false diff --git a/molecule/shared/molecule.yml b/molecule/shared/molecule.yml index 507a0b2771..bc0eaf5d7c 100644 --- a/molecule/shared/molecule.yml +++ b/molecule/shared/molecule.yml @@ -68,6 +68,7 @@ provisioner: all: # Ceph ceph_fsid: ${MOLECULE_CEPH_FSID:-"d1f9e453-f13a-4d30-9ab1-e5ae1d87b67f"} + ceph_public_network: 10.96.240.0/24 ceph_conf_overrides: - section: global option: osd crush chooseleaf type diff --git a/pkg/dag/dag.go b/pkg/dag/dag.go new file mode 100644 index 0000000000..541f85e713 --- /dev/null +++ b/pkg/dag/dag.go @@ -0,0 +1,350 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package dag + +import ( + "container/heap" + "context" + "fmt" + "sync" + + "golang.org/x/sync/errgroup" +) + +// Graph is a generic directed acyclic graph where nodes hold values of type T. +// Edges represent dependencies: an edge from A to B means A depends on B. +type Graph[T any] struct { + nodes map[string]T + edges map[string][]string // node -> nodes it depends on + reverse map[string][]string // node -> nodes that depend on it +} + +// NewGraph creates an empty graph. +func NewGraph[T any]() *Graph[T] { + return &Graph[T]{ + nodes: make(map[string]T), + edges: make(map[string][]string), + reverse: make(map[string][]string), + } +} + +// AddNode adds a node with the given ID and value. Returns an error if the ID +// already exists. +func (g *Graph[T]) AddNode(id string, value T) error { + if _, ok := g.nodes[id]; ok { + return fmt.Errorf("node %q already exists", id) + } + g.nodes[id] = value + return nil +} + +// AddEdge adds a dependency edge indicating that "from" depends on "to". +// Returns an error if either node does not exist. +func (g *Graph[T]) AddEdge(from, to string) error { + if _, ok := g.nodes[from]; !ok { + return fmt.Errorf("node %q not found", from) + } + if _, ok := g.nodes[to]; !ok { + return fmt.Errorf("node %q not found", to) + } + g.edges[from] = append(g.edges[from], to) + g.reverse[to] = append(g.reverse[to], from) + return nil +} + +// Waves performs a topological sort using Kahn's algorithm and returns node IDs +// grouped into waves. Nodes within the same wave have no dependencies on each +// other and can be processed concurrently. Returns an error if the graph +// contains a cycle. +func (g *Graph[T]) Waves() ([][]string, error) { + inDegree := make(map[string]int, len(g.nodes)) + for id := range g.nodes { + inDegree[id] = len(g.edges[id]) + } + + var queue []string + for id, deg := range inDegree { + if deg == 0 { + queue = append(queue, id) + } + } + + var waves [][]string + visited := 0 + + for len(queue) > 0 { + waves = append(waves, queue) + visited += len(queue) + + var next []string + for _, id := range queue { + for _, dep := range g.reverse[id] { + inDegree[dep]-- + if inDegree[dep] == 0 { + next = append(next, dep) + } + } + } + queue = next + } + + if visited != len(g.nodes) { + return nil, fmt.Errorf("cycle detected: visited %d of %d nodes", visited, len(g.nodes)) + } + return waves, nil +} + +// Subgraph extracts a new graph containing only the specified nodes, preserving +// edges between them. Returns an error if any node ID is not found. +func (g *Graph[T]) Subgraph(nodeIDs []string) (*Graph[T], error) { + sub := NewGraph[T]() + keep := make(map[string]bool, len(nodeIDs)) + + for _, id := range nodeIDs { + val, ok := g.nodes[id] + if !ok { + return nil, fmt.Errorf("node %q not found", id) + } + keep[id] = true + if err := sub.AddNode(id, val); err != nil { + return nil, err + } + } + + for from, deps := range g.edges { + if !keep[from] { + continue + } + for _, to := range deps { + if keep[to] { + if err := sub.AddEdge(from, to); err != nil { + return nil, err + } + } + } + } + return sub, nil +} + +// CriticalPath returns, for every node, the length (in unit-weighted node +// count including the node itself) of the longest downstream chain. Leaves +// have a value of 1. The result is stable across calls on the same graph +// and is used by Run to prioritise ready nodes when concurrency is capped: +// starting nodes with more work behind them first shortens the makespan +// under classic list-scheduling (HEFT) heuristics. +func (g *Graph[T]) CriticalPath() map[string]int { + memo := make(map[string]int, len(g.nodes)) + var visit func(id string) int + visit = func(id string) int { + if v, ok := memo[id]; ok { + return v + } + best := 0 + for _, dep := range g.reverse[id] { + if d := visit(dep); d > best { + best = d + } + } + memo[id] = best + 1 + return memo[id] + } + for id := range g.nodes { + visit(id) + } + return memo +} + +// Run executes fn for every node in topological order. Each node starts as +// soon as all of its direct dependencies have completed successfully, without +// waiting for the rest of its topological "wave" to finish. The concurrency +// parameter caps the number of nodes running at the same time across the +// whole graph; if <= 0 there is no limit. Execution stops on the first error: +// nodes already running continue, but dependents of any node (failed or not +// yet started) are cancelled via the context. +// +// When concurrency is bounded and multiple nodes are simultaneously ready to +// run, admission order follows CriticalPath: the ready node with the longest +// downstream chain is admitted first. This reduces tail latency compared to +// the arbitrary FIFO a plain channel semaphore would give. +func (g *Graph[T]) Run(ctx context.Context, concurrency int, fn func(ctx context.Context, id string, value T) error) error { + if _, err := g.Waves(); err != nil { + return err + } + + done := make(map[string]chan struct{}, len(g.nodes)) + for id := range g.nodes { + done[id] = make(chan struct{}) + } + + var sched *prioScheduler + if concurrency > 0 { + sched = newPrioScheduler(concurrency) + defer sched.stop() + } + + priorities := g.CriticalPath() + + eg, ctx := errgroup.WithContext(ctx) + for id, val := range g.nodes { + id, val := id, val + eg.Go(func() error { + for _, dep := range g.edges[id] { + select { + case <-done[dep]: + case <-ctx.Done(): + return ctx.Err() + } + } + + if sched != nil { + if err := sched.acquire(ctx, priorities[id]); err != nil { + return err + } + defer sched.release() + } + + if err := fn(ctx, id, val); err != nil { + return err + } + close(done[id]) + return nil + }) + } + return eg.Wait() +} + +// prioScheduler is a priority-aware concurrency limiter. Waiters register +// themselves with a priority; a single scheduler goroutine admits the +// highest-priority waiter whenever capacity becomes available. Ties are +// broken by arrival sequence so ordering remains deterministic under equal +// priorities. +type prioScheduler struct { + mu sync.Mutex + cap int + inFlight int + heap waiterHeap + seq uint64 + wake chan struct{} + quit chan struct{} +} + +func newPrioScheduler(capacity int) *prioScheduler { + s := &prioScheduler{ + cap: capacity, + wake: make(chan struct{}, 1), + quit: make(chan struct{}), + } + go s.loop() + return s +} + +func (s *prioScheduler) stop() { + close(s.quit) +} + +func (s *prioScheduler) acquire(ctx context.Context, priority int) error { + ready := make(chan struct{}) + w := &waiter{priority: priority, ready: ready} + + s.mu.Lock() + s.seq++ + w.seq = s.seq + heap.Push(&s.heap, w) + s.mu.Unlock() + s.signal() + + select { + case <-ready: + return nil + case <-ctx.Done(): + s.mu.Lock() + w.cancelled = true + // If the scheduler already admitted us concurrently, honour it + // and immediately release so capacity is not leaked. + select { + case <-ready: + s.inFlight-- + s.mu.Unlock() + s.signal() + default: + s.mu.Unlock() + } + return ctx.Err() + } +} + +func (s *prioScheduler) release() { + s.mu.Lock() + s.inFlight-- + s.mu.Unlock() + s.signal() +} + +func (s *prioScheduler) signal() { + select { + case s.wake <- struct{}{}: + default: + } +} + +func (s *prioScheduler) loop() { + for { + select { + case <-s.quit: + return + case <-s.wake: + } + s.mu.Lock() + for s.inFlight < s.cap && s.heap.Len() > 0 { + w := heap.Pop(&s.heap).(*waiter) + if w.cancelled { + continue + } + s.inFlight++ + close(w.ready) + } + s.mu.Unlock() + } +} + +type waiter struct { + priority int + seq uint64 + ready chan struct{} + cancelled bool + index int +} + +type waiterHeap []*waiter + +func (h waiterHeap) Len() int { return len(h) } + +func (h waiterHeap) Less(i, j int) bool { + if h[i].priority != h[j].priority { + return h[i].priority > h[j].priority + } + return h[i].seq < h[j].seq +} + +func (h waiterHeap) Swap(i, j int) { + h[i], h[j] = h[j], h[i] + h[i].index = i + h[j].index = j +} + +func (h *waiterHeap) Push(x any) { + w := x.(*waiter) + w.index = len(*h) + *h = append(*h, w) +} + +func (h *waiterHeap) Pop() any { + old := *h + n := len(old) + w := old[n-1] + old[n-1] = nil + w.index = -1 + *h = old[:n-1] + return w +} diff --git a/pkg/dag/dag_test.go b/pkg/dag/dag_test.go new file mode 100644 index 0000000000..efa66cba9a --- /dev/null +++ b/pkg/dag/dag_test.go @@ -0,0 +1,495 @@ +// Copyright (c) 2026 VEXXHOST, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package dag + +import ( + "context" + "errors" + "fmt" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestAddNode(t *testing.T) { + g := NewGraph[string]() + if err := g.AddNode("a", "A"); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if err := g.AddNode("a", "A2"); err == nil { + t.Fatal("expected error for duplicate node, got nil") + } +} + +func TestAddEdge(t *testing.T) { + g := NewGraph[string]() + _ = g.AddNode("a", "A") + + if err := g.AddEdge("a", "missing"); err == nil { + t.Fatal("expected error for missing target node") + } + if err := g.AddEdge("missing", "a"); err == nil { + t.Fatal("expected error for missing source node") + } + + _ = g.AddNode("b", "B") + if err := g.AddEdge("a", "b"); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestWaves_Simple(t *testing.T) { + // Diamond: A depends on B and C; B and C depend on D. + g := NewGraph[string]() + for _, id := range []string{"A", "B", "C", "D"} { + _ = g.AddNode(id, id) + } + _ = g.AddEdge("A", "B") + _ = g.AddEdge("A", "C") + _ = g.AddEdge("B", "D") + _ = g.AddEdge("C", "D") + + waves, err := g.Waves() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(waves) != 3 { + t.Fatalf("expected 3 waves, got %d: %v", len(waves), waves) + } + + // Wave 0: D (no deps), Wave 1: B and C, Wave 2: A + if waves[0][0] != "D" { + t.Errorf("wave 0: expected [D], got %v", waves[0]) + } + w1 := sortedCopy(waves[1]) + if len(w1) != 2 || w1[0] != "B" || w1[1] != "C" { + t.Errorf("wave 1: expected [B C], got %v", w1) + } + if waves[2][0] != "A" { + t.Errorf("wave 2: expected [A], got %v", waves[2]) + } +} + +func TestWaves_Linear(t *testing.T) { + // A → B → C (A depends on B, B depends on C) + g := NewGraph[string]() + for _, id := range []string{"A", "B", "C"} { + _ = g.AddNode(id, id) + } + _ = g.AddEdge("A", "B") + _ = g.AddEdge("B", "C") + + waves, err := g.Waves() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(waves) != 3 { + t.Fatalf("expected 3 waves, got %d", len(waves)) + } + for i, w := range waves { + if len(w) != 1 { + t.Errorf("wave %d: expected 1 node, got %d", i, len(w)) + } + } +} + +func TestWaves_Parallel(t *testing.T) { + // A, B, C with no edges → all in one wave + g := NewGraph[string]() + for _, id := range []string{"A", "B", "C"} { + _ = g.AddNode(id, id) + } + + waves, err := g.Waves() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(waves) != 1 { + t.Fatalf("expected 1 wave, got %d", len(waves)) + } + if len(waves[0]) != 3 { + t.Errorf("expected 3 nodes in wave, got %d", len(waves[0])) + } +} + +func TestWaves_Cycle(t *testing.T) { + g := NewGraph[string]() + _ = g.AddNode("A", "A") + _ = g.AddNode("B", "B") + _ = g.AddEdge("A", "B") + _ = g.AddEdge("B", "A") + + _, err := g.Waves() + if err == nil { + t.Fatal("expected cycle error, got nil") + } + if !strings.Contains(err.Error(), "cycle") { + t.Errorf("expected cycle in error message, got: %v", err) + } +} + +func TestSubgraph(t *testing.T) { + g := NewGraph[string]() + for _, id := range []string{"A", "B", "C", "D"} { + _ = g.AddNode(id, id) + } + _ = g.AddEdge("A", "B") + _ = g.AddEdge("B", "C") + _ = g.AddEdge("C", "D") + + sub, err := g.Subgraph([]string{"A", "B", "C"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Subgraph should have 3 nodes and preserve A→B and B→C edges, + // but not C→D since D is not in the subgraph. + if len(sub.nodes) != 3 { + t.Errorf("expected 3 nodes, got %d", len(sub.nodes)) + } + + waves, err := sub.Waves() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // C (no deps in sub), then B, then A → 3 waves + if len(waves) != 3 { + t.Fatalf("expected 3 waves in subgraph, got %d: %v", len(waves), waves) + } +} + +func TestSubgraph_MissingNode(t *testing.T) { + g := NewGraph[string]() + _ = g.AddNode("A", "A") + + _, err := g.Subgraph([]string{"A", "Z"}) + if err == nil { + t.Fatal("expected error for missing node, got nil") + } +} + +func TestRun_Parallel(t *testing.T) { + // Three independent nodes should run concurrently. + g := NewGraph[string]() + for _, id := range []string{"A", "B", "C"} { + _ = g.AddNode(id, id) + } + + var maxConcurrent atomic.Int64 + var current atomic.Int64 + + err := g.Run(context.Background(), 0, func(_ context.Context, _ string, _ string) error { + cur := current.Add(1) + for { + old := maxConcurrent.Load() + if cur <= old || maxConcurrent.CompareAndSwap(old, cur) { + break + } + } + time.Sleep(50 * time.Millisecond) + current.Add(-1) + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if maxConcurrent.Load() < 2 { + t.Errorf("expected concurrent execution, max concurrent was %d", maxConcurrent.Load()) + } +} + +func TestRun_Error(t *testing.T) { + // Wave 0: A (succeeds), Wave 1: B (fails), Wave 2: C (should not run) + g := NewGraph[string]() + for _, id := range []string{"A", "B", "C"} { + _ = g.AddNode(id, id) + } + _ = g.AddEdge("C", "B") + _ = g.AddEdge("B", "A") + + boom := errors.New("boom") + var ran sync.Map + + err := g.Run(context.Background(), 0, func(_ context.Context, id string, _ string) error { + ran.Store(id, true) + if id == "B" { + return boom + } + return nil + }) + if !errors.Is(err, boom) { + t.Fatalf("expected boom error, got: %v", err) + } + if _, ok := ran.Load("C"); ok { + t.Error("node C should not have run after B failed") + } +} + +func TestRun_Ordering(t *testing.T) { + // A depends on B depends on C. Execution order must be C, B, A. + g := NewGraph[string]() + for _, id := range []string{"A", "B", "C"} { + _ = g.AddNode(id, id) + } + _ = g.AddEdge("A", "B") + _ = g.AddEdge("B", "C") + + var mu sync.Mutex + var order []string + + err := g.Run(context.Background(), 1, func(_ context.Context, id string, _ string) error { + mu.Lock() + order = append(order, id) + mu.Unlock() + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + expected := "C,B,A" + got := fmt.Sprintf("%s,%s,%s", order[0], order[1], order[2]) + if got != expected { + t.Errorf("expected order %s, got %s", expected, got) + } +} + +func sortedCopy(s []string) []string { + c := make([]string, len(s)) + copy(c, s) + sort.Strings(c) + return c +} + +// TestRunShortNodeNotBlockedByUnrelatedLongNode asserts the event-driven +// scheduler lets a short node start as soon as its own deps are done, even +// if another (unrelated) node in the same Kahn wave is still running. +func TestRunShortNodeNotBlockedByUnrelatedLongNode(t *testing.T) { + g := NewGraph[string]() + for _, n := range []string{"root", "long", "short_dep", "short"} { + _ = g.AddNode(n, n) + } + // Both "long" and "short_dep" depend on "root" -> same wave. + _ = g.AddEdge("long", "root") + _ = g.AddEdge("short_dep", "root") + // "short" depends on "short_dep" only; it is in the next wave, but its + // single dep completes quickly. Under wave-barrier scheduling it would + // still wait for "long" to finish. + _ = g.AddEdge("short", "short_dep") + + var longEnd, shortStart time.Time + var mu sync.Mutex + + err := g.Run(context.Background(), 0, func(_ context.Context, id string, _ string) error { + switch id { + case "long": + time.Sleep(200 * time.Millisecond) + mu.Lock() + longEnd = time.Now() + mu.Unlock() + case "short_dep": + time.Sleep(20 * time.Millisecond) + case "short": + mu.Lock() + shortStart = time.Now() + mu.Unlock() + } + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + mu.Lock() + defer mu.Unlock() + if shortStart.IsZero() || longEnd.IsZero() { + t.Fatal("timestamps not captured") + } + // "short" must start before "long" ends; otherwise we still have a + // wave barrier. + if !shortStart.Before(longEnd) { + t.Errorf("short started at %v, expected before longEnd %v (wave barrier still in effect)", + shortStart, longEnd) + } +} + +// TestRunStopsDependentsOnError asserts that when a node fails, nodes that +// (transitively) depend on it are cancelled via context instead of hanging. +func TestRunStopsDependentsOnError(t *testing.T) { + g := NewGraph[string]() + for _, n := range []string{"a", "b", "c"} { + _ = g.AddNode(n, n) + } + _ = g.AddEdge("b", "a") + _ = g.AddEdge("c", "b") + + var ran atomic.Int32 + wantErr := errors.New("boom") + + err := g.Run(context.Background(), 0, func(ctx context.Context, id string, _ string) error { + ran.Add(1) + if id == "a" { + return wantErr + } + // b and c should never reach here. + return nil + }) + if !errors.Is(err, wantErr) { + t.Fatalf("expected wantErr, got %v", err) + } + if ran.Load() != 1 { + t.Errorf("expected only 'a' to run, got %d runs", ran.Load()) + } + _ = strings.TrimSpace // keep strings import used if trimmed in future +} + +// TestRunConcurrencyCap asserts the global concurrency cap limits in-flight +// nodes across the whole graph (not just within one wave). +func TestRunConcurrencyCap(t *testing.T) { + g := NewGraph[string]() + // 5 independent nodes — all roots, all ready at once. + for _, n := range []string{"a", "b", "c", "d", "e"} { + _ = g.AddNode(n, n) + } + + var inflight, peak atomic.Int32 + err := g.Run(context.Background(), 2, func(_ context.Context, _ string, _ string) error { + cur := inflight.Add(1) + for { + p := peak.Load() + if cur <= p || peak.CompareAndSwap(p, cur) { + break + } + } + time.Sleep(30 * time.Millisecond) + inflight.Add(-1) + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if peak.Load() > 2 { + t.Errorf("expected peak in-flight <= 2, got %d", peak.Load()) + } +} + +func TestCriticalPath(t *testing.T) { + // Graph: + // + // a -> b -> c -> d (chain of length 4) + // a -> e (side branch of length 2) + // + // Depths (longest chain of dependents starting at node, including self): + // + // d=1, c=2, b=3, e=1, a=4 + g := NewGraph[string]() + for _, n := range []string{"a", "b", "c", "d", "e"} { + _ = g.AddNode(n, n) + } + _ = g.AddEdge("b", "a") + _ = g.AddEdge("c", "b") + _ = g.AddEdge("d", "c") + _ = g.AddEdge("e", "a") + + got := g.CriticalPath() + want := map[string]int{"a": 4, "b": 3, "c": 2, "d": 1, "e": 1} + for k, v := range want { + if got[k] != v { + t.Errorf("CriticalPath[%s] = %d, want %d", k, got[k], v) + } + } +} + +func TestRunPriorityUnderConcurrencyCap(t *testing.T) { + // Test the priority scheduler directly: prime it so capacity is held, + // queue waiters with distinct priorities in a non-priority order, then + // release the held slot. Admission order must be priority-descending. + s := newPrioScheduler(1) + defer s.stop() + + ctx := context.Background() + if err := s.acquire(ctx, 0); err != nil { + t.Fatalf("prime acquire: %v", err) + } + + admitted := make(chan int, 3) + release := make(chan struct{}) + var wg sync.WaitGroup + queued := 0 + for _, p := range []int{1, 3, 2} { + p := p + wg.Add(1) + go func() { + defer wg.Done() + if err := s.acquire(ctx, p); err != nil { + t.Errorf("waiter p=%d acquire: %v", p, err) + return + } + admitted <- p + <-release + s.release() + }() + queued++ + // Wait for this waiter to enter the heap before queueing the next, + // so heap ordering rather than goroutine-start order decides admission. + for { + s.mu.Lock() + n := s.heap.Len() + s.mu.Unlock() + if n >= queued { + break + } + time.Sleep(time.Millisecond) + } + } + + // Release the primed slot so admission begins. + s.release() + + got := make([]int, 0, 3) + for i := 0; i < 3; i++ { + select { + case p := <-admitted: + got = append(got, p) + case <-time.After(time.Second): + t.Fatalf("timed out waiting for admission %d; got so far %v", i, got) + } + release <- struct{}{} + } + wg.Wait() + + want := []int{3, 2, 1} + for i := range want { + if got[i] != want[i] { + t.Fatalf("admission order = %v, want %v", got, want) + } + } +} + +func TestRunPriorityDoesNotBlockIndependentChains(t *testing.T) { + // Even with priority scheduling, once a higher-priority node is running + // a lower-priority ready node must be admitted as soon as capacity + // frees up; this guards against a deadlock where the scheduler only + // considers priority order and not current capacity. + g := NewGraph[string]() + for _, n := range []string{"x", "y"} { + _ = g.AddNode(n, n) + } + // no edges: both x and y are independent and ready immediately. + + var count atomic.Int32 + fn := func(_ context.Context, _ string, _ string) error { + count.Add(1) + return nil + } + if err := g.Run(context.Background(), 2, fn); err != nil { + t.Fatalf("Run failed: %v", err) + } + if count.Load() != 2 { + t.Fatalf("expected 2 runs, got %d", count.Load()) + } +} diff --git a/playbooks/image_warmup.yml b/playbooks/image_warmup.yml new file mode 100644 index 0000000000..7946cc5953 --- /dev/null +++ b/playbooks/image_warmup.yml @@ -0,0 +1,21 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- hosts: "{{ target | default('controllers:computes') }}" + become: true + gather_facts: false + tags: + - image-warmup + roles: + - role: vexxhost.atmosphere.image_warmup diff --git a/playbooks/kubernetes.yml b/playbooks/kubernetes.yml index 19b2a6ac8d..d9d0ac20db 100644 --- a/playbooks/kubernetes.yml +++ b/playbooks/kubernetes.yml @@ -63,3 +63,7 @@ ansible.builtin.package: name: unattended-upgrades state: absent + register: _uninstall_unattended + retries: 5 + delay: 10 + until: _uninstall_unattended is not failed diff --git a/playbooks/openstack.yml b/playbooks/openstack.yml index 871875e302..12773da104 100644 --- a/playbooks/openstack.yml +++ b/playbooks/openstack.yml @@ -53,6 +53,10 @@ tags: - glance + - role: glance_images + tags: + - glance-images + - role: staffeln when: atmosphere_staffeln_enabled | default(true) tags: @@ -137,11 +141,23 @@ tags: - heat + - role: octavia_pre + tags: + - octavia + - role: octavia + vars: + _pre_role_active: true tags: - octavia + - role: magnum_pre + tags: + - magnum + - role: magnum + vars: + _pre_role_active: true tags: - magnum diff --git a/playbooks/site.yml b/playbooks/site.yml index f65ef12feb..2bfb7032e8 100644 --- a/playbooks/site.yml +++ b/playbooks/site.yml @@ -18,6 +18,9 @@ - name: Install Kubernetes import_playbook: vexxhost.atmosphere.kubernetes +- name: Pre-pull container images + ansible.builtin.import_playbook: vexxhost.atmosphere.image_warmup + - name: Install CSI import_playbook: vexxhost.atmosphere.csi diff --git a/releasenotes/notes/dag-critical-path-scheduling-d2d5f1dc4ca5bf70.yaml b/releasenotes/notes/dag-critical-path-scheduling-d2d5f1dc4ca5bf70.yaml new file mode 100644 index 0000000000..8d188cf6dc --- /dev/null +++ b/releasenotes/notes/dag-critical-path-scheduling-d2d5f1dc4ca5bf70.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + The parallel deploy orchestrator now prioritises admission of + runnable components by the length of their longest downstream + dependency chain once the global concurrency cap saturates. + Components with more remaining work start ahead of shorter ones + that happen to arrive at the scheduler first, which shortens the + overall deployment time under bounded concurrency. diff --git a/releasenotes/notes/dag-edge-removal-phase1-439694be3dac4d84.yaml b/releasenotes/notes/dag-edge-removal-phase1-439694be3dac4d84.yaml new file mode 100644 index 0000000000..c26e742cf6 --- /dev/null +++ b/releasenotes/notes/dag-edge-removal-phase1-439694be3dac4d84.yaml @@ -0,0 +1,17 @@ +--- +features: + - | + The parallel deployment orchestrator now runs Magnum in parallel with + Barbican and Heat instead of after them. An audit showed that these + dependencies were configuration-only references and don't require the + dependent service to run at install time. Magnum retains install-time + dependencies on Keystone and Glance because it creates identity + resources and uploads cluster images during deployment. + - | + The Rook Ceph cluster role now depends on Keystone directly rather than + on Barbican. The previous Barbican dependency was mis-declared: the + role only creates an identity user and catalog service in Keystone. A + new readiness check on the ``keystone-api`` Deployment runs before the + OpenStack API calls, and the role now creates both the ``service`` + domain and the ``service`` project itself so it no longer relies on + another chart's ks-user job to create them first. diff --git a/releasenotes/notes/dag-event-driven-scheduler-297ecebcb0455852.yaml b/releasenotes/notes/dag-event-driven-scheduler-297ecebcb0455852.yaml new file mode 100644 index 0000000000..20960dda80 --- /dev/null +++ b/releasenotes/notes/dag-event-driven-scheduler-297ecebcb0455852.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + The parallel deploy orchestrator now schedules components using a + per-node readiness model instead of wave barriers. Each component + starts as soon as all its own dependencies have completed, even + when other unrelated components in the same topological level are + still running. This removes "wave gaps" where a short component + would otherwise wait for an unrelated long one to finish before + starting. diff --git a/releasenotes/notes/deploy-perf-tier1-d8f993d67122d060.yaml b/releasenotes/notes/deploy-perf-tier1-d8f993d67122d060.yaml new file mode 100644 index 0000000000..e928be0868 --- /dev/null +++ b/releasenotes/notes/deploy-perf-tier1-d8f993d67122d060.yaml @@ -0,0 +1,32 @@ +--- +features: + - | + Added ``PreRoleDependsOn`` to the parallel deploy orchestrator so a + component's pre-role can wait for different dependencies than the main + role. This lets the Keycloak realm setup gate the pre-role while + the Keystone Helm install runs in parallel with Keycloak startup. + - | + Split Keystone's Keycloak realm and OpenID configuration into a new + ``keystone_pre`` role. Under the parallel orchestrator this runs + concurrently with the Keystone Helm install. + - | + Extended the ``ResourceCoordinator`` in the parallel deploy + orchestrator with per-resource concurrency caps. A new ``k8s-api`` + resource (default cap ``6``) smooths the load on the Kubernetes API + server during wave fan-out without serializing OpenStack components. + - | + Added a dedicated semaphore (cap ``1``) for the Keycloak admin HTTP + endpoint. The orchestrator now serializes ``keystone`` and + ``kube-prometheus-stack`` whenever they create realms or clients, so + concurrent admin calls on the same Keycloak no longer race and fail. + - | + Moved Neutron's Helm install into a separate pre-role under the + parallel orchestrator. The install now overlaps with Nova; only the + post-install network creation, which hits an availability zone + check on the Neutron API, still waits on Nova. + - | + Split Glance image uploads into a separate component. Downstream + services that depend on Glance (Nova, Magnum, and others) now + wait only for the Glance API; image downloads continue in + parallel off the critical path. +upgrade: [] diff --git a/releasenotes/notes/deploy-perf-tier2-613c7bac4183c8bd.yaml b/releasenotes/notes/deploy-perf-tier2-613c7bac4183c8bd.yaml new file mode 100644 index 0000000000..0bdaeb862a --- /dev/null +++ b/releasenotes/notes/deploy-perf-tier2-613c7bac4183c8bd.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + Added the ``image_warmup`` role and component, which pre-pulls every + image listed in ``_atmosphere_images`` on every Kubernetes node once + the cluster reports READY. Subsequent component Helm installs + schedule pods that find their images already cached, eliminating + most ``ImagePulling`` waits from the critical path. Failures are + non-fatal: the ``kubelet`` falls back to on-demand pulls. +upgrade: [] diff --git a/releasenotes/notes/parallel-deployment-orchestrator-7808f907bf1ecd23.yaml b/releasenotes/notes/parallel-deployment-orchestrator-7808f907bf1ecd23.yaml new file mode 100644 index 0000000000..ad51d16dbc --- /dev/null +++ b/releasenotes/notes/parallel-deployment-orchestrator-7808f907bf1ecd23.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + Adds a parallel deployment orchestrator (``atmosphere deploy``) that + uses a DAG-based dependency graph to deploy components concurrently + across parallel waves. Components that have no direct dependency on + each other run in parallel instead of serially, which significantly + shortens full deployment time compared to the previous sequential + role-by-role flow. diff --git a/releasenotes/notes/selective-molecule-ci-96c6c8b19ed8ab7b.yaml b/releasenotes/notes/selective-molecule-ci-96c6c8b19ed8ab7b.yaml new file mode 100644 index 0000000000..813a136ccc --- /dev/null +++ b/releasenotes/notes/selective-molecule-ci-96c6c8b19ed8ab7b.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + The planner now maps pull request changes to focused Molecule deployment + and verification jobs. Selected jobs deploy the dependencies needed for a + fresh environment, while unrelated scenarios are skipped. Shared, unknown, + or unplannable changes continue to run the complete test set. diff --git a/roles/ceph_provisioners/tasks/main.yml b/roles/ceph_provisioners/tasks/main.yml index 5f555c97fa..260bcb3e8e 100644 --- a/roles/ceph_provisioners/tasks/main.yml +++ b/roles/ceph_provisioners/tasks/main.yml @@ -12,6 +12,16 @@ # License for the specific language governing permissions and limitations # under the License. +- name: Create namespace + run_once: true + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: "{{ ceph_provisioners_helm_release_namespace }}" + - name: Collect "ceph mon dump" output from a monitor delegate_to: "{{ groups[ceph_provisioners_ceph_mon_group][0] }}" run_once: true @@ -38,7 +48,7 @@ kind: Service metadata: name: ceph-mon - namespace: openstack + namespace: "{{ ceph_provisioners_helm_release_namespace }}" labels: application: ceph spec: @@ -62,7 +72,7 @@ kind: Endpoints metadata: name: ceph-mon - namespace: openstack + namespace: "{{ ceph_provisioners_helm_release_namespace }}" labels: application: ceph subsets: @@ -99,7 +109,7 @@ type: kubernetes.io/rbd metadata: name: pvc-ceph-client-key - namespace: openstack + namespace: "{{ ceph_provisioners_helm_release_namespace }}" labels: application: ceph stringData: diff --git a/roles/cluster_issuer/handlers/main.yml b/roles/cluster_issuer/handlers/main.yml index a4681d4a90..370d2c4a0f 100644 --- a/roles/cluster_issuer/handlers/main.yml +++ b/roles/cluster_issuer/handlers/main.yml @@ -17,6 +17,10 @@ ansible.builtin.package: name: openssl-perl state: present + register: _install_openssl_perl + retries: 5 + delay: 10 + until: _install_openssl_perl is not failed listen: Update CA certificates on host - name: Update CA certificates diff --git a/roles/glance/tasks/main.yml b/roles/glance/tasks/main.yml index 19c3f9616f..b1889810a8 100644 --- a/roles/glance/tasks/main.yml +++ b/roles/glance/tasks/main.yml @@ -35,19 +35,3 @@ openstack_helm_ingress_service_port: 9292 openstack_helm_ingress_annotations: "{{ _glance_ingress_annotations | combine(glance_ingress_annotations) }}" openstack_helm_ingress_class_name: "{{ glance_ingress_class_name }}" - -- name: Create images - ansible.builtin.include_role: - name: atmosphere.common.glance_image - loop: "{{ glance_images }}" - vars: - glance_image_name: "{{ item.name }}" - glance_image_url: "{{ item.url }}" - glance_image_min_disk: "{{ item.min_disk | default(omit) }}" - glance_image_min_ram: "{{ item.min_ram | default(omit) }}" - glance_image_container_format: "{{ item.container_format | default(omit) }}" - glance_image_disk_format: "{{ item.disk_format | default(omit) }}" - glance_image_properties: "{{ item.properties | default({}) }}" - glance_image_kernel: "{{ item.kernel | default(omit) }}" - glance_image_ramdisk: "{{ item.ramdisk | default(omit) }}" - glance_image_is_public: "{{ item.is_public | default(omit) }}" diff --git a/roles/glance_images/README.md b/roles/glance_images/README.md new file mode 100644 index 0000000000..8807b47637 --- /dev/null +++ b/roles/glance_images/README.md @@ -0,0 +1,6 @@ +# `glance_images` role + +Uploads Glance images defined in `glance_images` (see `glance` role +defaults). Split out from the main `glance` role so that downstream +services (Nova, Magnum, etc.) only wait for the Glance API to be +deployed and not for image downloads to finish. diff --git a/roles/glance_images/defaults/main.yml b/roles/glance_images/defaults/main.yml new file mode 120000 index 0000000000..7d94e40d57 --- /dev/null +++ b/roles/glance_images/defaults/main.yml @@ -0,0 +1 @@ +../../glance/defaults/main.yml \ No newline at end of file diff --git a/roles/glance_images/meta/main.yml b/roles/glance_images/meta/main.yml new file mode 100644 index 0000000000..ecf3828b99 --- /dev/null +++ b/roles/glance_images/meta/main.yml @@ -0,0 +1,30 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# SPDX-License-Identifier: Apache-2.0 + +galaxy_info: + author: VEXXHOST, Inc. + description: >- + Uploads Glance images. Runs as a separate component so that downstream + services (Nova, Magnum, etc.) only have to wait for the Glance API to + be installed (the `glance` component) and not for image downloads to + finish. + license: Apache-2.0 + min_ansible_version: 5.5.0 + standalone: false + platforms: + - name: EL + versions: + - "8" + - "9" + - name: Ubuntu + versions: + - focal + - jammy + +dependencies: + - role: defaults + - role: openstack_helm_endpoints + vars: + openstack_helm_endpoints_chart: glance + openstack_helm_endpoints_skip_cluster_creation: true + - role: openstacksdk diff --git a/roles/glance_images/tasks/main.yml b/roles/glance_images/tasks/main.yml new file mode 100644 index 0000000000..ad355d1f96 --- /dev/null +++ b/roles/glance_images/tasks/main.yml @@ -0,0 +1,18 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# SPDX-License-Identifier: Apache-2.0 + +- name: Create images + ansible.builtin.include_role: + name: atmosphere.common.glance_image + loop: "{{ glance_images }}" + vars: + glance_image_name: "{{ item.name }}" + glance_image_url: "{{ item.url }}" + glance_image_min_disk: "{{ item.min_disk | default(omit) }}" + glance_image_min_ram: "{{ item.min_ram | default(omit) }}" + glance_image_container_format: "{{ item.container_format | default(omit) }}" + glance_image_disk_format: "{{ item.disk_format | default(omit) }}" + glance_image_properties: "{{ item.properties | default({}) }}" + glance_image_kernel: "{{ item.kernel | default(omit) }}" + glance_image_ramdisk: "{{ item.ramdisk | default(omit) }}" + glance_image_is_public: "{{ item.is_public | default(omit) }}" diff --git a/roles/image_warmup/README.md b/roles/image_warmup/README.md new file mode 100644 index 0000000000..a9dc830612 --- /dev/null +++ b/roles/image_warmup/README.md @@ -0,0 +1,11 @@ +# `image_warmup` + +This role pre-pulls every container image listed in `_atmosphere_images` +on every Kubernetes node using `crictl`. Running it once the cluster is +ready ensures that subsequent component Helm installs schedule pods +whose images are already cached, removing most `ImagePulling` waits +from the deploy critical path. + +Pulls run with bounded concurrency and any failure is logged but +non-fatal so that on-demand pulls can still satisfy missing images +later. diff --git a/roles/image_warmup/defaults/main.yml b/roles/image_warmup/defaults/main.yml new file mode 100644 index 0000000000..4ec7953ce0 --- /dev/null +++ b/roles/image_warmup/defaults/main.yml @@ -0,0 +1,39 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Path to the containerd CRI socket used for `crictl pull`. +# +# image_warmup_containerd_socket: unix:///run/containerd/containerd.sock +image_warmup_containerd_socket: unix:///run/containerd/containerd.sock + +# Per-image pull timeout in seconds for `crictl pull`. The kubelet will pull +# on demand if warmup is slower, so this is purely a guard against runaway +# stuck pulls. +# +# image_warmup_pull_timeout: 600 +image_warmup_pull_timeout: 600 + +# Maximum number of parallel pulls per host. Containerd handles its own +# layer-level deduplication, but capping concurrency avoids saturating the +# image registry from every node at once. +# +# image_warmup_concurrency: 4 +image_warmup_concurrency: 4 + +# Optional explicit list of image references to pre-pull. When empty +# (default), the role pulls every value from `_atmosphere_images`. +# +# image_warmup_images: +# - quay.io/ceph/ceph:v18.2.7 +image_warmup_images: [] diff --git a/roles/image_warmup/meta/main.yml b/roles/image_warmup/meta/main.yml new file mode 100644 index 0000000000..d73ee634e4 --- /dev/null +++ b/roles/image_warmup/meta/main.yml @@ -0,0 +1,26 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +galaxy_info: + author: VEXXHOST, Inc. + description: >- + Pre-pulls all known Atmosphere container images on every node so that + later component Helm installs do not stall in ImagePulling. Best-effort: + failures are non-fatal because the kubelet will pull on demand if the + warmup did not complete in time. + license: Apache-2.0 + min_ansible_version: 5.5.0 + +dependencies: + - role: defaults diff --git a/roles/image_warmup/tasks/main.yml b/roles/image_warmup/tasks/main.yml new file mode 100644 index 0000000000..ada9fbdd9f --- /dev/null +++ b/roles/image_warmup/tasks/main.yml @@ -0,0 +1,58 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Resolve image list + ansible.builtin.set_fact: + _image_warmup_targets: >- + {{ + (image_warmup_images + if (image_warmup_images | length) > 0 + else (_atmosphere_images | dict2items | map(attribute='value') | list)) + | unique + }} + +- name: Check for crictl + ansible.builtin.command: + cmd: which crictl + register: _image_warmup_crictl + changed_when: false + failed_when: false + +- name: Pre-pull container images + when: _image_warmup_crictl.rc == 0 + ansible.builtin.command: + cmd: >- + crictl --runtime-endpoint {{ image_warmup_containerd_socket }} + --timeout {{ image_warmup_pull_timeout }}s + pull {{ item }} + loop: "{{ _image_warmup_targets }}" + loop_control: + label: "{{ item }}" + register: _image_warmup_pull + changed_when: false + failed_when: false + throttle: "{{ image_warmup_concurrency }}" + +- name: Report unreachable images + when: + - _image_warmup_crictl.rc == 0 + - _image_warmup_pull.results is defined + - item.rc != 0 + ansible.builtin.debug: + msg: >- + Warmup pull failed for {{ item.item }} (rc={{ item.rc }}); kubelet + will pull on demand. + loop: "{{ _image_warmup_pull.results | default([]) }}" + loop_control: + label: "{{ item.item | default('') }}" diff --git a/roles/iscsi/tasks/main.yml b/roles/iscsi/tasks/main.yml index 6391d3fdd5..7eac42524c 100644 --- a/roles/iscsi/tasks/main.yml +++ b/roles/iscsi/tasks/main.yml @@ -4,6 +4,10 @@ ansible.builtin.package: name: "{{ 'iscsi-initiator-utils' if ansible_os_family == 'RedHat' else 'open-iscsi' }}" state: present + register: _install_iscsi + retries: 5 + delay: 10 + until: _install_iscsi is not failed - name: Ensure iscsid is started ansible.builtin.service: diff --git a/roles/keepalived/tasks/main.yml b/roles/keepalived/tasks/main.yml index ceaa88ab0e..9aafa55362 100644 --- a/roles/keepalived/tasks/main.yml +++ b/roles/keepalived/tasks/main.yml @@ -18,6 +18,11 @@ kubernetes.core.k8s: state: present definition: + - apiVersion: v1 + kind: Namespace + metadata: + name: openstack + - apiVersion: v1 kind: Secret metadata: diff --git a/roles/keystone/.ci-selective-test b/roles/keystone/.ci-selective-test new file mode 100644 index 0000000000..7fefe230df --- /dev/null +++ b/roles/keystone/.ci-selective-test @@ -0,0 +1 @@ +This temporary marker exercises Keystone selective CI planning. diff --git a/roles/keystone/tasks/main.yml b/roles/keystone/tasks/main.yml index eef34564d9..4a039f4dda 100644 --- a/roles/keystone/tasks/main.yml +++ b/roles/keystone/tasks/main.yml @@ -16,6 +16,7 @@ no_log: true run_once: true changed_when: false + when: not (_pre_role_active | default(false)) community.general.keycloak_realm: # Keycloak settings auth_keycloak_url: "{{ item.keycloak_server_url }}" @@ -42,6 +43,7 @@ label: "{{ item.name }}" - name: Setup Keycloak Authentication Required Actions (MFA) + when: not (_pre_role_active | default(false)) community.general.keycloak_authentication_required_actions: # Keycloak settings auth_keycloak_url: "{{ item.keycloak_server_url }}" @@ -65,12 +67,14 @@ - name: Create ConfigMap with all OpenID connect configurations run_once: true + when: not (_pre_role_active | default(false)) kubernetes.core.k8s: template: configmap-openid-metadata.yml.j2 - name: Create Keycloak clients no_log: true run_once: true + when: not (_pre_role_active | default(false)) community.general.keycloak_client: # Keycloak settings auth_keycloak_url: "{{ item.keycloak_server_url }}" @@ -97,6 +101,7 @@ - name: Assign realm-management roles to service account no_log: true run_once: true + when: not (_pre_role_active | default(false)) community.general.keycloak_user_rolemapping: # Keycloak settings auth_keycloak_url: "{{ item.keycloak_server_url }}" @@ -115,6 +120,19 @@ loop_control: label: "{{ item.name }}" +- name: Wait for OpenID metadata ConfigMap + run_once: true + when: _pre_role_active | default(false) + kubernetes.core.k8s_info: + api_version: v1 + kind: ConfigMap + name: keystone-openid-metadata + namespace: "{{ keystone_helm_release_namespace }}" + register: _keystone_openid_metadata + retries: 300 + delay: 2 + until: _keystone_openid_metadata.resources | length > 0 + - name: Deploy Helm chart run_once: true kubernetes.core.helm: diff --git a/roles/keystone_pre/README.md b/roles/keystone_pre/README.md new file mode 100644 index 0000000000..dfc6483586 --- /dev/null +++ b/roles/keystone_pre/README.md @@ -0,0 +1,7 @@ +# `keystone_pre` + +This role performs the Keycloak-side setup required by Keystone — realm +creation, MFA configuration, OpenID Connect client registration, and the +OpenID metadata ConfigMap. It runs in parallel with the main `keystone` +role so the Keystone Helm install does not have to wait for Keycloak to +finish starting up. diff --git a/roles/keystone_pre/defaults/main.yml b/roles/keystone_pre/defaults/main.yml new file mode 120000 index 0000000000..849a4566c3 --- /dev/null +++ b/roles/keystone_pre/defaults/main.yml @@ -0,0 +1 @@ +../../keystone/defaults/main.yml \ No newline at end of file diff --git a/roles/keystone_pre/meta/main.yml b/roles/keystone_pre/meta/main.yml new file mode 100644 index 0000000000..3638050552 --- /dev/null +++ b/roles/keystone_pre/meta/main.yml @@ -0,0 +1,28 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# SPDX-License-Identifier: Apache-2.0 + +galaxy_info: + author: VEXXHOST, Inc. + description: >- + Pre-tasks for Keystone that configure Keycloak realms, MFA, OIDC + clients, and the OpenID metadata ConfigMap. Runs in parallel with the + main `keystone` role under the parallel deploy orchestrator. + license: Apache-2.0 + min_ansible_version: 5.5.0 + standalone: false + platforms: + - name: EL + versions: + - "8" + - "9" + - name: Ubuntu + versions: + - focal + - jammy + +dependencies: + - role: defaults + - role: openstack_helm_endpoints + vars: + openstack_helm_endpoints_chart: keystone + openstack_helm_endpoints_skip_cluster_creation: true diff --git a/roles/keystone_pre/tasks/main.yml b/roles/keystone_pre/tasks/main.yml new file mode 100644 index 0000000000..7ffb62204f --- /dev/null +++ b/roles/keystone_pre/tasks/main.yml @@ -0,0 +1,105 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# SPDX-License-Identifier: Apache-2.0 + +- name: Create Keycloak realms + no_log: true + run_once: true + changed_when: false + community.general.keycloak_realm: + # Keycloak settings + auth_keycloak_url: "{{ item.keycloak_server_url }}" + auth_realm: "{{ item.keycloak_user_realm_name }}" + auth_client_id: "{{ item.keycloak_admin_client_id }}" + auth_username: "{{ item.keycloak_admin_user }}" + auth_password: "{{ item.keycloak_admin_password }}" + validate_certs: "{{ cluster_issuer_type != 'self-signed' }}" + # Realm settings + id: "{{ item.keycloak_realm }}" + realm: "{{ item.keycloak_realm }}" + display_name: "{{ item.label }}" + enabled: true + password_policy: "{{ item.keycloak_password_policy | default(keystone_keycloak_realm_default_password_policy | default(omit)) }}" + brute_force_protected: "{{ item.keycloak_brute_force_protected | default(keystone_keycloak_realm_default_brute_force_protected | default(omit)) }}" + failure_factor: "{{ item.keycloak_brute_force_failure_factor | default(keystone_keycloak_realm_default_brute_force_failure_factor | default(omit)) }}" + wait_increment_seconds: "{{ item.keycloak_brute_force_wait_increment_seconds | default(keystone_keycloak_realm_default_brute_force_wait_increment_seconds | default(omit)) }}" + max_failure_wait_seconds: "{{ item.keycloak_brute_force_max_failure_wait_seconds | default(keystone_keycloak_realm_default_brute_force_max_failure_wait_seconds | default(omit)) }}" + max_delta_time_seconds: "{{ item.keycloak_brute_force_max_delta_time_seconds | default(keystone_keycloak_realm_default_brute_force_max_delta_time_seconds | default(omit)) }}" + minimum_quick_login_wait_seconds: "{{ item.keycloak_minimum_quick_login_wait_seconds | default(keystone_keycloak_realm_default_minimum_quick_login_wait_seconds | default(omit)) }}" + quick_login_check_milli_seconds: "{{ item.keycloak_quick_login_check_milli_seconds | default(keystone_keycloak_realm_default_quick_login_check_milli_seconds | default(omit)) }}" + loop: "{{ keystone_domains }}" + loop_control: + label: "{{ item.name }}" + +- name: Setup Keycloak Authentication Required Actions (MFA) + community.general.keycloak_authentication_required_actions: + # Keycloak settings + auth_keycloak_url: "{{ item.keycloak_server_url }}" + auth_realm: "{{ item.keycloak_user_realm_name }}" + auth_client_id: "{{ item.keycloak_admin_client_id }}" + auth_username: "{{ item.keycloak_admin_user }}" + auth_password: "{{ item.keycloak_admin_password }}" + validate_certs: "{{ cluster_issuer_type != 'self-signed' }}" + # Realm settings + realm: "{{ item.name }}" + required_actions: + - alias: "CONFIGURE_TOTP" + name: "Configure OTP" + providerId: "CONFIGURE_TOTP" + defaultAction: "{{ item.keycloak_totp_default_action | default(keystone_keycloak_realm_default_totp_default_action | default(omit)) }}" + enabled: true + state: present + loop: "{{ keystone_domains }}" + loop_control: + label: "{{ item.name }}" + +- name: Create ConfigMap with all OpenID connect configurations + run_once: true + kubernetes.core.k8s: + template: configmap-openid-metadata.yml.j2 + +- name: Create Keycloak clients + no_log: true + run_once: true + community.general.keycloak_client: + # Keycloak settings + auth_keycloak_url: "{{ item.keycloak_server_url }}" + auth_realm: "{{ item.keycloak_user_realm_name }}" + auth_client_id: "{{ item.keycloak_admin_client_id }}" + auth_username: "{{ item.keycloak_admin_user }}" + auth_password: "{{ item.keycloak_admin_password }}" + validate_certs: "{{ cluster_issuer_type != 'self-signed' }}" + # Realm settings + realm: "{{ item.keycloak_realm }}" + client_id: "{{ item.keycloak_client_id }}" + secret: "{{ item.keycloak_client_secret }}" + client_authenticator_type: client-secret + public_client: false + service_accounts_enabled: true + direct_access_grants_enabled: false + redirect_uris: + - "{{ keystone_oidc_redirect_uri }}" + - "https://{{ openstack_helm_endpoints_horizon_api_host }}/auth/logout/" + loop: "{{ keystone_domains }}" + loop_control: + label: "{{ item.name }}" + +- name: Assign realm-management roles to service account + no_log: true + run_once: true + community.general.keycloak_user_rolemapping: + # Keycloak settings + auth_keycloak_url: "{{ item.keycloak_server_url }}" + auth_realm: "{{ item.keycloak_user_realm_name }}" + auth_client_id: "{{ item.keycloak_admin_client_id }}" + auth_username: "{{ item.keycloak_admin_user }}" + auth_password: "{{ item.keycloak_admin_password }}" + validate_certs: "{{ cluster_issuer_type != 'self-signed' }}" + # Role mapping settings + realm: "{{ item.keycloak_realm }}" + service_account_user_client_id: "{{ item.keycloak_client_id }}" + client_id: realm-management + roles: + - name: view-users + loop: "{{ keystone_domains }}" + loop_control: + label: "{{ item.name }}" diff --git a/roles/keystone_pre/templates/configmap-openid-metadata.yml.j2 b/roles/keystone_pre/templates/configmap-openid-metadata.yml.j2 new file mode 100644 index 0000000000..4c2e653450 --- /dev/null +++ b/roles/keystone_pre/templates/configmap-openid-metadata.yml.j2 @@ -0,0 +1,13 @@ +apiVersion: v1 +metadata: + name: keystone-openid-metadata + namespace: "{{ keystone_helm_release_namespace }}" + labels: + application: keystone +kind: ConfigMap +data: +{% for domain in keystone_domains %} + {{ domain.name }}-oidc-client: '{"client_id":"{{ domain.keycloak_client_id }}","client_secret":"{{ domain.keycloak_client_secret }}","response_type":"id_token"}' + {{ domain.name }}-oidc-conf: '{"scope":"{{ domain.keycloak_scopes }}"}' + {{ domain.name }}-oidc-provider: '{{ lookup('url', domain.keycloak_server_url ~ "/realms/" ~ domain.keycloak_realm ~ "/.well-known/openid-configuration", validate_certs=keystone_oidc_ssl_validate_server) }}' +{% endfor %} diff --git a/roles/magnum/tasks/main.yml b/roles/magnum/tasks/main.yml index 51d0f8f004..86f2912c8b 100644 --- a/roles/magnum/tasks/main.yml +++ b/roles/magnum/tasks/main.yml @@ -222,3 +222,4 @@ glance_image_disk_format: "{{ magnum_image_disk_format }}" glance_image_properties: os_distro: "{{ item.distro }}" + when: not (_pre_role_active | default(false)) diff --git a/roles/magnum_pre/README.md b/roles/magnum_pre/README.md new file mode 100644 index 0000000000..45eece829c --- /dev/null +++ b/roles/magnum_pre/README.md @@ -0,0 +1,5 @@ +# `magnum_pre` + +This role uploads Glance images for Magnum. It runs in parallel with the +main `magnum` role during deployment to overlap image downloads with the +Helm install. diff --git a/roles/magnum_pre/defaults/main.yml b/roles/magnum_pre/defaults/main.yml new file mode 100644 index 0000000000..8e7967d494 --- /dev/null +++ b/roles/magnum_pre/defaults/main.yml @@ -0,0 +1,28 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Container format for Magnum images. +# +# magnum_image_container_format: bare +magnum_image_container_format: bare + +# Disk format for Magnum images. +# +# magnum_image_disk_format: raw +magnum_image_disk_format: raw + +# List of images to upload for Magnum. +# +# magnum_images: "{{ _magnum_images }}" +magnum_images: "{{ _magnum_images }}" diff --git a/roles/magnum_pre/meta/main.yml b/roles/magnum_pre/meta/main.yml new file mode 100644 index 0000000000..0ceb5e6246 --- /dev/null +++ b/roles/magnum_pre/meta/main.yml @@ -0,0 +1,29 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +galaxy_info: + author: VEXXHOST, Inc. + description: >- + Pre-tasks for Magnum that run in parallel with the main Helm deploy. + Handles Glance image uploads which are independent of the Magnum service. + license: Apache-2.0 + min_ansible_version: 5.5.0 + +dependencies: + - role: defaults + - role: openstack_helm_endpoints + vars: + openstack_helm_endpoints_chart: magnum + openstack_helm_endpoints_skip_cluster_creation: true + - role: openstacksdk diff --git a/roles/magnum_pre/tasks/main.yml b/roles/magnum_pre/tasks/main.yml new file mode 100644 index 0000000000..63042e17c5 --- /dev/null +++ b/roles/magnum_pre/tasks/main.yml @@ -0,0 +1,25 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Upload images + ansible.builtin.include_role: + name: atmosphere.common.glance_image + loop: "{{ magnum_images }}" + vars: + glance_image_name: "{{ item.name }}" + glance_image_url: "{{ item.url }}" + glance_image_container_format: "{{ magnum_image_container_format }}" + glance_image_disk_format: "{{ magnum_image_disk_format }}" + glance_image_properties: + os_distro: "{{ item.distro }}" diff --git a/roles/magnum_pre/vars/main.yml b/roles/magnum_pre/vars/main.yml new file mode 100644 index 0000000000..cb0b731dad --- /dev/null +++ b/roles/magnum_pre/vars/main.yml @@ -0,0 +1,24 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +_magnum_images: + - name: ubuntu-2204-kube-v1.32.10 + url: https://github.com/vexxhost/capo-image-elements/releases/download/2025.12-3/ubuntu-22.04-v1.32.10.qcow2 + distro: ubuntu + - name: ubuntu-2204-kube-v1.33.7 + url: https://github.com/vexxhost/capo-image-elements/releases/download/2025.12-3/ubuntu-22.04-v1.33.7.qcow2 + distro: ubuntu + - name: ubuntu-2204-kube-v1.34.3 + url: https://github.com/vexxhost/capo-image-elements/releases/download/2025.12-3/ubuntu-22.04-v1.34.3.qcow2 + distro: ubuntu diff --git a/roles/manila/tasks/main.yml b/roles/manila/tasks/main.yml index 540ce34aeb..2b8d18f39c 100644 --- a/roles/manila/tasks/main.yml +++ b/roles/manila/tasks/main.yml @@ -40,15 +40,39 @@ openstack_helm_ingress_annotations: "{{ manila_ingress_annotations }}" openstack_helm_ingress_class_name: "{{ manila_ingress_class_name }}" -- name: Update service tenant quotas - openstack.cloud.quota: - cloud: atmosphere - # NOTE(okozachenko): It uses project name instead of id. - name: service - instances: -1 - cores: -1 - ram: -1 - volumes: -1 - gigabytes: -1 - security_group: -1 - security_group_rule: -1 +- name: Update service tenant compute quotas + ansible.builtin.command: + cmd: >- + openstack quota set + --instances -1 + --cores -1 + --ram -1 + service + environment: + OS_CLOUD: atmosphere + register: _manila_compute_quota_set + changed_when: _manila_compute_quota_set.rc == 0 + +- name: Update service tenant volume quotas + ansible.builtin.command: + cmd: >- + openstack quota set + --volumes -1 + --gigabytes -1 + service + environment: + OS_CLOUD: atmosphere + register: _manila_volume_quota_set + changed_when: _manila_volume_quota_set.rc == 0 + +- name: Update service tenant network quotas + ansible.builtin.command: + cmd: >- + openstack quota set + --secgroups -1 + --secgroup-rules -1 + service + environment: + OS_CLOUD: atmosphere + register: _manila_network_quota_set + changed_when: _manila_network_quota_set.rc == 0 diff --git a/roles/multipathd/tasks/main.yml b/roles/multipathd/tasks/main.yml index a4acd90d34..3a888bc25b 100644 --- a/roles/multipathd/tasks/main.yml +++ b/roles/multipathd/tasks/main.yml @@ -25,6 +25,10 @@ ansible.builtin.package: name: "{{ 'multipath-tools' if ansible_os_family == 'Debian' else 'device-mapper-multipath' }}" state: latest # noqa: package-latest + register: _install_multipathd + retries: 5 + delay: 10 + until: _install_multipathd is not failed notify: - Restart "multipathd" diff --git a/roles/neutron/meta/main.yml b/roles/neutron/meta/main.yml index 535bd27436..48bff0d28c 100644 --- a/roles/neutron/meta/main.yml +++ b/roles/neutron/meta/main.yml @@ -32,9 +32,11 @@ dependencies: - role: defaults - role: openstacksdk - role: openstack_helm_endpoints + when: not (_pre_role_active | default(false)) vars: openstack_helm_endpoints_chart: neutron - role: vexxhost.kubernetes.upload_helm_chart + when: not (_pre_role_active | default(false)) vars: upload_helm_chart_src: "{{ neutron_helm_chart_path }}" upload_helm_chart_dest: "{{ neutron_helm_chart_ref }}" diff --git a/roles/neutron/tasks/main.yml b/roles/neutron/tasks/main.yml index e368e994b4..ba57aa5860 100644 --- a/roles/neutron/tasks/main.yml +++ b/roles/neutron/tasks/main.yml @@ -32,49 +32,52 @@ ) }} -- name: Set external_dns_driver - ansible.builtin.set_fact: - _neutron_external_dns_driver: "designate" - when: neutron_designate_integration_enabled | bool +- name: Deploy Neutron chart + when: not (_pre_role_active | default(false)) + block: + - name: Set external_dns_driver + ansible.builtin.set_fact: + _neutron_external_dns_driver: "designate" + when: neutron_designate_integration_enabled | bool -- name: Generate Helm values - ansible.builtin.set_fact: - _neutron_helm_values: "{{ __neutron_helm_values }}" + - name: Generate Helm values + ansible.builtin.set_fact: + _neutron_helm_values: "{{ __neutron_helm_values }}" -- name: Append Helm values - when: atmosphere_network_backend == 'ovn' - ansible.builtin.set_fact: - _neutron_helm_values: "{{ _neutron_helm_values | combine(__neutron_ovn_helm_values, recursive=True) }}" + - name: Append Helm values + when: atmosphere_network_backend == 'ovn' + ansible.builtin.set_fact: + _neutron_helm_values: "{{ _neutron_helm_values | combine(__neutron_ovn_helm_values, recursive=True) }}" -- name: Append Helm values (neutron_policy_server) - when: neutron_policy_server_integration_enabled | bool - ansible.builtin.set_fact: - _neutron_helm_values: "{{ _neutron_helm_values | combine(__neutron_policy_server_helm_values, recursive=True) }}" + - name: Append Helm values (neutron_policy_server) + when: neutron_policy_server_integration_enabled | bool + ansible.builtin.set_fact: + _neutron_helm_values: "{{ _neutron_helm_values | combine(__neutron_policy_server_helm_values, recursive=True) }}" -- name: Append Helm values (mount neutron genericswitch keys) - when: neutron_genericswitch_keys | length > 0 - ansible.builtin.set_fact: - _neutron_helm_values: "{{ _neutron_helm_values | combine(__neutron_generic_switch_keys_helm_values, recursive=True) }}" + - name: Append Helm values (mount neutron genericswitch keys) + when: neutron_genericswitch_keys | length > 0 + ansible.builtin.set_fact: + _neutron_helm_values: "{{ _neutron_helm_values | combine(__neutron_generic_switch_keys_helm_values, recursive=True) }}" -- name: Deploy Helm chart - run_once: true - kubernetes.core.helm: - name: "{{ neutron_helm_release_name }}" - chart_ref: "{{ neutron_helm_chart_ref }}" - release_namespace: "{{ neutron_helm_release_namespace }}" - create_namespace: true - kubeconfig: "{{ neutron_helm_kubeconfig }}" - values: "{{ _neutron_helm_values | combine(neutron_helm_values, recursive=True) }}" + - name: Deploy Helm chart + run_once: true + kubernetes.core.helm: + name: "{{ neutron_helm_release_name }}" + chart_ref: "{{ neutron_helm_chart_ref }}" + release_namespace: "{{ neutron_helm_release_namespace }}" + create_namespace: true + kubeconfig: "{{ neutron_helm_kubeconfig }}" + values: "{{ _neutron_helm_values | combine(neutron_helm_values, recursive=True) }}" -- name: Create Ingress - ansible.builtin.include_role: - name: openstack_helm_ingress - vars: - openstack_helm_ingress_endpoint: network - openstack_helm_ingress_service_name: neutron-server - openstack_helm_ingress_service_port: 9696 - openstack_helm_ingress_annotations: "{{ neutron_ingress_annotations }}" - openstack_helm_ingress_class_name: "{{ neutron_ingress_class_name }}" + - name: Create Ingress + ansible.builtin.include_role: + name: openstack_helm_ingress + vars: + openstack_helm_ingress_endpoint: network + openstack_helm_ingress_service_name: neutron-server + openstack_helm_ingress_service_port: 9696 + openstack_helm_ingress_annotations: "{{ neutron_ingress_annotations }}" + openstack_helm_ingress_class_name: "{{ neutron_ingress_class_name }}" - name: Create networks when: neutron_networks | length > 0 @@ -107,7 +110,7 @@ loop: "{{ neutron_networks }}" # NOTE(mnaser): This often fails since the SSL certificates are not # ready yet. We need to wait for them to be ready. - retries: 60 + retries: 120 delay: 5 register: _result until: _result is not failed @@ -134,7 +137,7 @@ - subnets # NOTE(mnaser): This often fails since the SSL certificates are not # ready yet. We need to wait for them to be ready. - retries: 60 + retries: 120 delay: 5 register: _result until: _result is not failed diff --git a/roles/neutron_pre/README.md b/roles/neutron_pre/README.md new file mode 100644 index 0000000000..853b87d825 --- /dev/null +++ b/roles/neutron_pre/README.md @@ -0,0 +1,8 @@ +# `neutron_pre` + +This role installs the Neutron Helm chart and creates the network +ingress. Under the parallel deploy orchestrator it runs in parallel with +the main `neutron` role so the heavy Helm install does not block on +Nova. Only the post-install "Create networks" task (which hits the +neutron-server AZ check that requires the `nova` availability zone) +remains gated on Nova in the main `neutron` role. diff --git a/roles/neutron_pre/defaults/main.yml b/roles/neutron_pre/defaults/main.yml new file mode 120000 index 0000000000..da17a16bbb --- /dev/null +++ b/roles/neutron_pre/defaults/main.yml @@ -0,0 +1 @@ +../../neutron/defaults/main.yml \ No newline at end of file diff --git a/roles/neutron_pre/meta/main.yml b/roles/neutron_pre/meta/main.yml new file mode 100644 index 0000000000..8d0e92926e --- /dev/null +++ b/roles/neutron_pre/meta/main.yml @@ -0,0 +1,34 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# SPDX-License-Identifier: Apache-2.0 + +galaxy_info: + author: VEXXHOST, Inc. + description: >- + Pre-tasks for Neutron that install the Helm chart and create the + network ingress. Runs in parallel with the main `neutron` role under + the parallel deploy orchestrator so that the heavy Helm install does + not have to wait for Nova to come up; only the post-install network + creation (which hits the AZ check) waits on Nova. + license: Apache-2.0 + min_ansible_version: 5.5.0 + standalone: false + platforms: + - name: EL + versions: + - "8" + - "9" + - name: Ubuntu + versions: + - focal + - jammy + +dependencies: + - role: defaults + - role: openstack_helm_endpoints + vars: + openstack_helm_endpoints_chart: neutron + openstack_helm_endpoints_skip_cluster_creation: true + - role: vexxhost.kubernetes.upload_helm_chart + vars: + upload_helm_chart_src: "{{ neutron_helm_chart_path }}" + upload_helm_chart_dest: "{{ neutron_helm_chart_ref }}" diff --git a/roles/neutron_pre/tasks/main.yml b/roles/neutron_pre/tasks/main.yml new file mode 100644 index 0000000000..cbe9206206 --- /dev/null +++ b/roles/neutron_pre/tasks/main.yml @@ -0,0 +1,47 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# SPDX-License-Identifier: Apache-2.0 + +- name: Configure OpenStack-Helm endpoints + ansible.builtin.include_role: + name: openstack_helm_endpoints + vars: + openstack_helm_endpoints_chart: neutron + +- name: Set external_dns_driver + ansible.builtin.set_fact: + _neutron_external_dns_driver: "designate" + when: neutron_designate_integration_enabled | bool + +- name: Generate Helm values + ansible.builtin.set_fact: + _neutron_helm_values: "{{ __neutron_helm_values }}" + +- name: Append Helm values + when: atmosphere_network_backend == 'ovn' + ansible.builtin.set_fact: + _neutron_helm_values: "{{ _neutron_helm_values | combine(__neutron_ovn_helm_values, recursive=True) }}" + +- name: Append Helm values (neutron_policy_server) + when: neutron_policy_server_integration_enabled | bool + ansible.builtin.set_fact: + _neutron_helm_values: "{{ _neutron_helm_values | combine(__neutron_policy_server_helm_values, recursive=True) }}" + +- name: Deploy Helm chart + run_once: true + kubernetes.core.helm: + name: "{{ neutron_helm_release_name }}" + chart_ref: "{{ neutron_helm_chart_ref }}" + release_namespace: "{{ neutron_helm_release_namespace }}" + create_namespace: true + kubeconfig: "{{ neutron_helm_kubeconfig }}" + values: "{{ _neutron_helm_values | combine(neutron_helm_values, recursive=True) }}" + +- name: Create Ingress + ansible.builtin.include_role: + name: openstack_helm_ingress + vars: + openstack_helm_ingress_endpoint: network + openstack_helm_ingress_service_name: neutron-server + openstack_helm_ingress_service_port: 9696 + openstack_helm_ingress_annotations: "{{ neutron_ingress_annotations }}" + openstack_helm_ingress_class_name: "{{ neutron_ingress_class_name }}" diff --git a/roles/neutron_pre/vars/main.yml b/roles/neutron_pre/vars/main.yml new file mode 120000 index 0000000000..904ce9d30a --- /dev/null +++ b/roles/neutron_pre/vars/main.yml @@ -0,0 +1 @@ +../../neutron/vars/main.yml \ No newline at end of file diff --git a/roles/octavia/tasks/main.yml b/roles/octavia/tasks/main.yml index 46fc6cddcb..d481023de0 100644 --- a/roles/octavia/tasks/main.yml +++ b/roles/octavia/tasks/main.yml @@ -17,6 +17,7 @@ file: generate_resources.yml - name: Create CAs & Issuers + when: not (_pre_role_active | default(false)) kubernetes.core.k8s: state: present definition: @@ -56,6 +57,7 @@ - octavia-server - name: Create certificate for Octavia clients + when: not (_pre_role_active | default(false)) kubernetes.core.k8s: state: present definition: @@ -84,6 +86,7 @@ size: {{ octavia_tls_client_private_key_size }} - name: Create admin compute quotaset + when: not (_pre_role_active | default(false)) openstack.cloud.quota: cloud: atmosphere # NOTE(okozachenko): It uses project name instead of id. @@ -96,6 +99,26 @@ security_group: -1 security_group_rule: -1 +- name: Wait for Octavia client certificate secret + # The Certificate resources above are issued asynchronously by cert-manager. + # When the pre-role runs in parallel with the main role, the Helm install + # below mounts these secrets and will stall pods in ContainerCreating if + # they are not present yet. Wait explicitly so the deploy is deterministic + # in both sequential and parallel modes. + run_once: true + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + name: "{{ item }}" + namespace: "{{ octavia_helm_release_namespace }}" + wait: true + wait_sleep: 2 + wait_timeout: 600 + loop: + - octavia-client-certs + - octavia-server-ca + - octavia-client-ca + - name: Deploy Helm chart run_once: true kubernetes.core.helm: diff --git a/roles/octavia_pre/README.md b/roles/octavia_pre/README.md new file mode 100644 index 0000000000..b7df63bec7 --- /dev/null +++ b/roles/octavia_pre/README.md @@ -0,0 +1,5 @@ +# `octavia_pre` + +This role creates cert-manager CAs, certificates, and admin quotas for +Octavia. It runs in parallel with the main `octavia` role during +deployment. diff --git a/roles/octavia_pre/defaults/main.yml b/roles/octavia_pre/defaults/main.yml new file mode 100644 index 0000000000..54a91305f2 --- /dev/null +++ b/roles/octavia_pre/defaults/main.yml @@ -0,0 +1,43 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Common name for the Octavia server TLS certificate. +# +# octavia_tls_server_common_name: octavia-server +octavia_tls_server_common_name: octavia-server + +# Private key algorithm for the Octavia server TLS certificate. +# +# octavia_tls_server_private_key_algorithm: ECDSA +octavia_tls_server_private_key_algorithm: ECDSA + +# Private key size for the Octavia server TLS certificate. +# +# octavia_tls_server_private_key_size: 256 +octavia_tls_server_private_key_size: 256 + +# Common name for the Octavia client TLS certificate. +# +# octavia_tls_client_common_name: octavia-client +octavia_tls_client_common_name: octavia-client + +# Private key algorithm for the Octavia client TLS certificate. +# +# octavia_tls_client_private_key_algorithm: ECDSA +octavia_tls_client_private_key_algorithm: ECDSA + +# Private key size for the Octavia client TLS certificate. +# +# octavia_tls_client_private_key_size: 256 +octavia_tls_client_private_key_size: 256 diff --git a/roles/octavia_pre/meta/main.yml b/roles/octavia_pre/meta/main.yml new file mode 100644 index 0000000000..401705ebb9 --- /dev/null +++ b/roles/octavia_pre/meta/main.yml @@ -0,0 +1,30 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +galaxy_info: + author: VEXXHOST, Inc. + description: >- + Pre-tasks for Octavia that run in parallel with the main role. + Handles cert-manager CAs, certificates, and admin quotas. + license: Apache-2.0 + min_ansible_version: 5.5.0 + +dependencies: + - role: defaults + - role: openstack_helm_endpoints + vars: + openstack_helm_endpoints_chart: octavia + openstack_helm_endpoints_skip_cluster_creation: true + - role: openstacksdk + - role: openstack_cli diff --git a/roles/octavia_pre/tasks/main.yml b/roles/octavia_pre/tasks/main.yml new file mode 100644 index 0000000000..035534d3f6 --- /dev/null +++ b/roles/octavia_pre/tasks/main.yml @@ -0,0 +1,93 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Create CAs & Issuers + kubernetes.core.k8s: + state: present + definition: + - apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: "{{ item }}-ca" + namespace: openstack + spec: + isCA: true + commonName: "{{ octavia_tls_server_common_name if item == 'octavia-server' else octavia_tls_client_common_name }}" + secretName: "{{ item }}-ca" + duration: 87600h0m0s + renewBefore: 720h0m0s + privateKey: "{{ private_key | from_yaml }}" + issuerRef: + name: self-signed + kind: ClusterIssuer + group: cert-manager.io + + - apiVersion: cert-manager.io/v1 + kind: Issuer + metadata: + name: "{{ item }}" + namespace: openstack + spec: + ca: + secretName: "{{ item }}-ca" + vars: + # NOTE(mnaser): Unfortunately, Ansible renders all variables as strings so + # we do this workaround to make sure the size is an integer. + private_key: | + algorithm: "{{ octavia_tls_server_private_key_algorithm if item == 'octavia-server' else octavia_tls_client_private_key_algorithm }}" + size: {{ octavia_tls_server_private_key_size if item == 'octavia-server' else octavia_tls_client_private_key_size }} + loop: + - octavia-client + - octavia-server + +- name: Create certificate for Octavia clients + kubernetes.core.k8s: + state: present + definition: + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: octavia-client-certs + namespace: openstack + spec: + commonName: "{{ octavia_tls_client_common_name }}" + secretName: octavia-client-certs + additionalOutputFormats: + - type: CombinedPEM + duration: 87600h0m0s + renewBefore: 720h0m0s + privateKey: "{{ private_key | from_yaml }}" + issuerRef: + name: octavia-client + kind: Issuer + group: cert-manager.io + vars: + # NOTE(mnaser): Unfortunately, Ansible renders all variables as strings so + # we do this workaround to make sure the size is an integer. + private_key: | + algorithm: "{{ octavia_tls_client_private_key_algorithm }}" + size: {{ octavia_tls_client_private_key_size }} + +- name: Create admin compute quotaset + openstack.cloud.quota: + cloud: atmosphere + # NOTE(okozachenko): It uses project name instead of id. + name: admin + instances: -1 + cores: -1 + ram: -1 + volumes: -1 + gigabytes: -1 + security_group: -1 + security_group_rule: -1 diff --git a/roles/openstack_cli/tasks/main.yml b/roles/openstack_cli/tasks/main.yml index 84c39a0081..06fe99ca21 100644 --- a/roles/openstack_cli/tasks/main.yml +++ b/roles/openstack_cli/tasks/main.yml @@ -16,12 +16,20 @@ ansible.builtin.package: name: "{{ openstack_cli_packages }}" state: absent + register: _uninstall_osc + retries: 5 + delay: 10 + until: _uninstall_osc is not failed when: ansible_facts['os_family'] in ['Debian'] - name: Uninstall Ubuntu Cloud Archive keyring ansible.builtin.apt: name: ubuntu-cloud-keyring state: absent + register: _uninstall_uca_keyring + retries: 5 + delay: 10 + until: _uninstall_uca_keyring is not failed when: ansible_facts['os_family'] in ['Debian'] - name: Remove Ubuntu Cloud Archive repository @@ -29,6 +37,10 @@ filename: ubuntu-cloud-archive repo: "{{ openstack_cli_cloud_archive_repo }}" state: absent + register: _remove_uca_repo + retries: 5 + delay: 10 + until: _remove_uca_repo is not failed when: ansible_facts['os_family'] in ['Debian'] - name: Generate OpenStack-Helm endpoints diff --git a/roles/openstack_helm_endpoints/tasks/main.yml b/roles/openstack_helm_endpoints/tasks/main.yml index 9170fd0e8a..5aec588bf9 100644 --- a/roles/openstack_helm_endpoints/tasks/main.yml +++ b/roles/openstack_helm_endpoints/tasks/main.yml @@ -19,70 +19,78 @@ when: - openstack_helm_endpoints_chart is defined -# NOTE(mnaser): Since we manage one-RabbitMQ per service, we create the RabbitMQ -# cluster here and then append the necessary values to be used -# inside the `oslo_messaging` section. -- name: Configure "oslo.messaging" - when: - - '"oslo_messaging" in openstack_helm_endpoints_list' +# NOTE: When invoked from a pre-role that only needs this role's defaults +# loaded (for example, so openstacksdk can render clouds.yaml), skip +# everything that touches Kubernetes resources or depends on them. +# This avoids racing with the main role on RabbitmqCluster/DB secret +# creation when pre-roles run in parallel with their main role. +- name: Configure endpoints and operator-managed resources + when: not (openstack_helm_endpoints_skip_cluster_creation | default(false)) block: - - name: Create RabbitMQ cluster - ansible.builtin.include_role: - name: rabbitmq - vars: - rabbitmq_cluster_name: "{{ openstack_helm_endpoints_chart }}" - rabbitmq_spec: "{{ lookup('vars', openstack_helm_endpoints_chart ~ '_rabbitmq_spec', default={}) }}" + # NOTE(mnaser): Since we manage one-RabbitMQ per service, we create the RabbitMQ + # cluster here and then append the necessary values to be used + # inside the `oslo_messaging` section. + - name: Configure "oslo.messaging" + when: + - '"oslo_messaging" in openstack_helm_endpoints_list' + block: + - name: Create RabbitMQ cluster + ansible.builtin.include_role: + name: rabbitmq + vars: + rabbitmq_cluster_name: "{{ openstack_helm_endpoints_chart }}" + rabbitmq_spec: "{{ lookup('vars', openstack_helm_endpoints_chart ~ '_rabbitmq_spec', default={}) }}" - - name: Grab RabbitMQ cluster secret - kubernetes.core.k8s_info: - api_version: v1 - kind: Secret - name: "rabbitmq-{{ openstack_helm_endpoints_chart }}-default-user" - namespace: openstack - register: _openstack_helm_endpoints_rabbitmq_cluster_secret + - name: Grab RabbitMQ cluster secret + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + name: "rabbitmq-{{ openstack_helm_endpoints_chart }}-default-user" + namespace: openstack + register: _openstack_helm_endpoints_rabbitmq_cluster_secret - - name: Cache fact with RabbitMQ cluster credentials - ansible.builtin.set_fact: - _openstack_helm_endpoints_rabbitmq_cluster_username: |- - {{ _openstack_helm_endpoints_rabbitmq_cluster_secret.resources[0]['data']['username'] | b64decode }} - _openstack_helm_endpoints_rabbitmq_cluster_password: |- - {{ _openstack_helm_endpoints_rabbitmq_cluster_secret.resources[0]['data']['password'] | b64decode }} + - name: Cache fact with RabbitMQ cluster credentials + ansible.builtin.set_fact: + _openstack_helm_endpoints_rabbitmq_cluster_username: |- + {{ _openstack_helm_endpoints_rabbitmq_cluster_secret.resources[0]['data']['username'] | b64decode }} + _openstack_helm_endpoints_rabbitmq_cluster_password: |- + {{ _openstack_helm_endpoints_rabbitmq_cluster_secret.resources[0]['data']['password'] | b64decode }} -# NOTE(mnaser): Since we deploy the database using the operator and we let it -# generate the root password, we look it up if the fact has not -# been cached from a previous run. -- name: Configure "oslo.db" - when: - - '"oslo_db" in openstack_helm_endpoints_list' - - openstack_helm_endpoints_mariadb_admin_password is not defined - block: - - name: Grab Percona XtraDB cluster secret - kubernetes.core.k8s_info: - api_version: v1 - kind: Secret - name: percona-xtradb - namespace: openstack - register: _openstack_helm_endpoints_oslo_db_secret + # NOTE(mnaser): Since we deploy the database using the operator and we let it + # generate the root password, we look it up if the fact has not + # been cached from a previous run. + - name: Configure "oslo.db" + when: + - '"oslo_db" in openstack_helm_endpoints_list' + - openstack_helm_endpoints_mariadb_admin_password is not defined + block: + - name: Grab Percona XtraDB cluster secret + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + name: percona-xtradb + namespace: openstack + register: _openstack_helm_endpoints_oslo_db_secret - - name: Cache fact with Percona XtraDB password - ansible.builtin.set_fact: - openstack_helm_endpoints_mariadb_admin_password: "{{ _openstack_helm_endpoints_oslo_db_secret.resources[0]['data']['root'] | b64decode }}" + - name: Cache fact with Percona XtraDB password + ansible.builtin.set_fact: + openstack_helm_endpoints_mariadb_admin_password: "{{ _openstack_helm_endpoints_oslo_db_secret.resources[0]['data']['root'] | b64decode }}" -- name: Reset value for OpenStack_Helm endpoints - ansible.builtin.set_fact: - openstack_helm_endpoints: "{{ openstack_helm_endpoints_config }}" + - name: Reset value for OpenStack_Helm endpoints + ansible.builtin.set_fact: + openstack_helm_endpoints: "{{ openstack_helm_endpoints_config }}" -- name: Generate OpenStack-Helm endpoints - ansible.builtin.set_fact: - openstack_helm_endpoints: | - {{ openstack_helm_endpoints | combine(lookup('vars', '_openstack_helm_endpoints_' + service), recursive=True) }} - loop: "{{ openstack_helm_endpoints_list }}" - loop_control: - loop_var: service + - name: Generate OpenStack-Helm endpoints + ansible.builtin.set_fact: + openstack_helm_endpoints: | + {{ openstack_helm_endpoints | combine(lookup('vars', '_openstack_helm_endpoints_' + service), recursive=True) }} + loop: "{{ openstack_helm_endpoints_list }}" + loop_control: + loop_var: service -# NOTE(mnaser): Since we use `openstack_helm_endpoints_list` to ensure that we -# have a common entry for endpoints and stay DRY, we need to -# reset the fact so it works for follow-up requests. -- name: Clean-up facts - ansible.builtin.set_fact: - openstack_helm_endpoints_list: + # NOTE(mnaser): Since we use `openstack_helm_endpoints_list` to ensure that we + # have a common entry for endpoints and stay DRY, we need to + # reset the fact so it works for follow-up requests. + - name: Clean-up facts + ansible.builtin.set_fact: + openstack_helm_endpoints_list: diff --git a/roles/reboot/tasks/main.yml b/roles/reboot/tasks/main.yml index efcd969d6b..8628d27bdd 100644 --- a/roles/reboot/tasks/main.yml +++ b/roles/reboot/tasks/main.yml @@ -15,6 +15,10 @@ - name: Install package for "community.general.pids" ansible.builtin.package: name: python3-psutil + register: _install_psutil + retries: 5 + delay: 10 + until: _install_psutil is not failed # NOTE(mnaser): These are all processes that we do not currently have a way to # safely evacuate *yet*. We should remove these as we add ways diff --git a/roles/rook_ceph_cluster/tasks/main.yml b/roles/rook_ceph_cluster/tasks/main.yml index 45916bfc84..71126903c9 100644 --- a/roles/rook_ceph_cluster/tasks/main.yml +++ b/roles/rook_ceph_cluster/tasks/main.yml @@ -108,6 +108,39 @@ kubeconfig: "{{ rook_ceph_cluster_helm_kubeconfig }}" values: "{{ _rook_ceph_cluster_helm_values | combine(rook_ceph_cluster_helm_values, recursive=True) }}" +# When the parallel orchestrator schedules rook-ceph-cluster shortly after +# keystone's Helm release is reported complete, the keystone-api pods can +# still be completing their rolling update. Wait explicitly so the +# subsequent openstack.cloud.* calls do not race keystone readiness. +- name: Wait for keystone API to be Available + run_once: true + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: Deployment + name: keystone-api + namespace: openstack + wait: true + wait_sleep: 5 + wait_timeout: 600 + wait_condition: + type: Available + status: "True" + +# The "service" domain is normally created on-demand by the first +# openstack-helm service chart's ks-user job. When rook-ceph-cluster +# runs in parallel with those charts, the domain might not yet exist, +# so create it idempotently here. +- name: Ensure "service" domain exists + run_once: true + openstack.cloud.identity_domain: + cloud: atmosphere + name: service + description: Service Domain + register: _rook_ceph_cluster_service_domain + until: _rook_ceph_cluster_service_domain is not failed + retries: 60 + delay: 5 + - name: Create OpenStack user openstack.cloud.identity_user: cloud: atmosphere @@ -115,18 +148,26 @@ password: "{{ openstack_helm_endpoints.identity.auth.rgw.password }}" domain: service +# The "service" project in the "service" domain is likewise created by +# the first openstack-helm ks-user job. Create it here so rook-ceph-cluster +# doesn't depend on another chart having run first. +- name: Ensure "service" project exists + run_once: true + openstack.cloud.project: + cloud: atmosphere + name: service + domain: service + description: Service Project + # NOTE(mnaser): https://storyboard.openstack.org/#!/story/2010579 - name: Grant access to "service" project - changed_when: false - ansible.builtin.command: - cmd: >- - openstack role add - --user-domain service - --project service - --user {{ openstack_helm_endpoints.identity.auth.rgw.username }} - admin - environment: - OS_CLOUD: atmosphere + openstack.cloud.role_assignment: + cloud: atmosphere + user: "{{ openstack_helm_endpoints.identity.auth.rgw.username }}" + user_domain: service + project: service + project_domain: service + role: admin - name: Create OpenStack service openstack.cloud.catalog_service: diff --git a/roles/tempest/defaults/main.yml b/roles/tempest/defaults/main.yml index e7e5bfc874..5abe6c2b8a 100644 --- a/roles/tempest/defaults/main.yml +++ b/roles/tempest/defaults/main.yml @@ -19,3 +19,20 @@ tempest_helm_chart_ref: /usr/local/src/tempest tempest_helm_release_namespace: openstack tempest_helm_kubeconfig: "{{ kubeconfig_path | default('/etc/kubernetes/admin.conf') }}" tempest_helm_values: {} + +# CI deployment mode used to limit Tempest to deployed services. +# +# tempest_ci_mode: selective +tempest_ci_mode: "{{ lookup('env', 'ATMOSPHERE_CI_MODE') | default('full', true) }}" + +# Components available to Tempest during a selective CI deployment. +# +# tempest_ci_components: +# - keystone +tempest_ci_components: >- + {{ + lookup('env', 'ATMOSPHERE_CI_COMPONENTS') + .split(',') + | reject('equalto', '') + | list + }} diff --git a/roles/tempest/tasks/main.yml b/roles/tempest/tasks/main.yml index 3440b2d57a..4637a0c68c 100644 --- a/roles/tempest/tasks/main.yml +++ b/roles/tempest/tasks/main.yml @@ -25,7 +25,9 @@ cloud: atmosphere image: cirros register: _tempest_test_image - when: tempest_helm_values.conf.tempest.compute.image_ref is not defined + when: + - tempest_ci_mode != 'selective' or 'glance' in tempest_ci_components + - tempest_helm_values.conf.tempest.compute.image_ref is not defined - name: Configure test image ref ansible.builtin.set_fact: @@ -36,6 +38,7 @@ compute: image_ref: "{{ _tempest_test_image.images.0.id }}" when: + - tempest_ci_mode != 'selective' or 'glance' in tempest_ci_components - tempest_helm_values.conf.tempest.compute.image_ref is not defined - _tempest_test_image.images | length > 0 @@ -44,7 +47,9 @@ cloud: atmosphere name: m1.tiny register: _tempest_test_flavor - when: tempest_helm_values.conf.tempest.compute.flavor_ref is not defined + when: + - tempest_ci_mode != 'selective' or 'nova' in tempest_ci_components + - tempest_helm_values.conf.tempest.compute.flavor_ref is not defined - name: Set test flavor ref ansible.builtin.set_fact: @@ -55,6 +60,7 @@ compute: flavor_ref: "{{ _tempest_test_flavor.flavors[0].id }}" when: + - tempest_ci_mode != 'selective' or 'nova' in tempest_ci_components - tempest_helm_values.conf.tempest.compute.flavor_ref is not defined - _tempest_test_flavor.flavors[0].id is defined @@ -63,7 +69,9 @@ cloud: atmosphere name: public register: _tempest_test_network - when: tempest_helm_values.conf.tempest.network.public_network_id is not defined + when: + - tempest_ci_mode != 'selective' or 'neutron' in tempest_ci_components + - tempest_helm_values.conf.tempest.network.public_network_id is not defined - name: Set test network ref ansible.builtin.set_fact: @@ -74,6 +82,7 @@ network: public_network_id: "{{ _tempest_test_network.networks[0].id }}" when: + - tempest_ci_mode != 'selective' or 'neutron' in tempest_ci_components - tempest_helm_values.conf.tempest.network.public_network_id is not defined - _tempest_test_network.networks[0].id is defined diff --git a/roles/tempest/vars/main.yml b/roles/tempest/vars/main.yml index 51ebd6e5e0..b115fc8701 100644 --- a/roles/tempest/vars/main.yml +++ b/roles/tempest/vars/main.yml @@ -12,6 +12,28 @@ # License for the specific language governing permissions and limitations # under the License. +_tempest_full_service_availability: + cinder: true + glance: true + horizon: true + neutron: true + nova: true + swift: true + +_tempest_ci_service_availability: + barbican: "{{ 'barbican' in tempest_ci_components }}" + cinder: "{{ 'cinder' in tempest_ci_components }}" + glance: "{{ 'glance' in tempest_ci_components }}" + heat: "{{ 'heat' in tempest_ci_components }}" + horizon: "{{ 'horizon' in tempest_ci_components }}" + load_balancer: "{{ 'octavia' in tempest_ci_components }}" + magnum: "{{ 'magnum' in tempest_ci_components }}" + manila: "{{ 'manila' in tempest_ci_components }}" + neutron: "{{ 'neutron' in tempest_ci_components }}" + nova: "{{ 'nova' in tempest_ci_components }}" + placement: "{{ 'placement' in tempest_ci_components }}" + swift: false + _tempest_helm_values: endpoints: "{{ openstack_helm_endpoints | combine(_tempest_endpoint_values, recursive=True) }}" images: @@ -25,13 +47,15 @@ _tempest_helm_values: include_app_kubernetes_io: false conf: tempest: - service_available: - cinder: true - glance: true - horizon: true - neutron: true - nova: true - swift: true + service_available: >- + {{ + _tempest_full_service_availability + | combine( + _tempest_ci_service_availability + if tempest_ci_mode == 'selective' + else {} + ) + }} compute: endpoint_type: internal fixed_network_name: public diff --git a/test-playbooks/go-test/run.yml b/test-playbooks/go-test/run.yml new file mode 100644 index 0000000000..d24e438765 --- /dev/null +++ b/test-playbooks/go-test/run.yml @@ -0,0 +1,21 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# SPDX-License-Identifier: Apache-2.0 + +- name: Install Go runtime on test host + hosts: all + roles: + - role: ensure-go + go_version: "1.24.4" + +- name: Run Atmosphere Go unit tests + hosts: all + tasks: + - name: Run Go unit tests + ansible.builtin.command: + cmd: go test ./pkg/... ./internal/... ./cmd/... + args: + chdir: "{{ zuul.project.src_dir }}" + environment: + CGO_ENABLED: "0" + PATH: "/usr/local/go/bin:{{ ansible_env.PATH | default(lookup('env', 'PATH')) }}" + changed_when: false diff --git a/test-playbooks/molecule/prepare.yml b/test-playbooks/molecule/prepare.yml new file mode 100644 index 0000000000..d151b5cba0 --- /dev/null +++ b/test-playbooks/molecule/prepare.yml @@ -0,0 +1,105 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# SPDX-License-Identifier: Apache-2.0 + +- name: Prepare Molecule dependencies + hosts: all + become: true + tasks: + - name: Install jq for log collection + when: ansible_facts["os_family"] | lower == "debian" + ansible.builtin.apt: + name: jq + update_cache: true + +- name: Install Go for the Atmosphere deployment binary + hosts: all + roles: + - role: ensure-go + go_version: "1.24.4" + +- name: Prepare selective Molecule execution plan + hosts: all + gather_facts: false + tasks: + - name: Build atmosphere binary + run_once: true + changed_when: false + ansible.builtin.command: + argv: + - go + - build + - -o + - ./bin/atmosphere + - ./cmd/atmosphere + args: + chdir: "{{ zuul.project.src_dir }}" + environment: + CGO_ENABLED: "0" + PATH: "/usr/local/go/bin:{{ ansible_env.PATH | default(lookup('env', 'PATH')) }}" + + - name: Generate selective CI plan + run_once: true + changed_when: false + failed_when: false + ansible.builtin.command: + argv: + - ./bin/atmosphere + - ci + - plan + - --base + - "{{ atmosphere_ci_plan_base }}" + - --head + - "{{ atmosphere_ci_plan_head | default('HEAD') }}" + - --format + - json + - --output + - ./ci-plan.json + args: + chdir: "{{ zuul.project.src_dir }}" + register: _atmosphere_ci_plan + + - name: Explain selective CI plan + when: _atmosphere_ci_plan.rc == 0 + run_once: true + changed_when: false + ansible.builtin.command: + argv: + - ./bin/atmosphere + - ci + - plan + - --base + - "{{ atmosphere_ci_plan_base }}" + - --head + - "{{ atmosphere_ci_plan_head | default('HEAD') }}" + args: + chdir: "{{ zuul.project.src_dir }}" + register: _atmosphere_ci_plan_explanation + + - name: Print selective CI plan + when: _atmosphere_ci_plan.rc == 0 + run_once: true + ansible.builtin.debug: + var: _atmosphere_ci_plan_explanation.stdout_lines + + - name: Report conservative fallback + when: _atmosphere_ci_plan.rc != 0 + run_once: true + ansible.builtin.debug: + msg: >- + Selective CI planning failed. Every Molecule job will run its + complete fallback: {{ _atmosphere_ci_plan.stderr }} + + - name: Create selective CI plan log directory + when: _atmosphere_ci_plan.rc == 0 + ansible.builtin.file: + path: /tmp/logs/system + state: directory + mode: "0755" + + - name: Publish selective CI plan with job logs + when: _atmosphere_ci_plan.rc == 0 + ansible.builtin.copy: + src: "{{ zuul.project.src_dir }}/ci-plan.json" + dest: /tmp/logs/system/ci-plan.json + remote_src: true + mode: "0644" diff --git a/test-playbooks/molecule/run.yml b/test-playbooks/molecule/run.yml new file mode 100644 index 0000000000..eba3c73c31 --- /dev/null +++ b/test-playbooks/molecule/run.yml @@ -0,0 +1,27 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# SPDX-License-Identifier: Apache-2.0 + +- hosts: all + become: true + tasks: + - name: Add IP address to "br-ex" + when: molecule_scenario | default('') in ['aio'] + changed_when: false + ansible.builtin.shell: + cmd: ip addr add 10.96.250.10/24 dev br-ex || true + + - name: Set "br-ex" interface to "up" + when: molecule_scenario | default('') in ['aio'] + changed_when: false + ansible.builtin.shell: + cmd: ip link set br-ex up || true + +- hosts: all + tasks: + - name: Run molecule verify + run_once: true + ansible.builtin.command: + cmd: uv run molecule verify -s {{ molecule_scenario }} + environment: "{{ molecule_environment | default({}) }}" + args: + chdir: "{{ zuul.project.src_dir }}" diff --git a/test-playbooks/molecule/selective-run.yml b/test-playbooks/molecule/selective-run.yml new file mode 100644 index 0000000000..63c77fe3c6 --- /dev/null +++ b/test-playbooks/molecule/selective-run.yml @@ -0,0 +1,262 @@ +# Copyright (c) 2026 VEXXHOST, Inc. +# SPDX-License-Identifier: Apache-2.0 + +- name: Run selected Molecule deployment and verification + hosts: all + gather_facts: false + tasks: + - name: Check for selective CI plan + run_once: true + ansible.builtin.stat: + path: "{{ zuul.project.src_dir }}/ci-plan.json" + register: _atmosphere_ci_plan_file + + - name: Validate selective CI plan + when: _atmosphere_ci_plan_file.stat.exists + run_once: true + changed_when: false + failed_when: false + ansible.builtin.command: + argv: + - jq + - empty + - "{{ zuul.project.src_dir }}/ci-plan.json" + register: _atmosphere_ci_plan_validation + + - name: Resolve selective CI plan availability + run_once: true + ansible.builtin.set_fact: + _atmosphere_ci_plan_available: >- + {{ + _atmosphere_ci_plan_file.stat.exists + and (_atmosphere_ci_plan_validation.rc | default(1)) == 0 + }} + + - name: Read selective CI plan + when: _atmosphere_ci_plan_available + run_once: true + ansible.builtin.slurp: + path: "{{ zuul.project.src_dir }}/ci-plan.json" + register: _atmosphere_ci_plan_raw + + - name: Load selective CI plan or full fallback + run_once: true + ansible.builtin.set_fact: + _atmosphere_ci_plan: >- + {{ + (_atmosphere_ci_plan_raw.content | b64decode | from_json) + if _atmosphere_ci_plan_available + else {'mode': 'full', 'jobs': []} + }} + + - name: Find this job in the selective CI plan + run_once: true + ansible.builtin.set_fact: + _atmosphere_ci_job_matches: >- + {{ + _atmosphere_ci_plan.jobs + | default([]) + | selectattr('name', 'equalto', atmosphere_ci_job) + | list + }} + + - name: Resolve this job or use full fallback + run_once: true + ansible.builtin.set_fact: + _atmosphere_ci_job_plan: >- + {{ + (_atmosphere_ci_job_matches | first) + if _atmosphere_ci_job_matches | length == 1 + else { + 'name': atmosphere_ci_job, + 'run': true, + 'components': [], + 'verification_profiles': ['full'], + 'reason': 'job decision is missing; running the full fallback' + } + }} + + - name: Resolve deployment and verification inputs + run_once: true + ansible.builtin.set_fact: + _atmosphere_ci_components: >- + {{ _atmosphere_ci_job_plan.components | default([]) }} + _atmosphere_ci_profiles: >- + {{ + _atmosphere_ci_job_plan.verification_profiles + | default(_atmosphere_ci_plan.verification_profiles | default(['full'])) + }} + _atmosphere_ci_mode: >- + {{ _atmosphere_ci_plan.mode | default('full') }} + _atmosphere_ci_network_backend: >- + {{ + _atmosphere_ci_job_plan.network_backend + | default(atmosphere_network_backend | default('openvswitch'), true) + }} + _atmosphere_ci_run: "{{ _atmosphere_ci_job_plan.run | default(true) | bool }}" + + - name: Resolve deployment tags + run_once: true + ansible.builtin.set_fact: + _atmosphere_ci_deploy_tags: >- + {{ + _atmosphere_ci_components + if ( + _atmosphere_ci_mode == 'selective' + and molecule_scenario == 'aio' + ) + else (atmosphere_deploy_tags | default([])) + }} + + - name: Report Molecule job decision + run_once: true + ansible.builtin.debug: + msg: + job: "{{ atmosphere_ci_job }}" + mode: "{{ _atmosphere_ci_mode }}" + run: "{{ _atmosphere_ci_run }}" + reason: "{{ _atmosphere_ci_job_plan.reason }}" + network_backend: "{{ _atmosphere_ci_network_backend }}" + components: "{{ _atmosphere_ci_components }}" + verification_profiles: "{{ _atmosphere_ci_profiles }}" + + - name: Run selected Molecule scenario + when: _atmosphere_ci_run + block: + - name: Copy inventory file for Zuul + run_once: true + ansible.builtin.copy: + src: "{{ zuul.executor.inventory_file }}" + dest: "{{ zuul.project.src_dir }}/inventory.yaml" + mode: "0644" + + - name: Switch ansible_host to private IP + run_once: true + ansible.builtin.replace: + path: "{{ zuul.project.src_dir }}/inventory.yaml" + regexp: '(^\s*ansible_host:\s*).*$' + replace: '\1"{% raw %}{{ nodepool.private_ipv4 }}{% endraw %}"' + + - name: Prepare Molecule scenario + run_once: true + changed_when: false + ansible.builtin.command: + argv: + - uv + - run + - molecule + - prepare + - -s + - "{{ molecule_scenario }}" + environment: "{{ molecule_environment | default({}) }}" + args: + chdir: "{{ zuul.project.src_dir }}" + + - name: Build deployment command + run_once: true + ansible.builtin.set_fact: + _atmosphere_ci_deploy_argv: >- + {{ + [ + 'uv', + 'run', + './bin/atmosphere', + 'deploy', + '--inventory', + './inventory.yaml', + '--concurrency', + ( + atmosphere_deploy_concurrency + | default((molecule_scenario == 'aio') | ternary(3, 0)) + | string + ), + '--dependency-option', + 'csi_driver=' + (csi_driver | default('rbd')), + '--dependency-option', + 'network_backend=' + + _atmosphere_ci_network_backend + ] + + ( + ['--tags', _atmosphere_ci_deploy_tags | join(',')] + if _atmosphere_ci_deploy_tags | length > 0 + else [] + ) + }} + _atmosphere_ci_deploy_environment: >- + {{ + molecule_environment + | default({}) + | combine({ + 'PATH': + zuul.project.src_dir + + '/.venv/bin:' + + (ansible_env.PATH | default(lookup('env', 'PATH'))) + }) + }} + + - name: Deploy with parallel orchestrator + run_once: true + changed_when: false + ansible.builtin.command: + argv: "{{ _atmosphere_ci_deploy_argv }}" + args: + chdir: "{{ zuul.project.src_dir }}" + environment: "{{ _atmosphere_ci_deploy_environment }}" + + - name: Re-deploy to verify idempotent execution + run_once: true + changed_when: false + ansible.builtin.command: + argv: "{{ _atmosphere_ci_deploy_argv }}" + args: + chdir: "{{ zuul.project.src_dir }}" + environment: "{{ _atmosphere_ci_deploy_environment }}" + + - name: Add IP address to br-ex + when: molecule_scenario == 'aio' + become: true + changed_when: false + ansible.builtin.command: + argv: + - ip + - address + - replace + - 10.96.250.10/24 + - dev + - br-ex + + - name: Set br-ex interface up + when: molecule_scenario == 'aio' + become: true + changed_when: false + ansible.builtin.command: + argv: + - ip + - link + - set + - br-ex + - up + + - name: Verify Molecule scenario + run_once: true + changed_when: false + ansible.builtin.command: + argv: + - uv + - run + - molecule + - verify + - -s + - "{{ molecule_scenario }}" + environment: >- + {{ + molecule_environment + | default({}) + | combine({ + 'ATMOSPHERE_CI_MODE': _atmosphere_ci_mode, + 'ATMOSPHERE_CI_COMPONENTS': _atmosphere_ci_components | join(','), + 'ATMOSPHERE_CI_VERIFICATION_PROFILES': _atmosphere_ci_profiles | join(',') + }) + }} + args: + chdir: "{{ zuul.project.src_dir }}"