From e7c54f47bec488282b84debd20f30a59a197838b Mon Sep 17 00:00:00 2001 From: Fabio Gallotti Date: Wed, 9 Sep 2026 16:58:24 +0200 Subject: [PATCH 1/5] Support multi-RHEL MicroShift bootc shipments rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- elliott/elliottlib/shipment_utils.py | 30 ++- elliott/tests/test_shipment_utils.py | 26 +++ .../pipelines/binary_release_konflux.py | 26 +-- .../pipelines/build_microshift_bootc.py | 198 ++++++++++++++---- pyartcd/pyartcd/shipment_utils.py | 87 ++++++++ .../pipelines/test_build_microshift_bootc.py | 101 ++++++++- pyartcd/tests/test_shipment_utils.py | 73 +++++++ 7 files changed, 467 insertions(+), 74 deletions(-) create mode 100644 pyartcd/pyartcd/shipment_utils.py create mode 100644 pyartcd/tests/test_shipment_utils.py diff --git a/elliott/elliottlib/shipment_utils.py b/elliott/elliottlib/shipment_utils.py index 3c2c985f75..d5e46b8e6c 100644 --- a/elliott/elliottlib/shipment_utils.py +++ b/elliott/elliottlib/shipment_utils.py @@ -211,7 +211,7 @@ def get_shipment_configs_from_mr( filename = file_path.split('/')[-1] parts = filename.replace('.yaml', '').replace('.yml', '') - kind = next((k for k in kinds if k in parts), None) + kind = _get_shipment_config_kind(parts, kinds) if not kind: continue @@ -228,6 +228,34 @@ def get_shipment_configs_from_mr( return shipment_configs +def _get_shipment_config_kind(filename_stem: str, kinds: Tuple[str, ...]) -> str | None: + """ + Extracts a shipment kind from a filename, preserving an optional RHEL suffix. + + New multi-RHEL shipment files use names such as ``image-el9`` and + ``microshift-bootc-el10``. The suffix must remain part of the returned key so + that multiple RHEL-specific configs can coexist in one merge request. + + Args: + filename_stem: Shipment filename without its YAML extension. + kinds: Base shipment kinds accepted by the caller. + Returns: + The matching base or RHEL-qualified shipment kind, if any. + """ + for kind in sorted(kinds, key=len, reverse=True): + qualified_match = re.search(rf"(?:^|\.)({re.escape(kind)}-el\d+)(?:\.|$)", filename_stem) + if qualified_match: + return qualified_match.group(1) + + base_match = re.search(rf"(?:^|\.){re.escape(kind)}(?:\.|$)", filename_stem) + if base_match: + return kind + + # Preserve the historical substring matching for unusual legacy filenames such as + # ``rpm-extra.yaml``. + return next((kind for kind in kinds if kind in filename_stem), None) + + def get_shipment_config_from_mr(mr_url: str, kind: str) -> ShipmentConfig | None: """Fetch a specific shipment config from a merge request URL.""" shipment_configs = get_shipment_configs_from_mr(mr_url) diff --git a/elliott/tests/test_shipment_utils.py b/elliott/tests/test_shipment_utils.py index 117a54c590..0edd70fa82 100644 --- a/elliott/tests/test_shipment_utils.py +++ b/elliott/tests/test_shipment_utils.py @@ -304,6 +304,32 @@ def test_get_shipment_configs_by_kind_all_default_kinds(self, mock_gitlab_class) expected_kinds = {"fbc", "image", "extras", "microshift-bootc", "metadata"} self.assertEqual(set(result.keys()), expected_kinds) + @patch('artcommonlib.gitlab.gitlab.Gitlab') + @patch.dict(os.environ, {'GITLAB_TOKEN': 'test-token'}) + def test_get_shipment_configs_preserves_rhel_qualified_kinds(self, mock_gitlab_class): + """RHEL-qualified shipment files remain distinct when parsed from one MR.""" + mock_gitlab = mock_gitlab_class.return_value + mock_gitlab.projects.get.side_effect = [self.mock_project, self.mock_source_project] + + self.mock_project.mergerequests.get.return_value = self.mock_mr + self.mock_mr.source_project_id = "source-project-id" + self.mock_mr.source_branch = "test-branch" + + self.mock_diff_info.id = "diff-id" + self.mock_mr.diffs.list.return_value = [self.mock_diff_info] + self.mock_mr.diffs.get.return_value = self.mock_diff + self.mock_diff.diffs = [ + {'new_path': 'microshift-bootc-el9.yaml', 'old_path': None}, + {'new_path': 'microshift-bootc-el10.yaml', 'old_path': None}, + ] + + self.mock_file_content.decode.return_value.decode.return_value = self.sample_yaml_content + self.mock_source_project.files.get.return_value = self.mock_file_content + + result = shipment_utils.get_shipment_configs_from_mr(self.test_mr_url) + + self.assertEqual(set(result), {'microshift-bootc-el9', 'microshift-bootc-el10'}) + class TestGroupFiltering(unittest.TestCase): """Test cases for group-based filtering in get_shipment_configs_from_mr""" diff --git a/pyartcd/pyartcd/pipelines/binary_release_konflux.py b/pyartcd/pyartcd/pipelines/binary_release_konflux.py index 19d957df86..321d9497e7 100644 --- a/pyartcd/pyartcd/pipelines/binary_release_konflux.py +++ b/pyartcd/pyartcd/pipelines/binary_release_konflux.py @@ -16,7 +16,6 @@ from artcommonlib.build_visibility import is_nvr_embargoed from artcommonlib.constants import SHIPMENT_DATA_URL_TEMPLATE from artcommonlib.gitlab import GitLabClient -from artcommonlib.release_util import isolate_el_version_in_release from artcommonlib.util import new_roundtrip_yaml_handler from elliottlib.shipment_model import ( Environments, @@ -32,6 +31,7 @@ from pyartcd.click_validators import validate_release_date from pyartcd.git import GitRepository from pyartcd.runtime import Runtime +from pyartcd.shipment_utils import get_release_plan_names, group_nvrs_by_rhel_version yaml = new_roundtrip_yaml_handler() @@ -237,14 +237,7 @@ def _group_nvrs_by_rhel_version(nvrs: List[str]) -> Dict[str, List[str]]: NVRs without a detectable .el* suffix go under the 'default' key. Returns an OrderedDict-like dict sorted by key for deterministic ordering. """ - groups: Dict[str, List[str]] = {} - for nvr in nvrs: - # The release field is the last hyphen-delimited segment of an NVR - release = nvr.rsplit('-', 1)[-1] if '-' in nvr else nvr - el_ver = isolate_el_version_in_release(release) - key = f"el{el_ver}" if el_ver is not None else "default" - groups.setdefault(key, []).append(nvr) - return dict(sorted(groups.items())) + return group_nvrs_by_rhel_version(nvrs) async def create_snapshot(self, builds: List[str]) -> Optional[Snapshot]: """ @@ -316,21 +309,8 @@ def create_shipment_config(self, snapshot: Snapshot, rhel_suffix: Optional[str] fbc=False, ) - stage_rpa = "n/a" - prod_rpa = "n/a" config_path = self.shipment_data_repo._directory / "config.yaml" - if config_path.exists(): - with open(config_path, 'r') as f: - shipment_config = stdlib_yaml.safe_load(f) or {} - applications = shipment_config.get("applications", {}) - # Try RHEL-versioned key first (e.g. 'oc-mirror-2-0-el9'), then fall back to the - # plain application name for products that don't split by RHEL version. - lookup_key = f"{application}-{rhel_suffix}" if rhel_suffix else application - app_env_config = (applications.get(lookup_key) or applications.get(application) or {}).get( - "environments", {} - ) - stage_rpa = app_env_config.get("stage", {}).get("releasePlan", "n/a") - prod_rpa = app_env_config.get("prod", {}).get("releasePlan", "n/a") + stage_rpa, prod_rpa = get_release_plan_names(config_path, application, rhel_suffix) if stage_rpa == "n/a" or prod_rpa == "n/a": effective_key = f"{application}-{rhel_suffix}" if rhel_suffix else application diff --git a/pyartcd/pyartcd/pipelines/build_microshift_bootc.py b/pyartcd/pyartcd/pipelines/build_microshift_bootc.py index 2500056230..f15ae12db1 100644 --- a/pyartcd/pyartcd/pipelines/build_microshift_bootc.py +++ b/pyartcd/pyartcd/pipelines/build_microshift_bootc.py @@ -42,7 +42,7 @@ ) from doozerlib.backend.konflux_client import API_VERSION, KIND_SNAPSHOT from doozerlib.util import isolate_git_commit_in_release -from elliottlib.shipment_model import ShipmentConfig, Snapshot, SnapshotSpec +from elliottlib.shipment_model import Environments, ShipmentConfig, ShipmentEnv, Snapshot, SnapshotSpec from github import GithubException from pyartcd import constants, jenkins @@ -50,6 +50,7 @@ from pyartcd.git import GitRepository from pyartcd.plashets import convert_plashet_config_to_new_style, plashet_config_for_major_minor from pyartcd.runtime import Runtime +from pyartcd.shipment_utils import get_release_plan_names, group_nvrs_by_rhel_version from pyartcd.util import ( default_release_suffix, get_assembly_type, @@ -871,7 +872,7 @@ async def _wait_for_pr_merge(self, pr): await asyncio.sleep(check_interval) async def _prepare_shipment(self, builds: dict[str, KonfluxBuildRecord]): - """Prepare shipment for microshift-bootc (all variants combined in one shipment MR). + """Prepare RHEL-specific microshift-bootc shipments in one shipment MR. Args: builds: Mapping of image_name -> KonfluxBuildRecord for each built variant. @@ -884,8 +885,8 @@ async def _prepare_shipment(self, builds: dict[str, KonfluxBuildRecord]): # Step 2: Setup shipment data repository first await self._setup_shipment_data_repo() - # Step 3: Check for existing shipment branch and try to load existing config - shipment_config = await self._load_or_init_shipment_config() + # Step 3: Reuse the existing shipment MR branch when one is configured. + await self._load_or_init_shipment_branch() # Check if there was an existing microshift_bootc_shipment URL assembly_shipment_config = self.assembly_group_config.get("microshift_bootc_shipment", {}) @@ -899,14 +900,11 @@ async def _prepare_shipment(self, builds: dict[str, KonfluxBuildRecord]): for image_name, build in builds.items(): self._logger.info("Using bootc build for %s: %s", image_name, build.nvr) - # Step 5: Create snapshot from all bootc builds - image_names = list(builds.keys()) - nvrs = [build.nvr for build in builds.values()] - snapshot = await self._create_snapshot(nvrs, image_names) - shipment_config.shipment.snapshot = snapshot + # Step 5: Create one snapshot and shipment config for each RHEL version. + shipment_configs = await self._create_shipment_configs(builds) - # Step 6: Create shipment MR - self.shipment_mr_url = await self._create_shipment_mr(shipment_config, env) + # Step 6: Create or update one shipment MR containing all RHEL-specific files. + self.shipment_mr_url = await self._create_shipment_mr(shipment_configs, env) if self.shipment_mr_url: await self.slack_client.say_in_thread(f"Shipment MR created: {self.shipment_mr_url}") @@ -918,6 +916,37 @@ async def _prepare_shipment(self, builds: dict[str, KonfluxBuildRecord]): else: await self.slack_client.say_in_thread("No changes in shipment data. MR was not created or updated.") + async def _create_shipment_configs(self, builds: dict[str, KonfluxBuildRecord]) -> dict[str, ShipmentConfig]: + """ + Create one snapshot and shipment config for each RHEL version in the builds. + + Args: + builds: Mapping of bootc image name to its Konflux build record. + Returns: + Shipment configurations keyed by the shipment filename kind. + """ + builds_by_nvr = {build.nvr: (image_name, build) for image_name, build in builds.items()} + nvr_groups = group_nvrs_by_rhel_version([build.nvr for build in builds.values()]) + multiple_rhel_versions = len(nvr_groups) > 1 + shipment_configs: dict[str, ShipmentConfig] = {} + + for rhel_suffix, group_nvrs in nvr_groups.items(): + image_names = [builds_by_nvr[nvr][0] for nvr in group_nvrs] + snapshot = await self._create_snapshot(group_nvrs, image_names) + if snapshot is None: + raise ValueError(f"No snapshot created for RHEL group {rhel_suffix}") + + shipment_config = await self._init_shipment_config(rhel_suffix if rhel_suffix != "default" else None) + shipment_config.shipment.snapshot = snapshot + + if multiple_rhel_versions: + shipment_kind = f"microshift-bootc-{rhel_suffix}" + else: + shipment_kind = "microshift-bootc" + shipment_configs[shipment_kind] = shipment_config + + return shipment_configs + def _resolve_shipment_env(self) -> str: """Resolve the target shipment environment (stage/prod) from the assembly definition. @@ -1088,35 +1117,33 @@ async def _create_or_update_build_data_pr(self) -> bool: await self._wait_for_pr_merge(pr) return True - async def _load_or_init_shipment_config(self) -> ShipmentConfig: - """Load existing shipment config from branch or initialize new one - - If a shipment MR already exists (URL in config), reuse the existing branch. - Otherwise, create a new branch with timestamp. + async def _load_or_init_shipment_branch(self) -> None: + """ + Switches to the existing shipment branch or prepares for a new branch. - Sets self._shipment_source_branch to the resolved branch name for reuse in _create_shipment_mr. + Existing shipment MRs must remain open so that a rerun can safely update + their source branch. """ - # Check if shipment already exists in assembly config assembly_shipment_config = self.assembly_group_config.get("microshift_bootc_shipment", {}) existing_mr_url = assembly_shipment_config.get("url") if existing_mr_url: - # Get the branch name from the existing MR (validates MR state) self._shipment_source_branch = self._get_shipment_mr_branch(existing_mr_url) - - self._logger.info('Found existing shipment MR, using branch: %s', self._shipment_source_branch) + self._logger.info("Found existing shipment MR, using branch: %s", self._shipment_source_branch) await self.shipment_data_repo.fetch_switch_branch(self._shipment_source_branch, remote="origin") - - # Initialize new config - we'll load the snapshot from existing files later if needed - return await self._init_shipment_config() else: - # No existing MR - this is the first run, initialize new config - self._logger.info('No existing shipment MR found, will initialize new shipment config') + self._logger.info("No existing shipment MR found, will initialize a new shipment branch") self._shipment_source_branch = None - return await self._init_shipment_config() - async def _init_shipment_config(self) -> ShipmentConfig: - """Initialize shipment configuration using elliott shipment init""" + async def _init_shipment_config(self, rhel_suffix: str | None = None) -> ShipmentConfig: + """ + Initialize shipment configuration and resolve its RHEL-specific ReleasePlans. + + Args: + rhel_suffix: Optional suffix such as ``el9`` or ``el10``. + Returns: + Initialized shipment configuration with resolved stage/prod ReleasePlans. + """ self._logger.info("Initializing shipment configuration for microshift-bootc...") create_cmd = self._elliott_base_command + [ @@ -1131,6 +1158,21 @@ async def _init_shipment_config(self) -> ShipmentConfig: out = Model(yaml.load(stdout)).primitive() shipment = ShipmentConfig(**out) + config_path = self.shipment_data_repo._directory / "config.yaml" + application = shipment.shipment.metadata.application + stage_release_plan, prod_release_plan = get_release_plan_names(config_path, application, rhel_suffix) + if stage_release_plan == "n/a" or prod_release_plan == "n/a": + effective_key = f"{application}-{rhel_suffix}" if rhel_suffix else application + raise ValueError( + f"stage/prod releasePlan is not registered for '{effective_key}' in {config_path}. " + "Cannot create a shipment MR with unresolved ReleasePlans." + ) + + shipment.shipment.environments = Environments( + stage=ShipmentEnv(releasePlan=stage_release_plan), + prod=ShipmentEnv(releasePlan=prod_release_plan), + ) + self._logger.info("Shipment configuration initialized") return shipment @@ -1190,8 +1232,8 @@ async def _setup_shipment_data_repo(self): self._logger.info("Shipment data repository setup completed") - async def _create_shipment_mr(self, shipment_config: ShipmentConfig, env: str = "prod") -> str | None: - """Create or update shipment MR with the given shipment config. Returns None if no changes. + async def _create_shipment_mr(self, shipments_by_kind: dict[str, ShipmentConfig], env: str = "prod") -> str | None: + """Create or update shipment MR with the given shipment configs. Returns None if no changes. :param env: The target environment (prod or stage) that determines the shipment file directory. """ @@ -1199,10 +1241,10 @@ async def _create_shipment_mr(self, shipment_config: ShipmentConfig, env: str = target_branch = "main" - # Use the cached branch name from _load_or_init_shipment_config (if available) + # Use the cached branch name from _load_or_init_shipment_branch (if available) cached_branch = getattr(self, '_shipment_source_branch', None) if cached_branch: - # Reusing existing MR branch (already switched in _load_or_init_shipment_config) + # Reusing existing MR branch (already switched in _load_or_init_shipment_branch) source_branch = cached_branch self._logger.info('Reusing existing shipment branch: %s', source_branch) else: @@ -1215,7 +1257,7 @@ async def _create_shipment_mr(self, shipment_config: ShipmentConfig, env: str = # Update shipment data repo with shipment config release_name = get_release_name_for_assembly(self.group, self.releases_config, self.assembly) commit_message = f"Add microshift-bootc shipment configuration for {release_name}" - updated = await self._update_shipment_data(shipment_config, commit_message, source_branch, env) + updated = await self._update_shipment_data(shipments_by_kind, commit_message, source_branch, env) if not updated: self._logger.info("No changes in shipment data. MR will not be created or updated.") return None @@ -1267,23 +1309,64 @@ def _get_project(url): return mr_url async def _update_shipment_data( - self, shipment_config: ShipmentConfig, commit_message: str, branch: str, env: str = "prod" + self, + shipments_by_kind: dict[str, ShipmentConfig], + commit_message: str, + branch: str, + env: str = "prod", ) -> bool: - """Update shipment data repo with the given shipment config file + """Update shipment data repo with the given shipment config files. - :param env: The target environment (prod or stage) that determines the shipment file directory. + Args: + shipments_by_kind: Shipment configurations keyed by filename kind. + commit_message: Git commit message for the shipment update. + branch: Shipment branch name containing the timestamp. + env: Target environment directory, either ``prod`` or ``stage``. """ # Extract timestamp from branch name (last segment after splitting by "-") # Branch format: prepare-microshift-bootc-shipment-{assembly}-{timestamp} timestamp = branch.split("-")[-1] - filename = f"{self.assembly}.microshift-bootc.{timestamp}.yaml" + + if len(shipments_by_kind) > 1: + await self._remove_legacy_combined_shipment_file(shipments_by_kind, timestamp, env) + + for shipment_kind, shipment_config in shipments_by_kind.items(): + await self._write_shipment_file(shipment_kind, shipment_config, timestamp, env) + + await self.shipment_data_repo.add_all() + await self.shipment_data_repo.log_diff() + + job_url = os.getenv('BUILD_URL') + if job_url and job_url not in commit_message: + commit_message += f"\n{job_url}" + + return await self.shipment_data_repo.commit_push(commit_message, safe=True) + + async def _write_shipment_file( + self, + shipment_kind: str, + shipment_config: ShipmentConfig, + timestamp: str, + env: str, + ) -> Path: + """ + Writes one shipment config file to the shipment-data repository. + + Args: + shipment_kind: Kind used in the shipment filename. + shipment_config: Shipment configuration to serialize. + timestamp: Timestamp shared by all files in the shipment MR. + env: Target environment directory, either ``prod`` or ``stage``. + Returns: + Relative path of the written shipment file. + """ product = shipment_config.shipment.metadata.product group = shipment_config.shipment.metadata.group application = shipment_config.shipment.metadata.application - relative_target_dir = Path("shipment") / product / group / application / env target_dir = self.shipment_data_repo._directory / relative_target_dir target_dir.mkdir(parents=True, exist_ok=True) + filename = f"{self.assembly}.{shipment_kind}.{timestamp}.yaml" filepath = relative_target_dir / filename self._logger.info("Updating shipment file: %s", filename) @@ -1291,15 +1374,38 @@ async def _update_shipment_data( out = StringIO() yaml.dump(shipment_dump, out) await self.shipment_data_repo.write_file(filepath, out.getvalue()) + return filepath - await self.shipment_data_repo.add_all() - await self.shipment_data_repo.log_diff() - - job_url = os.getenv('BUILD_URL') - if job_url and job_url not in commit_message: - commit_message += f"\n{job_url}" + async def _remove_legacy_combined_shipment_file( + self, + shipments_by_kind: dict[str, ShipmentConfig], + timestamp: str, + env: str, + ) -> None: + """ + Removes the old combined shipment file when migrating to RHEL-specific files. - return await self.shipment_data_repo.commit_push(commit_message, safe=True) + Args: + shipments_by_kind: New shipment configurations used to locate the target directory. + timestamp: Timestamp encoded in the existing shipment branch. + env: Target environment directory, either ``prod`` or ``stage``. + """ + shipment_config = next(iter(shipments_by_kind.values())) + product = shipment_config.shipment.metadata.product + group = shipment_config.shipment.metadata.group + application = shipment_config.shipment.metadata.application + legacy_relative_path = ( + Path("shipment") + / product + / group + / application + / env + / f"{self.assembly}.microshift-bootc.{timestamp}.yaml" + ) + legacy_path = self.shipment_data_repo._directory / legacy_relative_path + if legacy_path.exists(): + self._logger.info("Removing legacy combined shipment file: %s", legacy_relative_path) + legacy_path.unlink() @cached_property def _gitlab(self) -> GitLabClient: diff --git a/pyartcd/pyartcd/shipment_utils.py b/pyartcd/pyartcd/shipment_utils.py new file mode 100644 index 0000000000..a46de868eb --- /dev/null +++ b/pyartcd/pyartcd/shipment_utils.py @@ -0,0 +1,87 @@ +""" +Shared helpers for Konflux shipment pipelines. + +The helpers in this module keep RHEL-version handling consistent between +standalone binary releases and MicroShift bootc shipments. +""" + +from pathlib import Path + +import yaml as stdlib_yaml +from artcommonlib.release_util import isolate_el_version_in_release + + +def group_nvrs_by_rhel_version(nvrs: list[str]) -> dict[str, list[str]]: + """ + Groups NVRs by their RHEL version suffix. + + Args: + nvrs: Build NVRs whose final release segment may contain an ``elN`` suffix. + Returns: + Dictionary keyed by ``elN`` or ``default``, sorted by key for deterministic output. + """ + groups: dict[str, list[str]] = {} + for nvr in nvrs: + # The release field is the last hyphen-delimited segment of an NVR. + release = nvr.rsplit("-", 1)[-1] if "-" in nvr else nvr + el_version = isolate_el_version_in_release(release) + key = f"el{el_version}" if el_version is not None else "default" + groups.setdefault(key, []).append(nvr) + + return dict( + sorted( + groups.items(), + key=lambda item: -1 if item[0] == "default" else int(item[0].removeprefix("el")), + ) + ) + + +def get_release_plan_names( + config_path: Path, + application: str, + rhel_suffix: str | None = None, +) -> tuple[str, str]: + """ + Loads stage and production ReleasePlan names from shipment config.yaml. + + When a RHEL suffix is provided, the RHEL-specific application key is tried + first and the plain application key is used as a backwards-compatible fallback. + + Args: + config_path: Path to the shipment-data config.yaml file. + application: Base Konflux application name. + rhel_suffix: Optional suffix such as ``el9`` or ``el10``. + Returns: + Tuple containing the stage and production ReleasePlan names. Missing + values retain the existing ``n/a`` behavior. + """ + stage_release_plan = "n/a" + prod_release_plan = "n/a" + + if not config_path.exists(): + return stage_release_plan, prod_release_plan + + with config_path.open("r") as config_file: + shipment_config = stdlib_yaml.safe_load(config_file) or {} + + applications = shipment_config.get("applications", {}) + lookup_keys = [application] + if rhel_suffix: + rhel_number = rhel_suffix.removeprefix("el").removeprefix("rhel") + lookup_keys = [ + f"{application}-rhel{rhel_number}", + f"{application}-{rhel_suffix}", + application, + ] + + application_config = {} + for lookup_key in lookup_keys: + application_config = applications.get(lookup_key, {}) + if application_config: + break + + application_config = application_config.get("environments", {}) + stage_release_plan = application_config.get("stage", {}).get("releasePlan", "n/a") + prod_release_plan = application_config.get("prod", {}).get("releasePlan", "n/a") + + return stage_release_plan, prod_release_plan diff --git a/pyartcd/tests/pipelines/test_build_microshift_bootc.py b/pyartcd/tests/pipelines/test_build_microshift_bootc.py index be90758768..65e89c68c3 100644 --- a/pyartcd/tests/pipelines/test_build_microshift_bootc.py +++ b/pyartcd/tests/pipelines/test_build_microshift_bootc.py @@ -82,7 +82,7 @@ async def test_update_shipment_data_extracts_timestamp_from_branch(self, mock_ge source_branch = f"prepare-microshift-bootc-shipment-{self.assembly}-{existing_timestamp}" # when - await pipeline._update_shipment_data(mock_shipment_config, "Test commit", source_branch) + await pipeline._update_shipment_data({"microshift-bootc": mock_shipment_config}, "Test commit", source_branch) # then pipeline.shipment_data_repo.write_file.assert_called_once() @@ -126,7 +126,9 @@ async def test_update_shipment_data_uses_env_for_directory(self, mock_get_client source_branch = f"prepare-microshift-bootc-shipment-{self.assembly}-{timestamp}" # when - await pipeline._update_shipment_data(mock_shipment_config, "Test commit", source_branch, "stage") + await pipeline._update_shipment_data( + {"microshift-bootc": mock_shipment_config}, "Test commit", source_branch, "stage" + ) # then pipeline.shipment_data_repo.write_file.assert_called_once() @@ -235,7 +237,7 @@ async def test_create_shipment_mr_reuses_existing_timestamp(self, mock_get_clien # when with patch('pyartcd.pipelines.build_microshift_bootc.get_release_name_for_assembly', return_value="4.21.0"): - _ = await pipeline._create_shipment_mr(mock_shipment_config) + _ = await pipeline._create_shipment_mr({"microshift-bootc": mock_shipment_config}) # then pipeline.shipment_data_repo.write_file.assert_called_once() @@ -297,7 +299,7 @@ async def test_create_shipment_mr_generates_new_timestamp_for_new_shipment(self, # when with patch('pyartcd.pipelines.build_microshift_bootc.get_release_name_for_assembly', return_value="4.18.1"): - _ = await pipeline._create_shipment_mr(mock_shipment_config) + _ = await pipeline._create_shipment_mr({"microshift-bootc": mock_shipment_config}) # then # Verify a new branch with timestamp was created @@ -320,6 +322,27 @@ async def test_create_shipment_mr_generates_new_timestamp_for_new_shipment(self, self.assertEqual(len(timestamp_part), 14) self.assertTrue(timestamp_part.isdigit()) + async def test_load_or_init_shipment_branch_reuses_open_mr_branch(self): + """Reuses the source branch from the configured open shipment MR.""" + pipeline = self._make_pipeline(group="openshift-5.0", assembly="rc.1") + existing_mr_url = "https://gitlab.example.com/shipment-data/-/merge_requests/806" + existing_branch = "prepare-microshift-bootc-shipment-rc.1-20260904133832" + pipeline.releases_config = Model( + {"releases": {"rc.1": {"assembly": {"group": {"microshift_bootc_shipment": {"url": existing_mr_url}}}}}} + ) + pipeline.shipment_data_repo = Mock() + pipeline.shipment_data_repo.fetch_switch_branch = AsyncMock() + + mock_mr = Mock(source_branch=existing_branch, state="opened") + mock_gitlab = Mock() + mock_gitlab.get_mr_from_url.return_value = mock_mr + pipeline._gitlab = mock_gitlab + + await pipeline._load_or_init_shipment_branch() + + self.assertEqual(pipeline._shipment_source_branch, existing_branch) + pipeline.shipment_data_repo.fetch_switch_branch.assert_awaited_once_with(existing_branch, remote="origin") + @patch("pyartcd.pipelines.build_microshift_bootc.get_microshift_builds") async def test_get_microshift_rpm_commit_extracts_commit(self, mock_get_builds): """ @@ -643,6 +666,76 @@ def test_pin_image_nvr_multiple_variants(self): images[1]["metadata"]["is"]["nvr"], "microshift-bootc-rhel10-container-v4.22-202606081229.el10" ) + @patch.object(BuildMicroShiftBootcPipeline, "_init_shipment_config", new_callable=AsyncMock) + @patch.object(BuildMicroShiftBootcPipeline, "_create_snapshot", new_callable=AsyncMock) + async def test_create_shipment_configs_groups_builds_by_rhel_version( + self, mock_create_snapshot, mock_init_shipment_config + ): + """Creates one shipment config and snapshot for each RHEL version.""" + pipeline = self._make_pipeline(group="openshift-5.0", assembly="rc.1") + snapshot_el9 = Mock() + snapshot_el10 = Mock() + mock_create_snapshot.side_effect = [snapshot_el9, snapshot_el10] + + config_el9 = Mock() + config_el10 = Mock() + mock_init_shipment_config.side_effect = [config_el9, config_el10] + builds = { + "microshift-bootc": Mock(nvr="microshift-bootc-container-v5.0-1.el9"), + "microshift-bootc-rhel10": Mock(nvr="microshift-bootc-rhel10-container-v5.0-1.el10"), + } + + shipment_configs = await pipeline._create_shipment_configs(builds) + + self.assertEqual(list(shipment_configs), ["microshift-bootc-el9", "microshift-bootc-el10"]) + self.assertIs(shipment_configs["microshift-bootc-el9"], config_el9) + self.assertIs(shipment_configs["microshift-bootc-el10"], config_el10) + mock_init_shipment_config.assert_any_await("el9") + mock_init_shipment_config.assert_any_await("el10") + self.assertEqual(mock_create_snapshot.await_count, 2) + + async def test_update_shipment_data_writes_multiple_files_and_removes_legacy_file(self): + """Writes RHEL-specific files and removes the old combined shipment file.""" + pipeline = self._make_pipeline(group="openshift-5.0", assembly="rc.1") + pipeline.shipment_data_repo = Mock() + pipeline.shipment_data_repo._directory = Path(tempfile.mkdtemp()) + pipeline.shipment_data_repo.write_file = AsyncMock() + pipeline.shipment_data_repo.add_all = AsyncMock() + pipeline.shipment_data_repo.log_diff = AsyncMock() + pipeline.shipment_data_repo.commit_push = AsyncMock(return_value=True) + + legacy_dir = pipeline.shipment_data_repo._directory / "shipment/ocp/openshift-5.0/openshift-5-0/stage" + legacy_dir.mkdir(parents=True) + legacy_file = legacy_dir / "rc.1.microshift-bootc.20260904133832.yaml" + legacy_file.write_text("legacy") + + configs = {} + for kind in ("microshift-bootc-el9", "microshift-bootc-el10"): + config = Mock() + config.shipment.metadata.product = "ocp" + config.shipment.metadata.group = "openshift-5.0" + config.shipment.metadata.application = "openshift-5-0" + config.model_dump.return_value = {"shipment": {}} + configs[kind] = config + + updated = await pipeline._update_shipment_data( + configs, + "Update shipment", + "prepare-microshift-bootc-shipment-rc.1-20260904133832", + "stage", + ) + + self.assertTrue(updated) + self.assertFalse(legacy_file.exists()) + written_paths = [call.args[0] for call in pipeline.shipment_data_repo.write_file.call_args_list] + self.assertEqual( + {path.name for path in written_paths}, + { + "rc.1.microshift-bootc-el9.20260904133832.yaml", + "rc.1.microshift-bootc-el10.20260904133832.yaml", + }, + ) + def test_pin_image_nvr_updates_existing_entry(self): """ Test that _pin_image_nvr updates an existing pin entry instead of duplicating it. diff --git a/pyartcd/tests/test_shipment_utils.py b/pyartcd/tests/test_shipment_utils.py new file mode 100644 index 0000000000..b3ec464f44 --- /dev/null +++ b/pyartcd/tests/test_shipment_utils.py @@ -0,0 +1,73 @@ +""" +Tests for shared Konflux shipment helpers. +""" + +from pathlib import Path + +from pyartcd.shipment_utils import get_release_plan_names, group_nvrs_by_rhel_version + + +def test_group_nvrs_by_rhel_version_sorts_rhel_groups() -> None: + """Groups NVRs by their RHEL suffix in deterministic order.""" + nvrs = [ + "microshift-bootc-rhel10-container-v5.0-1.el10", + "microshift-bootc-container-v5.0-1.el9", + ] + + assert group_nvrs_by_rhel_version(nvrs) == { + "el10": ["microshift-bootc-rhel10-container-v5.0-1.el10"], + "el9": ["microshift-bootc-container-v5.0-1.el9"], + } + + +def test_group_nvrs_by_rhel_version_uses_default_without_suffix() -> None: + """Places NVRs without a detectable RHEL suffix in the default group.""" + assert group_nvrs_by_rhel_version(["microshift-bootc-container-v5.0-1"]) == { + "default": ["microshift-bootc-container-v5.0-1"] + } + + +def test_get_release_plan_names_prefers_rhel_specific_application(tmp_path: Path) -> None: + """Uses the RHEL-specific application entry when it exists.""" + config_path = tmp_path / "config.yaml" + config_path.write_text( + "applications:\n" + " openshift-5-0:\n" + " environments:\n" + " stage:\n" + " releasePlan: old-stage\n" + " prod:\n" + " releasePlan: old-prod\n" + " openshift-5-0-rhel10:\n" + " environments:\n" + " stage:\n" + " releasePlan: rhel10-stage\n" + " prod:\n" + " releasePlan: rhel10-prod\n" + ) + + assert get_release_plan_names(config_path, "openshift-5-0", "el10") == ( + "rhel10-stage", + "rhel10-prod", + ) + + +def test_get_release_plan_names_falls_back_to_plain_application(tmp_path: Path) -> None: + """Falls back to the plain application entry for older shipment data.""" + config_path = tmp_path / "config.yaml" + config_path.write_text( + "applications:\n" + " openshift-4-22:\n" + " environments:\n" + " stage:\n" + " releasePlan: stage\n" + " prod:\n" + " releasePlan: prod\n" + ) + + assert get_release_plan_names(config_path, "openshift-4-22", "el9") == ("stage", "prod") + + +def test_get_release_plan_names_returns_na_for_missing_config(tmp_path: Path) -> None: + """Returns the existing n/a defaults when no application is configured.""" + assert get_release_plan_names(tmp_path / "missing.yaml", "openshift-5-0", "el9") == ("n/a", "n/a") From f075f27ad42495725eb73f9deafd5f969f4e0ab6 Mon Sep 17 00:00:00 2001 From: Fabio Gallotti Date: Wed, 9 Sep 2026 17:19:09 +0200 Subject: [PATCH 2/5] Use configured ReleasePlans for RPA validation rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- elliott/elliottlib/cli/konflux_release_cli.py | 42 ++++++++++++++++--- elliott/tests/test_konflux_release_cli.py | 23 ++++++++++ .../pipelines/prepare_release_konflux.py | 12 +++++- 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/elliott/elliottlib/cli/konflux_release_cli.py b/elliott/elliottlib/cli/konflux_release_cli.py index 1d65afbc4a..6482484408 100644 --- a/elliott/elliottlib/cli/konflux_release_cli.py +++ b/elliott/elliottlib/cli/konflux_release_cli.py @@ -2,7 +2,7 @@ import re import sys from dataclasses import dataclass -from typing import List, Optional, Set +from typing import List, Mapping, Optional, Set import aiohttp import click @@ -87,7 +87,23 @@ async def _validate_snapshot_against_single_rpa(kind: str, rpa_name: str, snapsh ) -async def validate_snapshot_against_rpa(group: str, env: str, kind: str, snapshot_components: List[str]) -> None: +async def validate_snapshot_against_rpa( + group: str, + env: str, + kind: str, + snapshot_components: List[str], + release_plans: Mapping[str, str] | None = None, +) -> None: + """ + Validate snapshot components against the configured stage and prod RPAs. + + Args: + group: Build-data group used to determine whether RPA validation applies. + env: Environment being released first; the other environment is checked second. + kind: Shipment kind used by the legacy RPA naming fallback. + snapshot_components: Component names present in the shipment snapshot. + release_plans: Optional mapping of environment names to configured ReleasePlans. + """ match = re.fullmatch(r"openshift-(\d+)\.(\d+)", group) if group.startswith("openshift-") and not match: raise ValueError(f"Unrecognized openshift group format, refusing to skip RPA validation: {group!r}") @@ -108,7 +124,9 @@ async def validate_snapshot_against_rpa(group: str, env: str, kind: str, snapsho envs_to_check = [env] + [e for e in OCP_RPA_ENVS if e != env] for check_env in envs_to_check: - rpa_name = f"{OCP_RPA_KINDS[kind]}-{check_env}-{major}-{minor}" + rpa_name = (release_plans or {}).get(check_env) + if not rpa_name: + rpa_name = f"{OCP_RPA_KINDS[kind]}-{check_env}-{major}-{minor}" await _validate_snapshot_against_single_rpa(kind, rpa_name, snapshot_components) @@ -234,7 +252,17 @@ async def run(self) -> Optional[ResourceInstance]: if self.runtime.group.startswith("openshift-") and config.shipment.snapshot: LOGGER.info("Validating snapshot components against RPA...") component_names = [c.name for c in config.shipment.snapshot.spec.components] - await validate_snapshot_against_rpa(self.runtime.group, self.release_env, self.kind, component_names) + release_plans = { + "stage": config.shipment.environments.stage.releasePlan, + "prod": config.shipment.environments.prod.releasePlan, + } + await validate_snapshot_against_rpa( + self.runtime.group, + self.release_env, + self.kind, + component_names, + release_plans=release_plans, + ) # Create snapshot first using the spec from shipment config LOGGER.info("Creating snapshot from shipment config...") @@ -538,5 +566,9 @@ async def validate_rpa_cli(runtime: Runtime, config, env, kind): component_names = [c.name for c in shipment_config.shipment.snapshot.spec.components] LOGGER.info("Validating %d components against RPA for %s/%s...", len(component_names), env, kind) - await validate_snapshot_against_rpa(runtime.group, env, kind, component_names) + release_plans = { + "stage": shipment_config.shipment.environments.stage.releasePlan, + "prod": shipment_config.shipment.environments.prod.releasePlan, + } + await validate_snapshot_against_rpa(runtime.group, env, kind, component_names, release_plans=release_plans) LOGGER.info("Validation passed: all components are present in the RPA") diff --git a/elliott/tests/test_konflux_release_cli.py b/elliott/tests/test_konflux_release_cli.py index 392d32c3b1..8f495d5c3f 100644 --- a/elliott/tests/test_konflux_release_cli.py +++ b/elliott/tests/test_konflux_release_cli.py @@ -787,6 +787,29 @@ async def test_validate_rpa_checks_both_envs_stage_first(self, mock_fetch_rpa): calls = [c.args[0] for c in mock_fetch_rpa.await_args_list] self.assertEqual(calls, ["ocp-art-advisory-stage-4-18", "ocp-art-advisory-prod-4-18"]) + @patch("elliottlib.cli.konflux_release_cli.fetch_rpa", new_callable=AsyncMock) + async def test_validate_rpa_uses_configured_release_plans(self, mock_fetch_rpa): + """Uses RHEL-specific ReleasePlans from the shipment configuration when provided.""" + rpa_data = {"spec": {"data": {"mapping": {"components": [{"name": "comp1"}]}}}} + mock_fetch_rpa.return_value = rpa_data + + await validate_snapshot_against_rpa( + "openshift-5.0", + "stage", + "microshift-bootc", + ["comp1"], + release_plans={ + "stage": "ocp-art-advisory-stage-5-0-rhel9", + "prod": "ocp-art-advisory-prod-5-0-rhel9", + }, + ) + + calls = [c.args[0] for c in mock_fetch_rpa.await_args_list] + self.assertEqual( + calls, + ["ocp-art-advisory-stage-5-0-rhel9", "ocp-art-advisory-prod-5-0-rhel9"], + ) + @patch("elliottlib.cli.konflux_release_cli.fetch_rpa", new_callable=AsyncMock) async def test_validate_rpa_skipped_for_non_openshift(self, mock_fetch_rpa): await validate_snapshot_against_rpa("oadp-1.5", "prod", "image", ["comp1"]) diff --git a/pyartcd/pyartcd/pipelines/prepare_release_konflux.py b/pyartcd/pyartcd/pipelines/prepare_release_konflux.py index 9e31c41660..fb4b6b2ea3 100644 --- a/pyartcd/pyartcd/pipelines/prepare_release_konflux.py +++ b/pyartcd/pyartcd/pipelines/prepare_release_konflux.py @@ -676,7 +676,17 @@ async def prepare_shipment_builds(self) -> Optional[tuple]: for kind, shipment in shipments_by_kind.items(): if shipment.shipment.snapshot and shipment.shipment.snapshot.spec.components: component_names = [c.name for c in shipment.shipment.snapshot.spec.components] - await validate_snapshot_against_rpa(self.group, env, kind, component_names) + release_plans = { + "stage": shipment.shipment.environments.stage.releasePlan, + "prod": shipment.shipment.environments.prod.releasePlan, + } + await validate_snapshot_against_rpa( + self.group, + env, + kind, + component_names, + release_plans=release_plans, + ) # Update shipment MR with found builds await self.update_shipment_mr(shipments_by_kind, env, shipment_url) From 131ccd2ad93860b08bf02a37a475aa6c3410d76e Mon Sep 17 00:00:00 2001 From: Fabio Gallotti Date: Wed, 9 Sep 2026 17:50:03 +0200 Subject: [PATCH 3/5] Avoid collisions for multi-RHEL shipment objects rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- elliott/elliottlib/cli/konflux_release_cli.py | 16 ++++++++++++++- elliott/tests/test_konflux_release_cli.py | 20 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/elliott/elliottlib/cli/konflux_release_cli.py b/elliott/elliottlib/cli/konflux_release_cli.py index 6482484408..a45e7a302a 100644 --- a/elliott/elliottlib/cli/konflux_release_cli.py +++ b/elliott/elliottlib/cli/konflux_release_cli.py @@ -2,6 +2,7 @@ import re import sys from dataclasses import dataclass +from pathlib import Path from typing import List, Mapping, Optional, Set import aiohttp @@ -293,12 +294,25 @@ async def run(self) -> Optional[ResourceInstance]: def get_object_name(self) -> str: timestamp = get_utc_now_formatted_str() - raw_prefix = f"{self.runtime.product}-{self.release_env}-{self.runtime.assembly}-{self.kind}" + object_kind = self._get_object_kind() + raw_prefix = f"{self.runtime.product}-{self.release_env}-{self.runtime.assembly}-{object_kind}" prefix = normalize_k8s_dns_label(raw_prefix, max_length=63 - len(timestamp) - 1) if not prefix: raise ValueError(f"Release object name prefix {raw_prefix!r} cannot be normalized to a Kubernetes name") return f"{prefix}-{timestamp}" + def _get_object_kind(self) -> str: + """ + Return the shipment kind qualified by its RHEL version, when present. + + MicroShift bootc shipments have one configuration file per RHEL version, + so the version must be included in Kubernetes object names to avoid + collisions when the releases run concurrently. + """ + config_stem = Path(self.config_path).stem + match = re.search(rf"(?:^|\.)({re.escape(self.kind)}-el\d+)(?:\.|$)", config_stem) + return match.group(1) if match else self.kind + async def create_snapshot(self, shipment: Shipment) -> dict: """ Create a Konflux Snapshot manifest from the given shipment's snapshot spec. diff --git a/elliott/tests/test_konflux_release_cli.py b/elliott/tests/test_konflux_release_cli.py index 8f495d5c3f..797fa96d19 100644 --- a/elliott/tests/test_konflux_release_cli.py +++ b/elliott/tests/test_konflux_release_cli.py @@ -203,6 +203,26 @@ def test_object_name_is_normalized_and_preserves_timestamp(self, mock_konflux_cl self.assertEqual(name, "ocp-prod-4-18-2-rc-1-image-v2-20260805200004") self.assertLessEqual(len(name), 63) + @patch("elliottlib.cli.konflux_release_cli.get_utc_now_formatted_str", return_value="20260909154630") + @patch("doozerlib.backend.konflux_client.KonfluxClient.from_kubeconfig") + def test_object_name_includes_rhel_suffix_from_shipment_filename(self, mock_konflux_client_init, _mock_timestamp): + mock_konflux_client_init.return_value = self.konflux_client + self.runtime.assembly = "rc.1" + + cli = CreateReleaseCli( + runtime=self.runtime, + config_path=( + "shipment/ocp/openshift-5.0/openshift-5-0/stage/rc.1.microshift-bootc-el10.20260904133832.yaml" + ), + release_env="stage", + konflux_config=self.konflux_config, + image_repo_pull_secret=self.image_repo_pull_secret, + dry_run=self.dry_run, + kind="microshift-bootc", + ) + + self.assertEqual(cli.get_object_name(), "ocp-stage-rc-1-microshift-bootc-el10-20260909154630") + @patch("doozerlib.backend.konflux_client.KonfluxClient.from_kubeconfig") async def test_release_rejects_invalid_snapshot_reference(self, mock_konflux_client_init): mock_konflux_client_init.return_value = self.konflux_client From ce4a8e70659802ef00e441eff8f291da6aec3f44 Mon Sep 17 00:00:00 2001 From: Fabio Gallotti Date: Fri, 11 Sep 2026 14:09:05 +0200 Subject: [PATCH 4/5] Support RHEL-qualified shipment advisories rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- elliott/elliottlib/cli/konflux_release_cli.py | 9 +- .../elliottlib/cli/verify_docs_approval.py | 162 ++++++++++++++++-- elliott/elliottlib/cli/verify_payload.py | 11 +- elliott/elliottlib/shipment_utils.py | 93 +++++++++- elliott/tests/test_konflux_release_cli.py | 40 +++++ elliott/tests/test_shipment_utils.py | 92 +++++++++- elliott/tests/test_verify_docs_approval.py | 94 +++++++++- elliott/tests/test_verify_payload.py | 29 ++++ .../pipelines/prepare_release_konflux.py | 56 +++++- pyartcd/pyartcd/pipelines/promote.py | 6 +- .../pipelines/test_prepare_release_konflux.py | 117 ++++++++++++- 11 files changed, 666 insertions(+), 43 deletions(-) create mode 100644 elliott/tests/test_verify_payload.py diff --git a/elliott/elliottlib/cli/konflux_release_cli.py b/elliott/elliottlib/cli/konflux_release_cli.py index a45e7a302a..6d03ed0efe 100644 --- a/elliott/elliottlib/cli/konflux_release_cli.py +++ b/elliott/elliottlib/cli/konflux_release_cli.py @@ -117,7 +117,8 @@ async def validate_snapshot_against_rpa( LOGGER.info("Skipping RPA validation for FBC releases (different naming scheme)") return - if kind not in OCP_RPA_KINDS: + rpa_kind = re.sub(r"-el\d+$", "", kind) + if rpa_kind not in OCP_RPA_KINDS: raise ValueError(f"Unsupported release kind for RPA validation: {kind!r}. Supported: {sorted(OCP_RPA_KINDS)}") if env not in OCP_RPA_ENVS: @@ -127,7 +128,7 @@ async def validate_snapshot_against_rpa( for check_env in envs_to_check: rpa_name = (release_plans or {}).get(check_env) if not rpa_name: - rpa_name = f"{OCP_RPA_KINDS[kind]}-{check_env}-{major}-{minor}" + rpa_name = f"{OCP_RPA_KINDS[rpa_kind]}-{check_env}-{major}-{minor}" await _validate_snapshot_against_single_rpa(kind, rpa_name, snapshot_components) @@ -305,9 +306,9 @@ def _get_object_kind(self) -> str: """ Return the shipment kind qualified by its RHEL version, when present. - MicroShift bootc shipments have one configuration file per RHEL version, + RHEL-qualified shipments have one configuration file per RHEL version, so the version must be included in Kubernetes object names to avoid - collisions when the releases run concurrently. + collisions when releases run concurrently. """ config_stem = Path(self.config_path).stem match = re.search(rf"(?:^|\.)({re.escape(self.kind)}-el\d+)(?:\.|$)", config_stem) diff --git a/elliott/elliottlib/cli/verify_docs_approval.py b/elliott/elliottlib/cli/verify_docs_approval.py index 69a370fd54..048193f8bf 100644 --- a/elliott/elliottlib/cli/verify_docs_approval.py +++ b/elliott/elliottlib/cli/verify_docs_approval.py @@ -6,7 +6,8 @@ import json import re -from typing import Literal +from pathlib import Path +from typing import Literal, Sequence import click from artcommonlib import exectools @@ -17,6 +18,7 @@ from elliottlib import Runtime from elliottlib.cli.common import cli, click_coroutine from elliottlib.shipment_model import ReleaseNotes, ShipmentConfig +from elliottlib.shipment_utils import select_primary_image_shipment PUBLIC_ERRATA_URL = "https://access.redhat.com/errata" OCP_RELEASE_PULLSPEC_TEMPLATE = "quay.io/openshift-release-dev/ocp-release:{assembly}-{arch}" @@ -80,7 +82,22 @@ def contains_advisory_reference(text: str, advisory_type: str, live_id: int) -> Return Value(s): bool: True if text references TYPE-:live_id. """ - pattern = rf"{re.escape(advisory_type)}-\d{{4}}:{live_id}\b" + pattern = rf"{re.escape(advisory_type)}-\d{{4}}:0*{live_id}\b" + return re.search(pattern, text) is not None + + +def _contains_public_advisory_reference(text: str, advisory_type: str, live_id: int) -> bool: + """ + Check whether text contains a public Errata URL for an advisory. + + Args: + text: Freeform advisory or release-notes text to search. + advisory_type: RHSA/RHBA/RHEA. + live_id: The advisory's live ID. + Returns: + True when a public Errata URL references the advisory. + """ + pattern = rf"{re.escape(PUBLIC_ERRATA_URL)}/{re.escape(advisory_type)}-\d{{4}}:0*{live_id}\b" return re.search(pattern, text) is not None @@ -98,7 +115,7 @@ def extract_advisory_name(text: str, advisory_type: str, live_id: int) -> str | Return Value(s): str | None: the full display name (e.g. "RHSA-2026:51007"), or None if not found. """ - pattern = rf"({re.escape(advisory_type)}-\d{{4}}:{live_id})\b" + pattern = rf"({re.escape(advisory_type)}-\d{{4}}:0*{live_id})\b" match = re.search(pattern, text) return match.group(1) if match else None @@ -138,6 +155,48 @@ def check_extras_references_image( ) +def check_primary_image_references_secondary( + primary_image_release_notes: ReleaseNotes | None, + secondary_image_release_notes: Sequence[ReleaseNotes], +) -> CheckResult: + """ + Check that the principal image advisory references every secondary image advisory. + + Args: + primary_image_release_notes: Release notes for the principal image advisory. + secondary_image_release_notes: Release notes for additional image advisories. + Returns: + CheckResult: Pass, fail, or skip outcome for the cross-reference check. + """ + name = "primary image references secondary images" + if not secondary_image_release_notes: + return CheckResult(name=name, status="pass", detail="no secondary image advisories provided") + if primary_image_release_notes is None: + return CheckResult(name=name, status="fail", detail="principal image advisory not provided") + if primary_image_release_notes.description is None: + return CheckResult(name=name, status="fail", detail="principal image advisory has no description") + + missing = [] + for secondary in secondary_image_release_notes: + if not secondary.live_id or not _contains_public_advisory_reference( + primary_image_release_notes.description, secondary.type, secondary.live_id + ): + display = f"{secondary.type}:{secondary.live_id or 'missing-live-id'}" + missing.append(display) + + if missing: + return CheckResult( + name=name, + status="fail", + detail=f"principal image description does not reference secondary image advisory(s): {', '.join(missing)}", + ) + return CheckResult( + name=name, + status="pass", + detail=f"principal image description references {len(secondary_image_release_notes)} secondary image advisory(s)", + ) + + def check_image_references_rpm(image_release_notes: ReleaseNotes | None, rpm_errata_name: str | None) -> CheckResult: """ Check that the image shipment's description references the rpm advisory. @@ -373,13 +432,57 @@ def load_release_notes(runtime: Runtime, config_path: str) -> ReleaseNotes | Non Return Value(s): ReleaseNotes | None: the parsed release notes, or None if this shipment has none (e.g. FBC). """ - config_raw = runtime.shipment_gitdata.load_yaml_file(config_path) - config = ShipmentConfig.model_validate(config_raw) + config = load_shipment_config(runtime, config_path) if config.shipment.data is None: return None return config.shipment.data.releaseNotes +def load_shipment_config(runtime: Runtime, config_path: str) -> ShipmentConfig: + """ + Load and validate a shipment configuration from the shipment data repository. + + Args: + runtime: Elliott runtime with shipment data access configured. + config_path: Path to the shipment YAML file relative to the repository root. + Returns: + The validated shipment configuration. + """ + config_raw = runtime.shipment_gitdata.load_yaml_file(config_path) + return ShipmentConfig.model_validate(config_raw) + + +def _shipment_kind_from_config_path(config_path: str) -> str: + """ + Extract the image shipment kind from a shipment filename. + + Args: + config_path: Shipment YAML path, such as ``4.18.1.image-el10.123.yaml``. + Returns: + The image kind encoded in the filename, or ``image`` for legacy names. + """ + for part in Path(config_path).stem.split("."): + if re.fullmatch(r"image(?:-el\d+)?", part): + return part + return "image" + + +def _normalize_config_paths(config_paths: str | Sequence[str] | None) -> list[str]: + """ + Normalize one or more CLI configuration paths to a list. + + Args: + config_paths: A single path, an iterable of paths, or None. + Returns: + A list of non-empty paths. + """ + if config_paths is None: + return [] + if isinstance(config_paths, str): + return [config_paths] + return [path for path in config_paths if path] + + def get_advisory_id(runtime: Runtime, kind: str) -> int | None: """ Look up a classic Errata Tool advisory id for this assembly. @@ -446,19 +549,40 @@ def _build_advisory_text(erratum: Erratum) -> str: async def run_checks( - runtime: Runtime, image_config_path: str | None, extras_config_path: str | None + runtime: Runtime, + image_config_path: str | Sequence[str] | None, + extras_config_path: str | None, ) -> list[CheckResult]: """ - Run all six docs-approval checks for one shipment MR. + Run all seven docs-approval checks for one shipment MR. Arg(s): runtime (Runtime): the elliott Runtime, already initialized with shipment data access. - image_config_path (str | None): path to the changed image shipment file, if any. + image_config_path (str | Sequence[str] | None): path(s) to the changed image shipment file(s), if any. extras_config_path (str | None): path to the changed extras shipment file, if any. Return Value(s): list[CheckResult]: one result per check, in a fixed order. """ - image_release_notes = load_release_notes(runtime, image_config_path) if image_config_path else None + image_shipments = { + _shipment_kind_from_config_path(path): load_shipment_config(runtime, path) + for path in _normalize_config_paths(image_config_path) + } + primary_image = select_primary_image_shipment(image_shipments) + primary_image_kind = primary_image[0] if primary_image else None + image_release_notes = ( + primary_image[1].shipment.data.releaseNotes + if primary_image and primary_image[1].shipment.data is not None + else None + ) + secondary_image_release_notes = [ + shipment.shipment.data.releaseNotes + for kind, shipment in image_shipments.items() + if ( + kind != primary_image_kind + and shipment.shipment.data is not None + and shipment.shipment.data.releaseNotes is not None + ) + ] extras_release_notes = load_release_notes(runtime, extras_config_path) if extras_config_path else None rpm_advisory_id = get_advisory_id(runtime, "rpm") @@ -501,6 +625,7 @@ def _dropped(name: str, dropped_name: str) -> CheckResult: return CheckResult(name=name, status="skip", detail=f"advisory {dropped_name} is DROPPED_NO_SHIP") return [ + check_primary_image_references_secondary(image_release_notes, secondary_image_release_notes), check_extras_references_image(extras_release_notes, image_release_notes, image_errata_name), check_image_does_not_reference_dropped_rpm(image_release_notes, rpm_dropped_name) if rpm_dropped_name @@ -543,7 +668,8 @@ def format_report(results: list[CheckResult]) -> str: "image_config_path", metavar="PATH", default=None, - help="Path to the image shipment config for this release, if changed", + multiple=True, + help="Path to an image shipment config for this release; repeat for multiple RHEL variants", ) @click.option( "--extras-config", @@ -556,18 +682,20 @@ def format_report(results: list[CheckResult]) -> str: @click_coroutine async def verify_docs_approval(runtime, image_config_path, extras_config_path): """ - Verify advisory cross-references (extras->image, image->rpm, rpm->image, + Verify advisory cross-references (principal image->secondary images, + extras->image, image->rpm, rpm->image, rhcos->rpm) and per-arch release payload SHA values are correct for a shipment MR, automating the manual Docs approval review. \b Checks performed: - 1. extras (shipment MR) references image advisory - 2. image (shipment MR) references rpm advisory - 3. rpm advisory references image advisory - 4. image shipment payload SHAs match the published release payload - 5. rhcos advisory references rpm advisory - 6. rhcos advisory payload SHAs match image shipment SHAs + 1. principal image (shipment MR) references every secondary image advisory + 2. extras (shipment MR) references principal image advisory + 3. principal image (shipment MR) references rpm advisory + 4. rpm advisory references principal image advisory + 5. image shipment payload SHAs match the published release payload + 6. rhcos advisory references rpm advisory + 7. rhcos advisory payload SHAs match image shipment SHAs \b $ elliott -g openshift-4.20 --assembly=4.20.32 --shipment-path=. \\ diff --git a/elliott/elliottlib/cli/verify_payload.py b/elliott/elliottlib/cli/verify_payload.py index d1d6d9b3da..5a5662a5e0 100644 --- a/elliott/elliottlib/cli/verify_payload.py +++ b/elliott/elliottlib/cli/verify_payload.py @@ -11,7 +11,7 @@ from elliottlib.cli.common import cli, click_coroutine from elliottlib.constants import errata_url from elliottlib.errata import get_advisory_nvrs, get_brew_build, get_raw_erratum -from elliottlib.shipment_utils import get_builds_from_mr +from elliottlib.shipment_utils import get_shipment_config_from_mr from elliottlib.util import get_nvrs_from_release, parse_nvr @@ -165,8 +165,13 @@ async def get_shipment_nvrs(self) -> Dict[str, str]: if not mr_url: raise click.UsageError("Shipment block does not contain a 'url' field for the merge request") - builds_by_kind = get_builds_from_mr(mr_url) - return {parse_nvr(nvr)['name']: nvr for nvr in builds_by_kind['image']} + image_shipment = get_shipment_config_from_mr(mr_url, "image") + if image_shipment is None or image_shipment.shipment.snapshot is None: + raise click.UsageError("Could not find an image shipment config in the merge request") + return { + parse_nvr(nvr)["name"]: nvr + for nvr in image_shipment.shipment.snapshot.nvrs + } async def check_konflux_payload(self): self.all_advisory_nvrs = await self.get_shipment_nvrs() diff --git a/elliott/elliottlib/shipment_utils.py b/elliott/elliottlib/shipment_utils.py index d5e46b8e6c..28ab688cf6 100644 --- a/elliott/elliottlib/shipment_utils.py +++ b/elliott/elliottlib/shipment_utils.py @@ -1,7 +1,7 @@ import logging import re from datetime import datetime -from typing import Dict, Iterable, List, Tuple +from typing import Dict, Iterable, List, Mapping, Tuple from urllib.parse import urlparse from artcommonlib.assembly import assembly_config_struct @@ -20,9 +20,88 @@ # Single source of truth for the public errata URL. -# verify_docs_approval.py defines the same constant; import from here once that module lands. PUBLIC_ERRATA_URL = "https://access.redhat.com/errata" +_IMAGE_SHIPMENT_KIND_PATTERN = re.compile(r"^image(?:-el(?P\d+))?$") + + +def select_primary_image_shipment( + shipments_by_kind: Mapping[str, ShipmentConfig], +) -> tuple[str, ShipmentConfig] | None: + """ + Select the image shipment that represents the principal image advisory. + + The shipment with the most snapshot components is considered principal. If + multiple image shipments contain the same number of components, the one + with the highest RHEL version is selected. + + Args: + shipments_by_kind: Shipment configurations keyed by their filename kind. + Returns: + The principal shipment kind and configuration, or None when no image + shipment is present. + """ + image_shipments = [ + (kind, shipment) + for kind, shipment in shipments_by_kind.items() + if _IMAGE_SHIPMENT_KIND_PATTERN.fullmatch(kind) + ] + if not image_shipments: + return None + + def sort_key(item: tuple[str, ShipmentConfig]) -> tuple[int, int]: + kind, shipment = item + snapshot = shipment.shipment.snapshot + component_count = len(snapshot.spec.components) if snapshot else 0 + match = _IMAGE_SHIPMENT_KIND_PATTERN.fullmatch(kind) + rhel_version = int(match.group("rhel_version")) if match and match.group("rhel_version") else -1 + return component_count, rhel_version + + return max(image_shipments, key=sort_key) + + +def add_secondary_image_advisory_references(shipments_by_kind: Mapping[str, ShipmentConfig]) -> bool: + """ + Add public Errata references for secondary image advisories to the principal advisory. + + Args: + shipments_by_kind: Shipment configurations keyed by their filename kind. + Returns: + True when the principal image description was changed; otherwise False. + """ + primary = select_primary_image_shipment(shipments_by_kind) + if primary is None: + return False + + primary_kind, primary_shipment = primary + primary_release_notes = ( + primary_shipment.shipment.data.releaseNotes + if primary_shipment.shipment.data is not None + else None + ) + if primary_release_notes is None: + return False + + references: list[str] = [] + for kind, shipment in sorted(shipments_by_kind.items()): + if kind == primary_kind or not _IMAGE_SHIPMENT_KIND_PATTERN.fullmatch(kind): + continue + release_notes = shipment.shipment.data.releaseNotes if shipment.shipment.data is not None else None + if release_notes is None or not release_notes.live_id: + continue + advisory_id = get_full_advisory_id_from_shipment(shipment) + advisory_url = f"{PUBLIC_ERRATA_URL}/{advisory_id}" + if advisory_url not in (primary_release_notes.description or ""): + references.append(advisory_url) + + if not references: + return False + + description = primary_release_notes.description or "" + reference_block = "See the following advisory for additional container images:\n\n" + "\n".join(references) + primary_release_notes.description = f"{description.rstrip()}\n\n{reference_block}" if description else reference_block + return True + def strip_advisory_cross_reference(text: str, rpm_name: str) -> str: """ @@ -257,8 +336,16 @@ def _get_shipment_config_kind(filename_stem: str, kinds: Tuple[str, ...]) -> str def get_shipment_config_from_mr(mr_url: str, kind: str) -> ShipmentConfig | None: - """Fetch a specific shipment config from a merge request URL.""" + """ + Fetch a shipment config from a merge request URL. + + The base ``image`` kind resolves to the principal image shipment when the + merge request contains RHEL-qualified image variants. + """ shipment_configs = get_shipment_configs_from_mr(mr_url) + if kind == "image": + primary = select_primary_image_shipment(shipment_configs) + return primary[1] if primary else None return shipment_configs.get(kind) diff --git a/elliott/tests/test_konflux_release_cli.py b/elliott/tests/test_konflux_release_cli.py index 797fa96d19..593c4ee01b 100644 --- a/elliott/tests/test_konflux_release_cli.py +++ b/elliott/tests/test_konflux_release_cli.py @@ -223,6 +223,34 @@ def test_object_name_includes_rhel_suffix_from_shipment_filename(self, mock_konf self.assertEqual(cli.get_object_name(), "ocp-stage-rc-1-microshift-bootc-el10-20260909154630") + @patch("elliottlib.cli.konflux_release_cli.get_utc_now_formatted_str", return_value="20260909154630") + @patch("doozerlib.backend.konflux_client.KonfluxClient.from_kubeconfig") + def test_object_name_includes_rhel_suffix_for_image_extras_and_metadata( + self, mock_konflux_client_init, _mock_timestamp + ): + mock_konflux_client_init.return_value = self.konflux_client + self.runtime.assembly = "rc.1" + + for kind, rhel_version in (("image", "9"), ("extras", "10"), ("metadata", "9")): + with self.subTest(kind=kind, rhel_version=rhel_version): + cli = CreateReleaseCli( + runtime=self.runtime, + config_path=( + f"shipment/ocp/openshift-5.0/openshift-5-0/stage/rc.1." + f"{kind}-el{rhel_version}.20260904133832.yaml" + ), + release_env="stage", + konflux_config=self.konflux_config, + image_repo_pull_secret=self.image_repo_pull_secret, + dry_run=self.dry_run, + kind=kind, + ) + + self.assertEqual( + cli.get_object_name(), + f"ocp-stage-rc-1-{kind}-el{rhel_version}-20260909154630", + ) + @patch("doozerlib.backend.konflux_client.KonfluxClient.from_kubeconfig") async def test_release_rejects_invalid_snapshot_reference(self, mock_konflux_client_init): mock_konflux_client_init.return_value = self.konflux_client @@ -771,6 +799,18 @@ async def test_validate_rpa_success(self, mock_fetch_rpa): mock_fetch_rpa.assert_any_await("ocp-art-advisory-prod-4-18") mock_fetch_rpa.assert_any_await("ocp-art-advisory-stage-4-18") + @patch("elliottlib.cli.konflux_release_cli.fetch_rpa", new_callable=AsyncMock) + async def test_validate_rpa_accepts_rhel_qualified_image_extras_and_metadata(self, mock_fetch_rpa): + """RPA validation uses base kinds for RHEL-qualified shipment configs.""" + rpa_data = {"spec": {"data": {"mapping": {"components": [{"name": "comp1"}]}}}} + mock_fetch_rpa.return_value = rpa_data + + for kind in ("image-el9", "extras-el10", "metadata-el9"): + with self.subTest(kind=kind): + await validate_snapshot_against_rpa("openshift-4.18", "prod", kind, ["comp1"]) + + self.assertEqual(mock_fetch_rpa.await_count, 6) + @patch("elliottlib.cli.konflux_release_cli.fetch_rpa", new_callable=AsyncMock) async def test_validate_rpa_missing_components(self, mock_fetch_rpa): rpa_data = {"spec": {"data": {"mapping": {"components": [{"name": "test-rpm"}]}}}} diff --git a/elliott/tests/test_shipment_utils.py b/elliott/tests/test_shipment_utils.py index 0edd70fa82..9ced7ba2e8 100644 --- a/elliott/tests/test_shipment_utils.py +++ b/elliott/tests/test_shipment_utils.py @@ -6,13 +6,18 @@ from artcommonlib.model import Model from elliottlib import shipment_utils from elliottlib.shipment_model import ( + ComponentSource, Data, Environments, + GitSource, Metadata, ReleaseNotes, Shipment, ShipmentConfig, ShipmentEnv, + Snapshot, + SnapshotComponent, + SnapshotSpec, ) @@ -189,6 +194,19 @@ def test_get_builds_from_mr_success(self, mock_get_configs): # Verify the underlying function was called correctly mock_get_configs.assert_called_once_with(self.test_mr_url) + @patch("elliottlib.shipment_utils.get_shipment_configs_from_mr") + def test_get_shipment_config_image_returns_principal_rhel_variant(self, mock_get_configs): + """The generic image lookup selects the principal RHEL-qualified image shipment.""" + image_el9 = Mock() + image_el9.shipment.snapshot.spec.components = ["one"] + image_el10 = Mock() + image_el10.shipment.snapshot.spec.components = ["one", "two"] + mock_get_configs.return_value = {"image-el9": image_el9, "image-el10": image_el10} + + result = shipment_utils.get_shipment_config_from_mr(self.test_mr_url, "image") + + self.assertIs(result, image_el10) + def test_default_kinds_parameter(self): """Test that default kinds parameter works correctly""" with patch('artcommonlib.gitlab.gitlab.Gitlab') as mock_gitlab_class: @@ -319,8 +337,12 @@ def test_get_shipment_configs_preserves_rhel_qualified_kinds(self, mock_gitlab_c self.mock_mr.diffs.list.return_value = [self.mock_diff_info] self.mock_mr.diffs.get.return_value = self.mock_diff self.mock_diff.diffs = [ - {'new_path': 'microshift-bootc-el9.yaml', 'old_path': None}, - {'new_path': 'microshift-bootc-el10.yaml', 'old_path': None}, + {"new_path": "image-el9.yaml", "old_path": None}, + {"new_path": "image-el10.yaml", "old_path": None}, + {"new_path": "extras-el9.yaml", "old_path": None}, + {"new_path": "metadata-el10.yaml", "old_path": None}, + {"new_path": "microshift-bootc-el9.yaml", "old_path": None}, + {"new_path": "microshift-bootc-el10.yaml", "old_path": None}, ] self.mock_file_content.decode.return_value.decode.return_value = self.sample_yaml_content @@ -328,7 +350,10 @@ def test_get_shipment_configs_preserves_rhel_qualified_kinds(self, mock_gitlab_c result = shipment_utils.get_shipment_configs_from_mr(self.test_mr_url) - self.assertEqual(set(result), {'microshift-bootc-el9', 'microshift-bootc-el10'}) + self.assertEqual( + set(result), + {"image-el9", "image-el10", "extras-el9", "metadata-el10", "microshift-bootc-el9", "microshift-bootc-el10"}, + ) class TestGroupFiltering(unittest.TestCase): @@ -828,6 +853,67 @@ def test_formats_type_year_and_padded_id(self): ) +def _make_image_shipment_config(kind: str, component_count: int, live_id: int) -> ShipmentConfig: + """Build an image shipment with a controlled component count and advisory ID.""" + application = f"app-{kind}" + components = [ + SnapshotComponent( + name=f"image-{index}", + containerImage=f"quay.io/example/image-{index}:latest", + source=ComponentSource( + git=GitSource(url="https://github.com/example/image.git", revision="revision") + ), + ) + for index in range(component_count) + ] + return ShipmentConfig( + shipment=Shipment( + metadata=Metadata( + product="ocp", + application=application, + group="openshift-4.18", + assembly="4.18.1", + ), + environments=Environments( + stage=ShipmentEnv(releasePlan=f"rp-{kind}-stage"), + prod=ShipmentEnv(releasePlan=f"rp-{kind}-prod"), + ), + snapshot=Snapshot( + nvrs=[f"{kind}-nvr"], + spec=SnapshotSpec(application=application, components=components), + ), + data=Data(releaseNotes=ReleaseNotes(type="RHBA", live_id=live_id, description="Primary image advisory.")), + ) + ) + + +class TestPrimaryImageShipment(unittest.TestCase): + def test_selects_highest_rhel_version_when_image_counts_are_equal(self): + shipments = { + "image-el9": _make_image_shipment_config("image-el9", component_count=3, live_id=100), + "image-el10": _make_image_shipment_config("image-el10", component_count=3, live_id=101), + } + + kind, _ = shipment_utils.select_primary_image_shipment(shipments) + + self.assertEqual(kind, "image-el10") + + def test_adds_secondary_advisory_reference_to_primary_description(self): + shipments = { + "image-el9": _make_image_shipment_config("image-el9", component_count=2, live_id=100), + "image-el10": _make_image_shipment_config("image-el10", component_count=3, live_id=101), + } + + changed = shipment_utils.add_secondary_image_advisory_references(shipments) + + self.assertTrue(changed) + description = shipments["image-el10"].shipment.data.releaseNotes.description + self.assertIn("https://access.redhat.com/errata/RHBA-", description) + self.assertIn(":0100", description) + self.assertNotIn(":0101", description) + self.assertFalse(shipment_utils.add_secondary_image_advisory_references(shipments)) + + class TestStripAdvisoryCrossReference(unittest.TestCase): RPM = "RHBA-2026:44227" URL = f"https://access.redhat.com/errata/{RPM}" diff --git a/elliott/tests/test_verify_docs_approval.py b/elliott/tests/test_verify_docs_approval.py index fce7d8d27b..10675f0ec3 100644 --- a/elliott/tests/test_verify_docs_approval.py +++ b/elliott/tests/test_verify_docs_approval.py @@ -15,6 +15,7 @@ check_image_does_not_reference_dropped_rpm, check_image_references_rpm, check_payload_shas, + check_primary_image_references_secondary, check_rhcos_does_not_reference_dropped_rpm, check_rhcos_payload_shas, check_rhcos_references_rpm, @@ -64,6 +65,11 @@ def test_contains_advisory_reference_match_any_year(): assert contains_advisory_reference(text, "RHSA", 48676) is True +def test_contains_advisory_reference_accepts_zero_padded_live_id(): + text = "See https://access.redhat.com/errata/RHSA-2026:0100 for details." + assert contains_advisory_reference(text, "RHSA", 100) is True + + def test_contains_advisory_reference_wrong_live_id(): text = "See https://access.redhat.com/errata/RHSA-2026:11111 for details." assert contains_advisory_reference(text, "RHSA", 48676) is False @@ -150,6 +156,51 @@ def test_check_extras_references_image_skip_when_image_live_id_is_none(): assert result.status == "skip" +def test_check_primary_image_references_secondary_pass(): + primary = _release_notes( + "RHBA", + 48676, + description="See https://access.redhat.com/errata/RHBA-2026:48677 for additional images.", + ) + secondary = _release_notes("RHBA", 48677) + + result = check_primary_image_references_secondary(primary, [secondary]) + + assert result.status == "pass" + + +def test_check_primary_image_references_zero_padded_secondary_live_id(): + primary = _release_notes( + "RHBA", + 101, + description="See https://access.redhat.com/errata/RHBA-2026:0100 for additional images.", + ) + secondary = _release_notes("RHBA", 100) + + result = check_primary_image_references_secondary(primary, [secondary]) + + assert result.status == "pass" + + +def test_check_primary_image_references_secondary_requires_public_url(): + primary = _release_notes("RHBA", 101, description="See RHBA-2026:100 for additional images.") + secondary = _release_notes("RHBA", 100) + + result = check_primary_image_references_secondary(primary, [secondary]) + + assert result.status == "fail" + + +def test_check_primary_image_references_secondary_fails_when_reference_is_missing(): + primary = _release_notes("RHBA", 48676, description="No secondary advisory reference.") + secondary = _release_notes("RHBA", 48677) + + result = check_primary_image_references_secondary(primary, [secondary]) + + assert result.status == "fail" + assert "48677" in result.detail + + def test_check_rpm_references_image_skip_when_image_live_id_is_none(): image = ReleaseNotes(type="RHSA", live_id=48676) image.live_id = None # Force live_id to None @@ -479,9 +530,50 @@ async def test_run_checks_all_pass(self, mock_fetch, mock_erratum_cls, mock_asse results = await run_checks(runtime, "image.yaml", "extras.yaml") - assert len(results) == 6 + assert len(results) == 7 assert all(r.status == "pass" for r in results), results + @patch("elliottlib.cli.verify_docs_approval.get_advisory_id", return_value=None) + @patch("elliottlib.cli.verify_docs_approval.check_payload_shas", new_callable=AsyncMock) + async def test_run_checks_selects_principal_and_validates_secondary_image( + self, mock_check_payload_shas, _mock_get_advisory_id + ): + """The verify command selects the highest RHEL variant when image sizes tie.""" + image_config_raw = { + "shipment": { + "metadata": {"product": "ocp", "application": "app", "group": "openshift-4.20", "assembly": "4.20.32"}, + "environments": {"stage": {"releasePlan": "p"}, "prod": {"releasePlan": "p"}}, + "data": {"releaseNotes": {"type": "RHBA", "live_id": 100, "description": "Secondary image."}}, + } + } + principal_config_raw = { + "shipment": { + "metadata": {"product": "ocp", "application": "app", "group": "openshift-4.20", "assembly": "4.20.32"}, + "environments": {"stage": {"releasePlan": "p"}, "prod": {"releasePlan": "p"}}, + "data": { + "releaseNotes": { + "type": "RHBA", + "live_id": 101, + "description": "See https://access.redhat.com/errata/RHBA-2026:0100", + } + }, + } + } + runtime = self._make_runtime() + runtime.shipment_gitdata.load_yaml_file.side_effect = lambda path: { + "image-el9.yaml": image_config_raw, + "image-el10.yaml": principal_config_raw, + }[path] + mock_check_payload_shas.return_value = ( + CheckResult(name="image payload SHAs", status="skip", detail="not checked"), + {}, + ) + + results = await run_checks(runtime, ("image-el9.yaml", "image-el10.yaml"), None) + + assert results[0].status == "pass" + assert "secondary image advisory" in results[0].detail + if __name__ == "__main__": unittest.main() diff --git a/elliott/tests/test_verify_payload.py b/elliott/tests/test_verify_payload.py new file mode 100644 index 0000000000..641331a7eb --- /dev/null +++ b/elliott/tests/test_verify_payload.py @@ -0,0 +1,29 @@ +""" +Tests for Konflux payload verification shipment selection. +""" + +import unittest +from unittest.mock import MagicMock, patch + +from elliottlib.cli.verify_payload import VerifyPayloadPipeline + + +class TestVerifyPayloadPipeline(unittest.IsolatedAsyncioTestCase): + @patch("elliottlib.cli.verify_payload.get_shipment_config_from_mr") + async def test_get_shipment_nvrs_uses_principal_image_shipment(self, mock_get_shipment_config): + """Payload verification reads builds from the selected principal image shipment.""" + image_shipment = MagicMock() + image_shipment.shipment.snapshot.nvrs = ["test-container-v1.0.0-202312010000.p0.git12345"] + mock_get_shipment_config.return_value = image_shipment + + pipeline = VerifyPayloadPipeline(MagicMock(), "quay.io/example/release:4.20.1-x86_64") + pipeline.assembly_group_config = { + "shipment": {"url": "https://gitlab.example.com/project/-/merge_requests/1"} + } + + result = await pipeline.get_shipment_nvrs() + + self.assertEqual(result, {"test-container": "test-container-v1.0.0-202312010000.p0.git12345"}) + mock_get_shipment_config.assert_called_once_with( + "https://gitlab.example.com/project/-/merge_requests/1", "image" + ) diff --git a/pyartcd/pyartcd/pipelines/prepare_release_konflux.py b/pyartcd/pyartcd/pipelines/prepare_release_konflux.py index fb4b6b2ea3..a6d67bdd5f 100644 --- a/pyartcd/pyartcd/pipelines/prepare_release_konflux.py +++ b/pyartcd/pyartcd/pipelines/prepare_release_konflux.py @@ -53,9 +53,11 @@ from elliottlib.errata_async import AsyncErrataAPI from elliottlib.shipment_model import Issue, ReleaseNotes, ShipmentConfig, Snapshot, SnapshotSpec, Tools from elliottlib.shipment_utils import ( + add_secondary_image_advisory_references, get_full_advisory_id_from_shipment, get_shipment_configs_from_mr, patch_et_advisory_text, + select_primary_image_shipment, set_jira_bug_ids, ) from tenacity import retry, stop_after_attempt, wait_fixed @@ -76,6 +78,34 @@ yaml = new_roundtrip_yaml_handler() +def _get_base_shipment_kind(kind: str) -> str: + """ + Remove an optional RHEL suffix from a shipment kind. + + Args: + kind: Shipment kind, such as ``image-el9`` or ``metadata``. + Returns: + The base shipment kind used by build and advisory lookup commands. + """ + return re.sub(r"-el\d+$", "", kind) + + +def _get_shipment_builds(kind: str, shipment: ShipmentConfig, kind_to_builds: Dict[str, List[str]]) -> List[str]: + """ + Select builds for a shipment, preserving RHEL-qualified snapshot membership. + + Args: + kind: Shipment kind, optionally qualified by a RHEL version. + shipment: Shipment configuration that may contain an existing snapshot. + kind_to_builds: Builds grouped by base shipment kind. + Returns: + NVRs to use when creating the shipment snapshot. + """ + if re.search(r"-el\d+$", kind) and shipment.shipment.snapshot and shipment.shipment.snapshot.nvrs: + return shipment.shipment.snapshot.nvrs + return kind_to_builds[_get_base_shipment_kind(kind)] + + class PrepareReleaseKonfluxPipeline: def __init__( self, @@ -507,7 +537,7 @@ async def sweep_bugs(self, impetus_advisories: dict, shipment_data: Optional[tup for kind, shipment in shipments_by_kind.items(): if kind == "fbc": continue - bug_ids = bugs_by_kind.get(kind, []) + bug_ids = bugs_by_kind.get(_get_base_shipment_kind(kind), []) set_jira_bug_ids(shipment.shipment.data.releaseNotes, bug_ids) await self.update_shipment_mr(shipments_by_kind, env, shipment_url) @@ -634,7 +664,9 @@ async def prepare_shipment_builds(self) -> Optional[tuple]: # make sure that metadata shipment needs to be prepared # if so, build any missing bundle builds - if "metadata" in shipments_by_kind and kind_to_builds["olm_builds_not_found"]: + if any(_get_base_shipment_kind(kind) == "metadata" for kind in shipments_by_kind) and kind_to_builds[ + "olm_builds_not_found" + ]: bundle_nvrs, bundle_errors = await self.find_or_build_bundle_builds(kind_to_builds["olm_builds_not_found"]) kind_to_builds["metadata"] += bundle_nvrs if bundle_errors: @@ -670,7 +702,7 @@ async def prepare_shipment_builds(self) -> Optional[tuple]: # prepare snapshot from the found builds for kind, shipment in shipments_by_kind.items(): - shipment.shipment.snapshot = await self.get_snapshot(kind_to_builds[kind]) + shipment.shipment.snapshot = await self.get_snapshot(_get_shipment_builds(kind, shipment, kind_to_builds)) # Validate snapshot components against RPA before finalizing for kind, shipment in shipments_by_kind.items(): @@ -732,13 +764,13 @@ async def resolve_advisory_placeholders(self, impetus_advisories: dict, shipment image_advisory_id = None if shipment_data: shipments_by_kind, _, _ = shipment_data - image_shipment = (shipments_by_kind or {}).get("image") + primary_image = select_primary_image_shipment(shipments_by_kind or {}) if ( - image_shipment - and image_shipment.shipment.data - and isinstance(image_shipment.shipment.data.releaseNotes.live_id, int) + primary_image + and primary_image[1].shipment.data + and isinstance(primary_image[1].shipment.data.releaseNotes.live_id, int) ): - image_advisory_id = get_full_advisory_id_from_shipment(image_shipment) + image_advisory_id = get_full_advisory_id_from_shipment(primary_image[1]) if not rpm_advisory_id and not image_advisory_id: return @@ -826,6 +858,7 @@ async def _resolve_shipment_mr_placeholders( return modified: dict = {} + secondary_references_added = add_secondary_image_advisory_references(shipments_by_kind or {}) for kind, shipment in (shipments_by_kind or {}).items(): if kind == "fbc": continue @@ -846,6 +879,11 @@ async def _resolve_shipment_mr_placeholders( if changed: modified[kind] = shipment + if secondary_references_added: + primary_image = select_primary_image_shipment(shipments_by_kind or {}) + if primary_image: + modified[primary_image[0]] = primary_image[1] + if not modified: self.logger.info("No advisory ID placeholders found in shipment MR YAML, skipping") return @@ -1349,7 +1387,7 @@ async def attach_cve_flaws(self, kind: str, shipment: ShipmentConfig): attach_cve_flaws_command = self._elliott_base_command + [ 'attach-cve-flaws', - f'--use-default-advisory={kind}', + f'--use-default-advisory={_get_base_shipment_kind(kind)}', '--reconcile', '--output=yaml', ] diff --git a/pyartcd/pyartcd/pipelines/promote.py b/pyartcd/pyartcd/pipelines/promote.py index 4441a82b4b..a95804be81 100644 --- a/pyartcd/pyartcd/pipelines/promote.py +++ b/pyartcd/pyartcd/pipelines/promote.py @@ -48,6 +48,7 @@ get_shipment_config_from_mr, get_shipment_configs_from_mr, patch_et_advisory_text, + select_primary_image_shipment, strip_advisory_cross_reference, strip_et_advisory_rpm_reference, ) @@ -2794,7 +2795,8 @@ async def _prepare_shipment_templates( format_dict = {} # Add SHA digests for image shipments - if shipment_kind == "image": + primary_image = select_primary_image_shipment(shipments_by_kind) + if primary_image and shipment_kind == primary_image[0]: for arch, sha in payload_shas.items(): if arch == "multi": continue # Skip multi-arch as it's not a specific architecture @@ -2855,7 +2857,7 @@ async def _prepare_shipment_templates( self._logger.info("Found template placeholders in %s shipment: %s", shipment_kind, placeholders_found) # Validate template placeholders in both description and solution fields (for image shipments) - if shipment_kind == "image": + if primary_image and shipment_kind == primary_image[0]: # Check solution field for SHA digest placeholders if hasattr(shipment_config.shipment.data.releaseNotes, 'solution'): solution_text = shipment_config.shipment.data.releaseNotes.solution diff --git a/pyartcd/tests/pipelines/test_prepare_release_konflux.py b/pyartcd/tests/pipelines/test_prepare_release_konflux.py index 708a2b341a..0ff52674ad 100644 --- a/pyartcd/tests/pipelines/test_prepare_release_konflux.py +++ b/pyartcd/tests/pipelines/test_prepare_release_konflux.py @@ -27,13 +27,67 @@ SnapshotSpec, ) from pyartcd.git import GitRepository -from pyartcd.pipelines.prepare_release_konflux import PrepareReleaseKonfluxPipeline +from pyartcd.pipelines.prepare_release_konflux import PrepareReleaseKonfluxPipeline, _get_shipment_builds from pyartcd.runtime import Runtime from pyartcd.slack import SlackClient from pyartcd import constants +def _make_image_shipment( + kind: str, + component_count: int, + live_id: int, + description: str, + group: str, + assembly: str, +) -> ShipmentConfig: + """ + Build an image shipment with a controlled snapshot size and release notes. + + Args: + kind: Kind used to identify the image shipment. + component_count: Number of components in the snapshot. + live_id: Errata live ID in the release notes. + description: Image advisory description. + group: Build-data group for the shipment. + assembly: Release assembly for the shipment. + Returns: + A validated image shipment configuration. + """ + application = f"app-{kind}" + components = [ + SnapshotComponent( + name=f"image-{index}", + containerImage=f"quay.io/example/image-{index}:latest", + source=ComponentSource( + git=GitSource(url="https://github.com/example/image.git", revision="revision") + ), + ) + for index in range(component_count) + ] + return ShipmentConfig( + shipment=Shipment( + metadata=Metadata(product="ocp", group=group, assembly=assembly, application=application), + environments=Environments( + stage=ShipmentEnv(releasePlan=f"rp-{kind}-stage"), + prod=ShipmentEnv(releasePlan=f"rp-{kind}-prod"), + ), + snapshot=Snapshot( + nvrs=[f"{kind}-nvr"], + spec=SnapshotSpec(application=application, components=components), + ), + data=Data( + releaseNotes=ReleaseNotes( + type="RHBA", + live_id=live_id, + description=description, + ) + ), + ) + ) + + class TestPrepareReleaseKonfluxPipeline(unittest.IsolatedAsyncioTestCase): def setUp(self): self.runtime = Mock(spec=Runtime) @@ -56,6 +110,21 @@ def setUp(self): self.gitlab_token = "gl_token" self.job_url = "http://jenkins/job/test-job/1" + def test_get_shipment_builds_preserves_rhel_qualified_snapshot_membership(self): + """Qualified shipments keep their variant-specific builds during re-preparation.""" + shipment = _make_image_shipment( + "image-el9", + component_count=1, + live_id=100, + description="Image advisory.", + group=self.group, + assembly=self.assembly, + ) + + builds = _get_shipment_builds("image-el9", shipment, {"image": ["new-image-nvr"]}) + + self.assertEqual(builds, ["image-el9-nvr"]) + def test_init(self): pipeline = PrepareReleaseKonfluxPipeline( slack_client=self.mock_slack_client, @@ -1138,6 +1207,52 @@ async def test_resolve_advisory_placeholders_patches_classic_advisories(self, mo ) rpm_advisory.commit.assert_called_once() + async def test_resolve_advisory_placeholders_updates_principal_image_with_secondary_reference(self): + """The highest RHEL image shipment receives secondary advisory references.""" + pipeline = PrepareReleaseKonfluxPipeline( + slack_client=self.mock_slack_client, + runtime=self.runtime, + group=self.group, + assembly=self.assembly, + ) + pipeline.logger = Mock() + pipeline.update_shipment_mr = AsyncMock() + + shipments = { + "image-el9": _make_image_shipment( + "image-el9", + component_count=2, + live_id=100, + description="Secondary image advisory.", + group=self.group, + assembly=self.assembly, + ), + "image-el10": _make_image_shipment( + "image-el10", + component_count=2, + live_id=101, + description="See {IMAGE_ADVISORY}", + group=self.group, + assembly=self.assembly, + ), + } + + await pipeline._resolve_shipment_mr_placeholders( + shipments, + "prod", + "https://gitlab.example.com/x/-/merge_requests/1", + {"IMAGE_ADVISORY": "RHBA-2026:0101"}, + ) + + description = shipments["image-el10"].shipment.data.releaseNotes.description + self.assertIn("RHBA-2026:0101", description) + self.assertIn("https://access.redhat.com/errata/RHBA-2026:0100", description) + pipeline.update_shipment_mr.assert_awaited_once_with( + {"image-el10": shipments["image-el10"]}, + "prod", + "https://gitlab.example.com/x/-/merge_requests/1", + ) + @patch("elliottlib.shipment_utils.Erratum") @patch("pyartcd.pipelines.prepare_release_konflux.get_errata_live_id") async def test_resolve_advisory_placeholders_commit_failure_soft_fails(self, mock_get_live_id, mock_erratum_cls): From 46b716d9c43369ad2680de01b21b7a5173df57f2 Mon Sep 17 00:00:00 2001 From: Fabio Gallotti Date: Fri, 11 Sep 2026 14:51:47 +0200 Subject: [PATCH 5/5] Complete RHEL-qualified shipment workflow rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- artcommon/artcommonlib/constants.py | 3 + doozer/doozerlib/cli/release_gen_assembly.py | 31 ++++- doozer/tests/cli/test_gen_assembly.py | 69 ++++++++++ .../elliottlib/cli/attach_cve_flaws_cli.py | 14 +- elliott/elliottlib/cli/find_bugs_sweep_cli.py | 96 +++++++++++++- elliott/elliottlib/cli/shipment_cli.py | 53 ++++++-- elliott/elliottlib/cli/verify_payload.py | 5 +- elliott/elliottlib/shipment_utils.py | 65 ++++++++-- elliott/tests/test_find_bugs_sweep_cli.py | 88 +++++++++++++ elliott/tests/test_shipment_cli.py | 40 ++++++ elliott/tests/test_shipment_utils.py | 4 +- elliott/tests/test_verify_payload.py | 4 +- .../pipelines/prepare_release_konflux.py | 104 ++++++++++----- pyartcd/pyartcd/pipelines/promote.py | 6 +- pyartcd/pyartcd/shipment_utils.py | 21 +++ .../pipelines/test_prepare_release_konflux.py | 120 +++++++++++++++++- pyartcd/tests/test_shipment_utils.py | 40 +++++- 17 files changed, 681 insertions(+), 82 deletions(-) diff --git a/artcommon/artcommonlib/constants.py b/artcommon/artcommonlib/constants.py index 209ffcf8a2..32fb021589 100644 --- a/artcommon/artcommonlib/constants.py +++ b/artcommon/artcommonlib/constants.py @@ -60,6 +60,9 @@ REDHAT_GITLAB_URL = "https://gitlab.cee.redhat.com" SHIPMENT_DATA_URL_TEMPLATE = "https://gitlab.cee.redhat.com/hybrid-platforms/art/ocp-shipment-data" SHIPMENT_CONFIG_KINDS = ("image", "extras", "metadata", "fbc", "microshift-bootc") +SHIPMENT_CONFIG_KINDS_WITH_COMPOUNDS = SHIPMENT_CONFIG_KINDS + tuple( + f"{kind}-el{rhel_version}" for kind in SHIPMENT_CONFIG_KINDS if kind != "fbc" for rhel_version in (8, 9, 10) +) # Redis related vars REDIS_HOST = 'master.redis.gwprhd.use1.cache.amazonaws.com' diff --git a/doozer/doozerlib/cli/release_gen_assembly.py b/doozer/doozerlib/cli/release_gen_assembly.py index a93c590899..53adc39698 100644 --- a/doozer/doozerlib/cli/release_gen_assembly.py +++ b/doozer/doozerlib/cli/release_gen_assembly.py @@ -947,11 +947,16 @@ def _get_default_shipment(self, env: str = None) -> dict: if env and env not in ['stage', 'prod']: raise ValueError(f"Invalid environment: {env}") + shipment_kinds = ['image', 'extras', 'metadata'] + rhel_versions = self._get_rhel_versions() + if len(rhel_versions) > 1: + shipment_kinds = [ + f'{kind}-el{rhel_version}' for kind in shipment_kinds for rhel_version in sorted(rhel_versions) + ] + default_shipment = { 'advisories': [ - {'kind': 'image'}, - {'kind': 'extras'}, - {'kind': 'metadata'}, + *({'kind': kind} for kind in shipment_kinds), {'kind': 'fbc'}, ], } @@ -959,6 +964,26 @@ def _get_default_shipment(self, env: str = None) -> dict: default_shipment['env'] = env return default_shipment + def _get_rhel_versions(self) -> Set[int]: + """ + Get the RHEL versions represented by the selected Konflux image builds. + + Return Value(s): + Set[int]: The detected RHEL major versions. + """ + if self.runtime.build_system != 'konflux': + return set() + + rhel_versions = set() + for build_inspector in self.component_image_builds.values(): + release = build_inspector.get_release() + if not release: + continue + rhel_version = isolate_el_version_in_release(release) + if rhel_version is not None: + rhel_versions.add(rhel_version) + return rhel_versions + def _get_previous_shipment_info(self) -> dict: """ if this assembly is (e|r)c.X, then check if there is a previously defined (e|r)c.X-1 diff --git a/doozer/tests/cli/test_gen_assembly.py b/doozer/tests/cli/test_gen_assembly.py index 2c2dd39945..191524d960 100644 --- a/doozer/tests/cli/test_gen_assembly.py +++ b/doozer/tests/cli/test_gen_assembly.py @@ -376,6 +376,75 @@ def test_get_shipment_info(self): } self.assertEqual(expected, shipment) + def test_get_shipment_info_mixed_rhel_versions_uses_detected_compound_kinds(self): + """Mixed image RHEL streams generate one shipment reference per stream.""" + runtime = MagicMock(build_system='konflux') + runtime.get_releases_config.return_value = Model({'releases': {}}) + gacli = GenAssemblyCli(runtime=runtime, gen_assembly_name='4.23.52') + for index, release in enumerate(('1.el9', '2.el8', '3.el9')): + build = MagicMock() + build.get_release.return_value = release + gacli.component_image_builds[f'image-{index}'] = build + + shipment = gacli._get_shipment_info() + + self.assertEqual( + shipment, + { + 'advisories': [ + {'kind': 'image-el8'}, + {'kind': 'image-el9'}, + {'kind': 'extras-el8'}, + {'kind': 'extras-el9'}, + {'kind': 'metadata-el8'}, + {'kind': 'metadata-el9'}, + {'kind': 'fbc'}, + ], + }, + ) + + def test_get_shipment_info_detects_rhel_versions_without_allowlist(self): + """The detected stream names are not limited to a hardcoded RHEL version list.""" + runtime = MagicMock(build_system='konflux') + runtime.get_releases_config.return_value = Model({'releases': {}}) + gacli = GenAssemblyCli(runtime=runtime, gen_assembly_name='4.23.1') + for index, release in enumerate(('1.el10', '2.el9')): + build = MagicMock() + build.get_release.return_value = release + gacli.component_image_builds[f'image-{index}'] = build + + self.assertEqual( + gacli._get_shipment_info()['advisories'], + [ + {'kind': 'image-el9'}, + {'kind': 'image-el10'}, + {'kind': 'extras-el9'}, + {'kind': 'extras-el10'}, + {'kind': 'metadata-el9'}, + {'kind': 'metadata-el10'}, + {'kind': 'fbc'}, + ], + ) + + def test_get_shipment_info_single_rhel_version_keeps_plain_kinds(self): + """A single detected RHEL stream does not require compound shipment kinds.""" + runtime = MagicMock(build_system='konflux') + runtime.get_releases_config.return_value = Model({'releases': {}}) + gacli = GenAssemblyCli(runtime=runtime, gen_assembly_name='4.23.1') + build = MagicMock() + build.get_release.return_value = '1.el9' + gacli.component_image_builds['image'] = build + + self.assertEqual( + gacli._get_shipment_info()['advisories'], + [ + {'kind': 'image'}, + {'kind': 'extras'}, + {'kind': 'metadata'}, + {'kind': 'fbc'}, + ], + ) + def test_get_shipment_info_ec0(self): runtime = MagicMock(build_system='konflux') runtime.get_releases_config.return_value = Model({'releases': {}}) diff --git a/elliott/elliottlib/cli/attach_cve_flaws_cli.py b/elliott/elliottlib/cli/attach_cve_flaws_cli.py index 983dca2e4e..ebf67b1b7b 100644 --- a/elliott/elliottlib/cli/attach_cve_flaws_cli.py +++ b/elliott/elliottlib/cli/attach_cve_flaws_cli.py @@ -23,12 +23,12 @@ is_rhcos_pscomponent, sort_cve_bugs, ) -from elliottlib.cli.common import cli, click_coroutine, find_default_advisory, use_default_advisory_option +from elliottlib.cli.common import cli, click_coroutine, find_default_advisory from elliottlib.errata import is_security_advisory from elliottlib.errata_async import AsyncErrataAPI, AsyncErrataUtils from elliottlib.runtime import Runtime from elliottlib.shipment_model import CveAssociation, ReleaseNotes -from elliottlib.shipment_utils import get_shipment_config_from_mr, set_bugzilla_bug_ids +from elliottlib.shipment_utils import get_base_shipment_kind, get_shipment_config_from_mr, set_bugzilla_bug_ids from elliottlib.util import ( get_advisory_boilerplate, get_component_by_delivery_repo, @@ -195,7 +195,7 @@ def __init__( self.errata_config = self.runtime.get_errata_config() if default_advisory_type: - self.advisory_kind = default_advisory_type + self.advisory_kind = get_base_shipment_kind(default_advisory_type) elif advisory_id: self.advisory_kind = next( (k for k, v in self.runtime.group_config.advisories.items() if v == self.advisory_id), None @@ -698,7 +698,13 @@ def get_updated_advisory_rhsa(self, cve_boilerplate: dict, advisory: Erratum, fl is_flag=True, help="Print what would change, but don't change anything", ) -@use_default_advisory_option +@click.option( + "--use-default-advisory", + "default_advisory_type", + metavar="ADVISORY_TYPE", + type=click.STRING, + help="Use the default value from [group|releases].yml for ADVISORY_TYPE.", +) @click.option( "--into-default-advisories", is_flag=True, help='Run for all advisories values defined in [group|releases].yml' ) diff --git a/elliott/elliottlib/cli/find_bugs_sweep_cli.py b/elliott/elliottlib/cli/find_bugs_sweep_cli.py index ddd3e4a94c..fa67e8daf4 100644 --- a/elliott/elliottlib/cli/find_bugs_sweep_cli.py +++ b/elliott/elliottlib/cli/find_bugs_sweep_cli.py @@ -1,4 +1,5 @@ import json +import re import sys from datetime import datetime, timezone from typing import Dict, List, Optional, Set @@ -29,6 +30,76 @@ type_bug_set = Set[Bug] yaml = new_roundtrip_yaml_handler() +_COMPOUND_SHIPMENT_KIND_PATTERN = re.compile(r"^(?P.+)-el\d+$") + + +def _get_base_shipment_kind(kind: str) -> str: + """ + Return the unqualified shipment kind for a base or RHEL-qualified kind. + + Args: + kind: Shipment kind such as ``image`` or ``image-el9``. + Returns: + The base shipment kind. + """ + match = _COMPOUND_SHIPMENT_KIND_PATTERN.fullmatch(kind) + return match.group("base") if match else kind + + +def _get_compound_kinds(builds_by_advisory_kind: Dict[str, List[str]], base_kind: str) -> list[str]: + """ + Find configured RHEL-qualified variants for a base shipment kind. + + Args: + builds_by_advisory_kind: Build NVRs keyed by shipment kind. + base_kind: Unqualified shipment kind. + Returns: + Sorted RHEL-qualified shipment kinds. + """ + return sorted( + kind for kind in builds_by_advisory_kind if _get_base_shipment_kind(kind) == base_kind and kind != base_kind + ) + + +def _distribute_bugs_to_compound_kinds( + bugs: type_bug_set, + base_kind: str, + builds_by_advisory_kind: Dict[str, List[str]], + runtime: Runtime, +) -> Dict[str, type_bug_set]: + """ + Distribute non-tracker bugs to RHEL variants using their attached builds. + + Bugs whose component matches one variant's build package are routed to that + variant. Ambiguous bugs are retained in every variant so that a bug is not + silently omitted from a shipment. + + Args: + bugs: Bugs already categorized for the base advisory kind. + base_kind: Unqualified advisory kind. + builds_by_advisory_kind: Build NVRs keyed by shipment kind. + runtime: Elliott runtime used for component normalization. + Returns: + Bug sets keyed by the configured RHEL-qualified shipment kind. + """ + compound_kinds = _get_compound_kinds(builds_by_advisory_kind, base_kind) + if not compound_kinds: + return {base_kind: bugs} + + packages_by_kind = { + kind: {parse_nvr(nvr)["name"] for nvr in builds_by_advisory_kind.get(kind, [])} for kind in compound_kinds + } + distributed = {kind: set() for kind in compound_kinds} + for bug in bugs: + component = getattr(bug, "component", "") or "" + normalized_component = normalize_component_by_ocp_delivery_repo(runtime, component) if component else component + matching_kinds = [kind for kind, packages in packages_by_kind.items() if normalized_component in packages] + if not matching_kinds: + matching_kinds = compound_kinds + for kind in matching_kinds: + distributed[kind].add(bug) + return distributed + class FindBugsMode: def __init__(self, status: List, cve_only: bool = False, art_managed_trackers_only: bool = False): @@ -512,6 +583,14 @@ def categorize_bugs_by_type( operator_bundle_advisory: set(), "microshift": set(), } + compound_bases = { + _get_base_shipment_kind(kind) + for kind in (builds_by_advisory_kind or {}) + if _get_base_shipment_kind(kind) != kind + } + for base_kind in compound_bases: + for compound_kind in _get_compound_kinds(builds_by_advisory_kind, base_kind): + bugs_by_type[compound_kind] = set() # for 3.x, all bugs should go to the rpm advisory if int(major_version) < 4: @@ -553,6 +632,17 @@ def categorize_bugs_by_type( # remaining non-tracker bugs go to image advisory bugs_by_type["image"] = non_tracker_bugs + # Keep non-tracker bugs aligned with the shipment snapshot that contains + # their build when a base advisory has been split by RHEL. + for base_kind in ("extras", "image", operator_bundle_advisory): + if base_kind not in compound_bases or base_kind not in bugs_by_type: + continue + distributed = _distribute_bugs_to_compound_kinds( + bugs_by_type[base_kind], base_kind, builds_by_advisory_kind, runtime + ) + bugs_by_type[base_kind] = set() + bugs_by_type.update(distributed) + # Complain about fake trackers if fake_trackers: sorted_ids = sorted([t.id for t in fake_trackers]) @@ -604,12 +694,12 @@ def categorize_bugs_by_type( logger.info("Validating tracker bugs with builds in advisories..") found = set() for kind in bugs_by_type.keys(): - if len(found) == len(tracker_bugs): - break + if _get_base_shipment_kind(kind) in compound_bases and kind == _get_base_shipment_kind(kind): + continue attached_nvrs = builds_by_advisory_kind.get(kind, []) packages = {parse_nvr(nvr)["name"] for nvr in attached_nvrs} exception_packages = [] - if kind == 'image': + if _get_base_shipment_kind(kind) == 'image': # golang builder is a special tracker component # which applies to all our golang images exception_packages.append(constants.GOLANG_BUILDER_CVE_COMPONENT) diff --git a/elliott/elliottlib/cli/shipment_cli.py b/elliott/elliottlib/cli/shipment_cli.py index 8e79df8bb2..8b99413796 100644 --- a/elliott/elliottlib/cli/shipment_cli.py +++ b/elliott/elliottlib/cli/shipment_cli.py @@ -3,7 +3,6 @@ import click from artcommonlib import logutil from artcommonlib.assembly import AssemblyTypes -from artcommonlib.constants import SHIPMENT_CONFIG_KINDS from artcommonlib.gitdata import SafeFormatter from doozerlib.backend.konflux_fbc import KonfluxFbcBuilder from doozerlib.util import konflux_application_name @@ -20,6 +19,7 @@ ShipmentConfig, ShipmentEnv, ) +from elliottlib.shipment_utils import split_shipment_kind from elliottlib.util import get_advisory_boilerplate LOGGER = logutil.get_logger(__name__) @@ -30,6 +30,24 @@ yaml.indent(mapping=2, sequence=4, offset=2) +def _shipment_kind_type(value: str) -> str: + """ + Validate and return a base or RHEL-qualified shipment kind for Click. + + Args: + value: User-provided shipment kind. + Returns: + The validated shipment kind. + Raises: + click.BadParameter: If the shipment kind is unsupported. + """ + try: + split_shipment_kind(value) + except ValueError as error: + raise click.BadParameter(str(error)) from error + return value + + @cli.group("shipment", short_help="Commands for managing release Shipment config") def shipment_cli(): pass @@ -47,7 +65,9 @@ def __init__( async def run(self): self.runtime.initialize(build_system='konflux', with_shipment=True) - if self.kind == "fbc": + base_kind, rhel_suffix = split_shipment_kind(self.kind) + + if base_kind == "fbc": application = KonfluxFbcBuilder.get_application_name(self.runtime.group) else: application = konflux_application_name(self.runtime.group) @@ -56,17 +76,34 @@ async def run(self): # where defaults are set per application shipment_config = self.runtime.shipment_gitdata.load_yaml_file('config.yaml', strict=False) or {} app_env_config = shipment_config.get("applications", {}).get(application, {}).get("environments", {}) - stage_rpa = app_env_config.get("stage", {}).get("releasePlan", "n/a") - prod_rpa = app_env_config.get("prod", {}).get("releasePlan", "n/a") + + def get_release_plan(environment: str) -> str: + """ + Select the configured ReleasePlan for an environment. + + Args: + environment: Environment name, such as ``stage`` or ``prod``. + Returns: + The RHEL-specific plan when configured, otherwise the default plan. + """ + environment_config = app_env_config.get(environment, {}) + if rhel_suffix: + rhel_release_plan = environment_config.get(f"releasePlan-{rhel_suffix}") + if rhel_release_plan: + return rhel_release_plan + return environment_config.get("releasePlan", "n/a") + + stage_rpa = get_release_plan("stage") + prod_rpa = get_release_plan("prod") data = None - if self.kind != "fbc": + if base_kind != "fbc": et_data = self.runtime.get_errata_config() major, minor, patch = self.runtime.get_major_minor_patch() is_ga = self.runtime.assembly_type == AssemblyTypes.STANDARD and self.runtime.assembly.endswith(".0") errata_type = "RHEA" if is_ga else "RHBA" advisory_boilerplate = get_advisory_boilerplate( - runtime=self.runtime, et_data=et_data, art_advisory_key=self.kind, errata_type=errata_type + runtime=self.runtime, et_data=et_data, art_advisory_key=base_kind, errata_type=errata_type ) replace_vars = {"MAJOR": major, "MINOR": minor, "PATCH": patch} formatter = SafeFormatter() @@ -92,7 +129,7 @@ async def run(self): application=application, group=self.runtime.group, assembly=self.runtime.assembly, - fbc=self.kind == "fbc", + fbc=base_kind == "fbc", ), environments=Environments( stage=ShipmentEnv(releasePlan=stage_rpa), @@ -109,7 +146,7 @@ async def run(self): @click.argument( "kind", metavar="", - type=click.Choice(SHIPMENT_CONFIG_KINDS), + type=_shipment_kind_type, ) @click.pass_obj @click_coroutine diff --git a/elliott/elliottlib/cli/verify_payload.py b/elliott/elliottlib/cli/verify_payload.py index 5a5662a5e0..50a205b955 100644 --- a/elliott/elliottlib/cli/verify_payload.py +++ b/elliott/elliottlib/cli/verify_payload.py @@ -168,10 +168,7 @@ async def get_shipment_nvrs(self) -> Dict[str, str]: image_shipment = get_shipment_config_from_mr(mr_url, "image") if image_shipment is None or image_shipment.shipment.snapshot is None: raise click.UsageError("Could not find an image shipment config in the merge request") - return { - parse_nvr(nvr)["name"]: nvr - for nvr in image_shipment.shipment.snapshot.nvrs - } + return {parse_nvr(nvr)["name"]: nvr for nvr in image_shipment.shipment.snapshot.nvrs} async def check_konflux_payload(self): self.all_advisory_nvrs = await self.get_shipment_nvrs() diff --git a/elliott/elliottlib/shipment_utils.py b/elliott/elliottlib/shipment_utils.py index 28ab688cf6..27bf7f410a 100644 --- a/elliott/elliottlib/shipment_utils.py +++ b/elliott/elliottlib/shipment_utils.py @@ -5,7 +5,7 @@ from urllib.parse import urlparse from artcommonlib.assembly import assembly_config_struct -from artcommonlib.constants import SHIPMENT_CONFIG_KINDS +from artcommonlib.constants import SHIPMENT_CONFIG_KINDS, SHIPMENT_CONFIG_KINDS_WITH_COMPOUNDS from artcommonlib.gitlab import GitLabClient from artcommonlib.jira_config import JIRA_DOMAIN_NAME from artcommonlib.model import Model @@ -23,6 +23,44 @@ PUBLIC_ERRATA_URL = "https://access.redhat.com/errata" _IMAGE_SHIPMENT_KIND_PATTERN = re.compile(r"^image(?:-el(?P\d+))?$") +_RHEL_SHIPMENT_KIND_PATTERN = re.compile(r"^(?P.+)-(?Pel\d+)$") + + +def split_shipment_kind(kind: str) -> tuple[str, str | None]: + """ + Split a shipment kind into its base kind and optional RHEL suffix. + + Args: + kind: Shipment kind, such as ``image`` or ``image-el9``. + Returns: + A tuple containing the base kind and optional suffix. + Raises: + ValueError: If the kind is not a supported shipment kind. + """ + if kind in SHIPMENT_CONFIG_KINDS: + return kind, None + + match = _RHEL_SHIPMENT_KIND_PATTERN.fullmatch(kind) + if match and kind in SHIPMENT_CONFIG_KINDS_WITH_COMPOUNDS: + return match.group("base"), match.group("rhel_suffix") + + raise ValueError(f"Unsupported shipment kind: {kind}") + + +def get_base_shipment_kind(kind: str) -> str: + """ + Return the unqualified base kind for a shipment kind. + + Args: + kind: Shipment kind, optionally qualified by a RHEL suffix. + Returns: + The base shipment kind. + """ + try: + return split_shipment_kind(kind)[0] + except ValueError: + match = _RHEL_SHIPMENT_KIND_PATTERN.fullmatch(kind) + return match.group("base") if match else kind def select_primary_image_shipment( @@ -42,9 +80,7 @@ def select_primary_image_shipment( shipment is present. """ image_shipments = [ - (kind, shipment) - for kind, shipment in shipments_by_kind.items() - if _IMAGE_SHIPMENT_KIND_PATTERN.fullmatch(kind) + (kind, shipment) for kind, shipment in shipments_by_kind.items() if _IMAGE_SHIPMENT_KIND_PATTERN.fullmatch(kind) ] if not image_shipments: return None @@ -75,9 +111,7 @@ def add_secondary_image_advisory_references(shipments_by_kind: Mapping[str, Ship primary_kind, primary_shipment = primary primary_release_notes = ( - primary_shipment.shipment.data.releaseNotes - if primary_shipment.shipment.data is not None - else None + primary_shipment.shipment.data.releaseNotes if primary_shipment.shipment.data is not None else None ) if primary_release_notes is None: return False @@ -99,7 +133,9 @@ def add_secondary_image_advisory_references(shipments_by_kind: Mapping[str, Ship description = primary_release_notes.description or "" reference_block = "See the following advisory for additional container images:\n\n" + "\n".join(references) - primary_release_notes.description = f"{description.rstrip()}\n\n{reference_block}" if description else reference_block + primary_release_notes.description = ( + f"{description.rstrip()}\n\n{reference_block}" if description else reference_block + ) return True @@ -253,7 +289,7 @@ def patch_et_advisory_text( def get_shipment_configs_from_mr( mr_url: str, - kinds: Tuple[str, ...] = SHIPMENT_CONFIG_KINDS, + kinds: Tuple[str, ...] = SHIPMENT_CONFIG_KINDS_WITH_COMPOUNDS, group: str | None = None, ) -> Dict[str, ShipmentConfig]: """ @@ -322,14 +358,19 @@ def _get_shipment_config_kind(filename_stem: str, kinds: Tuple[str, ...]) -> str The matching base or RHEL-qualified shipment kind, if any. """ for kind in sorted(kinds, key=len, reverse=True): - qualified_match = re.search(rf"(?:^|\.)({re.escape(kind)}-el\d+)(?:\.|$)", filename_stem) - if qualified_match: - return qualified_match.group(1) + if kind != "fbc": + qualified_match = re.search(rf"(?:^|\.)({re.escape(kind)}-el\d+)(?:\.|$)", filename_stem) + if qualified_match and qualified_match.group(1) in SHIPMENT_CONFIG_KINDS_WITH_COMPOUNDS: + return qualified_match.group(1) base_match = re.search(rf"(?:^|\.){re.escape(kind)}(?:\.|$)", filename_stem) if base_match: return kind + supported_base_pattern = "|".join(re.escape(kind) for kind in SHIPMENT_CONFIG_KINDS) + if re.search(rf"(?:^|\.)(?:{supported_base_pattern})-el\d+(?:\.|$)", filename_stem): + return None + # Preserve the historical substring matching for unusual legacy filenames such as # ``rpm-extra.yaml``. return next((kind for kind in kinds if kind in filename_stem), None) diff --git a/elliott/tests/test_find_bugs_sweep_cli.py b/elliott/tests/test_find_bugs_sweep_cli.py index 1b9f76f38a..9dda4e4b51 100644 --- a/elliott/tests/test_find_bugs_sweep_cli.py +++ b/elliott/tests/test_find_bugs_sweep_cli.py @@ -512,6 +512,94 @@ def test_categorize_no_trackers(self): for kind in expected: self.assertEqual(expected[kind], set(b.id for b in bugs_by_kind[kind])) + def test_categorize_tracker_for_matching_rhel_variant(self): + """Tracker bugs are assigned to the compound advisory containing their build.""" + bug = flexmock( + id="OCPBUGS-EL8", + is_tracker_bug=lambda: True, + is_invalid_tracker_bug=lambda: False, + has_valid_target_version_in_summary=lambda *_: True, + whiteboard_component="ose-a-container", + component="", + summary="", + ) + flexmock(sweep_cli).should_receive("extras_bugs").and_return(set()) + builds_by_advisory_kind = { + "image-el8": {"ose-a-container-v4.17.0-1.el8"}, + "image-el9": {"ose-b-container-v4.17.0-1.el9"}, + "extras-el8": set(), + "extras-el9": set(), + } + + bugs_by_kind, issues = categorize_bugs_by_type( + runtime=self.runtime, + bugs=[bug], + builds_by_advisory_kind=builds_by_advisory_kind, + major_version=self.major_version, + minor_version=self.minor_version, + ) + + self.assertEqual(issues, []) + self.assertEqual({bug.id}, {item.id for item in bugs_by_kind["image-el8"]}) + self.assertEqual(set(), bugs_by_kind["image-el9"]) + + def test_categorize_non_tracker_for_matching_rhel_variant(self): + """Non-tracker bugs with a matching component stay with that variant.""" + bug = flexmock( + id="OCPBUGS-EL9", + is_tracker_bug=lambda: False, + is_invalid_tracker_bug=lambda: False, + component="ose-b-container", + summary="", + ) + flexmock(sweep_cli).should_receive("extras_bugs").and_return(set()) + flexmock(sweep_cli).should_receive("normalize_component_by_ocp_delivery_repo").and_return("ose-b-container") + builds_by_advisory_kind = { + "image-el8": {"ose-a-container-v4.17.0-1.el8"}, + "image-el9": {"ose-b-container-v4.17.0-1.el9"}, + } + + bugs_by_kind, issues = categorize_bugs_by_type( + runtime=self.runtime, + bugs=[bug], + builds_by_advisory_kind=builds_by_advisory_kind, + major_version=self.major_version, + minor_version=self.minor_version, + ) + + self.assertEqual(issues, []) + self.assertEqual(set(), bugs_by_kind["image-el8"]) + self.assertEqual({bug.id}, {item.id for item in bugs_by_kind["image-el9"]}) + + def test_categorize_shared_tracker_for_all_matching_rhel_variants(self): + """A tracker affecting builds in both streams is attached to both variants.""" + bug = flexmock( + id="OCPBUGS-BOTH", + is_tracker_bug=lambda: True, + is_invalid_tracker_bug=lambda: False, + has_valid_target_version_in_summary=lambda *_: True, + whiteboard_component="shared-container", + component="", + summary="", + ) + flexmock(sweep_cli).should_receive("extras_bugs").and_return(set()) + builds_by_advisory_kind = { + "image-el8": {"shared-container-v4.17.0-1.el8"}, + "image-el9": {"shared-container-v4.17.0-1.el9"}, + } + + bugs_by_kind, issues = categorize_bugs_by_type( + runtime=self.runtime, + bugs=[bug], + builds_by_advisory_kind=builds_by_advisory_kind, + major_version=self.major_version, + minor_version=self.minor_version, + ) + + self.assertEqual(issues, []) + self.assertEqual({bug.id}, {item.id for item in bugs_by_kind["image-el8"]}) + self.assertEqual({bug.id}, {item.id for item in bugs_by_kind["image-el9"]}) + def test_categorize_with_trackers_no_builds(self): bugs = [ # valid tracker diff --git a/elliott/tests/test_shipment_cli.py b/elliott/tests/test_shipment_cli.py index 0612a07d8f..07983b10bb 100644 --- a/elliott/tests/test_shipment_cli.py +++ b/elliott/tests/test_shipment_cli.py @@ -130,6 +130,46 @@ async def test_candidate_xy0_uses_rhba(self, _mock_app_name, mock_boilerplate): release_notes = result["shipment"]["data"]["releaseNotes"] self.assertEqual(release_notes["type"], "RHBA") + @patch("elliottlib.cli.shipment_cli.get_advisory_boilerplate") + @patch("elliottlib.cli.shipment_cli.konflux_application_name", return_value="test-app") + async def test_rhel_qualified_kind_uses_base_boilerplate_and_specific_release_plan( + self, _mock_app_name, mock_boilerplate + ): + """RHEL-qualified shipments keep the application name but select the qualified plans.""" + mock_boilerplate.return_value = { + "synopsis": "syn", + "topic": "top", + "description": "desc", + "solution": "sol", + } + runtime = self._make_runtime("4", "17", "52") + runtime.shipment_gitdata.load_yaml_file.return_value = { + "applications": { + "test-app": { + "environments": { + "stage": { + "releasePlan": "stage-default", + "releasePlan-el8": "stage-el8", + }, + "prod": { + "releasePlan": "prod-default", + "releasePlan-el8": "prod-el8", + }, + } + } + } + } + + result = await InitShipmentCli(runtime=runtime, kind="image-el8").run() + + mock_boilerplate.assert_called_once_with( + runtime=runtime, et_data={}, art_advisory_key="image", errata_type="RHBA" + ) + shipment = result["shipment"] + self.assertEqual(shipment["metadata"]["application"], "test-app") + self.assertEqual(shipment["environments"]["stage"]["releasePlan"], "stage-el8") + self.assertEqual(shipment["environments"]["prod"]["releasePlan"], "prod-el8") + if __name__ == "__main__": unittest.main() diff --git a/elliott/tests/test_shipment_utils.py b/elliott/tests/test_shipment_utils.py index 9ced7ba2e8..01829476a8 100644 --- a/elliott/tests/test_shipment_utils.py +++ b/elliott/tests/test_shipment_utils.py @@ -860,9 +860,7 @@ def _make_image_shipment_config(kind: str, component_count: int, live_id: int) - SnapshotComponent( name=f"image-{index}", containerImage=f"quay.io/example/image-{index}:latest", - source=ComponentSource( - git=GitSource(url="https://github.com/example/image.git", revision="revision") - ), + source=ComponentSource(git=GitSource(url="https://github.com/example/image.git", revision="revision")), ) for index in range(component_count) ] diff --git a/elliott/tests/test_verify_payload.py b/elliott/tests/test_verify_payload.py index 641331a7eb..a57d6c635d 100644 --- a/elliott/tests/test_verify_payload.py +++ b/elliott/tests/test_verify_payload.py @@ -17,9 +17,7 @@ async def test_get_shipment_nvrs_uses_principal_image_shipment(self, mock_get_sh mock_get_shipment_config.return_value = image_shipment pipeline = VerifyPayloadPipeline(MagicMock(), "quay.io/example/release:4.20.1-x86_64") - pipeline.assembly_group_config = { - "shipment": {"url": "https://gitlab.example.com/project/-/merge_requests/1"} - } + pipeline.assembly_group_config = {"shipment": {"url": "https://gitlab.example.com/project/-/merge_requests/1"}} result = await pipeline.get_shipment_nvrs() diff --git a/pyartcd/pyartcd/pipelines/prepare_release_konflux.py b/pyartcd/pyartcd/pipelines/prepare_release_konflux.py index a6d67bdd5f..0b40a31fb9 100644 --- a/pyartcd/pyartcd/pipelines/prepare_release_konflux.py +++ b/pyartcd/pyartcd/pipelines/prepare_release_konflux.py @@ -67,6 +67,7 @@ from pyartcd.git import GitRepository from pyartcd.jira_client import JIRAClient from pyartcd.runtime import Runtime +from pyartcd.shipment_utils import split_builds_by_shipment_kind from pyartcd.slack import SlackClient from pyartcd.util import ( get_assembly_basis, @@ -103,7 +104,24 @@ def _get_shipment_builds(kind: str, shipment: ShipmentConfig, kind_to_builds: Di """ if re.search(r"-el\d+$", kind) and shipment.shipment.snapshot and shipment.shipment.snapshot.nvrs: return shipment.shipment.snapshot.nvrs - return kind_to_builds[_get_base_shipment_kind(kind)] + return kind_to_builds.get(kind, kind_to_builds.get(_get_base_shipment_kind(kind), [])) + + +def _get_builds_for_base_kind(kind_to_builds: Dict[str, List[str]], base_kind: str) -> List[str]: + """ + Combine build lists for a base kind and all of its RHEL-qualified variants. + + Args: + kind_to_builds: Build lists keyed by shipment kind. + base_kind: Unqualified shipment kind. + Returns: + All builds belonging to the base kind. + """ + builds = [] + for kind, kind_builds in kind_to_builds.items(): + if _get_base_shipment_kind(kind) == base_kind: + builds.extend(kind_builds) + return builds class PrepareReleaseKonfluxPipeline: @@ -537,7 +555,9 @@ async def sweep_bugs(self, impetus_advisories: dict, shipment_data: Optional[tup for kind, shipment in shipments_by_kind.items(): if kind == "fbc": continue - bug_ids = bugs_by_kind.get(_get_base_shipment_kind(kind), []) + bug_ids = bugs_by_kind.get(kind) + if bug_ids is None: + bug_ids = bugs_by_kind.get(_get_base_shipment_kind(kind), []) set_jira_bug_ids(shipment.shipment.data.releaseNotes, bug_ids) await self.update_shipment_mr(shipments_by_kind, env, shipment_url) @@ -664,11 +684,12 @@ async def prepare_shipment_builds(self) -> Optional[tuple]: # make sure that metadata shipment needs to be prepared # if so, build any missing bundle builds - if any(_get_base_shipment_kind(kind) == "metadata" for kind in shipments_by_kind) and kind_to_builds[ - "olm_builds_not_found" - ]: + metadata_kinds = {kind for kind in shipments_by_kind if _get_base_shipment_kind(kind) == "metadata"} + if metadata_kinds and kind_to_builds["olm_builds_not_found"]: bundle_nvrs, bundle_errors = await self.find_or_build_bundle_builds(kind_to_builds["olm_builds_not_found"]) - kind_to_builds["metadata"] += bundle_nvrs + metadata_builds = split_builds_by_shipment_kind(bundle_nvrs, "metadata", metadata_kinds) + for kind, builds in metadata_builds.items(): + kind_to_builds.setdefault(kind, []).extend(builds) if bundle_errors: self.record_deferred_build_errors( "bundle", @@ -686,7 +707,7 @@ async def prepare_shipment_builds(self) -> Optional[tuple]: # find and build any missing fbc builds if "fbc" in shipments_by_kind: fbc_builds, fbc_errors = await self.find_or_build_fbc_builds( - kind_to_builds["extras"] + kind_to_builds["image"] + _get_builds_for_base_kind(kind_to_builds, "extras") + _get_builds_for_base_kind(kind_to_builds, "image") ) kind_to_builds["fbc"] = fbc_builds if fbc_errors: @@ -932,31 +953,39 @@ async def verify_attached_operators(self, kind_to_builds: Dict[str, List[str]]): attached image builds. """ self.logger.info("Verify_attached_operators ...") - olm_builds = kind_to_builds.get('metadata') - if not olm_builds: - # No metadata builds to verify, so the check passes. + metadata_kinds = [kind for kind in kind_to_builds if _get_base_shipment_kind(kind) == "metadata"] + if not metadata_kinds: return - image_builds = kind_to_builds['image'] + kind_to_builds['extras'] + kdb = KonfluxDb() kdb.bind(KonfluxBundleBuildRecord) - tasks = [ - kdb.get_latest_build(nvr=build, outcome=KonfluxBuildOutcome.SUCCESS, exclude_large_columns=True) - for build in olm_builds - ] - olm_records = await asyncio.gather(*tasks) missing_references = [] - for record in filter(None, olm_records): - # Check the main operator NVR - if record.operator_nvr not in image_builds: - missing_references.append( - f"Bundle {record.nvr} references operator {record.operator_nvr}, which is not in the release." - ) - # Check all operand NVRs - for operand in record.operand_nvrs: - if operand not in image_builds: + for metadata_kind in metadata_kinds: + rhel_suffix_match = re.search(r"-el\d+$", metadata_kind) + related_kinds = [ + kind + for kind in kind_to_builds + if _get_base_shipment_kind(kind) in ("image", "extras") + and (not rhel_suffix_match or kind.endswith(rhel_suffix_match.group(0))) + ] + image_builds = _get_builds_for_base_kind( + {kind: kind_to_builds[kind] for kind in related_kinds}, "image" + ) + _get_builds_for_base_kind({kind: kind_to_builds[kind] for kind in related_kinds}, "extras") + tasks = [ + kdb.get_latest_build(nvr=build, outcome=KonfluxBuildOutcome.SUCCESS, exclude_large_columns=True) + for build in kind_to_builds[metadata_kind] + ] + olm_records = await asyncio.gather(*tasks) + for record in filter(None, olm_records): + if record.operator_nvr not in image_builds: missing_references.append( - f"Bundle {record.nvr} references operand {operand}, which is not in the release." + f"Bundle {record.nvr} references operator {record.operator_nvr}, which is not in the release." ) + for operand in record.operand_nvrs: + if operand not in image_builds: + missing_references.append( + f"Bundle {record.nvr} references operand {operand}, which is not in the release." + ) if missing_references: error_details = "\n".join(missing_references) self.logger.warning("Verify_attached_operators check failed with the following errors:\n%s", error_details) @@ -1324,16 +1353,23 @@ async def find_builds_all(self) -> Dict[str, List[str]]: """ cmd = self._elliott_base_command + ["find-builds", "--kind=image", "--all-image-types", "--json=-"] stdout = await self.execute_command_with_logging(cmd) + shipment_kinds = { + advisory.get("kind") for advisory in self.shipment_config.get("advisories", []) if advisory.get("kind") + } if not stdout: self.logger.warning("No output received from find-builds command.") - return {"image": [], "extras": [], "metadata": [], "olm_builds_not_found": []} + kind_to_builds = {"olm_builds_not_found": []} + for base_kind in ("image", "extras", "metadata"): + kind_to_builds.update(split_builds_by_shipment_kind([], base_kind, shipment_kinds)) + return kind_to_builds out = json.loads(stdout) - kind_to_builds = { - "image": out.get("payload", []), - "extras": out.get("non_payload", []), - "metadata": out.get("olm_builds", []), - "olm_builds_not_found": out.get("olm_builds_not_found", []), - } + kind_to_builds = {"olm_builds_not_found": out.get("olm_builds_not_found", [])} + for base_kind, output_key in ( + ("image", "payload"), + ("extras", "non_payload"), + ("metadata", "olm_builds"), + ): + kind_to_builds.update(split_builds_by_shipment_kind(out.get(output_key, []), base_kind, shipment_kinds)) return kind_to_builds @@ -1387,7 +1423,7 @@ async def attach_cve_flaws(self, kind: str, shipment: ShipmentConfig): attach_cve_flaws_command = self._elliott_base_command + [ 'attach-cve-flaws', - f'--use-default-advisory={_get_base_shipment_kind(kind)}', + f'--use-default-advisory={kind}', '--reconcile', '--output=yaml', ] diff --git a/pyartcd/pyartcd/pipelines/promote.py b/pyartcd/pyartcd/pipelines/promote.py index a95804be81..cada6e6938 100644 --- a/pyartcd/pyartcd/pipelines/promote.py +++ b/pyartcd/pyartcd/pipelines/promote.py @@ -48,7 +48,6 @@ get_shipment_config_from_mr, get_shipment_configs_from_mr, patch_et_advisory_text, - select_primary_image_shipment, strip_advisory_cross_reference, strip_et_advisory_rpm_reference, ) @@ -2795,8 +2794,7 @@ async def _prepare_shipment_templates( format_dict = {} # Add SHA digests for image shipments - primary_image = select_primary_image_shipment(shipments_by_kind) - if primary_image and shipment_kind == primary_image[0]: + if re.fullmatch(r"image(?:-el\d+)?", shipment_kind): for arch, sha in payload_shas.items(): if arch == "multi": continue # Skip multi-arch as it's not a specific architecture @@ -2857,7 +2855,7 @@ async def _prepare_shipment_templates( self._logger.info("Found template placeholders in %s shipment: %s", shipment_kind, placeholders_found) # Validate template placeholders in both description and solution fields (for image shipments) - if primary_image and shipment_kind == primary_image[0]: + if re.fullmatch(r"image(?:-el\d+)?", shipment_kind): # Check solution field for SHA digest placeholders if hasattr(shipment_config.shipment.data.releaseNotes, 'solution'): solution_text = shipment_config.shipment.data.releaseNotes.solution diff --git a/pyartcd/pyartcd/shipment_utils.py b/pyartcd/pyartcd/shipment_utils.py index a46de868eb..957b553419 100644 --- a/pyartcd/pyartcd/shipment_utils.py +++ b/pyartcd/pyartcd/shipment_utils.py @@ -36,6 +36,27 @@ def group_nvrs_by_rhel_version(nvrs: list[str]) -> dict[str, list[str]]: ) +def split_builds_by_shipment_kind(builds: list[str], base_kind: str, shipment_kinds: set[str]) -> dict[str, list[str]]: + """ + Splits build NVRs into the RHEL-qualified shipment kinds requested by an assembly. + + Args: + builds: Build NVRs belonging to the base shipment kind. + base_kind: Unqualified shipment kind, such as ``image``. + shipment_kinds: Shipment kinds configured for the assembly. + Returns: + Build lists keyed by the configured shipment kind. Plain kinds retain all builds. + """ + qualified_kinds = sorted(kind for kind in shipment_kinds if kind.startswith(f"{base_kind}-el")) + split_builds = {base_kind: builds} if base_kind in shipment_kinds else {} + if not qualified_kinds: + return split_builds or {base_kind: builds} + + builds_by_rhel = group_nvrs_by_rhel_version(builds) + split_builds.update({kind: builds_by_rhel.get(kind.removeprefix(f"{base_kind}-"), []) for kind in qualified_kinds}) + return split_builds + + def get_release_plan_names( config_path: Path, application: str, diff --git a/pyartcd/tests/pipelines/test_prepare_release_konflux.py b/pyartcd/tests/pipelines/test_prepare_release_konflux.py index 0ff52674ad..7a557df0ba 100644 --- a/pyartcd/tests/pipelines/test_prepare_release_konflux.py +++ b/pyartcd/tests/pipelines/test_prepare_release_konflux.py @@ -60,9 +60,7 @@ def _make_image_shipment( SnapshotComponent( name=f"image-{index}", containerImage=f"quay.io/example/image-{index}:latest", - source=ComponentSource( - git=GitSource(url="https://github.com/example/image.git", revision="revision") - ), + source=ComponentSource(git=GitSource(url="https://github.com/example/image.git", revision="revision")), ) for index in range(component_count) ] @@ -125,6 +123,122 @@ def test_get_shipment_builds_preserves_rhel_qualified_snapshot_membership(self): self.assertEqual(builds, ["image-el9-nvr"]) + async def test_reserve_live_id_reserves_each_compound_advisory(self): + """Each non-FBC compound advisory receives its own live ID.""" + pipeline = PrepareReleaseKonfluxPipeline( + slack_client=self.mock_slack_client, + runtime=self.runtime, + group=self.group, + assembly=self.assembly, + ) + pipeline.updated_assembly_group_config = Model( + { + "shipment": { + "advisories": [ + {"kind": "image-el8"}, + {"kind": "image-el9"}, + {"kind": "extras-el8"}, + ] + } + } + ) + errata_api = AsyncMock() + errata_api.reserve_live_id.side_effect = [1001, 1002, 1003] + pipeline._errata_api = errata_api + + live_ids = [await pipeline.reserve_live_id({"kind": kind}) for kind in ("image-el8", "image-el9", "extras-el8")] + + self.assertEqual(live_ids, [1001, 1002, 1003]) + self.assertEqual( + [config.live_id for config in pipeline.updated_assembly_group_config.shipment.advisories], + [1001, 1002, 1003], + ) + + async def test_find_builds_all_splits_configured_compound_kinds(self): + """find-builds output is partitioned only for configured RHEL variants.""" + pipeline = PrepareReleaseKonfluxPipeline( + slack_client=self.mock_slack_client, + runtime=self.runtime, + group=self.group, + assembly=self.assembly, + ) + pipeline.releases_config = Model( + { + "releases": { + self.assembly: { + "assembly": { + "group": { + "shipment": { + "advisories": [ + {"kind": "image-el8"}, + {"kind": "image-el9"}, + {"kind": "extras"}, + ] + } + } + } + } + } + } + ) + pipeline.execute_command_with_logging = AsyncMock( + return_value=json.dumps( + { + "payload": [ + "image-a-container-v4.17.0-1.el8", + "image-b-container-v4.17.0-1.el9", + ], + "non_payload": ["extras-a-container-v4.17.0-1.el9"], + "olm_builds": [], + "olm_builds_not_found": [], + } + ) + ) + + builds = await pipeline.find_builds_all() + + self.assertEqual(builds["image-el8"], ["image-a-container-v4.17.0-1.el8"]) + self.assertEqual(builds["image-el9"], ["image-b-container-v4.17.0-1.el9"]) + self.assertEqual(builds["extras"], ["extras-a-container-v4.17.0-1.el9"]) + self.assertNotIn("image", builds) + + async def test_attach_cve_flaws_targets_exact_compound_kind(self): + """CVE reconciliation targets the RHEL-specific shipment advisory.""" + pipeline = PrepareReleaseKonfluxPipeline( + slack_client=self.mock_slack_client, + runtime=self.runtime, + group=self.group, + assembly=self.assembly, + ) + pipeline.execute_command_with_logging = AsyncMock(return_value="") + + await pipeline.attach_cve_flaws("image-el8", Mock()) + + command = pipeline.execute_command_with_logging.await_args.args[0] + self.assertIn("--use-default-advisory=image-el8", command) + + async def test_sweep_bugs_updates_each_compound_shipment_independently(self): + """Bug IDs are written to the matching RHEL-specific shipment file.""" + pipeline = PrepareReleaseKonfluxPipeline( + slack_client=self.mock_slack_client, + runtime=self.runtime, + group=self.group, + assembly=self.assembly, + ) + pipeline.find_bugs = AsyncMock(return_value={"image-el8": ["BUG-EL8"], "image-el9": ["BUG-EL9"]}) + pipeline.update_shipment_mr = AsyncMock() + shipments = { + "image-el8": _make_image_shipment("image-el8", 1, 1001, "el8", self.group, self.assembly), + "image-el9": _make_image_shipment("image-el9", 1, 1002, "el9", self.group, self.assembly), + } + + await pipeline.sweep_bugs({}, (shipments, "prod", "https://gitlab.example.com/mr/1")) + + el8_bugs = shipments["image-el8"].shipment.data.releaseNotes.issues.fixed + el9_bugs = shipments["image-el9"].shipment.data.releaseNotes.issues.fixed + self.assertEqual([issue.id for issue in el8_bugs], ["BUG-EL8"]) + self.assertEqual([issue.id for issue in el9_bugs], ["BUG-EL9"]) + def test_init(self): pipeline = PrepareReleaseKonfluxPipeline( slack_client=self.mock_slack_client, diff --git a/pyartcd/tests/test_shipment_utils.py b/pyartcd/tests/test_shipment_utils.py index b3ec464f44..1b6340e373 100644 --- a/pyartcd/tests/test_shipment_utils.py +++ b/pyartcd/tests/test_shipment_utils.py @@ -4,7 +4,11 @@ from pathlib import Path -from pyartcd.shipment_utils import get_release_plan_names, group_nvrs_by_rhel_version +from pyartcd.shipment_utils import ( + get_release_plan_names, + group_nvrs_by_rhel_version, + split_builds_by_shipment_kind, +) def test_group_nvrs_by_rhel_version_sorts_rhel_groups() -> None: @@ -27,6 +31,40 @@ def test_group_nvrs_by_rhel_version_uses_default_without_suffix() -> None: } +def test_split_builds_by_shipment_kind_uses_requested_rhel_variants() -> None: + """Only configured compound kinds receive RHEL-specific build lists.""" + builds = [ + "ose-a-container-v4.17.0-1.el8", + "ose-b-container-v4.17.0-1.el9", + ] + + assert split_builds_by_shipment_kind(builds, "image", {"image-el8", "image-el9"}) == { + "image-el8": ["ose-a-container-v4.17.0-1.el8"], + "image-el9": ["ose-b-container-v4.17.0-1.el9"], + } + + +def test_split_builds_by_shipment_kind_keeps_plain_kind_backward_compatible() -> None: + """Plain shipment kinds continue to receive the complete build list.""" + builds = ["ose-a-container-v4.18.0-1.el9"] + + assert split_builds_by_shipment_kind(builds, "image", {"image"}) == {"image": builds} + + +def test_split_builds_by_shipment_kind_preserves_plain_and_compound_kinds() -> None: + """A temporary mixed configuration does not lose the plain shipment build list.""" + builds = [ + "ose-a-container-v4.17.0-1.el8", + "ose-b-container-v4.17.0-1.el9", + ] + + assert split_builds_by_shipment_kind(builds, "image", {"image", "image-el8", "image-el9"}) == { + "image": builds, + "image-el8": ["ose-a-container-v4.17.0-1.el8"], + "image-el9": ["ose-b-container-v4.17.0-1.el9"], + } + + def test_get_release_plan_names_prefers_rhel_specific_application(tmp_path: Path) -> None: """Uses the RHEL-specific application entry when it exists.""" config_path = tmp_path / "config.yaml"