diff --git a/pyartcd/pyartcd/locks.py b/pyartcd/pyartcd/locks.py index af0c21dadc..14e9178abd 100644 --- a/pyartcd/pyartcd/locks.py +++ b/pyartcd/pyartcd/locks.py @@ -11,6 +11,8 @@ # Defines the pipeline locks managed by Redis class Lock(enum.Enum): + """Redis lock-name templates used to serialize pyartcd pipelines.""" + OLM_BUNDLE = 'lock:olm-bundle:{version}' OLM_BUNDLE_KONFLUX = 'lock:olm-bundle-konflux:{version}' MIRRORING_RPMS = 'lock:mirroring-rpms:{version}' @@ -35,6 +37,7 @@ class Lock(enum.Enum): SYNC_CI_IMAGES = 'lock:sync-ci-images:{version}' OPEN_RECONCILIATION_PRS = 'lock:open-reconciliation-prs:{version}' OPEN_RECONCILIATION_PRS_LAYERED = 'lock:open-reconciliation-prs-layered:{group}' + LAYERED_PRODUCT_SHIPMENT = 'lock:layered-product-shipment:{group}:{assembly}' class Keys(enum.Enum): @@ -173,6 +176,11 @@ class Keys(enum.Enum): 'retry_delay_min': 0.1, 'lock_timeout': DEFAULT_LOCK_TIMEOUT, }, + Lock.LAYERED_PRODUCT_SHIPMENT: { + 'retry_count': 36000, + 'retry_delay_min': 0.1, + 'lock_timeout': DEFAULT_LOCK_TIMEOUT, + }, } diff --git a/pyartcd/pyartcd/lp_shipment.py b/pyartcd/pyartcd/lp_shipment.py new file mode 100644 index 0000000000..af3f6fff83 --- /dev/null +++ b/pyartcd/pyartcd/lp_shipment.py @@ -0,0 +1,749 @@ +"""Helpers for safely creating and reusing layered-product shipment merge requests. + +This module owns the ``assembly.group.shipment.mr`` pointer in ``releases.yml`` +and reconciles generated shipment files with an existing GitLab merge request. +On reuse, the previous layered-product shipment files are discarded and rebuilt +from the current release inputs. +""" + +import re +from collections import Counter +from dataclasses import dataclass +from io import StringIO +from pathlib import Path +from typing import Dict +from urllib.parse import urlparse + +from artcommonlib import exectools +from artcommonlib.rpm_utils import parse_nvr +from artcommonlib.util import new_roundtrip_yaml_handler +from elliottlib.shipment_model import ShipmentConfig + +from pyartcd.fbc_util import extract_ocp_version_from_nvr +from pyartcd.git import GitRepository + +YAML = new_roundtrip_yaml_handler() +_TIMESTAMP_RE = re.compile(r"(\d{14})$") +_PROD_RELEASE_LABEL_PREFIX = "prod-release" +_STAGE_RELEASE_SUCCESS_LABEL = "stage-release-success" +_ACTIVE_CI_STATUSES = frozenset( + { + 'created', + 'waiting_for_resource', + 'preparing', + 'pending', + 'running', + 'scheduled', + 'canceling', + } +) +_TERMINAL_CI_STATUSES = frozenset({'success', 'failed', 'canceled', 'skipped', 'manual'}) +_UNTOUCHED_PROD_STATUSES = frozenset({'manual', 'skipped'}) + + +class ShipmentMRValidationError(ValueError): + """Indicate that a configured shipment MR is invalid or unrelated.""" + + +class ShipmentMRScopeError(ShipmentMRValidationError): + """Indicate that a shipment MR does not belong to the expected release scope.""" + + +class ShipmentMRProductionError(ValueError): + """Indicate that production history makes automated MR replacement unsafe.""" + + +class ShipmentMRActiveStageError(ValueError): + """Indicate that active stage work makes in-place MR reuse unsafe.""" + + +@dataclass(frozen=True) +class ShipmentMRCIState: + """Summarize Shipment CI state relevant to layered-product MR reuse. + + Attributes: + active_stage: Descriptions of active MR, stage bridge, or downstream + stage jobs. + prod_attempts: Descriptions proving that production was attempted. + """ + + active_stage: tuple[str, ...] + prod_attempts: tuple[str, ...] + + +def _project_path(url: str) -> str: + """Extract a GitLab project path from a repository URL. + + Args: + url: HTTPS or Git repository URL. + + Returns: + The normalized ``namespace/project`` path. + """ + return urlparse(url).path.strip('/').removesuffix('.git') + + +def get_shipment_mr_url(releases_config: dict, assembly: str) -> str | None: + """Read the layered-product shipment MR pointer for an assembly. + + Args: + releases_config: Parsed contents of ``releases.yml``. + assembly: Assembly name to inspect. + + Returns: + The configured shipment MR URL, or ``None`` when it is not present. + """ + return ( + (releases_config or {}) + .get('releases', {}) + .get(assembly, {}) + .get('assembly', {}) + .get('group', {}) + .get('shipment', {}) + .get('mr') + ) + + +async def update_shipment_mr_url( + repo: GitRepository, + group: str, + assembly: str, + mr_url: str, + expected_mr_url: str | None, + *, + create_as_stream: bool, +) -> bool: + """Persist a layered-product shipment MR pointer safely. + + The group branch is fetched immediately before editing. The write proceeds + only when the current pointer still equals ``expected_mr_url`` so that a + concurrent release cannot be overwritten. + + Args: + repo: Initialized ocp-build-data Git repository. + group: Group branch containing ``releases.yml``. + assembly: Assembly whose shipment pointer should be updated. + mr_url: Shipment MR URL to store. + expected_mr_url: Pointer value observed before the shipment work began. + create_as_stream: Create a missing assembly as an explicit stream + assembly. When false, the assembly must already exist. + + Returns: + Whether the commit was created and pushed. + + Raises: + RuntimeError: If the pointer changed concurrently or a required assembly + disappeared. + """ + await repo.fetch_switch_branch(group, remote="origin") + releases_path = repo._directory / "releases.yml" + releases_config = YAML.load(releases_path) if releases_path.exists() else None + releases_config = releases_config or {} + + current_mr_url = get_shipment_mr_url(releases_config, assembly) + if current_mr_url != expected_mr_url: + raise RuntimeError( + f"Shipment MR pointer changed concurrently from {expected_mr_url!r} to {current_mr_url!r}; " + "refusing to overwrite it" + ) + + releases = releases_config.setdefault('releases', {}) + if not create_as_stream and assembly not in releases: + raise RuntimeError(f"Assembly {assembly} disappeared before the shipment MR pointer could be stored") + assembly_entry = releases.setdefault(assembly, {}) + assembly_def = assembly_entry.setdefault('assembly', {}) + if create_as_stream and not assembly_def: + assembly_def['type'] = 'stream' + shipment = assembly_def.setdefault('group', {}).setdefault('shipment', {}) + shipment['mr'] = mr_url + YAML.dump(releases_config, releases_path) + return await repo.commit_push(f"Update assembly {assembly}: add shipment MR URL", safe=True) + + +async def verify_shipment_mr_url(repo: GitRepository, group: str, assembly: str, expected_mr_url: str) -> None: + """Confirm that ``releases.yml`` still points at the selected MR. + + Args: + repo: Initialized ocp-build-data Git repository. + group: Group branch containing ``releases.yml``. + assembly: Assembly whose pointer should be checked. + expected_mr_url: MR URL selected earlier in the release run. + + Raises: + RuntimeError: If another process changed the pointer. + """ + await repo.fetch_switch_branch(group, remote="origin") + releases_path = repo._directory / "releases.yml" + releases_config = YAML.load(releases_path) if releases_path.exists() else None + current_mr_url = get_shipment_mr_url(releases_config or {}, assembly) + if current_mr_url != expected_mr_url: + raise RuntimeError( + f"Shipment MR pointer changed concurrently from {expected_mr_url!r} to {current_mr_url!r}; " + "refusing to update the old MR" + ) + + +def _object_value(item, name: str, default=None): + """Read a field from a python-gitlab object or API response mapping. + + Args: + item: Python object or mapping returned by the GitLab API. + name: Field name to read. + default: Value returned when the field is absent. + + Returns: + The field value, or ``default`` when it is absent. + """ + if isinstance(item, dict): + return item.get(name, default) + return getattr(item, name, default) + + +def _checked_ci_status(item, context: str) -> str: + """Return a recognized GitLab CI status or fail closed. + + Args: + item: Python-gitlab object or response mapping containing ``status``. + context: Human-readable pipeline or job description. + + Returns: + The normalized GitLab CI status. + + Raises: + RuntimeError: If the status is absent or unknown. + """ + status = _object_value(item, 'status') + if status not in _ACTIVE_CI_STATUSES | _TERMINAL_CI_STATUSES: + raise RuntimeError(f"Cannot safely classify {context}: unknown GitLab CI status {status!r}") + return status + + +def inspect_shipment_mr_ci_state(gitlab_client, mr_url: str, mr) -> ShipmentMRCIState: + """Inspect all Shipment CI pipelines belonging to a merge request. + + Parent pipeline state, stage and production trigger bridges, and downstream + stage jobs are inspected with pagination enabled. A production bridge is + considered attempted once it leaves the untouched ``manual`` or ``skipped`` + states, or as soon as GitLab associates a downstream pipeline with it. + + Args: + gitlab_client: Authenticated ART GitLab client. + mr_url: URL of the shipment merge request. + mr: Python-gitlab merge request object. + + Returns: + Active stage work and evidence of production attempts. + + Raises: + RuntimeError: If GitLab returns incomplete or unrecognized pipeline + state. Callers must fail closed rather than mutate the MR. + """ + project_path, _ = gitlab_client._parse_mr_url(mr_url) + project = gitlab_client.get_project(project_path) + active_stage = [] + prod_attempts = [] + + mr_pipelines = mr.pipelines.list(get_all=True) + for mr_pipeline in mr_pipelines: + pipeline_id = _object_value(mr_pipeline, 'id') + if pipeline_id is None: + raise RuntimeError("Cannot safely inspect shipment MR CI state: a pipeline has no ID") + pipeline = project.pipelines.get(pipeline_id) + pipeline_url = _object_value(pipeline, 'web_url', f'pipeline {pipeline_id}') + pipeline_status = _checked_ci_status(pipeline, f'MR pipeline {pipeline_url}') + bridges = pipeline.bridges.list(get_all=True) + stage_bridge_found = False + + for bridge in bridges: + bridge_name = _object_value(bridge, 'name') + if bridge_name not in {'stage-job', 'prod-job'}: + continue + bridge_status = _checked_ci_status(bridge, f'{bridge_name} in {pipeline_url}') + downstream = _object_value(bridge, 'downstream_pipeline') + + if bridge_name == 'prod-job': + if bridge_status not in _UNTOUCHED_PROD_STATUSES or downstream: + prod_attempts.append(f"{pipeline_url} prod-job is {bridge_status}") + continue + + stage_bridge_found = True + if bridge_status in _ACTIVE_CI_STATUSES: + active_stage.append(f"{pipeline_url} stage-job is {bridge_status}") + + if not downstream: + continue + downstream_id = _object_value(downstream, 'id') + downstream_project_id = _object_value(downstream, 'project_id', _object_value(project, 'id')) + if downstream_id is None or downstream_project_id is None: + raise RuntimeError( + f"Cannot safely inspect {pipeline_url} stage-job: downstream pipeline identification is incomplete" + ) + downstream_project = ( + project + if downstream_project_id == _object_value(project, 'id') + else gitlab_client.get_project(downstream_project_id) + ) + downstream_pipeline = downstream_project.pipelines.get(downstream_id) + downstream_url = _object_value(downstream_pipeline, 'web_url', f'pipeline {downstream_id}') + downstream_status = _checked_ci_status(downstream_pipeline, f'downstream stage pipeline {downstream_url}') + if downstream_status in _ACTIVE_CI_STATUSES: + active_stage.append(f"downstream stage pipeline {downstream_url} is {downstream_status}") + + for job in downstream_pipeline.jobs.list(get_all=True, include_retried=True): + job_name = _object_value(job, 'name', 'unknown job') + job_status = _checked_ci_status(job, f'{job_name} in {downstream_url}') + if job_status in _ACTIVE_CI_STATUSES: + active_stage.append(f"{downstream_url} job {job_name!r} is {job_status}") + + # A ready MR pipeline can still be validating or generating its dynamic + # configuration before GitLab exposes the stage trigger bridge. + if pipeline_status in _ACTIVE_CI_STATUSES and not stage_bridge_found: + active_stage.append(f"{pipeline_url} is {pipeline_status} before its stage job is available") + + return ShipmentMRCIState(tuple(sorted(set(active_stage))), tuple(sorted(set(prod_attempts)))) + + +def validate_shipment_mr_ci_state( + gitlab_client, + mr_url: str, + mr, + *, + allow_active_stage: bool, +) -> ShipmentMRCIState: + """Validate whether Shipment CI state permits reuse or replacement. + + Args: + gitlab_client: Authenticated ART GitLab client. + mr_url: URL of the shipment merge request. + mr: Python-gitlab merge request object. + allow_active_stage: Permit stage-only activity for ``--force`` + replacement. Normal in-place reuse must pass ``False``. + + Returns: + The fully inspected Shipment CI state. + + Raises: + ShipmentMRProductionError: If production was attempted. + ShipmentMRActiveStageError: If stage is active during normal reuse. + RuntimeError: If CI state cannot be determined reliably. + """ + state = inspect_shipment_mr_ci_state(gitlab_client, mr_url, mr) + if state.prod_attempts: + raise ShipmentMRProductionError( + "Shipment MR has production pipeline history and cannot be reused or automatically replaced: " + f"{'; '.join(state.prod_attempts)}. Manual release recovery is required." + ) + if state.active_stage and not allow_active_stage: + raise ShipmentMRActiveStageError( + "Shipment MR still has active stage work and cannot be reused in place: " + f"{'; '.join(state.active_stage)}. Wait for stage to finish or use --force to create a replacement MR." + ) + return state + + +def validate_shipment_mr( + gitlab_client, + mr_url: str, + pull_url: str, + push_url: str, + *, + allowed_states: tuple[str, ...] = ('opened',), +): + """Validate a shipment MR referenced by ``releases.yml``. + + Args: + gitlab_client: Authenticated ART GitLab client. + mr_url: Referenced shipment MR URL. + pull_url: Configured canonical shipment-data repository URL. + push_url: Configured shipment-data push repository URL. + allowed_states: MR states accepted by the requested operation. Normal + reuse accepts only ``opened``; replacement inspection also accepts + ``closed``. + + Returns: + The GitLab merge request object when it satisfies the requested state + and repository constraints. + + Raises: + ShipmentMRValidationError: If the MR is missing, has a disallowed state, + points to the wrong project or target branch, or originates from the + wrong push repository. + ShipmentMRProductionError: If the MR is merged or has a production + release label. + """ + if urlparse(mr_url).netloc != urlparse(pull_url).netloc: + raise ShipmentMRValidationError( + f"Shipment MR host {urlparse(mr_url).netloc} does not match {urlparse(pull_url).netloc}. " + "Use --force to create a replacement MR." + ) + target_project_path, _ = gitlab_client._parse_mr_url(mr_url) + mr = gitlab_client.get_mr_from_url(mr_url) + if not mr: + raise ShipmentMRValidationError(f"Shipment MR {mr_url} was not found. Use --force to create a replacement MR.") + if mr.state == 'merged': + raise ShipmentMRProductionError( + f"Shipment MR {mr_url} is merged and cannot be reused or automatically replaced. Manual recovery is required." + ) + if mr.state not in allowed_states: + raise ShipmentMRValidationError( + f"Shipment MR state is {mr.state}, not one of {allowed_states}. Use --force to create a replacement MR." + ) + if target_project_path != _project_path(pull_url): + raise ShipmentMRValidationError( + f"Shipment MR target project {target_project_path} does not match {_project_path(pull_url)}. " + "Use --force to create a replacement MR." + ) + source_project_path = gitlab_client.get_project(mr.source_project_id).path_with_namespace + if source_project_path != _project_path(push_url): + raise ShipmentMRValidationError( + f"Shipment MR source project {source_project_path} does not match {_project_path(push_url)}. " + "Use --force to create a replacement MR." + ) + if mr.target_branch != "main": + raise ShipmentMRValidationError( + f"Shipment MR target branch is {mr.target_branch}, not main. Use --force to create a replacement MR." + ) + prod_labels = sorted( + label for label in (getattr(mr, 'labels', None) or []) if label.lower().startswith(_PROD_RELEASE_LABEL_PREFIX) + ) + if prod_labels: + raise ShipmentMRProductionError( + f"Shipment MR has production release label(s) {prod_labels} and must not be modified. " + "Automated replacement is disabled after a production attempt; manual recovery is required." + ) + return mr + + +def set_shipment_mr_draft(mr, dry_run: bool) -> None: + """Reset stage status and mark a reused or superseded MR as draft. + + For normal reuse, the success label describes the previous shipment files + and is removed before those files are replaced. For ``--force`` replacement, + marking an open previous MR draft prevents its manual production path from + proceeding while any already-started stage work finishes independently. + + Args: + mr: GitLab merge request object to update. + dry_run: Logically perform the transition without saving it remotely. + """ + changed = False + if not mr.title.startswith("Draft:"): + mr.title = f"Draft: {mr.title}" + changed = True + labels = list(getattr(mr, 'labels', None) or []) + if _STAGE_RELEASE_SUCCESS_LABEL in labels: + labels.remove(_STAGE_RELEASE_SUCCESS_LABEL) + mr.labels = labels + changed = True + if changed and not dry_run: + mr.save() + + +def _to_dict(config: ShipmentConfig) -> dict: + """Convert a shipment model to the mapping written to YAML.""" + return config.model_dump(exclude_unset=True, exclude_none=True) + + +def _identity(config: dict) -> tuple: + """Build the stable semantic identity for a shipment configuration. + + FBC identities include the component and target OCP version so multiple + operators and multiple OCP targets remain independently addressable. + + Args: + config: Parsed shipment configuration. + + Returns: + A tuple identifying the logical shipment independently of its filename. + + Raises: + ValueError: If an FBC shipment does not contain exactly one FBC NVR. + """ + shipment = config.get('shipment', {}) + metadata = shipment.get('metadata', {}) + base = ( + metadata.get('product'), + metadata.get('group'), + metadata.get('assembly'), + metadata.get('application'), + bool(metadata.get('fbc', False)), + ) + if not base[-1]: + return base + + nvrs = shipment.get('snapshot', {}).get('nvrs', []) + if len(nvrs) != 1: + raise ValueError(f"Expected one NVR in an FBC shipment, found {len(nvrs)}") + nvr = nvrs[0] + component = parse_nvr(nvr)['name'] + return (*base, component, extract_ocp_version_from_nvr(nvr)) + + +def _identity_sort_key(item: tuple) -> tuple[str, ...]: + """Return a deterministic ordering key for a shipment identity item.""" + return tuple("" if value is None else str(value) for value in item[0]) + + +def _shipment_path_matches( + path: str, + group: str, + assembly: str, + product: str | None = None, +) -> bool: + """Determine whether a shipment path belongs to a release scope. + + Path-based ownership lets a rerun remove a previously generated file even + when its YAML metadata was edited or damaged manually. + + Args: + path: Repository-relative shipment file path. + group: Expected layered-product group. + assembly: Expected layered-product assembly. + product: Optional expected shipment product. + + Returns: + Whether the path belongs to the requested release scope. + """ + parts = Path(path).parts + if len(parts) < 6 or parts[0] != 'shipment': + return False + if product is not None and parts[1] != product: + return False + return parts[2] == group and parts[-1].startswith(f"{assembly}.") and parts[-1].endswith(('.yaml', '.yml')) + + +async def validate_shipment_mr_reuse_state( + repo: GitRepository, + mr, + product: str, + group: str, + assembly: str, +) -> None: + """Validate the release scope and reject production-completed MR reuse. + + The GitLab success label is checked by :func:`validate_shipment_mr`. This + additional content check protects against a missing label or a partially + completed labeling job by inspecting both image advisory information and + FBC pipeline results. + + Args: + repo: Initialized shipment-data repository. + mr: Open GitLab merge request proposed for reuse. + product: Layered product expected in the shipment files. + group: Layered-product group expected in the shipment files. + assembly: Layered-product assembly expected in the shipment files. + + Raises: + ValueError: If the MR does not contain the expected release scope or a + matching shipment file records production release data. + RuntimeError: If GitLab truncates the MR change list. + """ + await repo.fetch_switch_branch(mr.source_branch, remote="origin") + change_data = mr.changes() + if change_data.get('overflow'): + raise RuntimeError("GitLab truncated the shipment MR change list; refusing an incomplete validation") + + matching_files = [] + shipment_paths = [] + for change in change_data.get('changes', []): + path = change['new_path'] + if _shipment_path_matches(path, group, assembly): + shipment_paths.append(path) + if not _shipment_path_matches(path, group, assembly, product=product): + continue + absolute_path = repo._directory / path + if not absolute_path.exists(): + continue + matching_files.append(path) + config = YAML.load(absolute_path) + if not isinstance(config, dict) or 'shipment' not in config: + raise ValueError(f"Cannot safely determine production release state from malformed shipment file {path}") + shipment = config['shipment'] + prod = shipment.get('environments', {}).get('prod', {}) or {} + advisory = prod.get('advisory') + pipeline = (prod.get('result') or {}).get('pipeline') + if advisory or pipeline: + markers = [] + if advisory: + markers.append('prod advisory') + if pipeline: + markers.append('prod pipeline result') + raise ShipmentMRProductionError( + f"Shipment MR file {path} contains {' and '.join(markers)} and must not be modified. " + "Automated replacement is disabled after a production attempt; manual recovery is required." + ) + + if not matching_files: + found = f" Found candidate files: {sorted(shipment_paths)}." if shipment_paths else "" + raise ShipmentMRScopeError( + f"Shipment MR does not contain shipment files for product {product!r}, group {group!r}, " + f"and assembly {assembly!r}; refusing to modify an unrelated MR.{found} " + "Correct the assembly shipment.mr pointer or use --force to create a replacement MR." + ) + + +async def validate_shipment_mr_for_operation( + gitlab_client, + repo: GitRepository, + mr_url: str, + pull_url: str, + push_url: str, + product: str, + group: str, + assembly: str, + *, + allowed_states: tuple[str, ...], + allow_active_stage: bool, +) -> tuple[object, ShipmentMRCIState]: + """Validate a layered-product shipment MR and its complete safety state. + + Args: + gitlab_client: Authenticated ART GitLab client. + repo: Initialized shipment-data repository. + mr_url: URL referenced by the assembly in ``releases.yml``. + pull_url: Configured canonical shipment-data repository URL. + push_url: Configured shipment-data push repository URL. + product: Expected layered-product name. + group: Expected layered-product group. + assembly: Expected layered-product assembly. + allowed_states: MR states accepted by the requested operation. + allow_active_stage: Permit an isolated ``--force`` replacement while + stage work on the previous MR continues. + + Returns: + The validated MR and its inspected Shipment CI state. + + Raises: + ShipmentMRValidationError: If the MR is invalid or unrelated. + ShipmentMRProductionError: If the MR is merged or production was + attempted. + ShipmentMRActiveStageError: If normal reuse encounters active stage + work. + RuntimeError: If GitLab state cannot be determined reliably. + """ + mr = validate_shipment_mr( + gitlab_client, + mr_url, + pull_url, + push_url, + allowed_states=allowed_states, + ) + await validate_shipment_mr_reuse_state(repo, mr, product, group, assembly) + state = validate_shipment_mr_ci_state( + gitlab_client, + mr_url, + mr, + allow_active_stage=allow_active_stage, + ) + return mr, state + + +async def _restore_from_main(repo: GitRepository, path: str) -> None: + """Restore a stale MR file to its content on the target branch. + + Args: + repo: Checked-out shipment-data repository. + path: Repository-relative shipment file path. + + Raises: + RuntimeError: If the file cannot be read from ``main``. + """ + rc, content, error = await exectools.cmd_gather_async( + ["git", "-C", str(repo._directory), "show", f"main:{path}"], env=repo._local_env() + ) + if rc: + raise RuntimeError(f"Unable to restore {path} from main: {error}") + await repo.write_file(path, content) + + +async def reconcile_shipment_mr( + repo: GitRepository, + mr, + shipments_by_kind: Dict[str, ShipmentConfig], + *, + include_fbc_ocp_version: bool, + dry_run: bool, +) -> bool: + """Replace a reusable MR's layered-product shipment files from scratch. + + Every MR-owned file for the generated product, group, and assembly is + removed (or restored from ``main``), then the current shipment set is + written with deterministic filenames. No content or CI mutation from the + previous files is carried into the replacement. + + Args: + repo: Initialized shipment-data repository. + mr: Validated GitLab merge request to update. + shipments_by_kind: Current generated shipment models keyed by kind. + include_fbc_ocp_version: Include the target OCP version in new FBC + filenames. + dry_run: Prepare and display changes without committing or pushing. + + Returns: + Whether reconciliation produced content changes. A no-change rerun + returns ``False`` and is considered successful. + + Raises: + ValueError: If the MR branch lacks a timestamp or desired files contain + duplicate semantic identities or scopes. + RuntimeError: If GitLab truncates the MR change list or a stale file + cannot be restored safely. + """ + timestamp_match = _TIMESTAMP_RE.search(mr.source_branch) + if not timestamp_match: + raise ValueError(f"Cannot determine shipment timestamp from MR branch {mr.source_branch}") + timestamp = timestamp_match.group(1) + + desired_items = [(kind, _to_dict(config)) for kind, config in shipments_by_kind.items()] + desired_identities = [_identity(config) for _, config in desired_items] + duplicates = [identity for identity, count in Counter(desired_identities).items() if count > 1] + if duplicates: + raise ValueError(f"Generated shipment configurations contain duplicate identities: {duplicates}") + desired_by_identity = {identity: item for identity, item in zip(desired_identities, desired_items)} + desired_scopes = {identity[:3] for identity in desired_identities} + if len(desired_scopes) != 1: + raise ValueError(f"Generated shipment configurations span multiple scopes: {sorted(desired_scopes)}") + desired_scope = next(iter(desired_scopes)) + + await repo.fetch_switch_branch(mr.source_branch, remote="origin") + change_data = mr.changes() + if change_data.get('overflow'): + raise RuntimeError("GitLab truncated the shipment MR change list; refusing an incomplete reconciliation") + changes = change_data.get('changes', []) + existing_files: list[tuple[str, bool]] = [] + for change in changes: + path = change['new_path'] + if not _shipment_path_matches(path, desired_scope[1], desired_scope[2], product=desired_scope[0]): + continue + absolute_path = repo._directory / path + if not absolute_path.exists(): + continue + is_new = change.get('new_file') is True or change.get('new_file') == 'true' + existing_files.append((path, is_new)) + + for path, is_new in existing_files: + if is_new: + (repo._directory / path).unlink() + else: + await _restore_from_main(repo, path) + + next_counter = 1 + for identity, (kind, desired) in sorted(desired_by_identity.items(), key=_identity_sort_key): + metadata = desired['shipment']['metadata'] + target_dir = Path('shipment') / metadata['product'] / metadata['group'] / metadata['application'] / 'prod' + if metadata.get('fbc'): + ocp_part = f".ocp{identity[-1]}" if include_fbc_ocp_version and identity[-1] else "" + filename = f"{metadata['assembly']}.fbc{ocp_part}.{timestamp}{next_counter:02d}.yaml" + next_counter += 1 + else: + filename = f"{metadata['assembly']}.{kind.rstrip('0123456789')}.{timestamp}.yaml" + out = StringIO() + YAML.dump(desired, out) + path = target_dir / filename + (repo._directory / path).parent.mkdir(parents=True, exist_ok=True) + await repo.write_file(path, out.getvalue()) + + await repo.log_diff() + if dry_run: + return True + return await repo.commit_push(f"Update shipment configurations for {desired_identities[0][2]}", safe=True) diff --git a/pyartcd/pyartcd/pipelines/prepare_release_lp.py b/pyartcd/pyartcd/pipelines/prepare_release_lp.py index 3b7bd1e855..22008913ab 100644 --- a/pyartcd/pyartcd/pipelines/prepare_release_lp.py +++ b/pyartcd/pyartcd/pipelines/prepare_release_lp.py @@ -38,10 +38,19 @@ from elliottlib.util import get_advisory_boilerplate from github import GithubException -from pyartcd import constants +from pyartcd import constants, locks from pyartcd.cli import cli, click_coroutine, pass_runtime from pyartcd.fbc_util import validate_fbc_related_images from pyartcd.git import GitRepository +from pyartcd.lp_shipment import ( + ShipmentMRValidationError, + get_shipment_mr_url, + reconcile_shipment_mr, + set_shipment_mr_draft, + update_shipment_mr_url, + validate_shipment_mr_for_operation, + verify_shipment_mr_url, +) from pyartcd.runtime import Runtime from pyartcd.util import load_group_config @@ -49,6 +58,17 @@ def _normalize_release_date(date_str: str) -> str: + """Normalize a supported release date to ``YYYY-Mon-DD``. + + Args: + date_str: Date in ``YYYY-Mon-DD`` or ``YYYY-MM-DD`` format. + + Returns: + The normalized date string. + + Raises: + click.ClickException: If the input is not in a supported format. + """ for fmt in ("%Y-%b-%d", "%Y-%m-%d"): try: return datetime.strptime(date_str.strip(), fmt).strftime("%Y-%b-%d") @@ -80,14 +100,29 @@ def __init__( build_data_repo_url: Optional[str] = None, shipment_data_repo_url: Optional[str] = None, create_mr: bool = False, + force: bool = False, jira_bugs: Optional[List[str]] = None, target_release_date: Optional[str] = None, ) -> None: + """Initialize a layered-product prepare-release pipeline. + + Args: + runtime: pyartcd runtime and configuration. + group: ocp-build-data group branch. + assembly: Named layered-product assembly to prepare. + build_data_repo_url: Optional ocp-build-data pull URL override. + shipment_data_repo_url: Optional shipment-data repository override. + create_mr: Create or reuse a shipment merge request. + force: Replace the configured shipment MR instead of reusing it. + jira_bugs: Jira issues to include in generated release notes. + target_release_date: Optional normalized target ship date. + """ self._logger = logging.getLogger(__name__) self.runtime = runtime self.group = group self.assembly = assembly self.create_mr = create_mr + self.force = force self.dry_run = self.runtime.dry_run self.jira_bugs = jira_bugs self.target_release_date = target_release_date @@ -101,6 +136,7 @@ def __init__( self.gitlab_url = self.runtime.config.get("gitlab_url", "https://gitlab.cee.redhat.com") self.gitlab_token: Optional[str] = None self.shipment_mr_url: Optional[str] = None + self._configured_shipment_mr_url: Optional[str] = None self.job_url: Optional[str] = None self.product: Optional[str] = None @@ -199,7 +235,15 @@ async def _load_mr_approvers_from_group_config(self) -> dict[str, list[str]]: return {} async def _load_assembly(self) -> Dict: - """Read assembly definition from ocp-build-data releases.yml.""" + """Read the named assembly definition from ocp-build-data. + + Returns: + The complete assembly mapping from ``releases.yml``. + + Raises: + click.ClickException: If ``releases.yml`` or the named assembly is + missing. + """ self._logger.info("Reading assembly '%s' from releases.yml...", self.assembly) build_data_path = self._working_dir / "ocp-build-data-read" @@ -211,11 +255,12 @@ async def _load_assembly(self) -> Dict: if not releases_yaml_path.exists(): raise click.ClickException(f"releases.yml not found in {self.build_data_repo_url} on branch {self.group}") - releases_config = yaml.load(releases_yaml_path) + releases_config = yaml.load(releases_yaml_path) or {} assembly = releases_config.get('releases', {}).get(self.assembly) if assembly is None: raise click.ClickException(f"Assembly '{self.assembly}' not found in releases.yml") + self._configured_shipment_mr_url = get_shipment_mr_url(releases_config, self.assembly) return assembly def _extract_operand_nvrs(self, assembly_config: Dict) -> List[str]: @@ -814,6 +859,12 @@ async def _update_assembly_with_shipment_url(self, shipment_mr_url: str) -> None Pushes a commit to ocp-build-data to record the shipment MR URL in the assembly's ``group.shipment.mr`` field. + + Args: + shipment_mr_url: Shipment merge request URL to persist. + + Raises: + RuntimeError: If the assembly pointer changed concurrently. """ ocp_build_data_repo_push_url = self.runtime.config["build_config"]["ocp_build_data_repo_push_url"] @@ -825,32 +876,34 @@ async def _update_assembly_with_shipment_url(self, shipment_mr_url: str) -> None ) return - build_data_path = self._working_dir / "ocp-build-data-push" - build_data = GitRepository(build_data_path, dry_run=self.dry_run) + build_data = GitRepository(self._working_dir / "ocp-build-data-push", dry_run=self.dry_run) await build_data.setup(ocp_build_data_repo_push_url) - await build_data.fetch_switch_branch(self.group) - - releases_yaml_path = build_data_path / "releases.yml" - if not releases_yaml_path.exists(): - self._logger.warning("releases.yml not found; skipping shipment URL update") - return - - releases_yaml = yaml.load(releases_yaml_path) - assembly_entry = releases_yaml.get('releases', {}).get(self.assembly, {}) - assembly_def = assembly_entry.get('assembly', {}) - group_info = assembly_def.setdefault('group', {}) - shipment_info = group_info.setdefault('shipment', {}) - shipment_info['mr'] = shipment_mr_url - - yaml.dump(releases_yaml, releases_yaml_path) - - pushed = await build_data.commit_push(f"Update assembly {self.assembly}: add shipment MR URL") + pushed = await update_shipment_mr_url( + build_data, + self.group, + self.assembly, + shipment_mr_url, + self._configured_shipment_mr_url, + create_as_stream=False, + ) if pushed: self._logger.info("Updated releases.yml with shipment MR URL: %s", shipment_mr_url) else: self._logger.warning("No changes to commit when updating shipment MR URL") + async def _verify_assembly_shipment_url(self) -> None: + """Verify that the assembly still points at the MR selected for reuse. + + Raises: + RuntimeError: If another release changed the pointer concurrently. + """ + push_url = self.runtime.config["build_config"]["ocp_build_data_repo_push_url"] + build_data = GitRepository(self._working_dir / "ocp-build-data-verify", dry_run=self.dry_run) + await build_data.setup(push_url) + await verify_shipment_mr_url(build_data, self.group, self.assembly, self._configured_shipment_mr_url) + async def run(self) -> None: + """Build release artifacts and create or reconcile their shipment MR.""" self._logger.info( "Starting prepare-release-lp for group=%s assembly=%s", self.group, @@ -867,6 +920,40 @@ async def run(self) -> None: await self._setup_shipment_repo() assembly_config = await self._load_assembly() + existing_mr = None + force_previous_mr = None + if self.create_mr and self._configured_shipment_mr_url: + try: + candidate_mr, ci_state = await validate_shipment_mr_for_operation( + self._gitlab, + self.shipment_data_repo, + self._configured_shipment_mr_url, + self.shipment_data_repo_pull_url, + self.shipment_data_repo_push_url, + self.product, + self.group, + self.assembly, + allowed_states=('opened', 'closed') if self.force else ('opened',), + allow_active_stage=self.force, + ) + if self.force: + force_previous_mr = candidate_mr + if ci_state.active_stage: + self._logger.warning( + "Previous shipment MR has active stage work that may continue during replacement: %s", + "; ".join(ci_state.active_stage), + ) + else: + existing_mr = candidate_mr + self._logger.info("Will reuse shipment MR: %s", self._configured_shipment_mr_url) + except ShipmentMRValidationError as exc: + if not self.force: + raise + self._logger.warning( + "Configured shipment MR cannot be associated safely with this release and will be left " + "unchanged while --force creates a replacement: %s", + exc, + ) operand_nvrs = self._extract_operand_nvrs(assembly_config) self._logger.info("Assembly contains %d pinned operand NVRs", len(operand_nvrs)) @@ -917,11 +1004,86 @@ async def run(self) -> None: if shipments_by_kind: if self.create_mr: - mr_url = await self._create_shipment_mr(shipments_by_kind) + if existing_mr: + # Revalidate around the draft transition because CI or MR state may have changed + # since the initial check performed before the expensive build work. + existing_mr, _ = await validate_shipment_mr_for_operation( + self._gitlab, + self.shipment_data_repo, + self._configured_shipment_mr_url, + self.shipment_data_repo_pull_url, + self.shipment_data_repo_push_url, + self.product, + self.group, + self.assembly, + allowed_states=('opened',), + allow_active_stage=False, + ) + await self._verify_assembly_shipment_url() + set_shipment_mr_draft(existing_mr, self.dry_run) + # Drafting mutates the MR; check again before replacing its shipment files. + await validate_shipment_mr_for_operation( + self._gitlab, + self.shipment_data_repo, + self._configured_shipment_mr_url, + self.shipment_data_repo_pull_url, + self.shipment_data_repo_push_url, + self.product, + self.group, + self.assembly, + allowed_states=('opened',), + allow_active_stage=False, + ) + await reconcile_shipment_mr( + self.shipment_data_repo, + existing_mr, + shipments_by_kind, + include_fbc_ocp_version=False, + dry_run=self.dry_run, + ) + mr_url = self._configured_shipment_mr_url + self.shipment_mr_url = mr_url + else: + if force_previous_mr: + await self._verify_assembly_shipment_url() + force_previous_mr, force_ci_state = await validate_shipment_mr_for_operation( + self._gitlab, + self.shipment_data_repo, + self._configured_shipment_mr_url, + self.shipment_data_repo_pull_url, + self.shipment_data_repo_push_url, + self.product, + self.group, + self.assembly, + allowed_states=('opened', 'closed'), + allow_active_stage=True, + ) + if force_previous_mr.state == 'opened': + set_shipment_mr_draft(force_previous_mr, self.dry_run) + _, force_ci_state = await validate_shipment_mr_for_operation( + self._gitlab, + self.shipment_data_repo, + self._configured_shipment_mr_url, + self.shipment_data_repo_pull_url, + self.shipment_data_repo_push_url, + self.product, + self.group, + self.assembly, + allowed_states=('opened',), + allow_active_stage=True, + ) + if force_ci_state.active_stage: + self._logger.warning( + "Replacing a shipment MR while its stage work is active; the old staging operation " + "may continue, but the old MR has been made draft and cannot proceed to production: %s", + "; ".join(force_ci_state.active_stage), + ) + mr_url = await self._create_shipment_mr(shipments_by_kind) if mr_url: self._logger.info("Shipment MR: %s", mr_url) + if not existing_mr: + await self._update_assembly_with_shipment_url(mr_url) await self._set_shipment_mr_ready() - await self._update_assembly_with_shipment_url(mr_url) else: timestamp = datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S') for kind, config in shipments_by_kind.items(): @@ -967,6 +1129,14 @@ async def run(self) -> None: is_flag=True, help="Create a merge request in the shipment data repository (requires GITLAB_TOKEN).", ) +@click.option( + "--force", + is_flag=True, + help=( + "Create a replacement shipment MR and update releases.yml. An open previous MR is made draft; replacement " + "is refused if it was merged or production was attempted." + ), +) @click.option( "--jira-bugs", default=None, @@ -986,6 +1156,7 @@ async def prepare_release_lp( build_data_repo_url: Optional[str], shipment_data_repo_url: Optional[str], create_mr: bool, + force: bool, jira_bugs: Optional[str], target_release_date: Optional[str], ): @@ -1005,6 +1176,9 @@ async def prepare_release_lp( $ artcd prepare-release-lp -g acm-2.17 --assembly 2.17.3 \\ --create-mr --jira-bugs ACM-1234,ACM-5678 """ + if force and not create_mr: + raise click.ClickException("--force requires --create-mr") + jira_bugs_list = None if jira_bugs: jira_bugs_list = [j.strip() for j in jira_bugs.split(',') if j.strip()] @@ -1022,8 +1196,17 @@ async def prepare_release_lp( build_data_repo_url=build_data_repo_url, shipment_data_repo_url=shipment_data_repo_url, create_mr=create_mr, + force=force, jira_bugs=jira_bugs_list, target_release_date=normalized_date, ) - await pipeline.run() + if create_mr and not runtime.dry_run: + lock_name = locks.Lock.LAYERED_PRODUCT_SHIPMENT.value.format(group=group, assembly=assembly) + await locks.run_with_lock( + coro=pipeline.run(), + lock=locks.Lock.LAYERED_PRODUCT_SHIPMENT, + lock_name=lock_name, + ) + else: + await pipeline.run() diff --git a/pyartcd/pyartcd/pipelines/release_from_fbc.py b/pyartcd/pyartcd/pipelines/release_from_fbc.py index 056802d98c..583384d88b 100644 --- a/pyartcd/pyartcd/pipelines/release_from_fbc.py +++ b/pyartcd/pyartcd/pipelines/release_from_fbc.py @@ -37,12 +37,21 @@ from github import GithubException from tenacity import retry, stop_after_attempt -from pyartcd import constants +from pyartcd import constants, locks from pyartcd.cli import cli, click_coroutine, pass_runtime from pyartcd.fbc_util import extract_fbc_labels as _extract_fbc_labels from pyartcd.fbc_util import extract_ocp_version_from_nvr from pyartcd.fbc_util import validate_fbc_related_images as _validate_fbc_related_images from pyartcd.git import GitRepository +from pyartcd.lp_shipment import ( + ShipmentMRValidationError, + get_shipment_mr_url, + reconcile_shipment_mr, + set_shipment_mr_draft, + update_shipment_mr_url, + validate_shipment_mr_for_operation, + verify_shipment_mr_url, +) from pyartcd.runtime import Runtime yaml = new_roundtrip_yaml_handler() @@ -97,7 +106,26 @@ def __init__( ocp_optional: bool = False, exclude_nvr_components: Optional[List[str]] = None, release_jira: Optional[str] = None, + force: bool = False, ) -> None: + """Initialize an FBC-based release pipeline. + + Args: + runtime: pyartcd runtime and configuration. + group: ocp-build-data group branch. + assembly: Release assembly name. + fbc_pullspecs: FBC images supplying release content. + create_mr: Create or reuse a shipment merge request. + shipment_data_repo_url: Optional shipment-data repository override. + shipment_path: Optional local output directory. + jira_bugs: Jira issues to include in release notes. + target_release_date: Optional normalized target ship date. + extra_image_nvrs: Additional image NVRs to ship. + ocp_optional: Use the independent OCP optional-operator workflow. + exclude_nvr_components: Components excluded from OCP optional mode. + release_jira: Jira release-request issue to link to the shipment MR. + force: Replace the configured layered-product shipment MR. + """ self.logger = logging.getLogger(__name__) self.runtime = runtime self.group = group @@ -105,6 +133,7 @@ def __init__( self.fbc_pullspecs = fbc_pullspecs self.extra_image_nvrs = extra_image_nvrs or [] self.create_mr = create_mr + self.force = force self.dry_run = self.runtime.dry_run self.ocp_optional = ocp_optional self.excluded_components = set(exclude_nvr_components) if exclude_nvr_components else set() @@ -119,6 +148,7 @@ def __init__( self.gitlab_url = self.runtime.config.get("gitlab_url", "https://gitlab.cee.redhat.com") self.gitlab_token = None self.shipment_mr_url = None + self._configured_shipment_mr_url: Optional[str] = None self.job_url = None # Product configuration - initialized to None, will be loaded from group config in run() @@ -192,6 +222,57 @@ def get_file_from_branch(self, branch: str, filename: str, data_path: str | None except GithubException as e: raise ValueError(f"Failed to fetch {filename} from {data_path} branch {branch}: {e}") + def _load_layered_product_shipment_mr(self) -> Optional[str]: + """Load the assembly's layered-product shipment MR pointer. + + Returns: + The configured MR URL, or ``None`` when the assembly or pointer is + absent. + """ + content = self.get_file_from_branch(self.group, "releases.yml") + releases_config = yaml.load(content.decode()) or {} + self._configured_shipment_mr_url = get_shipment_mr_url(releases_config, self.assembly) + return self._configured_shipment_mr_url + + async def _update_layered_product_shipment_mr(self, mr_url: str) -> None: + """Persist a newly created or replacement shipment MR pointer. + + Args: + mr_url: Shipment MR URL to store in ``releases.yml``. + + Raises: + RuntimeError: If the pointer changed concurrently. + """ + if self.dry_run: + self.logger.info("[DRY-RUN] Would store shipment MR in releases.yml: %s", mr_url) + return + push_url = self.runtime.config.get("build_config", {}).get( + "ocp_build_data_repo_push_url", constants.OCP_BUILD_DATA_URL + ) + build_data_repo = GitRepository(self.working_dir / "ocp-build-data-push", dry_run=self.dry_run) + await build_data_repo.setup(push_url) + await update_shipment_mr_url( + build_data_repo, + self.group, + self.assembly, + mr_url, + self._configured_shipment_mr_url, + create_as_stream=True, + ) + + async def _verify_layered_product_shipment_mr(self) -> None: + """Verify that the assembly still points at the selected reusable MR. + + Raises: + RuntimeError: If another release changed the pointer concurrently. + """ + push_url = self.runtime.config.get("build_config", {}).get( + "ocp_build_data_repo_push_url", constants.OCP_BUILD_DATA_URL + ) + build_data_repo = GitRepository(self.working_dir / "ocp-build-data-verify", dry_run=self.dry_run) + await build_data_repo.setup(push_url) + await verify_shipment_mr_url(build_data_repo, self.group, self.assembly, self._configured_shipment_mr_url) + def _load_release_notes_template(self, kind: str | None = None) -> dict | None: """ Load and populate release notes template from ocp-build-data advisory_templates.yml. @@ -1108,10 +1189,48 @@ async def run(self) -> None: if self.create_mr: await self.setup_shipment_repo() - # Load product from group configuration + # Load product before validating a reusable MR so an unrelated product's + # shipment cannot be modified through an incorrect releases.yml pointer. self.product = await self._load_product_from_group_config() self.logger.info(f"Loaded product '{self.product}' - continuing workflow for {self.product} {self.assembly}") + existing_mr = None + force_previous_mr = None + if self.create_mr and not self.ocp_optional: + configured_mr_url = self._load_layered_product_shipment_mr() + if configured_mr_url: + try: + candidate_mr, ci_state = await validate_shipment_mr_for_operation( + self._gitlab, + self.shipment_data_repo, + configured_mr_url, + self.shipment_data_repo_pull_url, + self.shipment_data_repo_push_url, + self.product, + self.group, + self.assembly, + allowed_states=('opened', 'closed') if self.force else ('opened',), + allow_active_stage=self.force, + ) + if self.force: + force_previous_mr = candidate_mr + if ci_state.active_stage: + self.logger.warning( + "Previous shipment MR has active stage work that may continue during replacement: %s", + "; ".join(ci_state.active_stage), + ) + else: + existing_mr = candidate_mr + self.logger.info("Will reuse shipment MR: %s", configured_mr_url) + except ShipmentMRValidationError as exc: + if not self.force: + raise + self.logger.warning( + "Configured shipment MR cannot be associated safely with this release and will be left " + "unchanged while --force creates a replacement: %s", + exc, + ) + related_nvrs = [] fbc_nvrs = [] @@ -1239,22 +1358,99 @@ async def run(self) -> None: # Create MR if requested if self.create_mr and shipments_by_kind: - try: - mr_url = await self.create_shipment_mr(shipments_by_kind, env="prod") - if mr_url: - self.logger.info(f"Created shipment MR: {mr_url}") - - if self.ocp_optional: - main_ocp_mr_url = self._get_main_ocp_shipment_url() - if main_ocp_mr_url: - await self._set_shipment_mr_dependency(main_ocp_mr_url) - + if self.ocp_optional: + try: + mr_url = await self.create_shipment_mr(shipments_by_kind, env="prod") + self.logger.info("Shipment MR: %s", mr_url) + main_ocp_mr_url = self._get_main_ocp_shipment_url() + if main_ocp_mr_url: + await self._set_shipment_mr_dependency(main_ocp_mr_url) self._update_jira_with_mr_link(mr_url) await self.set_shipment_mr_ready() - except Exception as e: - self.logger.exception(f"Failed to create MR: {e}") - if not self.dry_run: - self.logger.info("Continuing with local files only") + except Exception as e: + self.logger.exception("Failed to create MR: %s", e) + if not self.dry_run: + self.logger.info("Continuing with local files only") + else: + if existing_mr: + # Revalidate around the draft transition because CI or MR state may have changed + # since the initial check performed before the expensive build work. + existing_mr, _ = await validate_shipment_mr_for_operation( + self._gitlab, + self.shipment_data_repo, + self._configured_shipment_mr_url, + self.shipment_data_repo_pull_url, + self.shipment_data_repo_push_url, + self.product, + self.group, + self.assembly, + allowed_states=('opened',), + allow_active_stage=False, + ) + await self._verify_layered_product_shipment_mr() + set_shipment_mr_draft(existing_mr, self.dry_run) + # Drafting mutates the MR; check again before replacing its shipment files. + await validate_shipment_mr_for_operation( + self._gitlab, + self.shipment_data_repo, + self._configured_shipment_mr_url, + self.shipment_data_repo_pull_url, + self.shipment_data_repo_push_url, + self.product, + self.group, + self.assembly, + allowed_states=('opened',), + allow_active_stage=False, + ) + await reconcile_shipment_mr( + self.shipment_data_repo, + existing_mr, + shipments_by_kind, + include_fbc_ocp_version=True, + dry_run=self.dry_run, + ) + mr_url = self._configured_shipment_mr_url + self.shipment_mr_url = mr_url + else: + if force_previous_mr: + await self._verify_layered_product_shipment_mr() + force_previous_mr, force_ci_state = await validate_shipment_mr_for_operation( + self._gitlab, + self.shipment_data_repo, + self._configured_shipment_mr_url, + self.shipment_data_repo_pull_url, + self.shipment_data_repo_push_url, + self.product, + self.group, + self.assembly, + allowed_states=('opened', 'closed'), + allow_active_stage=True, + ) + if force_previous_mr.state == 'opened': + set_shipment_mr_draft(force_previous_mr, self.dry_run) + _, force_ci_state = await validate_shipment_mr_for_operation( + self._gitlab, + self.shipment_data_repo, + self._configured_shipment_mr_url, + self.shipment_data_repo_pull_url, + self.shipment_data_repo_push_url, + self.product, + self.group, + self.assembly, + allowed_states=('opened',), + allow_active_stage=True, + ) + if force_ci_state.active_stage: + self.logger.warning( + "Replacing a shipment MR while its stage work is active; the old staging operation " + "may continue, but the old MR has been made draft and cannot proceed to production: %s", + "; ".join(force_ci_state.active_stage), + ) + mr_url = await self.create_shipment_mr(shipments_by_kind, env="prod") + await self._update_layered_product_shipment_mr(mr_url) + self.logger.info("Shipment MR: %s", mr_url) + self._update_jira_with_mr_link(mr_url) + await self.set_shipment_mr_ready() # Generate completion message completion_msg = ( @@ -1302,6 +1498,14 @@ async def run(self) -> None: is_flag=True, help="Create a merge request in the shipment data repository (requires GITLAB_TOKEN environment variable)", ) +@click.option( + "--force", + is_flag=True, + help=( + "Create a replacement shipment MR and update releases.yml. An open previous MR is made draft; replacement " + "is refused if it was merged or production was attempted." + ), +) @click.option( '--shipment-data-repo-url', help='Shipment data repository URL for MR creation. If not provided, will use default based on configuration.', @@ -1353,6 +1557,7 @@ async def release_from_fbc( fbc_pullspecs: str, extra_image_nvrs: str, create_mr: bool, + force: bool, shipment_data_repo_url: Optional[str], shipment_path: Optional[str], jira_bugs: Optional[str], @@ -1407,6 +1612,11 @@ async def release_from_fbc( --exclude-nvr-components kube-rbac-proxy-container \\ --create-mr """ + if force and not create_mr: + raise click.ClickException("--force requires --create-mr") + if force and ocp_optional: + raise click.ClickException("--force is only supported for layered-product releases, not --ocp-optional") + fbc_pullspecs_list = [spec.strip() for spec in fbc_pullspecs.split(',') if spec.strip()] extra_image_nvrs_list = [nvr.strip() for nvr in extra_image_nvrs.split(',') if nvr.strip()] @@ -1446,6 +1656,15 @@ async def release_from_fbc( ocp_optional=ocp_optional, exclude_nvr_components=exclude_nvr_components_list, release_jira=release_jira, + force=force, ) - await pipeline.run() + if create_mr and not ocp_optional and not runtime.dry_run: + lock_name = locks.Lock.LAYERED_PRODUCT_SHIPMENT.value.format(group=group, assembly=assembly) + await locks.run_with_lock( + coro=pipeline.run(), + lock=locks.Lock.LAYERED_PRODUCT_SHIPMENT, + lock_name=lock_name, + ) + else: + await pipeline.run() diff --git a/pyartcd/tests/pipelines/test_prepare_release_lp.py b/pyartcd/tests/pipelines/test_prepare_release_lp.py index 4486a55b1b..aae43ec12f 100644 --- a/pyartcd/tests/pipelines/test_prepare_release_lp.py +++ b/pyartcd/tests/pipelines/test_prepare_release_lp.py @@ -3,6 +3,7 @@ import unittest from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch +from pyartcd.lp_shipment import ShipmentMRActiveStageError from pyartcd.pipelines.prepare_release_lp import PrepareReleaseLPPipeline @@ -108,6 +109,59 @@ def test_check_env_vars_requires_gitlab_token(self): with self.assertRaises(ValueError): pipeline._check_env_vars() + def test_cli_force_requires_create_mr(self): + """Reject force when shipment MR creation is disabled.""" + from click.testing import CliRunner + from pyartcd.pipelines.prepare_release_lp import prepare_release_lp + from pyartcd.runtime import Runtime + + runtime = MagicMock(spec=Runtime) + runtime.dry_run = False + runtime.working_dir = MagicMock() + runtime.config = {} + asyncio.set_event_loop(asyncio.new_event_loop()) + result = CliRunner().invoke( + prepare_release_lp, + ["--group", "acm-2.17", "--assembly", "2.17.3", "--force"], + obj=runtime, + standalone_mode=False, + ) + self.assertIn("--force requires --create-mr", str(result.exception)) + + @patch("pyartcd.pipelines.prepare_release_lp.locks.run_with_lock", new_callable=AsyncMock) + @patch("pyartcd.pipelines.prepare_release_lp.PrepareReleaseLPPipeline") + def test_cli_locks_layered_product_shipment_updates(self, pipeline_cls, run_with_lock): + """Serialize prepare-release shipment updates by group and assembly.""" + from click.testing import CliRunner + from pyartcd.pipelines.prepare_release_lp import prepare_release_lp + from pyartcd.runtime import Runtime + + async def await_pipeline(coro, **_kwargs): + """Execute the coroutine passed through the mocked lock.""" + return await coro + + run_with_lock.side_effect = await_pipeline + pipeline_cls.return_value.run = AsyncMock() + runtime = MagicMock(spec=Runtime) + runtime.dry_run = False + runtime.working_dir = MagicMock() + runtime.config = {} + asyncio.set_event_loop(asyncio.new_event_loop()) + + result = CliRunner().invoke( + prepare_release_lp, + ["--group", "acm-2.17", "--assembly", "2.17.3", "--create-mr"], + obj=runtime, + standalone_mode=False, + ) + + self.assertIsNone(result.exception) + run_with_lock.assert_awaited_once() + self.assertEqual( + run_with_lock.await_args.kwargs["lock_name"], + "lock:layered-product-shipment:acm-2.17:2.17.3", + ) + class TestPrepareReleaseLPMultiFBC(unittest.TestCase): """Tests for multi-FBC (multi-OCP-version) FBC builds in prepare-release-lp.""" @@ -443,6 +497,85 @@ def _make_pipeline(self, tmp_dir, **overrides): kwargs.update(overrides) return PrepareReleaseLPPipeline(**kwargs) + @patch('pyartcd.pipelines.prepare_release_lp.validate_shipment_mr_for_operation', new_callable=AsyncMock) + def test_active_stage_blocks_before_expensive_build_work(self, mock_validate): + """Reject unsafe reuse before bundle and FBC builds begin.""" + import tempfile + + mock_validate.side_effect = ShipmentMRActiveStageError("stage pipeline is running") + with tempfile.TemporaryDirectory() as tmp_dir: + pipeline = self._make_pipeline(tmp_dir, create_mr=True) + pipeline._configured_shipment_mr_url = "https://gitlab.example/project/-/merge_requests/42" + pipeline._check_env_vars = MagicMock() + pipeline._setup_working_dir = MagicMock() + pipeline._setup_shipment_repo = AsyncMock() + pipeline._load_product_from_group_config = AsyncMock(return_value="rhacm2") + pipeline._load_assembly = AsyncMock(return_value={'assembly': {'type': 'standard'}}) + pipeline._trigger_bundle_build = AsyncMock() + pipeline.__dict__['_gitlab'] = MagicMock() + + with self.assertRaisesRegex(ShipmentMRActiveStageError, "stage pipeline is running"): + asyncio.run(pipeline.run()) + + pipeline._trigger_bundle_build.assert_not_awaited() + + @patch('pyartcd.pipelines.prepare_release_lp.validate_shipment_mr_for_operation', new_callable=AsyncMock) + def test_force_makes_open_previous_mr_draft_before_replacement(self, mock_validate): + """Supersede an open stage-only MR without modifying its shipment files.""" + import tempfile + + previous_mr = MagicMock( + state='opened', + title='Shipment for rhacm2 2.17.3', + labels=['stage-release-success', 'reviewed'], + ) + ci_state = MagicMock(active_stage=('stage pipeline is running',), prod_attempts=()) + mock_validate.return_value = (previous_mr, ci_state) + + with tempfile.TemporaryDirectory() as tmp_dir: + pipeline = self._make_pipeline(tmp_dir, create_mr=True, force=True) + pipeline.dry_run = False + pipeline._configured_shipment_mr_url = "https://gitlab.example/project/-/merge_requests/42" + pipeline._check_env_vars = MagicMock() + pipeline._setup_working_dir = MagicMock() + pipeline._setup_shipment_repo = AsyncMock() + pipeline._load_product_from_group_config = AsyncMock(return_value="rhacm2") + pipeline._load_assembly = AsyncMock( + return_value={ + 'assembly': { + 'type': 'standard', + 'members': { + 'images': [ + { + 'distgit_key': 'search-v2-api-container', + 'metadata': {'is': {'nvr': 'search-v2-api-container-2.17.3-1'}}, + } + ] + }, + } + } + ) + pipeline._trigger_bundle_build = AsyncMock(return_value=([], [])) + pipeline._trigger_fbc_build = AsyncMock(return_value=([], [])) + pipeline._create_snapshot = AsyncMock(return_value=[MagicMock()]) + pipeline._create_shipment_config = MagicMock(return_value=MagicMock()) + pipeline._load_release_notes_template = MagicMock(return_value=None) + pipeline._verify_assembly_shipment_url = AsyncMock() + pipeline._create_shipment_mr = AsyncMock(return_value="https://gitlab.example/project/-/merge_requests/43") + pipeline._update_assembly_with_shipment_url = AsyncMock() + pipeline._set_shipment_mr_ready = AsyncMock() + pipeline.__dict__['_gitlab'] = MagicMock() + + asyncio.run(pipeline.run()) + + self.assertEqual(previous_mr.title, 'Draft: Shipment for rhacm2 2.17.3') + self.assertEqual(previous_mr.labels, ['reviewed']) + previous_mr.save.assert_called_once_with() + pipeline._create_shipment_mr.assert_awaited_once() + pipeline._update_assembly_with_shipment_url.assert_awaited_once_with( + "https://gitlab.example/project/-/merge_requests/43" + ) + @patch.object(PrepareReleaseLPPipeline, '_load_release_notes_template', return_value=None) @patch.object(PrepareReleaseLPPipeline, '_create_snapshot', new_callable=AsyncMock) @patch.object(PrepareReleaseLPPipeline, '_trigger_fbc_build', new_callable=AsyncMock) diff --git a/pyartcd/tests/pipelines/test_release_from_fbc.py b/pyartcd/tests/pipelines/test_release_from_fbc.py index 4a375807c2..27e0f46286 100644 --- a/pyartcd/tests/pipelines/test_release_from_fbc.py +++ b/pyartcd/tests/pipelines/test_release_from_fbc.py @@ -13,6 +13,7 @@ SnapshotComponent, SnapshotSpec, ) +from pyartcd.lp_shipment import ShipmentMRActiveStageError from pyartcd.pipelines.release_from_fbc import ReleaseFromFbcPipeline, _normalize_release_date @@ -861,6 +862,53 @@ def test_both_empty_raises_error(self): self.assertIsInstance(result.exception, click.ClickException) self.assertIn("At least one of", str(result.exception)) + def test_force_requires_create_mr(self): + """Reject force when shipment MR creation is disabled.""" + result = self._invoke(["--extra-image-nvrs", "foo-container-1.0-1.el9", "--force"]) + self.assertIsInstance(result.exception, click.ClickException) + self.assertIn("--force requires --create-mr", str(result.exception)) + + def test_force_rejected_for_ocp_optional(self): + """Keep replacement behavior out of the OCP optional path.""" + result = self._invoke( + ["--extra-image-nvrs", "foo-container-1.0-1.el9", "--create-mr", "--force", "--ocp-optional"] + ) + self.assertIsInstance(result.exception, click.ClickException) + self.assertIn("only supported for layered-product", str(result.exception)) + + @patch("pyartcd.pipelines.release_from_fbc.locks.run_with_lock", new_callable=AsyncMock) + @patch("pyartcd.pipelines.release_from_fbc.ReleaseFromFbcPipeline") + def test_layered_product_create_mr_uses_assembly_lock(self, pipeline_cls, run_with_lock): + """Serialize layered-product shipment updates by group and assembly.""" + + async def await_pipeline(coro, **_kwargs): + """Execute the coroutine passed through the mocked lock.""" + return await coro + + run_with_lock.side_effect = await_pipeline + pipeline_cls.return_value.run = AsyncMock() + + result = self._invoke(["--extra-image-nvrs", "foo-container-1.0-1.el9", "--create-mr"]) + + self.assertIsNone(result.exception) + run_with_lock.assert_awaited_once() + self.assertEqual( + run_with_lock.await_args.kwargs["lock_name"], + "lock:layered-product-shipment:oadp-1.5:1.5.3", + ) + + @patch("pyartcd.pipelines.release_from_fbc.locks.run_with_lock", new_callable=AsyncMock) + @patch("pyartcd.pipelines.release_from_fbc.ReleaseFromFbcPipeline") + def test_ocp_optional_create_mr_does_not_use_layered_product_lock(self, pipeline_cls, run_with_lock): + """Leave OCP optional shipment MR creation outside the LP lock path.""" + pipeline_cls.return_value.run = AsyncMock() + + result = self._invoke(["--extra-image-nvrs", "foo-container-1.0-1.el9", "--create-mr", "--ocp-optional"]) + + self.assertIsNone(result.exception) + run_with_lock.assert_not_awaited() + pipeline_cls.return_value.run.assert_awaited_once() + @patch("pyartcd.pipelines.release_from_fbc.ReleaseFromFbcPipeline") def test_fbc_only_does_not_raise(self, mock_pipeline_cls): """Providing only --fbc-pullspecs should pass CLI validation.""" @@ -1382,6 +1430,70 @@ def test_categorize_bundles_go_to_extras(self): # -- run() integration tests -- + @patch('pyartcd.pipelines.release_from_fbc.validate_shipment_mr_for_operation', new_callable=AsyncMock) + def test_active_stage_blocks_before_fbc_processing(self, mock_validate): + """Reject unsafe layered-product reuse before processing release inputs.""" + mock_validate.side_effect = ShipmentMRActiveStageError("stage pipeline is running") + pipeline = self._make_pipeline(ocp_optional=False, group="oadp-1.5", assembly="1.5.8") + pipeline.create_mr = True + pipeline.check_env_vars = MagicMock() + pipeline.setup_working_dir = MagicMock() + pipeline.setup_shipment_repo = AsyncMock() + pipeline._load_product_from_group_config = AsyncMock(return_value="oadp") + pipeline._load_layered_product_shipment_mr = MagicMock( + return_value="https://gitlab.example/project/-/merge_requests/42" + ) + pipeline.validate_fbc_related_images = AsyncMock() + pipeline.__dict__['_gitlab'] = MagicMock() + + with self.assertRaisesRegex(ShipmentMRActiveStageError, "stage pipeline is running"): + asyncio.run(pipeline.run()) + + pipeline.validate_fbc_related_images.assert_not_awaited() + + @patch('pyartcd.pipelines.release_from_fbc.validate_shipment_mr_for_operation', new_callable=AsyncMock) + def test_force_makes_open_previous_mr_draft_before_replacement(self, mock_validate): + """Draft an open stage-only MR before direct release creates its replacement.""" + previous_mr = MagicMock( + state='opened', + title='Shipment for oadp 1.5.8', + labels=['stage-release-success', 'reviewed'], + ) + ci_state = MagicMock(active_stage=('stage pipeline is running',), prod_attempts=()) + mock_validate.return_value = (previous_mr, ci_state) + pipeline = self._make_pipeline(ocp_optional=False, group="oadp-1.5", assembly="1.5.8") + pipeline.create_mr = True + pipeline.force = True + pipeline.fbc_pullspecs = [] + pipeline.extra_image_nvrs = ["oadp-container-v1.5.8-1.el9"] + pipeline._configured_shipment_mr_url = "https://gitlab.example/project/-/merge_requests/42" + pipeline.check_env_vars = MagicMock() + pipeline.setup_working_dir = MagicMock() + pipeline.setup_shipment_repo = AsyncMock() + pipeline._load_product_from_group_config = AsyncMock(return_value="oadp") + pipeline._load_layered_product_shipment_mr = MagicMock( + return_value="https://gitlab.example/project/-/merge_requests/42" + ) + pipeline.create_snapshot = AsyncMock(return_value=_make_snapshot(app="oadp-1-5")) + pipeline.create_shipment_config = MagicMock(return_value=MagicMock()) + pipeline._load_release_notes_template = MagicMock(return_value=None) + pipeline._verify_layered_product_shipment_mr = AsyncMock() + pipeline.create_shipment_mr = AsyncMock(return_value="https://gitlab.example/project/-/merge_requests/43") + pipeline._update_layered_product_shipment_mr = AsyncMock() + pipeline.set_shipment_mr_ready = AsyncMock() + pipeline.__dict__['_gitlab'] = MagicMock() + + with patch('pyartcd.pipelines.release_from_fbc.is_nvr_embargoed', return_value=False): + asyncio.run(pipeline.run()) + + self.assertEqual(previous_mr.title, 'Draft: Shipment for oadp 1.5.8') + self.assertEqual(previous_mr.labels, ['reviewed']) + previous_mr.save.assert_called_once_with() + pipeline.create_shipment_mr.assert_awaited_once() + pipeline._update_layered_product_shipment_mr.assert_awaited_once_with( + "https://gitlab.example/project/-/merge_requests/43" + ) + def test_extra_image_nvrs_merged_into_extras_key(self): """In OCP optional mode, extra_image_nvrs should merge into 'extras', not 'image'.""" pipeline = self._make_pipeline(ocp_optional=True) @@ -1554,6 +1666,8 @@ def test_mr_dependency_not_set_for_default_mode(self): pipeline.check_env_vars = MagicMock() pipeline.setup_working_dir = MagicMock() pipeline.setup_shipment_repo = AsyncMock() + pipeline._load_layered_product_shipment_mr = MagicMock(return_value=None) + pipeline._update_layered_product_shipment_mr = AsyncMock() pipeline._load_product_from_group_config = AsyncMock(return_value="oadp") pipeline._load_release_notes_template = MagicMock(return_value=None) pipeline.create_snapshot = AsyncMock(return_value=_make_snapshot(app="oadp-1-5")) diff --git a/pyartcd/tests/test_lp_shipment.py b/pyartcd/tests/test_lp_shipment.py new file mode 100644 index 0000000000..939b94ef78 --- /dev/null +++ b/pyartcd/tests/test_lp_shipment.py @@ -0,0 +1,544 @@ +import asyncio +from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from artcommonlib.util import new_roundtrip_yaml_handler +from elliottlib.shipment_model import ShipmentConfig +from pyartcd.git import GitRepository +from pyartcd.lp_shipment import ( + ShipmentMRActiveStageError, + ShipmentMRProductionError, + ShipmentMRScopeError, + ShipmentMRValidationError, + _identity, + get_shipment_mr_url, + inspect_shipment_mr_ci_state, + reconcile_shipment_mr, + set_shipment_mr_draft, + update_shipment_mr_url, + validate_shipment_mr, + validate_shipment_mr_ci_state, + validate_shipment_mr_reuse_state, +) + +YAML = new_roundtrip_yaml_handler() + + +def _shipment(*, fbc=False, nvr=None, release_notes=True): + """Build a minimal layered-product shipment mapping for tests.""" + shipment = { + 'metadata': { + 'product': 'openshift-logging', + 'application': 'fbc-logging-6-5' if fbc else 'logging-6-5', + 'group': 'logging-6.5', + 'assembly': '6.5.2', + 'fbc': fbc, + }, + 'environments': { + 'stage': {'releasePlan': 'stage-plan'}, + 'prod': {'releasePlan': 'prod-plan'}, + }, + 'snapshot': { + 'spec': {'application': 'fbc-logging-6-5' if fbc else 'logging-6-5', 'components': []}, + 'nvrs': [nvr] if nvr else ['logging-container-6.5.2-1.el9'], + }, + } + if release_notes: + shipment['data'] = {'releaseNotes': {'type': 'RHBA'}} + return {'shipment': shipment} + + +def test_get_shipment_mr_url(): + """Return the configured MR pointer and tolerate a missing assembly.""" + config = {'releases': {'6.5.2': {'assembly': {'group': {'shipment': {'mr': 'https://example/mr/1'}}}}}} + assert get_shipment_mr_url(config, '6.5.2') == 'https://example/mr/1' + assert get_shipment_mr_url({}, '6.5.2') is None + + +def test_fbc_identity_uses_component_and_ocp_target(): + """Distinguish FBC shipments by both operator and target OCP version.""" + first = _identity(_shipment(fbc=True, nvr='cluster-logging-operator-fbc-6.5.2-1.ocp4.19')) + second = _identity(_shipment(fbc=True, nvr='cluster-logging-operator-fbc-6.5.2-2.ocp4.20')) + other_operator = _identity(_shipment(fbc=True, nvr='loki-operator-fbc-6.5.2-1.ocp4.19')) + assert first != second + assert first != other_operator + + +def test_validate_shipment_mr(): + """Accept an open MR with the configured source and target repositories.""" + client = MagicMock() + client._parse_mr_url.return_value = ('hybrid-platforms/art/ocp-shipment-data', '42') + mr = MagicMock(state='opened', source_project_id=10, target_branch='main', labels=[]) + client.get_mr_from_url.return_value = mr + client.get_project.return_value.path_with_namespace = 'openshift-eng/ocp-shipment-data' + + assert ( + validate_shipment_mr( + client, + 'https://gitlab.example/hybrid-platforms/art/ocp-shipment-data/-/merge_requests/42', + 'https://gitlab.example/hybrid-platforms/art/ocp-shipment-data.git', + 'https://gitlab.example/openshift-eng/ocp-shipment-data.git', + ) + is mr + ) + + +def test_validate_shipment_mr_rejects_closed_mr(): + """Reject a closed shipment MR and direct the operator to use force.""" + client = MagicMock() + client._parse_mr_url.return_value = ('hybrid-platforms/art/ocp-shipment-data', '42') + client.get_mr_from_url.return_value = MagicMock(state='closed') + + with pytest.raises(ShipmentMRValidationError, match='--force'): + validate_shipment_mr( + client, + 'https://gitlab.example/hybrid-platforms/art/ocp-shipment-data/-/merge_requests/42', + 'https://gitlab.example/hybrid-platforms/art/ocp-shipment-data.git', + 'https://gitlab.example/openshift-eng/ocp-shipment-data.git', + ) + + +def test_validate_shipment_mr_rejects_prod_release_label(): + """Reject an open MR after shipment CI records production success.""" + client = MagicMock() + client._parse_mr_url.return_value = ('hybrid-platforms/art/ocp-shipment-data', '42') + mr = MagicMock( + state='opened', + source_project_id=10, + target_branch='main', + labels=['stage-release-success', 'prod-release-success'], + ) + client.get_mr_from_url.return_value = mr + client.get_project.return_value.path_with_namespace = 'openshift-eng/ocp-shipment-data' + + with pytest.raises(ShipmentMRProductionError, match='prod-release-success') as exc_info: + validate_shipment_mr( + client, + 'https://gitlab.example/hybrid-platforms/art/ocp-shipment-data/-/merge_requests/42', + 'https://gitlab.example/hybrid-platforms/art/ocp-shipment-data.git', + 'https://gitlab.example/openshift-eng/ocp-shipment-data.git', + ) + assert 'manual recovery' in str(exc_info.value) + + +def _shipment_ci_graph( + *, + parent_status='manual', + stage_status='success', + downstream_stage_status='success', + stage_job_status='success', + prod_status='manual', + prod_downstream=None, +): + """Build a minimal python-gitlab object graph for Shipment CI tests.""" + client = MagicMock() + client._parse_mr_url.return_value = ('hybrid-platforms/art/ocp-shipment-data', '42') + + project = SimpleNamespace(id=10, pipelines=MagicMock()) + parent_pipeline = SimpleNamespace( + id=100, + status=parent_status, + web_url='https://gitlab.example/pipelines/100', + bridges=MagicMock(), + ) + stage_downstream = {'id': 200, 'project_id': 10} + stage_bridge = SimpleNamespace(name='stage-job', status=stage_status, downstream_pipeline=stage_downstream) + prod_bridge = SimpleNamespace(name='prod-job', status=prod_status, downstream_pipeline=prod_downstream) + parent_pipeline.bridges.list.return_value = [stage_bridge, prod_bridge] + + stage_pipeline = SimpleNamespace( + id=200, + status=downstream_stage_status, + web_url='https://gitlab.example/pipelines/200', + jobs=MagicMock(), + ) + stage_pipeline.jobs.list.return_value = [SimpleNamespace(name='shipment-watch-stage', status=stage_job_status)] + + def get_pipeline(pipeline_id): + return parent_pipeline if pipeline_id == 100 else stage_pipeline + + project.pipelines.get.side_effect = get_pipeline + client.get_project.return_value = project + mr = SimpleNamespace(pipelines=MagicMock()) + mr.pipelines.list.return_value = [SimpleNamespace(id=100)] + return client, mr, parent_pipeline, stage_pipeline + + +def test_inspect_shipment_mr_ci_state_accepts_terminal_stage_and_manual_prod(): + """Allow the normal reuse window after stage and before manual prod starts.""" + client, mr, parent, stage = _shipment_ci_graph() + + state = inspect_shipment_mr_ci_state(client, 'https://gitlab.example/project/-/merge_requests/42', mr) + + assert state.active_stage == () + assert state.prod_attempts == () + mr.pipelines.list.assert_called_once_with(get_all=True) + parent.bridges.list.assert_called_once_with(get_all=True) + stage.jobs.list.assert_called_once_with(get_all=True, include_retried=True) + + +@pytest.mark.parametrize( + 'status', ['created', 'waiting_for_resource', 'preparing', 'pending', 'running', 'scheduled', 'canceling'] +) +def test_validate_shipment_mr_ci_state_rejects_active_stage(status): + """Reject every nonterminal stage state during in-place reuse.""" + client, mr, _, _ = _shipment_ci_graph(stage_status=status, downstream_stage_status=status, stage_job_status=status) + + with pytest.raises(ShipmentMRActiveStageError, match='active stage work'): + validate_shipment_mr_ci_state( + client, + 'https://gitlab.example/project/-/merge_requests/42', + mr, + allow_active_stage=False, + ) + + +def test_validate_shipment_mr_ci_state_allows_active_stage_for_force(): + """Allow force replacement to isolate a new MR while old stage continues.""" + client, mr, _, _ = _shipment_ci_graph( + parent_status='running', + stage_status='running', + downstream_stage_status='running', + stage_job_status='running', + ) + + state = validate_shipment_mr_ci_state( + client, + 'https://gitlab.example/project/-/merge_requests/42', + mr, + allow_active_stage=True, + ) + + assert state.active_stage + assert not state.prod_attempts + + +@pytest.mark.parametrize('status', ['created', 'pending', 'running', 'success', 'failed', 'canceled']) +def test_validate_shipment_mr_ci_state_rejects_any_prod_attempt(status): + """Block replacement once the production bridge leaves its untouched state.""" + client, mr, _, _ = _shipment_ci_graph(prod_status=status) + + with pytest.raises(ShipmentMRProductionError, match='Manual release recovery'): + validate_shipment_mr_ci_state( + client, + 'https://gitlab.example/project/-/merge_requests/42', + mr, + allow_active_stage=True, + ) + + +def test_validate_shipment_mr_ci_state_rejects_prod_downstream_from_manual_bridge(): + """Treat any associated production child pipeline as an attempted release.""" + client, mr, _, _ = _shipment_ci_graph( + prod_status='manual', + prod_downstream={'id': 300, 'project_id': 10}, + ) + + with pytest.raises(ShipmentMRProductionError, match='production pipeline history'): + validate_shipment_mr_ci_state( + client, + 'https://gitlab.example/project/-/merge_requests/42', + mr, + allow_active_stage=True, + ) + + +def test_inspect_shipment_mr_ci_state_fails_closed_on_unknown_status(): + """Refuse mutation when GitLab returns a state the implementation cannot classify.""" + client, mr, _, _ = _shipment_ci_graph(stage_status='mystery') + + with pytest.raises(RuntimeError, match='unknown GitLab CI status'): + inspect_shipment_mr_ci_state(client, 'https://gitlab.example/project/-/merge_requests/42', mr) + + +def test_validate_shipment_mr_allows_closed_for_replacement_inspection(): + """Inspect a closed, unmerged MR when force replacement is requested.""" + client = MagicMock() + client._parse_mr_url.return_value = ('hybrid-platforms/art/ocp-shipment-data', '42') + mr = MagicMock(state='closed', source_project_id=10, target_branch='main', labels=[]) + client.get_mr_from_url.return_value = mr + client.get_project.return_value.path_with_namespace = 'openshift-eng/ocp-shipment-data' + + result = validate_shipment_mr( + client, + 'https://gitlab.example/hybrid-platforms/art/ocp-shipment-data/-/merge_requests/42', + 'https://gitlab.example/hybrid-platforms/art/ocp-shipment-data.git', + 'https://gitlab.example/openshift-eng/ocp-shipment-data.git', + allowed_states=('opened', 'closed'), + ) + + assert result is mr + + +def test_validate_shipment_mr_rejects_merged_for_replacement(): + """Never automate a replacement for a merged shipment record.""" + client = MagicMock() + client._parse_mr_url.return_value = ('hybrid-platforms/art/ocp-shipment-data', '42') + client.get_mr_from_url.return_value = MagicMock(state='merged') + + with pytest.raises(ShipmentMRProductionError, match='merged'): + validate_shipment_mr( + client, + 'https://gitlab.example/hybrid-platforms/art/ocp-shipment-data/-/merge_requests/42', + 'https://gitlab.example/hybrid-platforms/art/ocp-shipment-data.git', + 'https://gitlab.example/openshift-eng/ocp-shipment-data.git', + allowed_states=('opened', 'closed'), + ) + + +def test_set_shipment_mr_draft_clears_stage_success_label(): + """Reset stale stage success when preparing an allowed MR rerun.""" + mr = MagicMock(title='Shipment for logging 6.5.2', labels=['stage-release-success', 'reviewed']) + + set_shipment_mr_draft(mr, dry_run=False) + + assert mr.title == 'Draft: Shipment for logging 6.5.2' + assert mr.labels == ['reviewed'] + mr.save.assert_called_once_with() + + +def test_validate_shipment_mr_reuse_state_rejects_prod_advisory(): + """Reject reuse when an image file records a production advisory.""" + with TemporaryDirectory() as directory: + repo = GitRepository(directory) + repo.fetch_switch_branch = AsyncMock() + path = Path(directory, 'shipment/openshift-logging/logging-6.5/logging-6-5/prod/6.5.2.image.yaml') + path.parent.mkdir(parents=True) + existing = _shipment() + existing['shipment']['environments']['prod']['advisory'] = {'url': 'prod-advisory'} + YAML.dump(existing, path) + mr = MagicMock(source_branch='prepare-shipment-6.5.2-20260817161645') + mr.changes.return_value = {'changes': [{'new_path': str(path.relative_to(directory))}]} + + with pytest.raises(ShipmentMRProductionError, match='prod advisory') as exc_info: + asyncio.run(validate_shipment_mr_reuse_state(repo, mr, 'openshift-logging', 'logging-6.5', '6.5.2')) + assert 'manual recovery' in str(exc_info.value) + + +def test_validate_shipment_mr_reuse_state_rejects_prod_fbc_result(): + """Reject reuse when an FBC file records a production pipeline result.""" + with TemporaryDirectory() as directory: + repo = GitRepository(directory) + repo.fetch_switch_branch = AsyncMock() + path = Path(directory, 'shipment/openshift-logging/logging-6.5/fbc-logging-6-5/prod/6.5.2.fbc.yaml') + path.parent.mkdir(parents=True) + existing = _shipment(fbc=True, nvr='cluster-logging-operator-fbc-6.5.2-1.ocp4.19') + existing['shipment']['environments']['prod']['result'] = {'pipeline': 'prod-ci'} + YAML.dump(existing, path) + mr = MagicMock(source_branch='prepare-shipment-6.5.2-20260817161645') + mr.changes.return_value = {'changes': [{'new_path': str(path.relative_to(directory))}]} + + with pytest.raises(ShipmentMRProductionError, match='prod pipeline result') as exc_info: + asyncio.run(validate_shipment_mr_reuse_state(repo, mr, 'openshift-logging', 'logging-6.5', '6.5.2')) + assert 'manual recovery' in str(exc_info.value) + + +def test_validate_shipment_mr_reuse_state_rejects_wrong_product(): + """Reject an MR whose shipment files belong to another product.""" + with TemporaryDirectory() as directory: + repo = GitRepository(directory) + repo.fetch_switch_branch = AsyncMock() + path = Path(directory, 'shipment/openshift-logging/logging-6.5/logging-6-5/prod/6.5.2.image.yaml') + path.parent.mkdir(parents=True) + YAML.dump(_shipment(), path) + mr = MagicMock(source_branch='prepare-shipment-6.5.2-20260817161645') + mr.changes.return_value = {'changes': [{'new_path': str(path.relative_to(directory))}]} + + with pytest.raises(ShipmentMRScopeError, match='refusing to modify an unrelated MR') as exc_info: + asyncio.run(validate_shipment_mr_reuse_state(repo, mr, 'oadp', 'logging-6.5', '6.5.2')) + message = str(exc_info.value) + assert "product 'oadp'" in message + assert str(path.relative_to(directory)) in message + assert '--force' in message + + +def test_update_shipment_mr_url_creates_explicit_stream_assembly(): + """Create a minimal explicit-stream assembly for a direct release.""" + with TemporaryDirectory() as directory: + repo = GitRepository(directory) + repo.fetch_switch_branch = AsyncMock() + repo.commit_push = AsyncMock(return_value=True) + Path(directory, 'releases.yml').write_text('releases: {}\n') + + asyncio.run( + update_shipment_mr_url( + repo, + 'logging-6.5', + '6.5.2', + 'https://gitlab.example/mr/42', + None, + create_as_stream=True, + ) + ) + + result = YAML.load(Path(directory, 'releases.yml')) + assembly = result['releases']['6.5.2']['assembly'] + assert assembly['type'] == 'stream' + assert assembly['group']['shipment']['mr'] == 'https://gitlab.example/mr/42' + repo.commit_push.assert_awaited_once() + + +def test_update_shipment_mr_url_preserves_full_standard_assembly(): + """Add the MR pointer without replacing standard assembly fields.""" + with TemporaryDirectory() as directory: + repo = GitRepository(directory) + repo.fetch_switch_branch = AsyncMock() + repo.commit_push = AsyncMock(return_value=True) + Path(directory, 'releases.yml').write_text( + "releases:\n" + " 2.17.3:\n" + " assembly:\n" + " type: standard\n" + " basis:\n" + " fbc_pullspecs: quay.io/example/fbc\n" + " members:\n" + " images: []\n" + ) + + asyncio.run( + update_shipment_mr_url( + repo, + 'acm-2.17', + '2.17.3', + 'https://gitlab.example/mr/42', + None, + create_as_stream=False, + ) + ) + + assembly = YAML.load(Path(directory, 'releases.yml'))['releases']['2.17.3']['assembly'] + assert assembly['type'] == 'standard' + assert assembly['basis']['fbc_pullspecs'] == 'quay.io/example/fbc' + assert assembly['members']['images'] == [] + assert assembly['group']['shipment']['mr'] == 'https://gitlab.example/mr/42' + + +def test_update_shipment_mr_url_rejects_concurrent_pointer_change(): + """Refuse to overwrite a pointer changed by another release run.""" + with TemporaryDirectory() as directory: + repo = GitRepository(directory) + repo.fetch_switch_branch = AsyncMock() + repo.commit_push = AsyncMock(return_value=True) + Path(directory, 'releases.yml').write_text( + "releases:\n" + " 6.5.2:\n" + " assembly:\n" + " group:\n" + " shipment:\n" + " mr: https://gitlab.example/mr/concurrent\n" + ) + + try: + asyncio.run( + update_shipment_mr_url( + repo, + 'logging-6.5', + '6.5.2', + 'https://gitlab.example/mr/new', + 'https://gitlab.example/mr/old', + create_as_stream=True, + ) + ) + except RuntimeError as exc: + assert 'changed concurrently' in str(exc) + else: + raise AssertionError("Expected a concurrent pointer update to be rejected") + repo.commit_push.assert_not_awaited() + + +def test_reconcile_recreates_existing_fbc_from_scratch(): + """Discard existing FBC content and write only the current generated data.""" + with TemporaryDirectory() as directory: + repo = GitRepository(directory) + repo.fetch_switch_branch = AsyncMock() + repo.log_diff = AsyncMock() + repo.commit_push = AsyncMock(return_value=True) + + async def write_file(relative_path, content): + """Write reconciled content in the temporary repository.""" + destination = Path(directory, relative_path) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(content) + return destination + + repo.write_file = AsyncMock(side_effect=write_file) + path = Path( + directory, + 'shipment/openshift-logging/logging-6.5/fbc-logging-6-5/prod/', + '6.5.2.fbc.ocp4.19.2026081716164501.yaml', + ) + path.parent.mkdir(parents=True) + existing = _shipment(fbc=True, nvr='cluster-logging-operator-fbc-6.5.2-1.ocp4.19') + existing['shipment']['environments']['stage']['result'] = {'pipeline': 'stage-ci'} + existing['shipment']['metadata']['group'] = 'manually-edited' + existing['shipment']['manual'] = {'field': 'discarded'} + YAML.dump(existing, path) + + desired = _shipment(fbc=True, nvr='cluster-logging-operator-fbc-6.5.2-2.ocp4.19', release_notes=False) + mr = MagicMock(source_branch='prepare-shipment-6.5.2-20260817161645') + mr.changes.return_value = {'changes': [{'new_path': str(path.relative_to(directory)), 'new_file': True}]} + + changed = asyncio.run( + reconcile_shipment_mr( + repo, + mr, + {'fbc01': ShipmentConfig(**desired)}, + include_fbc_ocp_version=True, + dry_run=False, + ) + ) + + assert changed + result = YAML.load(path) + assert result['shipment']['snapshot']['nvrs'] == ['cluster-logging-operator-fbc-6.5.2-2.ocp4.19'] + assert result == desired + repo.commit_push.assert_awaited_once() + + +def test_reconcile_creates_new_shipment_directory_inside_repository(monkeypatch): + """Create directories for new shipments relative to the repository root.""" + with TemporaryDirectory() as directory, TemporaryDirectory() as working_directory: + monkeypatch.chdir(working_directory) + repo = GitRepository(directory) + repo.fetch_switch_branch = AsyncMock() + repo.log_diff = AsyncMock() + repo.commit_push = AsyncMock(return_value=True) + + async def write_file(relative_path, content): + """Assert that reconciliation created the repository directory.""" + destination = Path(directory, relative_path) + assert destination.parent.is_dir() + destination.write_text(content) + return destination + + repo.write_file = AsyncMock(side_effect=write_file) + mr = MagicMock(source_branch='prepare-shipment-6.5.2-20260817161645') + mr.changes.return_value = {'changes': []} + desired = _shipment( + fbc=True, + nvr='cluster-logging-operator-fbc-6.5.2-1.ocp4.19', + release_notes=False, + ) + + changed = asyncio.run( + reconcile_shipment_mr( + repo, + mr, + {'fbc01': ShipmentConfig(**desired)}, + include_fbc_ocp_version=True, + dry_run=False, + ) + ) + + expected = Path( + directory, + 'shipment/openshift-logging/logging-6.5/fbc-logging-6-5/prod/', + '6.5.2.fbc.ocp4.19.2026081716164501.yaml', + ) + assert changed + assert expected.is_file() + assert not Path(working_directory, 'shipment').exists() + repo.commit_push.assert_awaited_once()