diff --git a/doozer/doozerlib/cli/images_streams.py b/doozer/doozerlib/cli/images_streams.py index 39a4fbab66..7f7f039ea5 100644 --- a/doozer/doozerlib/cli/images_streams.py +++ b/doozer/doozerlib/cli/images_streams.py @@ -386,6 +386,8 @@ def mirror_image(cmd_start: str, upstream_dest: str): # upstream_image to all the upstream_image_mirror destinations so they all get the same version. if config.upstream_image_mirror is not Missing: for upstream_image_mirror_dest in config.upstream_image_mirror: + if live_test_mode: + upstream_image_mirror_dest += '.test' # Mirror to each destination only if not in only-if-missing mode OR destination doesn't exist if not only_if_missing or not destinations_to_check.get(upstream_image_mirror_dest, False): priv_cmd = f'oc image mirror {config.upstream_image}' diff --git a/doozer/doozerlib/cli/scan_sources_konflux.py b/doozer/doozerlib/cli/scan_sources_konflux.py index aead3d74e6..80099fd363 100644 --- a/doozer/doozerlib/cli/scan_sources_konflux.py +++ b/doozer/doozerlib/cli/scan_sources_konflux.py @@ -563,8 +563,9 @@ async def find_latest_image_builds(self, image_names: List[str]): self.latest_image_build_records_map.update((zip(image_names, latest_image_builds))) async def scan_images(self, image_names: List[str]): - # Filter to only enabled images (variant-aware filtering is handled by _is_image_enabled) - image_names = filter(lambda name: self._is_image_enabled(self.runtime.image_map[name]), image_names) + # Filter to only enabled images (variant-aware filtering is handled by _is_image_enabled_for_scan, + # which also honors --load-disabled -- unlike _is_image_enabled alone) + image_names = filter(lambda name: self._is_image_enabled_for_scan(self.runtime.image_map[name]), image_names) # Do not scan images that have already been requested for rebuild image_names = list(filter(lambda name: name not in self.changing_image_names, image_names)) diff --git a/ocp-build-data-validator/tests/test_schema/test_group_schema.py b/ocp-build-data-validator/tests/test_schema/test_group_schema.py index db5c7145d4..9ce58f3f8b 100644 --- a/ocp-build-data-validator/tests/test_schema/test_group_schema.py +++ b/ocp-build-data-validator/tests/test_schema/test_group_schema.py @@ -102,3 +102,33 @@ def test_validate_with_invalid_okd_enabled_flag(self): }, } self.assertIn("'yes' is not of type 'boolean'", group_schema.validate("group.yml", invalid_data)) + + def test_validate_with_distinct_go_version_vars(self): + valid_data = { + "name": "openshift-4.22", + "vars": {"MAJOR": 4, "MINOR": 22, "GO_LATEST": "1.25", "GO_EXTRA": "1.24", "GO_PREVIOUS": "1.23"}, + } + self.assertEqual("", group_schema.validate("group.yml", valid_data)) + + def test_validate_with_duplicate_go_version_vars(self): + invalid_data = { + "name": "openshift-4.22", + "vars": {"MAJOR": 4, "MINOR": 22, "GO_LATEST": "1.25", "GO_EXTRA": "1.25"}, + } + result = group_schema.validate("group.yml", invalid_data) + self.assertIn("GO_EXTRA and GO_LATEST cannot both resolve to the same major.minor version (1.25)", result) + + def test_validate_with_duplicate_go_version_vars_different_patch(self): + invalid_data = { + "name": "openshift-4.22", + "vars": {"MAJOR": 4, "MINOR": 22, "GO_LATEST": "1.25.1", "GO_PREVIOUS": "1.25.0"}, + } + result = group_schema.validate("group.yml", invalid_data) + self.assertIn("GO_PREVIOUS and GO_LATEST cannot both resolve to the same major.minor version (1.25)", result) + + def test_validate_with_invalid_go_version_var(self): + invalid_data = { + "name": "openshift-4.22", + "vars": {"MAJOR": 4, "MINOR": 22, "GO_LATEST": "not-a-version"}, + } + self.assertIn("Invalid GO_LATEST value: not-a-version", group_schema.validate("group.yml", invalid_data)) diff --git a/ocp-build-data-validator/validator/schema/group_schema.py b/ocp-build-data-validator/validator/schema/group_schema.py index 2d1ad3ef4c..e98f4d7aed 100644 --- a/ocp-build-data-validator/validator/schema/group_schema.py +++ b/ocp-build-data-validator/validator/schema/group_schema.py @@ -5,6 +5,7 @@ """ import json +import re import sys from artcommonlib.util import validate_bridge_release_basis_group @@ -14,6 +15,38 @@ from validator.support import replace_vars +GO_VERSION_VARS = ("GO_LATEST", "GO_EXTRA", "GO_PREVIOUS") + + +def _go_major_minor(var_name, value): + match = re.fullmatch(r"(\d+)\.(\d+)(?:\.\d+)?", str(value)) + if not match: + raise SchemaError(f"Invalid {var_name} value: {value}") + return f"{match[1]}.{match[2]}" + + +def _validate_go_version_vars(vars_map): + """ + GO_LATEST/GO_EXTRA/GO_PREVIOUS must all resolve to distinct major.minor versions. + Consumers (e.g. pyartcd's update_golang pipeline) rely on this to identify a single + matching variant for a given golang build, rather than having to check every variant + that could share the same major.minor. + """ + major_minors = {} + for var_name in GO_VERSION_VARS: + value = vars_map.get(var_name) + if value is None: + continue + major_minor = _go_major_minor(var_name, value) + if major_minor in major_minors: + return ( + f"{var_name} and {major_minors[major_minor]} cannot both resolve to the same " + f"major.minor version ({major_minor})" + ) + major_minors[major_minor] = var_name + return '' + + if sys.version_info < (3, 9): # importlib.resources either doesn't exist or lacks the files() # function, so use the PyPI version: @@ -95,4 +128,11 @@ def validate(_, data): except ValueError as e: return str(e) + try: + go_version_error = _validate_go_version_vars(vars_map) + except SchemaError as e: + return str(e) + if go_version_error: + return go_version_error + return '' diff --git a/pyartcd/pyartcd/jenkins.py b/pyartcd/pyartcd/jenkins.py index aded16fcab..fdc8e0e200 100644 --- a/pyartcd/pyartcd/jenkins.py +++ b/pyartcd/pyartcd/jenkins.py @@ -52,7 +52,8 @@ class Jobs(Enum): SCAN_PLASHET_RPMS = 'scanning/scanning%2Fplashet-rpms' BUILD_CONFORMA_VERIFY = 'aos-cd-builds/build%2Fbuild-conforma-verify' SCAN_OPERATOR = 'aos-cd-builds/build%2Fscan-operator' - SYNC_CI_IMAGES = 'aos-cd-builds/build%2Fsync-ci-images' + # TODO TEST + SYNC_CI_IMAGES = 'hack/bvizi/add-load-disabled' OPEN_RECONCILIATION_PRS = 'aos-cd-builds/build%2Fopen-reconciliation-prs' OPEN_RECONCILIATION_PRS_LAYERED = 'aos-cd-builds/build%2Fopen-reconciliation-prs-layered-products' @@ -531,10 +532,25 @@ def start_rhcos(build_version: str, new_build: bool, job_name: str = 'build', ** ) -def start_sync_ci_images(version: str, **kwargs) -> Optional[str]: +def start_sync_ci_images( + version: str, + assembly: str = 'stream', + image_list: list = None, + dry_run: bool = False, + load_disabled: bool = False, + live_test_mode: bool = False, + **kwargs, +) -> Optional[str]: params = { 'VERSION': version, } + if image_list: + params['IMAGES'] = ','.join(image_list) + params['ART_TOOLS_COMMIT'] = 'kopero2000@ci-golang-builder-from-update-golang-ART-21958' + params['DRY_RUN'] = dry_run + params['LOAD_DISABLED'] = load_disabled + params['LIVE_TEST_MODE'] = live_test_mode + params['ASSEMBLY'] = assembly return start_build( job=Jobs.SYNC_CI_IMAGES, params=params, diff --git a/pyartcd/pyartcd/pipelines/sync_ci_images.py b/pyartcd/pyartcd/pipelines/sync_ci_images.py index 1a5c8e3de2..31dfca00e0 100644 --- a/pyartcd/pyartcd/pipelines/sync_ci_images.py +++ b/pyartcd/pyartcd/pipelines/sync_ci_images.py @@ -57,6 +57,8 @@ def __init__( skip_waits: bool = False, force_run: bool = False, update_images_only_when_missing: bool = False, + load_disabled: bool = False, + live_test_mode: bool = False, ) -> None: """ Initialize sync-ci-images pipeline. @@ -84,7 +86,8 @@ def __init__( self.skip_waits = skip_waits self.force_run = force_run self.update_images_only_when_missing = update_images_only_when_missing - + self.load_disabled = load_disabled + self.live_test_mode = live_test_mode # Validate parameters self._validate_parameters() @@ -512,6 +515,8 @@ def _build_doozer_options(self, group_dir: Path, auth_file: str) -> str: f"--build-system {self.BUILD_SYSTEM} " f"--registry-config {auth_file}" ) + if self.load_disabled: + doozer_opts += " --load-disabled" return doozer_opts @property @@ -547,6 +552,8 @@ async def _mirror_images_to_ci(self, doozer_opts: str, auth_file: str) -> None: mirror_args += "--only-if-missing " if self.runtime.dry_run: mirror_args += "--dry-run" + if self.live_test_mode: + mirror_args += "--live-test-mode" await self._run_doozer_command(doozer_opts, "images:streams mirror", mirror_args.strip()) async def _trigger_ci_builds(self, doozer_opts: str, auth_file: str) -> None: @@ -669,6 +676,18 @@ async def run(self) -> int: default=False, help='Pass --only-if-missing to doozer mirror (update only missing images)', ) +@click.option( + '--load-disabled', + is_flag=True, + default=False, + help='Pass --load-disabled to doozer mirror', +) +@click.option( + '--live-test-mode', + is_flag=True, + default=False, + help='Pass --live-test-mode to doozer mirror', +) @pass_runtime @click_coroutine async def sync_ci_images_cli( @@ -682,6 +701,8 @@ async def sync_ci_images_cli( skip_waits: bool, force_run: bool, update_images_only_when_missing: bool, + load_disabled: bool, + live_test_mode: bool, ): """ CLI entrypoint for sync-ci-images pipeline. @@ -709,6 +730,8 @@ async def sync_ci_images_cli( skip_waits=skip_waits, force_run=force_run, update_images_only_when_missing=update_images_only_when_missing, + load_disabled=load_disabled, + live_test_mode=live_test_mode, ) # Run with per-version lock diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index b971b4208e..d6723fa165 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -31,7 +31,7 @@ from pyartcd import constants, jenkins from pyartcd.cli import cli, click_coroutine, pass_runtime from pyartcd.runtime import Runtime -from pyartcd.util import default_release_suffix, kinit +from pyartcd.util import default_release_suffix, get_changes, kinit _LOGGER = logging.getLogger(__name__) yaml = new_roundtrip_yaml_handler() @@ -41,6 +41,8 @@ "Brew builds for the test assembly are not supported because Brew floating tags are updated after every " "successful build. Use --build-system konflux for test assembly builds." ) +CI_GOLANG_BUILDER_IMAGE_PREFIX = "ci-openshift-golang-builder-" +CI_BUILD_ROOT_IMAGE_PREFIX = "ci-openshift-build-root-" def is_latest(ocp_version: str, el_v: int, nvr: str, koji_session) -> bool: @@ -230,10 +232,10 @@ def __init__( # GitHub auth is handled by get_github_client_for_org() with App auth / PAT fallback - # Initialize KonfluxDb for Konflux build system - if build_system in ('konflux', 'both'): - self.konflux_db = KonfluxDb() - self.konflux_db.bind(KonfluxBuildRecord) + # Initialize KonfluxDb. Needed for Konflux build systems, and also for the CI golang + # builder image reconciliation stage, which always checks/rebuilds via Konflux. + self.konflux_db = KonfluxDb() + self.konflux_db.bind(KonfluxBuildRecord) @property def is_production_assembly(self) -> bool: @@ -260,15 +262,17 @@ def _load_yaml_from_repo(repo, path: str, ref: str): def _get_upstream_ocp_build_data_repo(self): return get_github_client_for_org("openshift-eng").get_repo("openshift-eng/ocp-build-data") - def _get_ocp_build_data_repo_and_branch(self, default_branch): + def _get_ocp_build_data_repo_and_branch(self, default_branch, data_path=None, data_gitref=None): """Get the ocp-build-data repo and branch, respecting data_path/data_gitref overrides.""" - if self.data_path: - match = re.search(r'github\.com[:/](.+?)(?:\.git)?$', self.data_path) + data_path = self.data_path if data_path is None else data_path + data_gitref = self.data_gitref if data_gitref is None else data_gitref + if data_path: + match = re.search(r'github\.com[:/](.+?)(?:\.git)?$', data_path) if match: repo_name = match.group(1) org = repo_name.split('/')[0] repo = get_github_client_for_org(org).get_repo(repo_name) - branch = self.data_gitref or default_branch + branch = data_gitref or default_branch return repo, branch return self._get_upstream_ocp_build_data_repo(), default_branch @@ -556,6 +560,8 @@ async def run(self): else: _LOGGER.info("No Konflux golang builder images found; streams.yml will not be updated.") + await self._refresh_ci_images(build_major_minor, allowed_major_minors, el_nvr_map_for_images) + if self.is_production_assembly: await move_golang_bugs( ocp_version=self.ocp_version, @@ -1163,6 +1169,203 @@ async def _rebase_and_build_konflux(self, el_v, go_version, go_nvr: str): await self._rebase_konflux(el_v, go_version, go_nvr) await self._build_konflux(el_v, go_version) + async def _get_ci_image_keys(self) -> List[str]: + """ + List the `ci-openshift-golang-builder-*` and `ci-openshift-build-root-*` doozer image keys + defined for this OCP version. Golang-builder images pull their parent directly from a + golang stream in streams.yml (e.g. rhel-9-golang-{GO_LATEST}); build-root images pull + their parent via a `member` reference to the corresponding golang-builder CI image. Both + need to be rebuilt whenever their respective parent changes. + """ + branch = f"openshift-{self.ocp_version}" + repo, branch = self._get_ocp_build_data_repo_and_branch(branch) + contents = repo.get_contents("images", ref=branch) + ci_image_keys = sorted( + content.name[: -len(".yml")] + for content in contents + if content.name.endswith(".yml") + and ( + content.name.startswith(CI_GOLANG_BUILDER_IMAGE_PREFIX) + or content.name.startswith(CI_BUILD_ROOT_IMAGE_PREFIX) + ) + ) + return ci_image_keys + + async def _scan_stale_ci_images(self, image_keys: List[str]) -> List[str]: + """ + Determine which of the given CI image keys need rebuilding, by invoking doozer's + `beta:config:konflux:scan-sources` -- the same builder-staleness check the standing ocp4 + scan-sources job uses. For each image's declared builder/parent (resolving `stream:` + references against streams.yml), it queries the parent's actual current build and flags + the image if that build is newer than the image's own last build. + """ + group = self._get_ci_group() + cmd = [ + "doozer", + f"--working-dir={self._doozer_working_dir}-ci-scan", + "--build-system=konflux", + ] + if self.data_path: + cmd.append(f"--data-path={self.data_path}") + cmd.extend(["--group", group]) + # These images are `mode: disabled` in ocp-build-data so the standing ocp4-scan job (which + # scans the whole group without --load-disabled) leaves their lifecycle to this pipeline. + # scan-sources applies that same enabled-filter even to explicitly `-i`-named images, so + # --load-disabled is required here or this scan would always report them as not stale. + cmd.append("--load-disabled") + cmd.extend(self._get_doozer_assembly_args()) + cmd.extend(["-i", ",".join(image_keys)]) + # These CI images declare `scan_sources: exempt_rpms: - '*'`, so RPM changes never affect + # their staleness anyway. --skip-rpms avoids loading/cloning every RPM source in the whole + # group (mode='images' instead of 'both' in doozer), which is otherwise unconditional and + # unrelated to the -i image filter above. + cmd.extend(["beta:config:konflux:scan-sources", "--yaml", "--skip-rpms"]) + # --ci-kubeconfig is for looking at release-controller imagestreams on app.ci, which is a + # different cluster/identity than self.kubeconfig (the Konflux SA kubeconfig). + ci_kubeconfig = os.environ.get('KUBECONFIG') + if ci_kubeconfig: + cmd.append(f"--ci-kubeconfig={ci_kubeconfig}") + + rc, out, _ = await exectools.cmd_gather_async(cmd, env=self._doozer_env_vars, stderr=None, check=False) + if rc != 0: + raise RuntimeError(f"doozer scan-sources failed with exit code {rc} for {', '.join(image_keys)}:\n{out}") + + report = yaml.load(out) or {} + changes = get_changes(report) + return [image_key for image_key in changes.get('images', []) if image_key in image_keys] + + CI_VARIANT_BY_GROUP_VAR = { + "GO_LATEST": "latest", + "GO_EXTRA": "extra", + "GO_PREVIOUS": "previous", + } + + async def _refresh_ci_images( + self, + build_major_minor: str, + allowed_major_minors: dict[str, str], + el_nvr_map_for_images: dict[int, str], + ): + """ + Standing reconciliation check: make sure the ci-openshift-golang-builder-* and + ci-openshift-build-root-* image(s) for the variant (GO_LATEST/GO_EXTRA/GO_PREVIOUS) + matching this run's golang version are built against their current declared parent. The + ocp-build-data-validator enforces that GO_LATEST/GO_EXTRA/GO_PREVIOUS never share a + major.minor, so at most one variant can match here. Both families are scanned together in + a single `_scan_stale_ci_images` call (see `beta:config:konflux:scan-sources`) -- the same + builder-staleness check used by the standing ocp4 scan-sources job -- so build-root's own + source changes are caught directly. Loading both together also means doozer's own change + propagation (a changing image marks its `member`-referencing descendants as changing too) + naturally covers the case where build-root itself hasn't changed but its golang-builder + parent has. Whatever comes back stale is rebased+built together in one batch by triggering + the standing `ocp4-konflux` job (see `jenkins.start_ocp4_konflux`), scoped to just these + image(s) via `IMAGE_LIST` -- doozer only resolves a `from: member:` reference correctly + when both images are loaded in the same run. Once done, every image considered this run + (not just what was rebuilt) is synced to CI in a single final step by triggering the + standing `sync-ci-images` job (see `jenkins.start_sync_ci_images`) -- re-mirroring an + already-current image is cheap for a handful of images, and it keeps CI in sync with the + latest successful build even when nothing needed rebuilding. For the test assembly, + `live_test_mode` is passed so that job publishes to the `.test`-suffixed CI imagestream + tag instead of the real one, so test-assembly runs never overwrite what production CI + actually consumes. + """ + variant = next( + ( + self.CI_VARIANT_BY_GROUP_VAR[var_name] + for var_name, major_minor in allowed_major_minors.items() + if major_minor == build_major_minor + ), + None, + ) + if not variant: + _LOGGER.info( + "Golang %s does not match any of GO_LATEST/GO_EXTRA/GO_PREVIOUS for openshift-%s; " + "skipping CI golang builder/build-root reconciliation", + build_major_minor, + self.ocp_version, + ) + return + + all_image_keys = set(await self._get_ci_image_keys()) + # (el_v, image_key) for every rhel version sharing this golang version that has a CI golang builder image defined + builder_targets = [ + (el_v, f"{CI_GOLANG_BUILDER_IMAGE_PREFIX}{variant}.rhel{el_v}") + for el_v in el_nvr_map_for_images + if f"{CI_GOLANG_BUILDER_IMAGE_PREFIX}{variant}.rhel{el_v}" in all_image_keys + ] + if not builder_targets: + _LOGGER.info( + "No CI golang builder images found for variant %s on openshift-%s; " + "skipping CI golang builder/build-root reconciliation", + variant, + self.ocp_version, + ) + return + + # Build-root counterpart for every el_v that has a golang-builder image, scanned alongside + # it regardless of the builder's own staleness -- both build-root's own source changes and + # "parent golang-builder changed" (via doozer's descendant-propagation once both images are + # loaded together) are caught by this one scan. + build_root_targets = [ + (el_v, f"{CI_BUILD_ROOT_IMAGE_PREFIX}{variant}.rhel{el_v}") + for el_v, _ in builder_targets + if f"{CI_BUILD_ROOT_IMAGE_PREFIX}{variant}.rhel{el_v}" in all_image_keys + ] + if not build_root_targets: + _LOGGER.info( + "No CI build-root images found for variant %s on openshift-%s; scanning golang builder image(s) only", + variant, + self.ocp_version, + ) + + scan_targets = builder_targets + build_root_targets + scan_keys = [image_key for _, image_key in scan_targets] + stale_image_keys = set(await self._scan_stale_ci_images(scan_keys)) + rebuilt_image_keys = [image_key for image_key in scan_keys if image_key in stale_image_keys] + + if rebuilt_image_keys: + await self._slack_client.say_in_thread( + f":construction: Rebuilding CI golang builder/build-root image(s) for " + f"{self.ocp_version}: {', '.join(rebuilt_image_keys)}" + ) + + build_result = jenkins.start_ocp4_konflux( + build_version=self.ocp_version, + assembly=self.assembly, + image_list=rebuilt_image_keys, + dry_run=self.dry_run, + block_until_complete=True, + ) + if build_result != "SUCCESS": + raise RuntimeError(f"CI image build for {self.ocp_version} failed with result: {build_result}") + + await self._slack_client.say_in_thread( + f":white_check_mark: Rebuilt CI golang builder/build-root image(s): {', '.join(rebuilt_image_keys)}" + ) + else: + _LOGGER.info( + "All CI golang builder/build-root images for openshift-%s already use their current parent image;", + self.ocp_version, + ) + + # Sync every image considered this run, not just what was just rebuilt -- mirroring an + # already-current image is a no-op cost-wise (a handful of images at most), and it keeps CI + # in sync with the latest successful build even on runs where nothing needed rebuilding. + + sync_result = jenkins.start_sync_ci_images( + version=self.ocp_version, + block_until_complete=True, + assembly=self.assembly, + image_list=scan_keys, + dry_run=self.dry_run, + load_disabled=True, + live_test_mode=not self.is_production_assembly, + ) + if sync_result != "SUCCESS": + raise RuntimeError(f"CI image sync for {self.ocp_version} failed with result: {sync_result}") + + await self._slack_client.say_in_thread(f":white_check_mark: Synced CI image(s): {', '.join(scan_keys)}") + GOLANG_DATA_BRANCH = 'golang' @staticmethod @@ -1182,6 +1385,16 @@ def _get_doozer_group_and_image(self, el_v, go_version): group += f'@{self.data_gitref}' return group, image_key + def _get_ci_group(self) -> str: + """The openshift-{version} doozer group used by the CI golang-builder/build-root + methods, honoring --data-gitref so a fork's non-default-named branch (e.g. + openshift-5.0-test) is actually checked out instead of doozer defaulting to a branch + literally named openshift-{version}.""" + group = f"openshift-{self.ocp_version}" + if self.data_gitref: + group += f'@{self.data_gitref}' + return group + def verify_golang_builder_repo(self, el_v, go_version): default_branch = self.GOLANG_DATA_BRANCH filename = 'group.yml' diff --git a/pyartcd/tests/pipelines/test_update_golang.py b/pyartcd/tests/pipelines/test_update_golang.py index e7f42324e6..96840968b0 100644 --- a/pyartcd/tests/pipelines/test_update_golang.py +++ b/pyartcd/tests/pipelines/test_update_golang.py @@ -1131,11 +1131,13 @@ async def test_run_brew_only_skips_updating_streams(self, mock_konflux_db, move_ return_value={9: "openshift-golang-builder-container-v1.25.8-202604150744.p2.gf28329a.el9"} ) pipeline.update_golang_streams = AsyncMock() + pipeline._refresh_ci_images = AsyncMock() await pipeline.run() mock_kinit.assert_awaited_once() pipeline.update_golang_streams.assert_not_awaited() + pipeline._refresh_ci_images.assert_awaited_once() move_golang_bugs.assert_awaited_once() slack_messages = [call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list] self.assertTrue( @@ -1165,6 +1167,7 @@ async def test_run_test_assembly_skips_production_operations(self, mock_konflux_ pipeline.get_existing_builders_konflux = AsyncMock(return_value={8: builder_record}) pipeline._get_builder_pullspec = Mock() pipeline.update_golang_streams = AsyncMock() + pipeline._refresh_ci_images = AsyncMock() await pipeline.run() @@ -1174,6 +1177,9 @@ async def test_run_test_assembly_skips_production_operations(self, mock_konflux_ self.assertEqual(list(pipeline._build_golang_plashets.await_args.args[1]), [8]) pipeline._get_builder_pullspec.assert_not_called() pipeline.update_golang_streams.assert_not_awaited() + # CI image reconciliation still runs for the test assembly (it publishes to the + # .test-suffixed CI tag via --live-test-mode instead of skipping entirely). + pipeline._refresh_ci_images.assert_awaited_once() move_golang_bugs.assert_not_awaited() slack_messages = [call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list] self.assertTrue(any("test assembly" in message for message in slack_messages), slack_messages) @@ -1204,6 +1210,7 @@ async def test_run_go_extra_reuses_builder_without_processing_rpm( pipeline.get_existing_builders_konflux = AsyncMock(return_value={8: builder_record}) pipeline._get_builder_pullspec = Mock(return_value="registry.example.com/golang-builder:v1.25.11-el8") pipeline.update_golang_streams = AsyncMock() + pipeline._refresh_ci_images = AsyncMock() await pipeline.run() @@ -1217,6 +1224,7 @@ async def test_run_go_extra_reuses_builder_without_processing_rpm( "1.25.11", {8: "registry.example.com/golang-builder:v1.25.11-el8"}, ) + pipeline._refresh_ci_images.assert_awaited_once() move_golang_bugs.assert_awaited_once() @patch("pyartcd.pipelines.update_golang.kinit", new_callable=AsyncMock) @@ -1283,6 +1291,7 @@ async def test_run_go_extra_external_rpms_builds_missing_builder( pipeline._rebase_and_build_konflux = AsyncMock() pipeline._get_builder_pullspec = Mock(return_value="registry.example.com/golang-builder:v1.25.11-el8") pipeline.update_golang_streams = AsyncMock() + pipeline._refresh_ci_images = AsyncMock() await pipeline.run() @@ -1296,6 +1305,7 @@ async def test_run_go_extra_external_rpms_builds_missing_builder( ) self.assertEqual(pipeline.get_existing_builders_konflux.await_count, 2) pipeline.update_golang_streams.assert_awaited_once() + pipeline._refresh_ci_images.assert_awaited_once() move_golang_bugs.assert_awaited_once() @patch("pyartcd.pipelines.update_golang.KonfluxDb") @@ -2432,6 +2442,419 @@ async def test_dry_run_passes_dry_run_flag(self, mock_konflux_db, mock_jenkins): self.assertTrue(mock_jenkins.start_build_plashets.call_args.kwargs["dry_run"]) +class TestReconcileCiImages(IsolatedAsyncioTestCase): + """Test the CI golang builder / build-root image reconciliation stage""" + + def _make_pipeline(self, dry_run=False, assembly=DEFAULT_GOLANG_ASSEMBLY): + mock_slack = Mock() + mock_slack.say_in_thread = AsyncMock() + mock_runtime = Mock(dry_run=dry_run, working_dir=Path("/tmp/working")) + mock_runtime.new_slack_client.return_value = mock_slack + return UpdateGolangPipeline( + runtime=mock_runtime, + ocp_version="4.18", + cves=None, + force_update_tracker=False, + go_nvrs=["golang-1.22.9-1.el9"], + art_jira="ART-1234", + tag_builds=(assembly == DEFAULT_GOLANG_ASSEMBLY), + build_system="konflux", + assembly=assembly, + ) + + @staticmethod + def _content(name): + content = Mock() + content.name = name + return content + + @patch("pyartcd.pipelines.update_golang.get_github_client_for_org") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_get_ci_image_keys_filters_by_prefix(self, mock_konflux_db, mock_get_github_client): + pipeline = self._make_pipeline() + upstream_repo = Mock() + upstream_repo.get_contents.return_value = [ + self._content("ci-openshift-golang-builder-latest.rhel9.yml"), + self._content("ci-openshift-golang-builder-extra.rhel8.yml"), + self._content("ci-openshift-build-root-latest.rhel9.yml"), + self._content("openshift-enterprise-ansible-operator.yml"), + ] + mock_get_github_client.return_value.get_repo.return_value = upstream_repo + + image_keys = await pipeline._get_ci_image_keys() + + self.assertEqual( + image_keys, + [ + "ci-openshift-build-root-latest.rhel9", + "ci-openshift-golang-builder-extra.rhel8", + "ci-openshift-golang-builder-latest.rhel9", + ], + ) + upstream_repo.get_contents.assert_called_once_with("images", ref="openshift-4.18") + + @patch("pyartcd.pipelines.update_golang.get_github_client_for_org") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_get_ci_image_keys_returns_empty_when_none_found(self, mock_konflux_db, mock_get_github_client): + pipeline = self._make_pipeline() + upstream_repo = Mock() + upstream_repo.get_contents.return_value = [self._content("openshift-enterprise-ansible-operator.yml")] + mock_get_github_client.return_value.get_repo.return_value = upstream_repo + + image_keys = await pipeline._get_ci_image_keys() + + self.assertEqual(image_keys, []) + + # Common args for _refresh_ci_images: a GO_LATEST bump on rhel9. + RECONCILE_ARGS = ("1.22", {"GO_LATEST": "1.22"}, {9: "golang-1.22.9-1.el9"}) + + @patch("pyartcd.pipelines.update_golang.jenkins") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_noops_when_variant_unmatched(self, mock_konflux_db, mock_jenkins): + """No GO_LATEST/GO_EXTRA/GO_PREVIOUS var matches this build; nothing to check.""" + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock() + + await pipeline._refresh_ci_images("1.23", {"GO_LATEST": "1.22"}, {9: "golang"}) + + pipeline._get_ci_image_keys.assert_not_awaited() + mock_jenkins.start_ocp4_konflux.assert_not_called() + mock_jenkins.start_sync_ci_images.assert_not_called() + pipeline._slack_client.say_in_thread.assert_not_awaited() + + @patch("pyartcd.pipelines.update_golang.jenkins") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_noops_when_no_matching_images_found(self, mock_konflux_db, mock_jenkins): + """GO_LATEST matches, but no ci-openshift-golang-builder-latest.rhel* image exists.""" + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock(return_value=[]) + pipeline._scan_stale_ci_images = AsyncMock() + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + pipeline._scan_stale_ci_images.assert_not_awaited() + mock_jenkins.start_ocp4_konflux.assert_not_called() + mock_jenkins.start_sync_ci_images.assert_not_called() + pipeline._slack_client.say_in_thread.assert_not_awaited() + + @patch("pyartcd.pipelines.update_golang.jenkins") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_syncs_but_does_not_rebuild_when_ci_image_already_current( + self, mock_konflux_db, mock_jenkins + ): + """ + No image was stale, so the ocp4-konflux build is never triggered -- but the sync-ci-images + job still runs unconditionally, to keep CI in sync with the latest successful build even + when nothing needed rebuilding this run. + """ + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._scan_stale_ci_images = AsyncMock(return_value=[]) + mock_jenkins.start_sync_ci_images.return_value = "SUCCESS" + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + pipeline._scan_stale_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) + mock_jenkins.start_ocp4_konflux.assert_not_called() + mock_jenkins.start_sync_ci_images.assert_called_once_with( + version=pipeline.ocp_version, + block_until_complete=True, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + load_disabled=True, + live_test_mode=False, + ) + slack_messages = [call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list] + self.assertFalse(any("Rebuilding CI golang builder/build-root image" in m for m in slack_messages)) + self.assertTrue(any("Synced CI image" in m for m in slack_messages)) + + @patch("pyartcd.pipelines.update_golang.jenkins") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_rebuilds_stale_image(self, mock_konflux_db, mock_jenkins): + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + mock_jenkins.start_ocp4_konflux.return_value = "SUCCESS" + mock_jenkins.start_sync_ci_images.return_value = "SUCCESS" + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + mock_jenkins.start_ocp4_konflux.assert_called_once_with( + build_version=pipeline.ocp_version, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + block_until_complete=True, + ) + mock_jenkins.start_sync_ci_images.assert_called_once_with( + version=pipeline.ocp_version, + block_until_complete=True, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + load_disabled=True, + live_test_mode=False, + ) + self.assertTrue( + any( + "Rebuilding CI golang builder/build-root image" in call.args[0] + for call in pipeline._slack_client.say_in_thread.await_args_list + ) + ) + self.assertTrue( + any("Synced CI image" in call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list) + ) + + @patch("pyartcd.pipelines.update_golang.jenkins") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_raises_when_scan_fails(self, mock_konflux_db, mock_jenkins): + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._scan_stale_ci_images = AsyncMock(side_effect=RuntimeError("doozer scan-sources failed")) + + with self.assertRaisesRegex(RuntimeError, "scan-sources failed"): + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + mock_jenkins.start_ocp4_konflux.assert_not_called() + mock_jenkins.start_sync_ci_images.assert_not_called() + + @patch("pyartcd.pipelines.update_golang.jenkins") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_raises_when_rebuild_fails(self, mock_konflux_db, mock_jenkins): + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + mock_jenkins.start_ocp4_konflux.return_value = "FAILURE" + + with self.assertRaisesRegex(RuntimeError, "CI image build .* failed with result: FAILURE"): + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + # Sync only happens after the build step completes, so a build failure means nothing + # gets mirrored to CI this run. + mock_jenkins.start_sync_ci_images.assert_not_called() + + @patch("pyartcd.pipelines.update_golang.jenkins") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_triggers_build_root_after_golang_builder_rebuild(self, mock_konflux_db, mock_jenkins): + """ + Build-root images pull their parent via a `member` reference to the golang-builder CI + image. Both are scanned together in one `_scan_stale_ci_images` call, so doozer's own + change propagation (a changing image marks its `member`-referencing descendants as + changing too) reports build-root as stale right alongside its golang-builder parent. + """ + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock( + return_value=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + ) + pipeline._scan_stale_ci_images = AsyncMock( + return_value=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + ) + mock_jenkins.start_ocp4_konflux.return_value = "SUCCESS" + mock_jenkins.start_sync_ci_images.return_value = "SUCCESS" + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + # Both families are scanned together in a single call. + pipeline._scan_stale_ci_images.assert_awaited_once_with( + ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + ) + # Golang builder and build-root are rebuilt together in a single batch call, so doozer + # resolves the `from: member:` reference between them in-process. + mock_jenkins.start_ocp4_konflux.assert_called_once_with( + build_version=pipeline.ocp_version, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"], + dry_run=pipeline.dry_run, + block_until_complete=True, + ) + # Both families are synced together in a single call, after the rebuild batch completes. + mock_jenkins.start_sync_ci_images.assert_called_once_with( + version=pipeline.ocp_version, + block_until_complete=True, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"], + dry_run=pipeline.dry_run, + load_disabled=True, + live_test_mode=False, + ) + slack_messages = [call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list] + self.assertTrue(any("Rebuilding CI golang builder/build-root image" in m for m in slack_messages)) + self.assertTrue(any("Synced CI image" in m for m in slack_messages)) + + @patch("pyartcd.pipelines.update_golang.jenkins") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_skips_build_root_when_golang_builder_already_fresh(self, mock_konflux_db, mock_jenkins): + """If golang builder wasn't rebuilt, its build-root sibling is left untouched.""" + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock( + return_value=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + ) + pipeline._scan_stale_ci_images = AsyncMock(return_value=[]) + mock_jenkins.start_sync_ci_images.return_value = "SUCCESS" + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + mock_jenkins.start_ocp4_konflux.assert_not_called() + slack_messages = [call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list] + self.assertFalse(any("Rebuilding CI golang builder/build-root image" in m for m in slack_messages)) + + @patch("pyartcd.pipelines.update_golang.jenkins") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_skips_build_root_when_none_defined_for_variant(self, mock_konflux_db, mock_jenkins): + """Golang builder rebuilds fine even when no matching build-root image is defined.""" + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + mock_jenkins.start_ocp4_konflux.return_value = "SUCCESS" + mock_jenkins.start_sync_ci_images.return_value = "SUCCESS" + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + mock_jenkins.start_ocp4_konflux.assert_called_once_with( + build_version=pipeline.ocp_version, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + block_until_complete=True, + ) + mock_jenkins.start_sync_ci_images.assert_called_once_with( + version=pipeline.ocp_version, + block_until_complete=True, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + load_disabled=True, + live_test_mode=False, + ) + + @patch("pyartcd.pipelines.update_golang.jenkins") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_raises_when_build_root_rebuild_fails(self, mock_konflux_db, mock_jenkins): + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock( + return_value=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + ) + pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + mock_jenkins.start_ocp4_konflux.return_value = "FAILURE" + + with self.assertRaisesRegex(RuntimeError, "CI image build .* failed with result: FAILURE"): + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + # Sync only happens after the rebuild batch completes, so a build-root failure means + # nothing gets mirrored to CI this run -- even though the batch call is single-shot. + mock_jenkins.start_sync_ci_images.assert_not_called() + + @patch("pyartcd.pipelines.update_golang.jenkins") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_syncs_for_test_assembly(self, mock_konflux_db, mock_jenkins): + """ + Test-assembly builds get rebuilt and synced too -- start_sync_ci_images itself is + responsible for redirecting the publish to the .test-suffixed CI tag via + `live_test_mode`, so _refresh_ci_images doesn't need to skip the call for the test + assembly. + """ + pipeline = self._make_pipeline(assembly="test") + pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + mock_jenkins.start_ocp4_konflux.return_value = "SUCCESS" + mock_jenkins.start_sync_ci_images.return_value = "SUCCESS" + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + mock_jenkins.start_ocp4_konflux.assert_called_once_with( + build_version=pipeline.ocp_version, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + block_until_complete=True, + ) + mock_jenkins.start_sync_ci_images.assert_called_once_with( + version=pipeline.ocp_version, + block_until_complete=True, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + load_disabled=True, + live_test_mode=True, + ) + self.assertTrue( + any("Synced CI image" in call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list) + ) + + +class TestScanStaleCiImages(IsolatedAsyncioTestCase): + """Test the doozer scan-sources wrapper used to detect stale CI images""" + + def _make_pipeline(self, kubeconfig=None): + mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) + mock_runtime.new_slack_client.return_value = Mock() + return UpdateGolangPipeline( + runtime=mock_runtime, + ocp_version="4.18", + cves=None, + force_update_tracker=False, + go_nvrs=["golang-1.22.9-1.el9"], + art_jira="ART-1234", + tag_builds=True, + build_system="konflux", + kubeconfig=kubeconfig, + ) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + @patch("artcommonlib.exectools.cmd_gather_async") + async def test_builds_command_and_returns_changed_images(self, mock_cmd_gather, mock_konflux_db): + # --ci-kubeconfig must come from the app.ci KUBECONFIG env var, not self.kubeconfig + # (which holds the Konflux SA kubeconfig and lacks RBAC on app.ci imagestreams). + pipeline = self._make_pipeline(kubeconfig="/tmp/konflux-kubeconfig") + report = ( + "images:\n" + "- name: ci-openshift-golang-builder-latest.rhel9\n" + " changed: true\n" + "- name: ci-openshift-golang-builder-extra.rhel8\n" + " changed: false\n" + ) + mock_cmd_gather.return_value = (0, report, "") + + with patch.dict(os.environ, {"KUBECONFIG": "/tmp/ci-kubeconfig"}): + stale = await pipeline._scan_stale_ci_images( + ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-golang-builder-extra.rhel8"] + ) + + self.assertEqual(stale, ["ci-openshift-golang-builder-latest.rhel9"]) + cmd = mock_cmd_gather.call_args.args[0] + self.assertIn("beta:config:konflux:scan-sources", cmd) + self.assertIn("--yaml", cmd) + self.assertEqual(cmd[cmd.index("--assembly") + 1], "stream") + self.assertEqual( + cmd[cmd.index("-i") + 1], + "ci-openshift-golang-builder-latest.rhel9,ci-openshift-golang-builder-extra.rhel8", + ) + self.assertIn("--ci-kubeconfig=/tmp/ci-kubeconfig", cmd) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + @patch("artcommonlib.exectools.cmd_gather_async") + async def test_omits_ci_kubeconfig_when_not_set(self, mock_cmd_gather, mock_konflux_db): + pipeline = self._make_pipeline(kubeconfig=None) + mock_cmd_gather.return_value = (0, "images: []\n", "") + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("KUBECONFIG", None) + await pipeline._scan_stale_ci_images(["ci-openshift-golang-builder-latest.rhel9"]) + + cmd = mock_cmd_gather.call_args.args[0] + self.assertFalse(any(arg.startswith("--ci-kubeconfig") for arg in cmd)) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + @patch("artcommonlib.exectools.cmd_gather_async") + async def test_raises_when_doozer_command_fails(self, mock_cmd_gather, mock_konflux_db): + pipeline = self._make_pipeline() + mock_cmd_gather.return_value = (1, "", "boom") + + with self.assertRaisesRegex(RuntimeError, "doozer scan-sources failed"): + await pipeline._scan_stale_ci_images(["ci-openshift-golang-builder-latest.rhel9"]) + + class TestMonobranchDispatch(IsolatedAsyncioTestCase): """Test that doozer group and image methods work with monobranch"""