From c488f8bdc97d420063be538503ee7d3cfd0b6fcc Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Mon, 17 Aug 2026 15:27:41 +0200 Subject: [PATCH 01/18] golang builder job triggers building for ci-golang-builders, and syncs it as well rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED build ci-build-root as well rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED update doozer command with assembly rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED run ocp4 scan for ci builder rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED if not stream assembly add live-test-mode arg for ci sync doozer command rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- .../tests/test_schema/test_group_schema.py | 30 ++ .../validator/schema/group_schema.py | 40 ++ pyartcd/pyartcd/pipelines/update_golang.py | 339 +++++++++++- pyartcd/tests/pipelines/test_update_golang.py | 493 ++++++++++++++++++ 4 files changed, 897 insertions(+), 5 deletions(-) diff --git a/ocp-build-data-validator/tests/test_schema/test_group_schema.py b/ocp-build-data-validator/tests/test_schema/test_group_schema.py index db5c7145d4..9ce58f3f8b 100644 --- a/ocp-build-data-validator/tests/test_schema/test_group_schema.py +++ b/ocp-build-data-validator/tests/test_schema/test_group_schema.py @@ -102,3 +102,33 @@ def test_validate_with_invalid_okd_enabled_flag(self): }, } self.assertIn("'yes' is not of type 'boolean'", group_schema.validate("group.yml", invalid_data)) + + def test_validate_with_distinct_go_version_vars(self): + valid_data = { + "name": "openshift-4.22", + "vars": {"MAJOR": 4, "MINOR": 22, "GO_LATEST": "1.25", "GO_EXTRA": "1.24", "GO_PREVIOUS": "1.23"}, + } + self.assertEqual("", group_schema.validate("group.yml", valid_data)) + + def test_validate_with_duplicate_go_version_vars(self): + invalid_data = { + "name": "openshift-4.22", + "vars": {"MAJOR": 4, "MINOR": 22, "GO_LATEST": "1.25", "GO_EXTRA": "1.25"}, + } + result = group_schema.validate("group.yml", invalid_data) + self.assertIn("GO_EXTRA and GO_LATEST cannot both resolve to the same major.minor version (1.25)", result) + + def test_validate_with_duplicate_go_version_vars_different_patch(self): + invalid_data = { + "name": "openshift-4.22", + "vars": {"MAJOR": 4, "MINOR": 22, "GO_LATEST": "1.25.1", "GO_PREVIOUS": "1.25.0"}, + } + result = group_schema.validate("group.yml", invalid_data) + self.assertIn("GO_PREVIOUS and GO_LATEST cannot both resolve to the same major.minor version (1.25)", result) + + def test_validate_with_invalid_go_version_var(self): + invalid_data = { + "name": "openshift-4.22", + "vars": {"MAJOR": 4, "MINOR": 22, "GO_LATEST": "not-a-version"}, + } + self.assertIn("Invalid GO_LATEST value: not-a-version", group_schema.validate("group.yml", invalid_data)) diff --git a/ocp-build-data-validator/validator/schema/group_schema.py b/ocp-build-data-validator/validator/schema/group_schema.py index 2d1ad3ef4c..e98f4d7aed 100644 --- a/ocp-build-data-validator/validator/schema/group_schema.py +++ b/ocp-build-data-validator/validator/schema/group_schema.py @@ -5,6 +5,7 @@ """ import json +import re import sys from artcommonlib.util import validate_bridge_release_basis_group @@ -14,6 +15,38 @@ from validator.support import replace_vars +GO_VERSION_VARS = ("GO_LATEST", "GO_EXTRA", "GO_PREVIOUS") + + +def _go_major_minor(var_name, value): + match = re.fullmatch(r"(\d+)\.(\d+)(?:\.\d+)?", str(value)) + if not match: + raise SchemaError(f"Invalid {var_name} value: {value}") + return f"{match[1]}.{match[2]}" + + +def _validate_go_version_vars(vars_map): + """ + GO_LATEST/GO_EXTRA/GO_PREVIOUS must all resolve to distinct major.minor versions. + Consumers (e.g. pyartcd's update_golang pipeline) rely on this to identify a single + matching variant for a given golang build, rather than having to check every variant + that could share the same major.minor. + """ + major_minors = {} + for var_name in GO_VERSION_VARS: + value = vars_map.get(var_name) + if value is None: + continue + major_minor = _go_major_minor(var_name, value) + if major_minor in major_minors: + return ( + f"{var_name} and {major_minors[major_minor]} cannot both resolve to the same " + f"major.minor version ({major_minor})" + ) + major_minors[major_minor] = var_name + return '' + + if sys.version_info < (3, 9): # importlib.resources either doesn't exist or lacks the files() # function, so use the PyPI version: @@ -95,4 +128,11 @@ def validate(_, data): except ValueError as e: return str(e) + try: + go_version_error = _validate_go_version_vars(vars_map) + except SchemaError as e: + return str(e) + if go_version_error: + return go_version_error + return '' diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index b971b4208e..1ac0328c96 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -14,11 +14,18 @@ BREW_HUB, GOLANG_BUILDER_IMAGE_NAME, GOLANG_NVR_LABEL, + KONFLUX_DEFAULT_FBC_REPO, + KONFLUX_DEFAULT_IMAGE_SHARE_REPO, PRODUCT_NAMESPACE_MAP, + REGISTRY_CI_OPENSHIFT, + REGISTRY_QUAY_OCP_RELEASE_DEV, + REGISTRY_QUAY_OPENSHIFT, + REGISTRY_REDHAT_IO, ) from artcommonlib.github_auth import get_github_client_for_org from artcommonlib.konflux.konflux_build_record import ArtifactType, Engine, KonfluxBuildOutcome, KonfluxBuildRecord from artcommonlib.konflux.konflux_db import KonfluxDb +from artcommonlib.registry_config import RegistryConfig, RegistryCredential from artcommonlib.release_util import isolate_assembly_in_release, isolate_el_version_in_release from artcommonlib.rpm_utils import parse_nvr from artcommonlib.util import new_roundtrip_yaml_handler @@ -31,7 +38,7 @@ from pyartcd import constants, jenkins from pyartcd.cli import cli, click_coroutine, pass_runtime from pyartcd.runtime import Runtime -from pyartcd.util import default_release_suffix, kinit +from pyartcd.util import default_release_suffix, get_changes, kinit _LOGGER = logging.getLogger(__name__) yaml = new_roundtrip_yaml_handler() @@ -41,6 +48,8 @@ "Brew builds for the test assembly are not supported because Brew floating tags are updated after every " "successful build. Use --build-system konflux for test assembly builds." ) +CI_GOLANG_BUILDER_IMAGE_PREFIX = "ci-openshift-golang-builder-" +CI_BUILD_ROOT_IMAGE_PREFIX = "ci-openshift-build-root-" def is_latest(ocp_version: str, el_v: int, nvr: str, koji_session) -> bool: @@ -230,10 +239,10 @@ def __init__( # GitHub auth is handled by get_github_client_for_org() with App auth / PAT fallback - # Initialize KonfluxDb for Konflux build system - if build_system in ('konflux', 'both'): - self.konflux_db = KonfluxDb() - self.konflux_db.bind(KonfluxBuildRecord) + # Initialize KonfluxDb. Needed for Konflux build systems, and also for the CI golang + # builder image reconciliation stage, which always checks/rebuilds via Konflux. + self.konflux_db = KonfluxDb() + self.konflux_db.bind(KonfluxBuildRecord) @property def is_production_assembly(self) -> bool: @@ -556,6 +565,8 @@ async def run(self): else: _LOGGER.info("No Konflux golang builder images found; streams.yml will not be updated.") + await self._refresh_ci_images(build_major_minor, allowed_major_minors, el_nvr_map_for_images) + if self.is_production_assembly: await move_golang_bugs( ocp_version=self.ocp_version, @@ -1163,6 +1174,324 @@ async def _rebase_and_build_konflux(self, el_v, go_version, go_nvr: str): await self._rebase_konflux(el_v, go_version, go_nvr) await self._build_konflux(el_v, go_version) + async def _get_ci_image_keys(self) -> List[str]: + """ + List the `ci-openshift-golang-builder-*` and `ci-openshift-build-root-*` doozer image keys + defined for this OCP version. Golang-builder images pull their parent directly from a + golang stream in streams.yml (e.g. rhel-9-golang-{GO_LATEST}); build-root images pull + their parent via a `member` reference to the corresponding golang-builder CI image. Both + need to be rebuilt whenever their respective parent changes. + """ + branch = f"openshift-{self.ocp_version}" + repo, branch = self._get_ocp_build_data_repo_and_branch(branch) + contents = repo.get_contents("images", ref=branch) + ci_image_keys = sorted( + content.name[: -len(".yml")] + for content in contents + if content.name.endswith(".yml") + and ( + content.name.startswith(CI_GOLANG_BUILDER_IMAGE_PREFIX) + or content.name.startswith(CI_BUILD_ROOT_IMAGE_PREFIX) + ) + ) + return ci_image_keys + + async def _scan_stale_ci_images(self, image_keys: List[str]) -> List[str]: + """ + Determine which of the given CI image keys need rebuilding, by invoking doozer's + `beta:config:konflux:scan-sources` -- the same builder-staleness check the standing ocp4 + scan-sources job uses. For each image's declared builder/parent (resolving `stream:` + references against streams.yml), it queries the parent's actual current build and flags + the image if that build is newer than the image's own last build. + """ + group = f"openshift-{self.ocp_version}" + cmd = [ + "doozer", + f"--working-dir={self._doozer_working_dir}-ci-scan", + "--build-system=konflux", + ] + if self.data_path: + cmd.append(f"--data-path={self.data_path}") + cmd.extend(["--group", group]) + cmd.extend(self._get_doozer_assembly_args()) + cmd.extend(["-i", ",".join(image_keys)]) + cmd.extend(["beta:config:konflux:scan-sources", "--yaml"]) + if self.kubeconfig: + cmd.append(f"--ci-kubeconfig={self.kubeconfig}") + + rc, out, _ = await exectools.cmd_gather_async(cmd, env=self._doozer_env_vars, stderr=None, check=False) + if rc != 0: + raise RuntimeError(f"doozer scan-sources failed with exit code {rc} for {', '.join(image_keys)}:\n{out}") + + report = yaml.load(out) or {} + changes = get_changes(report) + return [image_key for image_key in changes.get('images', []) if image_key in image_keys] + + async def _rebase_ci_image(self, image_key: str, version: str, release: str): + _LOGGER.info("Rebasing %s for Konflux...", image_key) + group = f"openshift-{self.ocp_version}" + cmd = [ + "doozer", + f"--working-dir={self._doozer_working_dir}-ci-{image_key}", + "--build-system=konflux", + ] + cmd.extend(self._get_doozer_assembly_args()) + if self.data_path: + cmd.append(f"--data-path={self.data_path}") + cmd.extend( + [ + "--group", + group, + "-i", + image_key, + "beta:images:konflux:rebase", + "--version", + version, + "--release", + release, + "--message", + f"Bump golang parent image for {image_key}", + ] + ) + if self.network_mode: + cmd.extend(["--network-mode", self.network_mode]) + if not self.dry_run: + cmd.append("--push") + await exectools.cmd_assert_async(cmd, env=self._doozer_env_vars) + + async def _build_ci_image(self, image_key: str): + _LOGGER.info("Building %s on Konflux...", image_key) + group = f"openshift-{self.ocp_version}" + konflux_namespace = PRODUCT_NAMESPACE_MAP["ocp"] + cmd = [ + "doozer", + f"--working-dir={self._doozer_working_dir}-ci-{image_key}", + "--build-system=konflux", + ] + cmd.extend(self._get_doozer_assembly_args()) + if self.data_path: + cmd.append(f"--data-path={self.data_path}") + cmd.extend( + [ + "--group", + group, + "-i", + image_key, + "beta:images:konflux:build", + f"--konflux-namespace={konflux_namespace}", + "--skip-ec-verify", + ] + ) + if self.kubeconfig: + cmd.extend(['--konflux-kubeconfig', self.kubeconfig]) + if self.network_mode: + cmd.extend(["--network-mode", self.network_mode]) + if self.dry_run: + cmd.append("--dry-run") + await exectools.cmd_assert_async(cmd, env=self._doozer_env_vars, log_stdout=True) + + async def _rebase_and_build_ci_images(self, image_keys: List[str]): + """ + Rebase+build each CI image directly via doozer, without touching streams.yml or creating + an ocp-build-data PR. + """ + version = f"v{self.ocp_version}.0" + release = default_release_suffix() + + async def _rebase_and_build(image_key: str): + await self._rebase_ci_image(image_key, version, release) + await self._build_ci_image(image_key) + + results = await asyncio.gather( + *[_rebase_and_build(image_key) for image_key in image_keys], + return_exceptions=True, + ) + failed = [(image_key, r) for image_key, r in zip(image_keys, results) if isinstance(r, Exception)] + if failed: + summary = "\n".join(f" ❌ {image_key}: {err}" for image_key, err in failed) + raise RuntimeError(f"Failed to rebuild {len(failed)}/{len(image_keys)} CI image(s):\n{summary}") + + @staticmethod + def _get_required_env(var_name: str) -> str: + value = os.getenv(var_name) + if not value: + raise ValueError(f"Required environment variable {var_name} not set") + return value + + def _create_ci_sync_registry_config(self) -> RegistryConfig: + """ + Build registry credentials for pushing a newly built CI image straight to CI, using the + same env vars (and QCI credential setup) as the scheduled sync-ci-images job. + """ + quay_auth_file = self._get_required_env('QUAY_AUTH_FILE') + kubeconfig = self._get_required_env('KUBECONFIG') + qci_user = self._get_required_env('QCI_USER') + qci_password = self._get_required_env('QCI_PASSWORD') + + return RegistryConfig( + source_files=[quay_auth_file], + kubeconfig=kubeconfig, + registries=[ + REGISTRY_CI_OPENSHIFT, + REGISTRY_QUAY_OCP_RELEASE_DEV, + KONFLUX_DEFAULT_IMAGE_REPO, + KONFLUX_DEFAULT_IMAGE_SHARE_REPO, + KONFLUX_DEFAULT_FBC_REPO, + REGISTRY_REDHAT_IO, + ], + credentials=[ + RegistryCredential(REGISTRY_QUAY_OPENSHIFT, qci_user, qci_password), + ], + ) + + async def _sync_ci_images(self, image_keys: List[str]): + """ + Mirror newly rebuilt CI image(s) straight to CI (the same `images:streams mirror` doozer + verb the scheduled sync-ci-images job uses), so CI does not have to wait for that job's + next run to pick up the rebuild. For the test assembly, `--live-test-mode` is passed so + doozer publishes to the `.test`-suffixed CI imagestream tag instead of the real one -- + `images:streams mirror` resolves its destination entirely from each image's + ci_alignment.upstream_image config, not from anything on this command line, so there is no + other way to redirect it away from the production tag. + """ + _LOGGER.info("Syncing CI image(s) to CI: %s", ", ".join(image_keys)) + group = f"openshift-{self.ocp_version}" + with self._create_ci_sync_registry_config() as auth_file: + cmd = [ + "doozer", + f"--working-dir={self._doozer_working_dir}-ci-sync", + "--build-system=konflux", + ] + cmd.extend(self._get_doozer_assembly_args()) + if self.data_path: + cmd.append(f"--data-path={self.data_path}") + cmd.extend(["--group", group]) + for image_key in image_keys: + cmd.extend(["--image", image_key]) + cmd.extend(["images:streams", "mirror", "--registry-auth", auth_file]) + if not self.is_production_assembly: + cmd.append("--live-test-mode") + if self.dry_run: + cmd.append("--dry-run") + await exectools.cmd_assert_async(cmd, env=self._doozer_env_vars, log_stdout=True) + + CI_VARIANT_BY_GROUP_VAR = { + "GO_LATEST": "latest", + "GO_EXTRA": "extra", + "GO_PREVIOUS": "previous", + } + + async def _refresh_ci_images( + self, + build_major_minor: str, + allowed_major_minors: dict[str, str], + el_nvr_map_for_images: dict[int, str], + ): + """ + Standing reconciliation check: make sure the ci-openshift-golang-builder-* image(s) for + the variant (GO_LATEST/GO_EXTRA/GO_PREVIOUS) matching this run's golang version are built + against their current declared parent. The ocp-build-data-validator enforces that + GO_LATEST/GO_EXTRA/GO_PREVIOUS never share a major.minor, so at most one variant can match + here. Staleness is determined via doozer's `beta:config:konflux:scan-sources` (see + `_scan_stale_ci_images`) -- the same builder-staleness check used by the standing ocp4 + scan-sources job. Rebuilds (via doozer directly, without touching streams.yml or creating + an ocp-build-data PR) any golang-builder CI image found to be stale. ci-openshift-build-root-* + images pull their parent via a `member` reference to the golang-builder CI image (not a + golang stream or the raw golang-builder container), so once a golang-builder image is + rebuilt, its corresponding build-root image is unconditionally rebuilt right after, to + pick up the new base image. Both rebuild steps must complete before either image is + mirrored to CI, so everything rebuilt this run is synced together in a single final step. + For the test assembly, `_sync_ci_images` passes `--live-test-mode`, which publishes to + the `.test`-suffixed CI imagestream tag instead of the real one, so test-assembly runs + never overwrite what production CI actually consumes. + """ + variant = next( + ( + self.CI_VARIANT_BY_GROUP_VAR[var_name] + for var_name, major_minor in allowed_major_minors.items() + if major_minor == build_major_minor + ), + None, + ) + if not variant: + _LOGGER.info( + "Golang %s does not match any of GO_LATEST/GO_EXTRA/GO_PREVIOUS for openshift-%s; " + "skipping CI golang builder/build-root reconciliation", + build_major_minor, + self.ocp_version, + ) + return + + all_image_keys = set(await self._get_ci_image_keys()) + # (el_v, image_key) for every rhel version sharing this golang version that has a CI golang builder image defined + builder_targets = [ + (el_v, f"{CI_GOLANG_BUILDER_IMAGE_PREFIX}{variant}.rhel{el_v}") + for el_v in el_nvr_map_for_images + if f"{CI_GOLANG_BUILDER_IMAGE_PREFIX}{variant}.rhel{el_v}" in all_image_keys + ] + if not builder_targets: + _LOGGER.info( + "No CI golang builder images found for variant %s on openshift-%s; " + "skipping CI golang builder/build-root reconciliation", + variant, + self.ocp_version, + ) + return + + builder_image_keys = [image_key for _, image_key in builder_targets] + stale_image_keys = set(await self._scan_stale_ci_images(builder_image_keys)) + stale_builder_targets = [ + (el_v, image_key) for el_v, image_key in builder_targets if image_key in stale_image_keys + ] + + if not stale_builder_targets: + _LOGGER.info( + "All CI golang builder images for openshift-%s already use their current parent image", + self.ocp_version, + ) + return + + stale_builder_keys = [image_key for _, image_key in stale_builder_targets] + await self._slack_client.say_in_thread( + f":construction: Rebuilding CI golang builder image(s) for " + f"{self.ocp_version}: {', '.join(stale_builder_keys)}" + ) + await self._rebase_and_build_ci_images(stale_builder_keys) + await self._slack_client.say_in_thread( + f":white_check_mark: Rebuilt CI golang builder image(s): {', '.join(stale_builder_keys)}" + ) + + # Trigger the build-root image(s) whose golang-builder parent was just rebuilt above. + build_root_keys = [ + f"{CI_BUILD_ROOT_IMAGE_PREFIX}{variant}.rhel{el_v}" + for el_v, _ in stale_builder_targets + if f"{CI_BUILD_ROOT_IMAGE_PREFIX}{variant}.rhel{el_v}" in all_image_keys + ] + if build_root_keys: + await self._slack_client.say_in_thread( + f":construction: Rebuilding CI build-root image(s) for {self.ocp_version}: {', '.join(build_root_keys)}" + ) + await self._rebase_and_build_ci_images(build_root_keys) + await self._slack_client.say_in_thread( + f":white_check_mark: Rebuilt CI build-root image(s): {', '.join(build_root_keys)}" + ) + else: + _LOGGER.info( + "No CI build-root images found for variant %s on openshift-%s; skipping CI build-root rebuild", + variant, + self.ocp_version, + ) + + rebuilt_image_keys = stale_builder_keys + build_root_keys + try: + await self._sync_ci_images(rebuilt_image_keys) + except Exception as e: + raise RuntimeError(f"Failed to sync CI image(s) to CI: {e}") from e + + await self._slack_client.say_in_thread( + f":white_check_mark: Synced CI image(s): {', '.join(rebuilt_image_keys)}" + ) + GOLANG_DATA_BRANCH = 'golang' @staticmethod diff --git a/pyartcd/tests/pipelines/test_update_golang.py b/pyartcd/tests/pipelines/test_update_golang.py index e7f42324e6..ae07dff3ec 100644 --- a/pyartcd/tests/pipelines/test_update_golang.py +++ b/pyartcd/tests/pipelines/test_update_golang.py @@ -1,4 +1,5 @@ import asyncio +import contextlib import os import tempfile import unittest @@ -1131,11 +1132,13 @@ async def test_run_brew_only_skips_updating_streams(self, mock_konflux_db, move_ return_value={9: "openshift-golang-builder-container-v1.25.8-202604150744.p2.gf28329a.el9"} ) pipeline.update_golang_streams = AsyncMock() + pipeline._refresh_ci_images = AsyncMock() await pipeline.run() mock_kinit.assert_awaited_once() pipeline.update_golang_streams.assert_not_awaited() + pipeline._refresh_ci_images.assert_awaited_once() move_golang_bugs.assert_awaited_once() slack_messages = [call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list] self.assertTrue( @@ -1165,6 +1168,7 @@ async def test_run_test_assembly_skips_production_operations(self, mock_konflux_ pipeline.get_existing_builders_konflux = AsyncMock(return_value={8: builder_record}) pipeline._get_builder_pullspec = Mock() pipeline.update_golang_streams = AsyncMock() + pipeline._refresh_ci_images = AsyncMock() await pipeline.run() @@ -1174,6 +1178,9 @@ async def test_run_test_assembly_skips_production_operations(self, mock_konflux_ self.assertEqual(list(pipeline._build_golang_plashets.await_args.args[1]), [8]) pipeline._get_builder_pullspec.assert_not_called() pipeline.update_golang_streams.assert_not_awaited() + # CI image reconciliation still runs for the test assembly (it publishes to the + # .test-suffixed CI tag via --live-test-mode instead of skipping entirely). + pipeline._refresh_ci_images.assert_awaited_once() move_golang_bugs.assert_not_awaited() slack_messages = [call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list] self.assertTrue(any("test assembly" in message for message in slack_messages), slack_messages) @@ -1204,6 +1211,7 @@ async def test_run_go_extra_reuses_builder_without_processing_rpm( pipeline.get_existing_builders_konflux = AsyncMock(return_value={8: builder_record}) pipeline._get_builder_pullspec = Mock(return_value="registry.example.com/golang-builder:v1.25.11-el8") pipeline.update_golang_streams = AsyncMock() + pipeline._refresh_ci_images = AsyncMock() await pipeline.run() @@ -1217,6 +1225,7 @@ async def test_run_go_extra_reuses_builder_without_processing_rpm( "1.25.11", {8: "registry.example.com/golang-builder:v1.25.11-el8"}, ) + pipeline._refresh_ci_images.assert_awaited_once() move_golang_bugs.assert_awaited_once() @patch("pyartcd.pipelines.update_golang.kinit", new_callable=AsyncMock) @@ -1283,6 +1292,7 @@ async def test_run_go_extra_external_rpms_builds_missing_builder( pipeline._rebase_and_build_konflux = AsyncMock() pipeline._get_builder_pullspec = Mock(return_value="registry.example.com/golang-builder:v1.25.11-el8") pipeline.update_golang_streams = AsyncMock() + pipeline._refresh_ci_images = AsyncMock() await pipeline.run() @@ -1296,6 +1306,7 @@ async def test_run_go_extra_external_rpms_builds_missing_builder( ) self.assertEqual(pipeline.get_existing_builders_konflux.await_count, 2) pipeline.update_golang_streams.assert_awaited_once() + pipeline._refresh_ci_images.assert_awaited_once() move_golang_bugs.assert_awaited_once() @patch("pyartcd.pipelines.update_golang.KonfluxDb") @@ -2147,6 +2158,112 @@ async def test_build_konflux_dry_run(self, mock_cmd_assert, mock_konflux_db): cmd = mock_cmd_assert.call_args[0][0] self.assertIn("--dry-run", cmd) + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + @patch("artcommonlib.exectools.cmd_assert_async") + async def test_rebase_ci_image_includes_assembly(self, mock_cmd_assert, mock_konflux_db): + """Test _rebase_ci_image passes --assembly for the pipeline's assembly""" + mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) + mock_runtime.new_slack_client.return_value = Mock() + + pipeline = UpdateGolangPipeline( + runtime=mock_runtime, + ocp_version="4.18", + cves=None, + force_update_tracker=False, + go_nvrs=["golang-1.22.9-1.el9"], + art_jira="ART-1234", + tag_builds=False, + build_system="konflux", + assembly="test", + ) + + await pipeline._rebase_ci_image("ci-openshift-golang-builder-latest.rhel9", "v4.18.0", "1") + + cmd = mock_cmd_assert.call_args[0][0] + self.assertIn("beta:images:konflux:rebase", cmd) + self.assertEqual(cmd[cmd.index("--assembly") + 1], "test") + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + @patch("artcommonlib.exectools.cmd_assert_async") + async def test_build_ci_image_includes_assembly(self, mock_cmd_assert, mock_konflux_db): + """Test _build_ci_image passes --assembly for the pipeline's assembly""" + mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) + mock_runtime.new_slack_client.return_value = Mock() + + pipeline = UpdateGolangPipeline( + runtime=mock_runtime, + ocp_version="4.18", + cves=None, + force_update_tracker=False, + go_nvrs=["golang-1.22.9-1.el9"], + art_jira="ART-1234", + tag_builds=True, + build_system="konflux", + ) + + await pipeline._build_ci_image("ci-openshift-golang-builder-latest.rhel9") + + cmd = mock_cmd_assert.call_args[0][0] + self.assertIn("beta:images:konflux:build", cmd) + self.assertEqual(cmd[cmd.index("--assembly") + 1], "stream") + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + @patch("artcommonlib.exectools.cmd_assert_async") + async def test_sync_ci_images_includes_assembly(self, mock_cmd_assert, mock_konflux_db): + """Test _sync_ci_images passes --assembly for the pipeline's assembly""" + mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) + mock_runtime.new_slack_client.return_value = Mock() + + pipeline = UpdateGolangPipeline( + runtime=mock_runtime, + ocp_version="4.18", + cves=None, + force_update_tracker=False, + go_nvrs=["golang-1.22.9-1.el9"], + art_jira="ART-1234", + tag_builds=True, + build_system="konflux", + ) + pipeline._create_ci_sync_registry_config = Mock(return_value=contextlib.nullcontext("/tmp/auth.json")) + + await pipeline._sync_ci_images(["ci-openshift-golang-builder-latest.rhel9"]) + + cmd = mock_cmd_assert.call_args[0][0] + self.assertIn("images:streams", cmd) + self.assertEqual(cmd[cmd.index("--assembly") + 1], "stream") + self.assertNotIn("--live-test-mode", cmd) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + @patch("artcommonlib.exectools.cmd_assert_async") + async def test_sync_ci_images_uses_live_test_mode_for_test_assembly(self, mock_cmd_assert, mock_konflux_db): + """ + For the test assembly, doozer must publish to the .test-suffixed CI imagestream tag + instead of the real one -- images:streams mirror resolves its destination purely from + each image's ci_alignment.upstream_image config, so --live-test-mode is the only way to + redirect it away from what production CI actually consumes. + """ + mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) + mock_runtime.new_slack_client.return_value = Mock() + + pipeline = UpdateGolangPipeline( + runtime=mock_runtime, + ocp_version="4.18", + cves=None, + force_update_tracker=False, + go_nvrs=["golang-1.22.9-1.el9"], + art_jira="ART-1234", + tag_builds=False, + build_system="konflux", + assembly="test", + ) + pipeline._create_ci_sync_registry_config = Mock(return_value=contextlib.nullcontext("/tmp/auth.json")) + + await pipeline._sync_ci_images(["ci-openshift-golang-builder-latest.rhel9"]) + + cmd = mock_cmd_assert.call_args[0][0] + self.assertEqual(cmd[cmd.index("--assembly") + 1], "test") + self.assertIn("--live-test-mode", cmd) + @patch("pyartcd.pipelines.update_golang.KonfluxDb") @patch("artcommonlib.exectools.cmd_assert_async") async def test_test_assembly_dry_run_commands_for_konflux(self, mock_cmd_assert, mock_konflux_db): @@ -2432,6 +2549,382 @@ async def test_dry_run_passes_dry_run_flag(self, mock_konflux_db, mock_jenkins): self.assertTrue(mock_jenkins.start_build_plashets.call_args.kwargs["dry_run"]) +class TestReconcileCiImages(IsolatedAsyncioTestCase): + """Test the CI golang builder / build-root image reconciliation stage""" + + def _make_pipeline(self, dry_run=False, assembly=DEFAULT_GOLANG_ASSEMBLY): + mock_slack = Mock() + mock_slack.say_in_thread = AsyncMock() + mock_runtime = Mock(dry_run=dry_run, working_dir=Path("/tmp/working")) + mock_runtime.new_slack_client.return_value = mock_slack + return UpdateGolangPipeline( + runtime=mock_runtime, + ocp_version="4.18", + cves=None, + force_update_tracker=False, + go_nvrs=["golang-1.22.9-1.el9"], + art_jira="ART-1234", + tag_builds=(assembly == DEFAULT_GOLANG_ASSEMBLY), + build_system="konflux", + assembly=assembly, + ) + + @staticmethod + def _content(name): + content = Mock() + content.name = name + return content + + @patch("pyartcd.pipelines.update_golang.get_github_client_for_org") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_get_ci_image_keys_filters_by_prefix(self, mock_konflux_db, mock_get_github_client): + pipeline = self._make_pipeline() + upstream_repo = Mock() + upstream_repo.get_contents.return_value = [ + self._content("ci-openshift-golang-builder-latest.rhel9.yml"), + self._content("ci-openshift-golang-builder-extra.rhel8.yml"), + self._content("ci-openshift-build-root-latest.rhel9.yml"), + self._content("openshift-enterprise-ansible-operator.yml"), + ] + mock_get_github_client.return_value.get_repo.return_value = upstream_repo + + image_keys = await pipeline._get_ci_image_keys() + + self.assertEqual( + image_keys, + [ + "ci-openshift-build-root-latest.rhel9", + "ci-openshift-golang-builder-extra.rhel8", + "ci-openshift-golang-builder-latest.rhel9", + ], + ) + upstream_repo.get_contents.assert_called_once_with("images", ref="openshift-4.18") + + @patch("pyartcd.pipelines.update_golang.get_github_client_for_org") + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_get_ci_image_keys_returns_empty_when_none_found(self, mock_konflux_db, mock_get_github_client): + pipeline = self._make_pipeline() + upstream_repo = Mock() + upstream_repo.get_contents.return_value = [self._content("openshift-enterprise-ansible-operator.yml")] + mock_get_github_client.return_value.get_repo.return_value = upstream_repo + + image_keys = await pipeline._get_ci_image_keys() + + self.assertEqual(image_keys, []) + + # Common args for _refresh_ci_images: a GO_LATEST bump on rhel9. + RECONCILE_ARGS = ("1.22", {"GO_LATEST": "1.22"}, {9: "golang-1.22.9-1.el9"}) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_noops_when_variant_unmatched(self, mock_konflux_db): + """No GO_LATEST/GO_EXTRA/GO_PREVIOUS var matches this build; nothing to check.""" + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock() + pipeline._rebase_and_build_ci_images = AsyncMock() + + await pipeline._refresh_ci_images("1.23", {"GO_LATEST": "1.22"}, {9: "golang"}) + + pipeline._get_ci_image_keys.assert_not_awaited() + pipeline._rebase_and_build_ci_images.assert_not_awaited() + pipeline._slack_client.say_in_thread.assert_not_awaited() + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_noops_when_no_matching_images_found(self, mock_konflux_db): + """GO_LATEST matches, but no ci-openshift-golang-builder-latest.rhel* image exists.""" + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock(return_value=[]) + pipeline._scan_stale_ci_images = AsyncMock() + pipeline._rebase_and_build_ci_images = AsyncMock() + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + pipeline._scan_stale_ci_images.assert_not_awaited() + pipeline._rebase_and_build_ci_images.assert_not_awaited() + pipeline._slack_client.say_in_thread.assert_not_awaited() + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_noops_when_ci_image_already_current(self, mock_konflux_db): + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._scan_stale_ci_images = AsyncMock(return_value=[]) + pipeline._rebase_and_build_ci_images = AsyncMock() + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + pipeline._scan_stale_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._rebase_and_build_ci_images.assert_not_awaited() + pipeline._slack_client.say_in_thread.assert_not_awaited() + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_rebuilds_stale_image(self, mock_konflux_db): + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._rebase_and_build_ci_images = AsyncMock() + pipeline._sync_ci_images = AsyncMock() + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + pipeline._rebase_and_build_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._sync_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) + self.assertTrue( + any( + "Rebuilding CI golang builder image" in call.args[0] + for call in pipeline._slack_client.say_in_thread.await_args_list + ) + ) + self.assertTrue( + any("Synced CI image" in call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list) + ) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_raises_when_scan_fails(self, mock_konflux_db): + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._scan_stale_ci_images = AsyncMock(side_effect=RuntimeError("doozer scan-sources failed")) + pipeline._rebase_and_build_ci_images = AsyncMock() + + with self.assertRaisesRegex(RuntimeError, "scan-sources failed"): + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + pipeline._rebase_and_build_ci_images.assert_not_awaited() + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_raises_when_rebuild_fails(self, mock_konflux_db): + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._rebase_and_build_ci_images = AsyncMock(side_effect=RuntimeError("Failed to rebuild 1/1")) + + with self.assertRaisesRegex(RuntimeError, "Failed to rebuild"): + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_triggers_build_root_after_golang_builder_rebuild(self, mock_konflux_db): + """ + Build-root images pull their parent via a `member` reference to the golang-builder CI + image, not via the golang-builder staleness scan, so once golang builder is rebuilt its + build-root sibling is triggered unconditionally -- no extra staleness check for build-root. + """ + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock( + return_value=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + ) + pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._rebase_and_build_ci_images = AsyncMock() + pipeline._sync_ci_images = AsyncMock() + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + # Only the golang-builder image is scanned; build-root is triggered directly, without a + # staleness check of its own. + pipeline._scan_stale_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) + # Golang builder and build-root are rebuilt as two separate, ordered batch calls. + self.assertEqual( + [call.args[0] for call in pipeline._rebase_and_build_ci_images.await_args_list], + [["ci-openshift-golang-builder-latest.rhel9"], ["ci-openshift-build-root-latest.rhel9"]], + ) + # Both families are synced together in a single call, after both rebuild steps complete. + pipeline._sync_ci_images.assert_awaited_once_with( + ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + ) + slack_messages = [call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list] + self.assertTrue(any("Rebuilding CI golang builder image" in m for m in slack_messages)) + self.assertTrue(any("Rebuilding CI build-root image" in m for m in slack_messages)) + self.assertTrue(any("Synced CI image" in m for m in slack_messages)) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_skips_build_root_when_golang_builder_already_fresh(self, mock_konflux_db): + """If golang builder wasn't rebuilt, its build-root sibling is left untouched.""" + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock( + return_value=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + ) + pipeline._scan_stale_ci_images = AsyncMock(return_value=[]) + pipeline._rebase_and_build_ci_images = AsyncMock() + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + pipeline._rebase_and_build_ci_images.assert_not_awaited() + pipeline._slack_client.say_in_thread.assert_not_awaited() + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_skips_build_root_when_none_defined_for_variant(self, mock_konflux_db): + """Golang builder rebuilds fine even when no matching build-root image is defined.""" + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._rebase_and_build_ci_images = AsyncMock() + pipeline._sync_ci_images = AsyncMock() + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + pipeline._rebase_and_build_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._sync_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_raises_when_build_root_rebuild_fails(self, mock_konflux_db): + pipeline = self._make_pipeline() + pipeline._get_ci_image_keys = AsyncMock( + return_value=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + ) + pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._rebase_and_build_ci_images = AsyncMock( + side_effect=[None, RuntimeError("Failed to rebuild 1/1")], + ) + pipeline._sync_ci_images = AsyncMock() + + with self.assertRaisesRegex(RuntimeError, "Failed to rebuild"): + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + # Sync only happens after both rebuild stages complete, so a build-root failure means + # nothing gets mirrored to CI this run -- even though golang-builder itself rebuilt fine. + pipeline._sync_ci_images.assert_not_awaited() + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_reconcile_syncs_for_test_assembly(self, mock_konflux_db): + """ + Test-assembly builds get rebuilt and synced too -- _sync_ci_images itself is responsible + for redirecting the publish to the .test-suffixed CI tag via --live-test-mode, so + _refresh_ci_images doesn't need to skip the call for the test assembly. + """ + pipeline = self._make_pipeline(assembly="test") + pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._rebase_and_build_ci_images = AsyncMock() + pipeline._sync_ci_images = AsyncMock() + + await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + + pipeline._rebase_and_build_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) + pipeline._sync_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) + self.assertTrue( + any("Synced CI image" in call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list) + ) + + +class TestScanStaleCiImages(IsolatedAsyncioTestCase): + """Test the doozer scan-sources wrapper used to detect stale CI images""" + + def _make_pipeline(self, kubeconfig=None): + mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) + mock_runtime.new_slack_client.return_value = Mock() + return UpdateGolangPipeline( + runtime=mock_runtime, + ocp_version="4.18", + cves=None, + force_update_tracker=False, + go_nvrs=["golang-1.22.9-1.el9"], + art_jira="ART-1234", + tag_builds=True, + build_system="konflux", + kubeconfig=kubeconfig, + ) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + @patch("artcommonlib.exectools.cmd_gather_async") + async def test_builds_command_and_returns_changed_images(self, mock_cmd_gather, mock_konflux_db): + pipeline = self._make_pipeline(kubeconfig="/tmp/kubeconfig") + report = ( + "images:\n" + "- name: ci-openshift-golang-builder-latest.rhel9\n" + " changed: true\n" + "- name: ci-openshift-golang-builder-extra.rhel8\n" + " changed: false\n" + ) + mock_cmd_gather.return_value = (0, report, "") + + stale = await pipeline._scan_stale_ci_images( + ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-golang-builder-extra.rhel8"] + ) + + self.assertEqual(stale, ["ci-openshift-golang-builder-latest.rhel9"]) + cmd = mock_cmd_gather.call_args.args[0] + self.assertIn("beta:config:konflux:scan-sources", cmd) + self.assertIn("--yaml", cmd) + self.assertEqual(cmd[cmd.index("--assembly") + 1], "stream") + self.assertEqual( + cmd[cmd.index("-i") + 1], + "ci-openshift-golang-builder-latest.rhel9,ci-openshift-golang-builder-extra.rhel8", + ) + self.assertIn("--ci-kubeconfig=/tmp/kubeconfig", cmd) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + @patch("artcommonlib.exectools.cmd_gather_async") + async def test_omits_ci_kubeconfig_when_not_set(self, mock_cmd_gather, mock_konflux_db): + pipeline = self._make_pipeline(kubeconfig=None) + # Force this regardless of ambient env vars other tests in this process may have leaked + # (KONFLUX_SA_KUBECONFIG is a fallback the pipeline reads at construction time). + pipeline.kubeconfig = None + mock_cmd_gather.return_value = (0, "images: []\n", "") + + await pipeline._scan_stale_ci_images(["ci-openshift-golang-builder-latest.rhel9"]) + + cmd = mock_cmd_gather.call_args.args[0] + self.assertFalse(any(arg.startswith("--ci-kubeconfig") for arg in cmd)) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + @patch("artcommonlib.exectools.cmd_gather_async") + async def test_raises_when_doozer_command_fails(self, mock_cmd_gather, mock_konflux_db): + pipeline = self._make_pipeline() + mock_cmd_gather.return_value = (1, "", "boom") + + with self.assertRaisesRegex(RuntimeError, "doozer scan-sources failed"): + await pipeline._scan_stale_ci_images(["ci-openshift-golang-builder-latest.rhel9"]) + + +class TestRebaseAndBuildCiImages(IsolatedAsyncioTestCase): + """Test the low-level rebase+build batch helper used by CI image reconciliation""" + + def _make_pipeline(self): + mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) + mock_runtime.new_slack_client.return_value = Mock() + return UpdateGolangPipeline( + runtime=mock_runtime, + ocp_version="4.18", + cves=None, + force_update_tracker=False, + go_nvrs=["golang-1.22.9-1.el9"], + art_jira="ART-1234", + tag_builds=True, + ) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_rebases_and_builds_each_image(self, mock_konflux_db): + pipeline = self._make_pipeline() + pipeline._rebase_ci_image = AsyncMock() + pipeline._build_ci_image = AsyncMock() + + await pipeline._rebase_and_build_ci_images( + ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + ) + + self.assertEqual( + {call.args[0] for call in pipeline._rebase_ci_image.await_args_list}, + {"ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"}, + ) + self.assertEqual( + {call.args[0] for call in pipeline._build_ci_image.await_args_list}, + {"ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"}, + ) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_raises_aggregated_error_when_some_images_fail(self, mock_konflux_db): + pipeline = self._make_pipeline() + pipeline._rebase_ci_image = AsyncMock() + + async def _build_side_effect(image_key): + if image_key == "ci-openshift-build-root-latest.rhel9": + raise ChildProcessError("build failed") + + pipeline._build_ci_image = AsyncMock(side_effect=_build_side_effect) + + with self.assertRaisesRegex(RuntimeError, r"Failed to rebuild 1/2 CI image\(s\)"): + await pipeline._rebase_and_build_ci_images( + ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + ) + + class TestMonobranchDispatch(IsolatedAsyncioTestCase): """Test that doozer group and image methods work with monobranch""" From b20ed33e0bee9f33b950896816f42dda47a463ac Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Thu, 27 Aug 2026 16:32:50 +0200 Subject: [PATCH 02/18] separate app ci kubeconfig rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- pyartcd/pyartcd/pipelines/update_golang.py | 7 +++++-- pyartcd/tests/pipelines/test_update_golang.py | 20 ++++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index 1ac0328c96..57fc2db372 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -1216,8 +1216,11 @@ async def _scan_stale_ci_images(self, image_keys: List[str]) -> List[str]: cmd.extend(self._get_doozer_assembly_args()) cmd.extend(["-i", ",".join(image_keys)]) cmd.extend(["beta:config:konflux:scan-sources", "--yaml"]) - if self.kubeconfig: - cmd.append(f"--ci-kubeconfig={self.kubeconfig}") + # --ci-kubeconfig is for looking at release-controller imagestreams on app.ci, which is a + # different cluster/identity than self.kubeconfig (the Konflux SA kubeconfig). + ci_kubeconfig = os.environ.get('KUBECONFIG') + if ci_kubeconfig: + cmd.append(f"--ci-kubeconfig={ci_kubeconfig}") rc, out, _ = await exectools.cmd_gather_async(cmd, env=self._doozer_env_vars, stderr=None, check=False) if rc != 0: diff --git a/pyartcd/tests/pipelines/test_update_golang.py b/pyartcd/tests/pipelines/test_update_golang.py index ae07dff3ec..e6358c8d1b 100644 --- a/pyartcd/tests/pipelines/test_update_golang.py +++ b/pyartcd/tests/pipelines/test_update_golang.py @@ -2824,7 +2824,9 @@ def _make_pipeline(self, kubeconfig=None): @patch("pyartcd.pipelines.update_golang.KonfluxDb") @patch("artcommonlib.exectools.cmd_gather_async") async def test_builds_command_and_returns_changed_images(self, mock_cmd_gather, mock_konflux_db): - pipeline = self._make_pipeline(kubeconfig="/tmp/kubeconfig") + # --ci-kubeconfig must come from the app.ci KUBECONFIG env var, not self.kubeconfig + # (which holds the Konflux SA kubeconfig and lacks RBAC on app.ci imagestreams). + pipeline = self._make_pipeline(kubeconfig="/tmp/konflux-kubeconfig") report = ( "images:\n" "- name: ci-openshift-golang-builder-latest.rhel9\n" @@ -2834,9 +2836,10 @@ async def test_builds_command_and_returns_changed_images(self, mock_cmd_gather, ) mock_cmd_gather.return_value = (0, report, "") - stale = await pipeline._scan_stale_ci_images( - ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-golang-builder-extra.rhel8"] - ) + with patch.dict(os.environ, {"KUBECONFIG": "/tmp/ci-kubeconfig"}): + stale = await pipeline._scan_stale_ci_images( + ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-golang-builder-extra.rhel8"] + ) self.assertEqual(stale, ["ci-openshift-golang-builder-latest.rhel9"]) cmd = mock_cmd_gather.call_args.args[0] @@ -2847,18 +2850,17 @@ async def test_builds_command_and_returns_changed_images(self, mock_cmd_gather, cmd[cmd.index("-i") + 1], "ci-openshift-golang-builder-latest.rhel9,ci-openshift-golang-builder-extra.rhel8", ) - self.assertIn("--ci-kubeconfig=/tmp/kubeconfig", cmd) + self.assertIn("--ci-kubeconfig=/tmp/ci-kubeconfig", cmd) @patch("pyartcd.pipelines.update_golang.KonfluxDb") @patch("artcommonlib.exectools.cmd_gather_async") async def test_omits_ci_kubeconfig_when_not_set(self, mock_cmd_gather, mock_konflux_db): pipeline = self._make_pipeline(kubeconfig=None) - # Force this regardless of ambient env vars other tests in this process may have leaked - # (KONFLUX_SA_KUBECONFIG is a fallback the pipeline reads at construction time). - pipeline.kubeconfig = None mock_cmd_gather.return_value = (0, "images: []\n", "") - await pipeline._scan_stale_ci_images(["ci-openshift-golang-builder-latest.rhel9"]) + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("KUBECONFIG", None) + await pipeline._scan_stale_ci_images(["ci-openshift-golang-builder-latest.rhel9"]) cmd = mock_cmd_gather.call_args.args[0] self.assertFalse(any(arg.startswith("--ci-kubeconfig") for arg in cmd)) From 2083a775700d6d3a561726d78dd42b8f2362a229 Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Tue, 1 Sep 2026 17:47:22 +0200 Subject: [PATCH 03/18] consider live-test-mode for upstream mirror dest rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- doozer/doozerlib/cli/images_streams.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doozer/doozerlib/cli/images_streams.py b/doozer/doozerlib/cli/images_streams.py index 39a4fbab66..7f7f039ea5 100644 --- a/doozer/doozerlib/cli/images_streams.py +++ b/doozer/doozerlib/cli/images_streams.py @@ -386,6 +386,8 @@ def mirror_image(cmd_start: str, upstream_dest: str): # upstream_image to all the upstream_image_mirror destinations so they all get the same version. if config.upstream_image_mirror is not Missing: for upstream_image_mirror_dest in config.upstream_image_mirror: + if live_test_mode: + upstream_image_mirror_dest += '.test' # Mirror to each destination only if not in only-if-missing mode OR destination doesn't exist if not only_if_missing or not destinations_to_check.get(upstream_image_mirror_dest, False): priv_cmd = f'oc image mirror {config.upstream_image}' From b48e9bd9a4ce3a96f5c2c18528ef46396cb3f5c1 Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Wed, 2 Sep 2026 12:19:32 +0200 Subject: [PATCH 04/18] buildroot and ci golang builder built at the same time rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- doozer/doozerlib/backend/rebaser.py | 44 +++++- pyartcd/pyartcd/pipelines/update_golang.py | 149 ++++++++++-------- pyartcd/tests/pipelines/test_update_golang.py | 76 ++++----- 3 files changed, 159 insertions(+), 110 deletions(-) diff --git a/doozer/doozerlib/backend/rebaser.py b/doozer/doozerlib/backend/rebaser.py index fad63746e7..2a5536c038 100644 --- a/doozer/doozerlib/backend/rebaser.py +++ b/doozer/doozerlib/backend/rebaser.py @@ -556,6 +556,13 @@ async def _resolve_member_parent(self, member: str, original_parent: str): if parent_metadata.should_trigger_base_image_release(): released_pullspec = (build.released_pullspec or "").strip() if released_pullspec: + self._logger.info( + "member-resolve: %s NOT loaded in this run; late-resolved via latest Konflux " + "build (nvr=%s) -> %s", + member, + build.nvr, + released_pullspec, + ) return released_pullspec, build.embargoed self._logger.warning( "Late-resolved parent %s: Konflux latest build has empty released_pullspec (nvr=%s); " @@ -565,10 +572,29 @@ async def _resolve_member_parent(self, member: str, original_parent: str): ) rh_pullspec = util.rh_art_images_base_pullspec(build.nvr) if await self._registry_pullspec_exists(rh_pullspec): + self._logger.info( + "member-resolve: %s NOT loaded in this run; late-resolved via latest Konflux " + "build (nvr=%s) -> %s", + member, + build.nvr, + rh_pullspec, + ) return rh_pullspec, build.embargoed raise IOError(f"Late-resolved parent {member}: art-images-base tag unreachable at {rh_pullspec}") + self._logger.info( + "member-resolve: %s NOT loaded in this run; late-resolved via latest Konflux build (nvr=%s) -> %s", + member, + build.nvr, + build.image_pullspec, + ) return build.image_pullspec, build.embargoed + self._logger.info( + "member-resolve: %s NOT loaded in this run; --latest-parent-version not set, leaving FROM " + "unchanged -> %s", + member, + original_parent, + ) return original_parent, False else: if not self.image_repo: @@ -582,9 +608,23 @@ async def _resolve_member_parent(self, member: str, original_parent: str): private_fix = parent_metadata.private_fix if parent_metadata.should_trigger_base_image_release(): parent_nvr = self._rebased_member_image_nvr(parent_metadata) - return util.rh_art_images_base_pullspec(parent_nvr), private_fix + pullspec = util.rh_art_images_base_pullspec(parent_nvr) + self._logger.info( + "member-resolve: %s loaded in this run; resolved via rebased NVR (%s) -> %s", + member, + parent_nvr, + pullspec, + ) + return pullspec, private_fix parent_image_repo = parent_metadata.get_konflux_image_repo(default=self.image_repo) - return f"{parent_image_repo}:{parent_metadata.image_name_short}-{self.uuid_tag}", private_fix + pullspec = f"{parent_image_repo}:{parent_metadata.image_name_short}-{self.uuid_tag}" + self._logger.info( + "member-resolve: %s loaded in this run; resolved via shared run uuid_tag (%s) -> %s", + member, + self.uuid_tag, + pullspec, + ) + return pullspec, private_fix @start_as_current_span_async(TRACER, "rebase.resolve_stream_parent") async def _resolve_stream_parent( diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index 57fc2db372..2a2ee98871 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -269,15 +269,17 @@ def _load_yaml_from_repo(repo, path: str, ref: str): def _get_upstream_ocp_build_data_repo(self): return get_github_client_for_org("openshift-eng").get_repo("openshift-eng/ocp-build-data") - def _get_ocp_build_data_repo_and_branch(self, default_branch): + def _get_ocp_build_data_repo_and_branch(self, default_branch, data_path=None, data_gitref=None): """Get the ocp-build-data repo and branch, respecting data_path/data_gitref overrides.""" - if self.data_path: - match = re.search(r'github\.com[:/](.+?)(?:\.git)?$', self.data_path) + data_path = self.data_path if data_path is None else data_path + data_gitref = self.data_gitref if data_gitref is None else data_gitref + if data_path: + match = re.search(r'github\.com[:/](.+?)(?:\.git)?$', data_path) if match: repo_name = match.group(1) org = repo_name.split('/')[0] repo = get_github_client_for_org(org).get_repo(repo_name) - branch = self.data_gitref or default_branch + branch = data_gitref or default_branch return repo, branch return self._get_upstream_ocp_build_data_repo(), default_branch @@ -1183,7 +1185,11 @@ async def _get_ci_image_keys(self) -> List[str]: need to be rebuilt whenever their respective parent changes. """ branch = f"openshift-{self.ocp_version}" - repo, branch = self._get_ocp_build_data_repo_and_branch(branch) + # TODO: revert -- forcing the test fork/branch below for CI rebuild testing; restore + # `repo, branch = self._get_ocp_build_data_repo_and_branch(branch)`. + repo, branch = self._get_ocp_build_data_repo_and_branch( + branch, data_path=self._CI_TEST_DATA_PATH, data_gitref=self._CI_TEST_DATA_GITREF + ) contents = repo.get_contents("images", ref=branch) ci_image_keys = sorted( content.name[: -len(".yml")] @@ -1204,14 +1210,15 @@ async def _scan_stale_ci_images(self, image_keys: List[str]) -> List[str]: references against streams.yml), it queries the parent's actual current build and flags the image if that build is newer than the image's own last build. """ - group = f"openshift-{self.ocp_version}" + group = self._get_ci_group() cmd = [ "doozer", f"--working-dir={self._doozer_working_dir}-ci-scan", "--build-system=konflux", ] - if self.data_path: - cmd.append(f"--data-path={self.data_path}") + # TODO: revert -- forcing the test data_path below for CI rebuild testing; restore + # `if self.data_path: cmd.append(f"--data-path={self.data_path}")`. + cmd.append(f"--data-path={self._CI_TEST_DATA_PATH}") cmd.extend(["--group", group]) cmd.extend(self._get_doozer_assembly_args()) cmd.extend(["-i", ",".join(image_keys)]) @@ -1230,30 +1237,37 @@ async def _scan_stale_ci_images(self, image_keys: List[str]) -> List[str]: changes = get_changes(report) return [image_key for image_key in changes.get('images', []) if image_key in image_keys] - async def _rebase_ci_image(self, image_key: str, version: str, release: str): - _LOGGER.info("Rebasing %s for Konflux...", image_key) - group = f"openshift-{self.ocp_version}" + async def _rebase_ci_images(self, image_keys: List[str], version: str, release: str): + _LOGGER.info("Rebasing %s for Konflux...", ", ".join(image_keys)) + group = self._get_ci_group() cmd = [ "doozer", - f"--working-dir={self._doozer_working_dir}-ci-{image_key}", + f"--working-dir={self._doozer_working_dir}-ci-rebase", "--build-system=konflux", ] cmd.extend(self._get_doozer_assembly_args()) - if self.data_path: - cmd.append(f"--data-path={self.data_path}") + # TODO: revert -- forcing the test data_path below for CI rebuild testing; restore + # `if self.data_path: cmd.append(f"--data-path={self.data_path}")`. + cmd.append(f"--data-path={self._CI_TEST_DATA_PATH}") cmd.extend( [ "--group", group, + # All CI images being rebuilt this run are rebased together in one doozer + # invocation (matching ocp4_konflux's approach), so a `from: member:` reference + # between them (e.g. ci-openshift-build-root-* -> ci-openshift-golang-builder-*) + # is resolved in-process. --latest-parent-version is kept as a fallback for a + # member that isn't part of this batch (e.g. only the build-root side went stale). + "--latest-parent-version", "-i", - image_key, + ",".join(image_keys), "beta:images:konflux:rebase", "--version", version, "--release", release, "--message", - f"Bump golang parent image for {image_key}", + f"Bump golang parent image for {', '.join(image_keys)}", ] ) if self.network_mode: @@ -1262,24 +1276,26 @@ async def _rebase_ci_image(self, image_key: str, version: str, release: str): cmd.append("--push") await exectools.cmd_assert_async(cmd, env=self._doozer_env_vars) - async def _build_ci_image(self, image_key: str): - _LOGGER.info("Building %s on Konflux...", image_key) - group = f"openshift-{self.ocp_version}" + async def _build_ci_images(self, image_keys: List[str]): + _LOGGER.info("Building %s on Konflux...", ", ".join(image_keys)) + group = self._get_ci_group() konflux_namespace = PRODUCT_NAMESPACE_MAP["ocp"] cmd = [ "doozer", - f"--working-dir={self._doozer_working_dir}-ci-{image_key}", + f"--working-dir={self._doozer_working_dir}-ci-build", "--build-system=konflux", ] cmd.extend(self._get_doozer_assembly_args()) - if self.data_path: - cmd.append(f"--data-path={self.data_path}") + # TODO: revert -- forcing the test data_path below for CI rebuild testing; restore + # `if self.data_path: cmd.append(f"--data-path={self.data_path}")`. + cmd.append(f"--data-path={self._CI_TEST_DATA_PATH}") cmd.extend( [ "--group", group, + "--latest-parent-version", "-i", - image_key, + ",".join(image_keys), "beta:images:konflux:build", f"--konflux-namespace={konflux_namespace}", "--skip-ec-verify", @@ -1295,24 +1311,13 @@ async def _build_ci_image(self, image_key: str): async def _rebase_and_build_ci_images(self, image_keys: List[str]): """ - Rebase+build each CI image directly via doozer, without touching streams.yml or creating - an ocp-build-data PR. + Rebase, then build, all given CI images together -- one doozer invocation per phase -- + directly via doozer, without touching streams.yml or creating an ocp-build-data PR. """ version = f"v{self.ocp_version}.0" release = default_release_suffix() - - async def _rebase_and_build(image_key: str): - await self._rebase_ci_image(image_key, version, release) - await self._build_ci_image(image_key) - - results = await asyncio.gather( - *[_rebase_and_build(image_key) for image_key in image_keys], - return_exceptions=True, - ) - failed = [(image_key, r) for image_key, r in zip(image_keys, results) if isinstance(r, Exception)] - if failed: - summary = "\n".join(f" ❌ {image_key}: {err}" for image_key, err in failed) - raise RuntimeError(f"Failed to rebuild {len(failed)}/{len(image_keys)} CI image(s):\n{summary}") + await self._rebase_ci_images(image_keys, version, release) + await self._build_ci_images(image_keys) @staticmethod def _get_required_env(var_name: str) -> str: @@ -1358,7 +1363,7 @@ async def _sync_ci_images(self, image_keys: List[str]): other way to redirect it away from the production tag. """ _LOGGER.info("Syncing CI image(s) to CI: %s", ", ".join(image_keys)) - group = f"openshift-{self.ocp_version}" + group = self._get_ci_group() with self._create_ci_sync_registry_config() as auth_file: cmd = [ "doozer", @@ -1366,8 +1371,9 @@ async def _sync_ci_images(self, image_keys: List[str]): "--build-system=konflux", ] cmd.extend(self._get_doozer_assembly_args()) - if self.data_path: - cmd.append(f"--data-path={self.data_path}") + # TODO: revert -- forcing the test data_path below for CI rebuild testing; restore + # `if self.data_path: cmd.append(f"--data-path={self.data_path}")`. + cmd.append(f"--data-path={self._CI_TEST_DATA_PATH}") cmd.extend(["--group", group]) for image_key in image_keys: cmd.extend(["--image", image_key]) @@ -1384,6 +1390,12 @@ async def _sync_ci_images(self, image_keys: List[str]): "GO_PREVIOUS": "previous", } + # TODO: revert -- temporary hardcoded fork/branch for testing the CI golang-builder/build-root + # rebuild path end-to-end. Every site tagged "TODO: revert" below forces this instead of the + # normal --data-path/--data-gitref values; remove them all once testing is done. + _CI_TEST_DATA_PATH = "https://github.com/kopero2000/ocp-build-data" + _CI_TEST_DATA_GITREF = "openshift-5.0-test" + async def _refresh_ci_images( self, build_major_minor: str, @@ -1401,12 +1413,13 @@ async def _refresh_ci_images( an ocp-build-data PR) any golang-builder CI image found to be stale. ci-openshift-build-root-* images pull their parent via a `member` reference to the golang-builder CI image (not a golang stream or the raw golang-builder container), so once a golang-builder image is - rebuilt, its corresponding build-root image is unconditionally rebuilt right after, to - pick up the new base image. Both rebuild steps must complete before either image is - mirrored to CI, so everything rebuilt this run is synced together in a single final step. - For the test assembly, `_sync_ci_images` passes `--live-test-mode`, which publishes to - the `.test`-suffixed CI imagestream tag instead of the real one, so test-assembly runs - never overwrite what production CI actually consumes. + stale, its corresponding build-root image is unconditionally rebuilt alongside it, in the + same rebase/build batch (see `_rebase_and_build_ci_images`) -- doozer only resolves a + `from: member:` reference correctly when both images are loaded in the same run. Once + both rebuild, everything is synced together in a single final step. For the test + assembly, `_sync_ci_images` passes `--live-test-mode`, which publishes to the + `.test`-suffixed CI imagestream tag instead of the real one, so test-assembly runs never + overwrite what production CI actually consumes. """ variant = next( ( @@ -1455,30 +1468,16 @@ async def _refresh_ci_images( return stale_builder_keys = [image_key for _, image_key in stale_builder_targets] - await self._slack_client.say_in_thread( - f":construction: Rebuilding CI golang builder image(s) for " - f"{self.ocp_version}: {', '.join(stale_builder_keys)}" - ) - await self._rebase_and_build_ci_images(stale_builder_keys) - await self._slack_client.say_in_thread( - f":white_check_mark: Rebuilt CI golang builder image(s): {', '.join(stale_builder_keys)}" - ) - # Trigger the build-root image(s) whose golang-builder parent was just rebuilt above. + # Build-root image(s) whose golang-builder parent is about to be rebuilt. Rebuilt in the + # same batch as their golang-builder parent (not a separate call) so doozer resolves the + # `from: member:` reference in-process. build_root_keys = [ f"{CI_BUILD_ROOT_IMAGE_PREFIX}{variant}.rhel{el_v}" for el_v, _ in stale_builder_targets if f"{CI_BUILD_ROOT_IMAGE_PREFIX}{variant}.rhel{el_v}" in all_image_keys ] - if build_root_keys: - await self._slack_client.say_in_thread( - f":construction: Rebuilding CI build-root image(s) for {self.ocp_version}: {', '.join(build_root_keys)}" - ) - await self._rebase_and_build_ci_images(build_root_keys) - await self._slack_client.say_in_thread( - f":white_check_mark: Rebuilt CI build-root image(s): {', '.join(build_root_keys)}" - ) - else: + if not build_root_keys: _LOGGER.info( "No CI build-root images found for variant %s on openshift-%s; skipping CI build-root rebuild", variant, @@ -1486,6 +1485,15 @@ async def _refresh_ci_images( ) rebuilt_image_keys = stale_builder_keys + build_root_keys + await self._slack_client.say_in_thread( + f":construction: Rebuilding CI golang builder/build-root image(s) for " + f"{self.ocp_version}: {', '.join(rebuilt_image_keys)}" + ) + await self._rebase_and_build_ci_images(rebuilt_image_keys) + await self._slack_client.say_in_thread( + f":white_check_mark: Rebuilt CI golang builder/build-root image(s): {', '.join(rebuilt_image_keys)}" + ) + try: await self._sync_ci_images(rebuilt_image_keys) except Exception as e: @@ -1514,6 +1522,17 @@ def _get_doozer_group_and_image(self, el_v, go_version): group += f'@{self.data_gitref}' return group, image_key + def _get_ci_group(self) -> str: + """The openshift-{version} doozer group used by the CI golang-builder/build-root + methods, honoring --data-gitref so a fork's non-default-named branch (e.g. + openshift-5.0-test) is actually checked out instead of doozer defaulting to a branch + literally named openshift-{version}.""" + group = f"openshift-{self.ocp_version}" + # TODO: revert -- forcing the test data_gitref below for CI rebuild testing; restore + # `if self.data_gitref: group += f'@{self.data_gitref}'`. + group += f'@{self._CI_TEST_DATA_GITREF}' + return group + def verify_golang_builder_repo(self, el_v, go_version): default_branch = self.GOLANG_DATA_BRANCH filename = 'group.yml' diff --git a/pyartcd/tests/pipelines/test_update_golang.py b/pyartcd/tests/pipelines/test_update_golang.py index e6358c8d1b..0a484fda4f 100644 --- a/pyartcd/tests/pipelines/test_update_golang.py +++ b/pyartcd/tests/pipelines/test_update_golang.py @@ -2160,8 +2160,8 @@ async def test_build_konflux_dry_run(self, mock_cmd_assert, mock_konflux_db): @patch("pyartcd.pipelines.update_golang.KonfluxDb") @patch("artcommonlib.exectools.cmd_assert_async") - async def test_rebase_ci_image_includes_assembly(self, mock_cmd_assert, mock_konflux_db): - """Test _rebase_ci_image passes --assembly for the pipeline's assembly""" + async def test_rebase_ci_images_includes_assembly(self, mock_cmd_assert, mock_konflux_db): + """Test _rebase_ci_images passes --assembly for the pipeline's assembly""" mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) mock_runtime.new_slack_client.return_value = Mock() @@ -2177,7 +2177,7 @@ async def test_rebase_ci_image_includes_assembly(self, mock_cmd_assert, mock_kon assembly="test", ) - await pipeline._rebase_ci_image("ci-openshift-golang-builder-latest.rhel9", "v4.18.0", "1") + await pipeline._rebase_ci_images(["ci-openshift-golang-builder-latest.rhel9"], "v4.18.0", "1") cmd = mock_cmd_assert.call_args[0][0] self.assertIn("beta:images:konflux:rebase", cmd) @@ -2185,8 +2185,8 @@ async def test_rebase_ci_image_includes_assembly(self, mock_cmd_assert, mock_kon @patch("pyartcd.pipelines.update_golang.KonfluxDb") @patch("artcommonlib.exectools.cmd_assert_async") - async def test_build_ci_image_includes_assembly(self, mock_cmd_assert, mock_konflux_db): - """Test _build_ci_image passes --assembly for the pipeline's assembly""" + async def test_build_ci_images_includes_assembly(self, mock_cmd_assert, mock_konflux_db): + """Test _build_ci_images passes --assembly for the pipeline's assembly""" mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) mock_runtime.new_slack_client.return_value = Mock() @@ -2201,7 +2201,7 @@ async def test_build_ci_image_includes_assembly(self, mock_cmd_assert, mock_konf build_system="konflux", ) - await pipeline._build_ci_image("ci-openshift-golang-builder-latest.rhel9") + await pipeline._build_ci_images(["ci-openshift-golang-builder-latest.rhel9"]) cmd = mock_cmd_assert.call_args[0][0] self.assertIn("beta:images:konflux:build", cmd) @@ -2669,7 +2669,7 @@ async def test_reconcile_rebuilds_stale_image(self, mock_konflux_db): pipeline._sync_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) self.assertTrue( any( - "Rebuilding CI golang builder image" in call.args[0] + "Rebuilding CI golang builder/build-root image" in call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list ) ) @@ -2703,7 +2703,7 @@ async def test_reconcile_raises_when_rebuild_fails(self, mock_konflux_db): async def test_reconcile_triggers_build_root_after_golang_builder_rebuild(self, mock_konflux_db): """ Build-root images pull their parent via a `member` reference to the golang-builder CI - image, not via the golang-builder staleness scan, so once golang builder is rebuilt its + image, not via the golang-builder staleness scan, so once golang builder is stale its build-root sibling is triggered unconditionally -- no extra staleness check for build-root. """ pipeline = self._make_pipeline() @@ -2719,18 +2719,17 @@ async def test_reconcile_triggers_build_root_after_golang_builder_rebuild(self, # Only the golang-builder image is scanned; build-root is triggered directly, without a # staleness check of its own. pipeline._scan_stale_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) - # Golang builder and build-root are rebuilt as two separate, ordered batch calls. - self.assertEqual( - [call.args[0] for call in pipeline._rebase_and_build_ci_images.await_args_list], - [["ci-openshift-golang-builder-latest.rhel9"], ["ci-openshift-build-root-latest.rhel9"]], + # Golang builder and build-root are rebuilt together in a single batch call, so doozer + # resolves the `from: member:` reference between them in-process. + pipeline._rebase_and_build_ci_images.assert_awaited_once_with( + ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] ) - # Both families are synced together in a single call, after both rebuild steps complete. + # Both families are synced together in a single call, after the rebuild batch completes. pipeline._sync_ci_images.assert_awaited_once_with( ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] ) slack_messages = [call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list] - self.assertTrue(any("Rebuilding CI golang builder image" in m for m in slack_messages)) - self.assertTrue(any("Rebuilding CI build-root image" in m for m in slack_messages)) + self.assertTrue(any("Rebuilding CI golang builder/build-root image" in m for m in slack_messages)) self.assertTrue(any("Synced CI image" in m for m in slack_messages)) @patch("pyartcd.pipelines.update_golang.KonfluxDb") @@ -2769,16 +2768,14 @@ async def test_reconcile_raises_when_build_root_rebuild_fails(self, mock_konflux return_value=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] ) pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) - pipeline._rebase_and_build_ci_images = AsyncMock( - side_effect=[None, RuntimeError("Failed to rebuild 1/1")], - ) + pipeline._rebase_and_build_ci_images = AsyncMock(side_effect=RuntimeError("Failed to rebuild 1/1")) pipeline._sync_ci_images = AsyncMock() with self.assertRaisesRegex(RuntimeError, "Failed to rebuild"): await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) - # Sync only happens after both rebuild stages complete, so a build-root failure means - # nothing gets mirrored to CI this run -- even though golang-builder itself rebuilt fine. + # Sync only happens after the rebuild batch completes, so a build-root failure means + # nothing gets mirrored to CI this run -- even though the batch call is single-shot. pipeline._sync_ci_images.assert_not_awaited() @patch("pyartcd.pipelines.update_golang.KonfluxDb") @@ -2892,36 +2889,29 @@ def _make_pipeline(self): ) @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_rebases_and_builds_each_image(self, mock_konflux_db): + async def test_rebases_then_builds_the_batch_together(self, mock_konflux_db): + """ + All given image keys are rebased in one doozer call, then built in one doozer call -- + not per-image -- so a `from: member:` reference between them (e.g. + ci-openshift-build-root-* -> ci-openshift-golang-builder-*) resolves in-process. + """ pipeline = self._make_pipeline() - pipeline._rebase_ci_image = AsyncMock() - pipeline._build_ci_image = AsyncMock() + pipeline._rebase_ci_images = AsyncMock() + pipeline._build_ci_images = AsyncMock() - await pipeline._rebase_and_build_ci_images( - ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] - ) + image_keys = ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + await pipeline._rebase_and_build_ci_images(image_keys) - self.assertEqual( - {call.args[0] for call in pipeline._rebase_ci_image.await_args_list}, - {"ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"}, - ) - self.assertEqual( - {call.args[0] for call in pipeline._build_ci_image.await_args_list}, - {"ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"}, - ) + self.assertEqual(pipeline._rebase_ci_images.await_args.args[0], image_keys) + pipeline._build_ci_images.assert_awaited_once_with(image_keys) @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_raises_aggregated_error_when_some_images_fail(self, mock_konflux_db): + async def test_raises_when_build_fails(self, mock_konflux_db): pipeline = self._make_pipeline() - pipeline._rebase_ci_image = AsyncMock() - - async def _build_side_effect(image_key): - if image_key == "ci-openshift-build-root-latest.rhel9": - raise ChildProcessError("build failed") - - pipeline._build_ci_image = AsyncMock(side_effect=_build_side_effect) + pipeline._rebase_ci_images = AsyncMock() + pipeline._build_ci_images = AsyncMock(side_effect=ChildProcessError("build failed")) - with self.assertRaisesRegex(RuntimeError, r"Failed to rebuild 1/2 CI image\(s\)"): + with self.assertRaises(ChildProcessError): await pipeline._rebase_and_build_ci_images( ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] ) From 4802e7bbe6b28f30c70df7b56776ec73e499bda0 Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Wed, 2 Sep 2026 14:20:44 +0200 Subject: [PATCH 05/18] load disabled for building ci images rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- pyartcd/pyartcd/pipelines/update_golang.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index 2a2ee98871..0e176d4f67 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -1220,6 +1220,11 @@ async def _scan_stale_ci_images(self, image_keys: List[str]) -> List[str]: # `if self.data_path: cmd.append(f"--data-path={self.data_path}")`. cmd.append(f"--data-path={self._CI_TEST_DATA_PATH}") cmd.extend(["--group", group]) + # These images are `mode: disabled` in ocp-build-data so the standing ocp4-scan job (which + # scans the whole group without --load-disabled) leaves their lifecycle to this pipeline. + # scan-sources applies that same enabled-filter even to explicitly `-i`-named images, so + # --load-disabled is required here or this scan would always report them as not stale. + cmd.append("--load-disabled") cmd.extend(self._get_doozer_assembly_args()) cmd.extend(["-i", ",".join(image_keys)]) cmd.extend(["beta:config:konflux:scan-sources", "--yaml"]) From cc73175a0b8386d117ebb9bf1ba052cb80718c53 Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Wed, 2 Sep 2026 14:39:39 +0200 Subject: [PATCH 06/18] consider --load-disabled, test rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- doozer/doozerlib/cli/scan_sources_konflux.py | 5 ++-- pyartcd/pyartcd/pipelines/update_golang.py | 24 ++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/doozer/doozerlib/cli/scan_sources_konflux.py b/doozer/doozerlib/cli/scan_sources_konflux.py index aead3d74e6..80099fd363 100644 --- a/doozer/doozerlib/cli/scan_sources_konflux.py +++ b/doozer/doozerlib/cli/scan_sources_konflux.py @@ -563,8 +563,9 @@ async def find_latest_image_builds(self, image_names: List[str]): self.latest_image_build_records_map.update((zip(image_names, latest_image_builds))) async def scan_images(self, image_names: List[str]): - # Filter to only enabled images (variant-aware filtering is handled by _is_image_enabled) - image_names = filter(lambda name: self._is_image_enabled(self.runtime.image_map[name]), image_names) + # Filter to only enabled images (variant-aware filtering is handled by _is_image_enabled_for_scan, + # which also honors --load-disabled -- unlike _is_image_enabled alone) + image_names = filter(lambda name: self._is_image_enabled_for_scan(self.runtime.image_map[name]), image_names) # Do not scan images that have already been requested for rebuild image_names = list(filter(lambda name: name not in self.changing_image_names, image_names)) diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index 0e176d4f67..2537b4e7be 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -1426,6 +1426,13 @@ async def _refresh_ci_images( `.test`-suffixed CI imagestream tag instead of the real one, so test-assembly runs never overwrite what production CI actually consumes. """ + # TODO: revert -- re-deriving allowed_major_minors from the test fork/branch below. + # _get_allowed_go_major_minors (which produced the `allowed_major_minors` param) always + # reads the real upstream ocp-build-data regardless of --data-path, so without this the + # variant match below would use production GO_LATEST/GO_EXTRA/GO_PREVIOUS values while + # every doozer call in this method reads from the test fork. Restore by deleting this + # line so `allowed_major_minors` (the param) is used directly below. + allowed_major_minors = self._get_ci_allowed_go_major_minors() variant = next( ( self.CI_VARIANT_BY_GROUP_VAR[var_name] @@ -1538,6 +1545,23 @@ def _get_ci_group(self) -> str: group += f'@{self._CI_TEST_DATA_GITREF}' return group + # TODO: revert -- delete this whole method once CI rebuild testing is done. + def _get_ci_allowed_go_major_minors(self) -> dict[str, str]: + """CI-scoped re-lookup of GO_LATEST/GO_EXTRA/GO_PREVIOUS against the test fork/branch, for + _refresh_ci_images' variant matching only -- see the TODO where this is called.""" + branch = f"openshift-{self.ocp_version}" + repo, branch = self._get_ocp_build_data_repo_and_branch( + branch, data_path=self._CI_TEST_DATA_PATH, data_gitref=self._CI_TEST_DATA_GITREF + ) + group_content = self._load_yaml_from_repo(repo, "group.yml", branch) + vars_content = group_content.get("vars", {}) + return { + var_name: extract_major_minor(var_value, f"group.yml {var_name}") + for var_name in ("GO_LATEST", "GO_EXTRA", "GO_PREVIOUS") + for var_value in [vars_content.get(var_name)] + if var_value + } + def verify_golang_builder_repo(self, el_v, go_version): default_branch = self.GOLANG_DATA_BRANCH filename = 'group.yml' From 4237bf266622c18f7772dcc7739557f5038a8dad Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Wed, 2 Sep 2026 14:51:36 +0200 Subject: [PATCH 07/18] skip scanning rpms rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- pyartcd/pyartcd/pipelines/update_golang.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index 2537b4e7be..6e5a562543 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -1227,7 +1227,11 @@ async def _scan_stale_ci_images(self, image_keys: List[str]) -> List[str]: cmd.append("--load-disabled") cmd.extend(self._get_doozer_assembly_args()) cmd.extend(["-i", ",".join(image_keys)]) - cmd.extend(["beta:config:konflux:scan-sources", "--yaml"]) + # These CI images declare `scan_sources: exempt_rpms: - '*'`, so RPM changes never affect + # their staleness anyway. --skip-rpms avoids loading/cloning every RPM source in the whole + # group (mode='images' instead of 'both' in doozer), which is otherwise unconditional and + # unrelated to the -i image filter above. + cmd.extend(["beta:config:konflux:scan-sources", "--yaml", "--skip-rpms"]) # --ci-kubeconfig is for looking at release-controller imagestreams on app.ci, which is a # different cluster/identity than self.kubeconfig (the Konflux SA kubeconfig). ci_kubeconfig = os.environ.get('KUBECONFIG') From 4e1b78b5e8cc302a39a73cbb41ba5e6f878d7f85 Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Wed, 2 Sep 2026 15:57:04 +0200 Subject: [PATCH 08/18] scan everything rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- pyartcd/pyartcd/pipelines/update_golang.py | 72 ++++++++++------------ 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index 6e5a562543..5de53ac2da 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -1412,23 +1412,22 @@ async def _refresh_ci_images( el_nvr_map_for_images: dict[int, str], ): """ - Standing reconciliation check: make sure the ci-openshift-golang-builder-* image(s) for - the variant (GO_LATEST/GO_EXTRA/GO_PREVIOUS) matching this run's golang version are built - against their current declared parent. The ocp-build-data-validator enforces that - GO_LATEST/GO_EXTRA/GO_PREVIOUS never share a major.minor, so at most one variant can match - here. Staleness is determined via doozer's `beta:config:konflux:scan-sources` (see - `_scan_stale_ci_images`) -- the same builder-staleness check used by the standing ocp4 - scan-sources job. Rebuilds (via doozer directly, without touching streams.yml or creating - an ocp-build-data PR) any golang-builder CI image found to be stale. ci-openshift-build-root-* - images pull their parent via a `member` reference to the golang-builder CI image (not a - golang stream or the raw golang-builder container), so once a golang-builder image is - stale, its corresponding build-root image is unconditionally rebuilt alongside it, in the - same rebase/build batch (see `_rebase_and_build_ci_images`) -- doozer only resolves a - `from: member:` reference correctly when both images are loaded in the same run. Once - both rebuild, everything is synced together in a single final step. For the test - assembly, `_sync_ci_images` passes `--live-test-mode`, which publishes to the - `.test`-suffixed CI imagestream tag instead of the real one, so test-assembly runs never - overwrite what production CI actually consumes. + Standing reconciliation check: make sure the ci-openshift-golang-builder-* and + ci-openshift-build-root-* image(s) for the variant (GO_LATEST/GO_EXTRA/GO_PREVIOUS) + matching this run's golang version are built against their current declared parent. The + ocp-build-data-validator enforces that GO_LATEST/GO_EXTRA/GO_PREVIOUS never share a + major.minor, so at most one variant can match here. Both families are scanned together in + a single `_scan_stale_ci_images` call (see `beta:config:konflux:scan-sources`) -- the same + builder-staleness check used by the standing ocp4 scan-sources job -- so build-root's own + source changes are caught directly. Loading both together also means doozer's own change + propagation (a changing image marks its `member`-referencing descendants as changing too) + naturally covers the case where build-root itself hasn't changed but its golang-builder + parent has. Whatever comes back stale is rebased+built together in one batch (see + `_rebase_and_build_ci_images`) -- doozer only resolves a `from: member:` reference + correctly when both images are loaded in the same run. Once done, everything rebuilt is + synced together in a single final step. For the test assembly, `_sync_ci_images` passes + `--live-test-mode`, which publishes to the `.test`-suffixed CI imagestream tag instead of + the real one, so test-assembly runs never overwrite what production CI actually consumes. """ # TODO: revert -- re-deriving allowed_major_minors from the test fork/branch below. # _get_allowed_go_major_minors (which produced the `allowed_major_minors` param) always @@ -1470,37 +1469,34 @@ async def _refresh_ci_images( ) return - builder_image_keys = [image_key for _, image_key in builder_targets] - stale_image_keys = set(await self._scan_stale_ci_images(builder_image_keys)) - stale_builder_targets = [ - (el_v, image_key) for el_v, image_key in builder_targets if image_key in stale_image_keys + # Build-root counterpart for every el_v that has a golang-builder image, scanned alongside + # it regardless of the builder's own staleness -- both build-root's own source changes and + # "parent golang-builder changed" (via doozer's descendant-propagation once both images are + # loaded together) are caught by this one scan. + build_root_targets = [ + (el_v, f"{CI_BUILD_ROOT_IMAGE_PREFIX}{variant}.rhel{el_v}") + for el_v, _ in builder_targets + if f"{CI_BUILD_ROOT_IMAGE_PREFIX}{variant}.rhel{el_v}" in all_image_keys ] - - if not stale_builder_targets: + if not build_root_targets: _LOGGER.info( - "All CI golang builder images for openshift-%s already use their current parent image", + "No CI build-root images found for variant %s on openshift-%s; scanning golang builder image(s) only", + variant, self.ocp_version, ) - return - stale_builder_keys = [image_key for _, image_key in stale_builder_targets] + scan_targets = builder_targets + build_root_targets + scan_keys = [image_key for _, image_key in scan_targets] + stale_image_keys = set(await self._scan_stale_ci_images(scan_keys)) + rebuilt_image_keys = [image_key for image_key in scan_keys if image_key in stale_image_keys] - # Build-root image(s) whose golang-builder parent is about to be rebuilt. Rebuilt in the - # same batch as their golang-builder parent (not a separate call) so doozer resolves the - # `from: member:` reference in-process. - build_root_keys = [ - f"{CI_BUILD_ROOT_IMAGE_PREFIX}{variant}.rhel{el_v}" - for el_v, _ in stale_builder_targets - if f"{CI_BUILD_ROOT_IMAGE_PREFIX}{variant}.rhel{el_v}" in all_image_keys - ] - if not build_root_keys: + if not rebuilt_image_keys: _LOGGER.info( - "No CI build-root images found for variant %s on openshift-%s; skipping CI build-root rebuild", - variant, + "All CI golang builder/build-root images for openshift-%s already use their current parent image", self.ocp_version, ) + return - rebuilt_image_keys = stale_builder_keys + build_root_keys await self._slack_client.say_in_thread( f":construction: Rebuilding CI golang builder/build-root image(s) for " f"{self.ocp_version}: {', '.join(rebuilt_image_keys)}" From c6b5fc3115bf687e84544ab9797878c0329d3279 Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Thu, 3 Sep 2026 14:38:58 +0200 Subject: [PATCH 09/18] add KONFLUX_DEFAULT_IMAGE_REPO rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- pyartcd/pyartcd/pipelines/update_golang.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index 5de53ac2da..165a4808ba 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -21,6 +21,7 @@ REGISTRY_QUAY_OCP_RELEASE_DEV, REGISTRY_QUAY_OPENSHIFT, REGISTRY_REDHAT_IO, + KONFLUX_DEFAULT_IMAGE_REPO, ) from artcommonlib.github_auth import get_github_client_for_org from artcommonlib.konflux.konflux_build_record import ArtifactType, Engine, KonfluxBuildOutcome, KonfluxBuildRecord From 4a8ade9cdf49f8439c45829c2908ccca8565ba50 Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Thu, 3 Sep 2026 16:37:48 +0200 Subject: [PATCH 10/18] always sync ci images rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- pyartcd/pyartcd/pipelines/update_golang.py | 44 ++++++++++++---------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index 165a4808ba..5d426caa82 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -1425,10 +1425,13 @@ async def _refresh_ci_images( naturally covers the case where build-root itself hasn't changed but its golang-builder parent has. Whatever comes back stale is rebased+built together in one batch (see `_rebase_and_build_ci_images`) -- doozer only resolves a `from: member:` reference - correctly when both images are loaded in the same run. Once done, everything rebuilt is - synced together in a single final step. For the test assembly, `_sync_ci_images` passes - `--live-test-mode`, which publishes to the `.test`-suffixed CI imagestream tag instead of - the real one, so test-assembly runs never overwrite what production CI actually consumes. + correctly when both images are loaded in the same run. Once done, every image considered + this run (not just what was rebuilt) is synced to CI in a single final step -- re-mirroring + an already-current image is cheap for a handful of images, and it keeps CI in sync with the + latest successful build even when nothing needed rebuilding. For the test assembly, + `_sync_ci_images` passes `--live-test-mode`, which publishes to the `.test`-suffixed CI + imagestream tag instead of the real one, so test-assembly runs never overwrite what + production CI actually consumes. """ # TODO: revert -- re-deriving allowed_major_minors from the test fork/branch below. # _get_allowed_go_major_minors (which produced the `allowed_major_minors` param) always @@ -1491,30 +1494,31 @@ async def _refresh_ci_images( stale_image_keys = set(await self._scan_stale_ci_images(scan_keys)) rebuilt_image_keys = [image_key for image_key in scan_keys if image_key in stale_image_keys] - if not rebuilt_image_keys: + if rebuilt_image_keys: + await self._slack_client.say_in_thread( + f":construction: Rebuilding CI golang builder/build-root image(s) for " + f"{self.ocp_version}: {', '.join(rebuilt_image_keys)}" + ) + await self._rebase_and_build_ci_images(rebuilt_image_keys) + await self._slack_client.say_in_thread( + f":white_check_mark: Rebuilt CI golang builder/build-root image(s): {', '.join(rebuilt_image_keys)}" + ) + else: _LOGGER.info( - "All CI golang builder/build-root images for openshift-%s already use their current parent image", + "All CI golang builder/build-root images for openshift-%s already use their current parent " + "image;", self.ocp_version, ) - return - - await self._slack_client.say_in_thread( - f":construction: Rebuilding CI golang builder/build-root image(s) for " - f"{self.ocp_version}: {', '.join(rebuilt_image_keys)}" - ) - await self._rebase_and_build_ci_images(rebuilt_image_keys) - await self._slack_client.say_in_thread( - f":white_check_mark: Rebuilt CI golang builder/build-root image(s): {', '.join(rebuilt_image_keys)}" - ) + # Sync every image considered this run, not just what was just rebuilt -- mirroring an + # already-current image is a no-op cost-wise (a handful of images at most), and it keeps CI + # in sync with the latest successful build even on runs where nothing needed rebuilding. try: - await self._sync_ci_images(rebuilt_image_keys) + await self._sync_ci_images(scan_keys) except Exception as e: raise RuntimeError(f"Failed to sync CI image(s) to CI: {e}") from e - await self._slack_client.say_in_thread( - f":white_check_mark: Synced CI image(s): {', '.join(rebuilt_image_keys)}" - ) + await self._slack_client.say_in_thread(f":white_check_mark: Synced CI image(s): {', '.join(scan_keys)}") GOLANG_DATA_BRANCH = 'golang' From b124cd5a06fbfce5a01b4d4ee00b6280d84248d8 Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Thu, 3 Sep 2026 16:58:45 +0200 Subject: [PATCH 11/18] fix image stream mirror command rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- pyartcd/pyartcd/pipelines/update_golang.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index 5d426caa82..547a04b5cd 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -1385,9 +1385,9 @@ async def _sync_ci_images(self, image_keys: List[str]): # `if self.data_path: cmd.append(f"--data-path={self.data_path}")`. cmd.append(f"--data-path={self._CI_TEST_DATA_PATH}") cmd.extend(["--group", group]) + cmd.extend(["images:streams", "mirror", "--registry-auth", auth_file]) for image_key in image_keys: cmd.extend(["--image", image_key]) - cmd.extend(["images:streams", "mirror", "--registry-auth", auth_file]) if not self.is_production_assembly: cmd.append("--live-test-mode") if self.dry_run: @@ -1505,8 +1505,7 @@ async def _refresh_ci_images( ) else: _LOGGER.info( - "All CI golang builder/build-root images for openshift-%s already use their current parent " - "image;", + "All CI golang builder/build-root images for openshift-%s already use their current parent image;", self.ocp_version, ) From 7468a056f9c804d3b285752e6c676fe6a436dbcd Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Fri, 4 Sep 2026 14:10:18 +0200 Subject: [PATCH 12/18] add -load-disabled rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- pyartcd/pyartcd/pipelines/update_golang.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index 547a04b5cd..4bb5d5be1f 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -1385,6 +1385,11 @@ async def _sync_ci_images(self, image_keys: List[str]): # `if self.data_path: cmd.append(f"--data-path={self.data_path}")`. cmd.append(f"--data-path={self._CI_TEST_DATA_PATH}") cmd.extend(["--group", group]) + # These images are `mode: disabled` in ocp-build-data (see _scan_stale_ci_images), and + # unlike that command this one doesn't use the global `-i` image filter (its `--image` + # below is a subcommand-local option), so the runtime's default enabled-only filter + # would otherwise drop them before `images:streams mirror` can look them up. + cmd.append("--load-disabled") cmd.extend(["images:streams", "mirror", "--registry-auth", auth_file]) for image_key in image_keys: cmd.extend(["--image", image_key]) From 57f1891ebd0e0e13bc6c153d1de207ff942be9ea Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Fri, 4 Sep 2026 15:50:09 +0200 Subject: [PATCH 13/18] Revert TESTs, debugs rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- doozer/doozerlib/backend/rebaser.py | 44 +--------------- pyartcd/pyartcd/pipelines/update_golang.py | 61 ++++------------------ 2 files changed, 13 insertions(+), 92 deletions(-) diff --git a/doozer/doozerlib/backend/rebaser.py b/doozer/doozerlib/backend/rebaser.py index 2a5536c038..fad63746e7 100644 --- a/doozer/doozerlib/backend/rebaser.py +++ b/doozer/doozerlib/backend/rebaser.py @@ -556,13 +556,6 @@ async def _resolve_member_parent(self, member: str, original_parent: str): if parent_metadata.should_trigger_base_image_release(): released_pullspec = (build.released_pullspec or "").strip() if released_pullspec: - self._logger.info( - "member-resolve: %s NOT loaded in this run; late-resolved via latest Konflux " - "build (nvr=%s) -> %s", - member, - build.nvr, - released_pullspec, - ) return released_pullspec, build.embargoed self._logger.warning( "Late-resolved parent %s: Konflux latest build has empty released_pullspec (nvr=%s); " @@ -572,29 +565,10 @@ async def _resolve_member_parent(self, member: str, original_parent: str): ) rh_pullspec = util.rh_art_images_base_pullspec(build.nvr) if await self._registry_pullspec_exists(rh_pullspec): - self._logger.info( - "member-resolve: %s NOT loaded in this run; late-resolved via latest Konflux " - "build (nvr=%s) -> %s", - member, - build.nvr, - rh_pullspec, - ) return rh_pullspec, build.embargoed raise IOError(f"Late-resolved parent {member}: art-images-base tag unreachable at {rh_pullspec}") - self._logger.info( - "member-resolve: %s NOT loaded in this run; late-resolved via latest Konflux build (nvr=%s) -> %s", - member, - build.nvr, - build.image_pullspec, - ) return build.image_pullspec, build.embargoed - self._logger.info( - "member-resolve: %s NOT loaded in this run; --latest-parent-version not set, leaving FROM " - "unchanged -> %s", - member, - original_parent, - ) return original_parent, False else: if not self.image_repo: @@ -608,23 +582,9 @@ async def _resolve_member_parent(self, member: str, original_parent: str): private_fix = parent_metadata.private_fix if parent_metadata.should_trigger_base_image_release(): parent_nvr = self._rebased_member_image_nvr(parent_metadata) - pullspec = util.rh_art_images_base_pullspec(parent_nvr) - self._logger.info( - "member-resolve: %s loaded in this run; resolved via rebased NVR (%s) -> %s", - member, - parent_nvr, - pullspec, - ) - return pullspec, private_fix + return util.rh_art_images_base_pullspec(parent_nvr), private_fix parent_image_repo = parent_metadata.get_konflux_image_repo(default=self.image_repo) - pullspec = f"{parent_image_repo}:{parent_metadata.image_name_short}-{self.uuid_tag}" - self._logger.info( - "member-resolve: %s loaded in this run; resolved via shared run uuid_tag (%s) -> %s", - member, - self.uuid_tag, - pullspec, - ) - return pullspec, private_fix + return f"{parent_image_repo}:{parent_metadata.image_name_short}-{self.uuid_tag}", private_fix @start_as_current_span_async(TRACER, "rebase.resolve_stream_parent") async def _resolve_stream_parent( diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index 4bb5d5be1f..90bc38557a 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -1186,11 +1186,7 @@ async def _get_ci_image_keys(self) -> List[str]: need to be rebuilt whenever their respective parent changes. """ branch = f"openshift-{self.ocp_version}" - # TODO: revert -- forcing the test fork/branch below for CI rebuild testing; restore - # `repo, branch = self._get_ocp_build_data_repo_and_branch(branch)`. - repo, branch = self._get_ocp_build_data_repo_and_branch( - branch, data_path=self._CI_TEST_DATA_PATH, data_gitref=self._CI_TEST_DATA_GITREF - ) + repo, branch = self._get_ocp_build_data_repo_and_branch(branch) contents = repo.get_contents("images", ref=branch) ci_image_keys = sorted( content.name[: -len(".yml")] @@ -1217,9 +1213,8 @@ async def _scan_stale_ci_images(self, image_keys: List[str]) -> List[str]: f"--working-dir={self._doozer_working_dir}-ci-scan", "--build-system=konflux", ] - # TODO: revert -- forcing the test data_path below for CI rebuild testing; restore - # `if self.data_path: cmd.append(f"--data-path={self.data_path}")`. - cmd.append(f"--data-path={self._CI_TEST_DATA_PATH}") + if self.data_path: + cmd.append(f"--data-path={self.data_path}") cmd.extend(["--group", group]) # These images are `mode: disabled` in ocp-build-data so the standing ocp4-scan job (which # scans the whole group without --load-disabled) leaves their lifecycle to this pipeline. @@ -1256,9 +1251,8 @@ async def _rebase_ci_images(self, image_keys: List[str], version: str, release: "--build-system=konflux", ] cmd.extend(self._get_doozer_assembly_args()) - # TODO: revert -- forcing the test data_path below for CI rebuild testing; restore - # `if self.data_path: cmd.append(f"--data-path={self.data_path}")`. - cmd.append(f"--data-path={self._CI_TEST_DATA_PATH}") + if self.data_path: + cmd.append(f"--data-path={self.data_path}") cmd.extend( [ "--group", @@ -1296,9 +1290,8 @@ async def _build_ci_images(self, image_keys: List[str]): "--build-system=konflux", ] cmd.extend(self._get_doozer_assembly_args()) - # TODO: revert -- forcing the test data_path below for CI rebuild testing; restore - # `if self.data_path: cmd.append(f"--data-path={self.data_path}")`. - cmd.append(f"--data-path={self._CI_TEST_DATA_PATH}") + if self.data_path: + cmd.append(f"--data-path={self.data_path}") cmd.extend( [ "--group", @@ -1381,9 +1374,8 @@ async def _sync_ci_images(self, image_keys: List[str]): "--build-system=konflux", ] cmd.extend(self._get_doozer_assembly_args()) - # TODO: revert -- forcing the test data_path below for CI rebuild testing; restore - # `if self.data_path: cmd.append(f"--data-path={self.data_path}")`. - cmd.append(f"--data-path={self._CI_TEST_DATA_PATH}") + if self.data_path: + cmd.append(f"--data-path={self.data_path}") cmd.extend(["--group", group]) # These images are `mode: disabled` in ocp-build-data (see _scan_stale_ci_images), and # unlike that command this one doesn't use the global `-i` image filter (its `--image` @@ -1405,12 +1397,6 @@ async def _sync_ci_images(self, image_keys: List[str]): "GO_PREVIOUS": "previous", } - # TODO: revert -- temporary hardcoded fork/branch for testing the CI golang-builder/build-root - # rebuild path end-to-end. Every site tagged "TODO: revert" below forces this instead of the - # normal --data-path/--data-gitref values; remove them all once testing is done. - _CI_TEST_DATA_PATH = "https://github.com/kopero2000/ocp-build-data" - _CI_TEST_DATA_GITREF = "openshift-5.0-test" - async def _refresh_ci_images( self, build_major_minor: str, @@ -1438,13 +1424,6 @@ async def _refresh_ci_images( imagestream tag instead of the real one, so test-assembly runs never overwrite what production CI actually consumes. """ - # TODO: revert -- re-deriving allowed_major_minors from the test fork/branch below. - # _get_allowed_go_major_minors (which produced the `allowed_major_minors` param) always - # reads the real upstream ocp-build-data regardless of --data-path, so without this the - # variant match below would use production GO_LATEST/GO_EXTRA/GO_PREVIOUS values while - # every doozer call in this method reads from the test fork. Restore by deleting this - # line so `allowed_major_minors` (the param) is used directly below. - allowed_major_minors = self._get_ci_allowed_go_major_minors() variant = next( ( self.CI_VARIANT_BY_GROUP_VAR[var_name] @@ -1549,28 +1528,10 @@ def _get_ci_group(self) -> str: openshift-5.0-test) is actually checked out instead of doozer defaulting to a branch literally named openshift-{version}.""" group = f"openshift-{self.ocp_version}" - # TODO: revert -- forcing the test data_gitref below for CI rebuild testing; restore - # `if self.data_gitref: group += f'@{self.data_gitref}'`. - group += f'@{self._CI_TEST_DATA_GITREF}' + if self.data_gitref: + group += f'@{self.data_gitref}' return group - # TODO: revert -- delete this whole method once CI rebuild testing is done. - def _get_ci_allowed_go_major_minors(self) -> dict[str, str]: - """CI-scoped re-lookup of GO_LATEST/GO_EXTRA/GO_PREVIOUS against the test fork/branch, for - _refresh_ci_images' variant matching only -- see the TODO where this is called.""" - branch = f"openshift-{self.ocp_version}" - repo, branch = self._get_ocp_build_data_repo_and_branch( - branch, data_path=self._CI_TEST_DATA_PATH, data_gitref=self._CI_TEST_DATA_GITREF - ) - group_content = self._load_yaml_from_repo(repo, "group.yml", branch) - vars_content = group_content.get("vars", {}) - return { - var_name: extract_major_minor(var_value, f"group.yml {var_name}") - for var_name in ("GO_LATEST", "GO_EXTRA", "GO_PREVIOUS") - for var_value in [vars_content.get(var_name)] - if var_value - } - def verify_golang_builder_repo(self, el_v, go_version): default_branch = self.GOLANG_DATA_BRANCH filename = 'group.yml' From 1495c65053fb2a768e9d49baa9ea2a78ecf63544 Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Mon, 7 Sep 2026 15:18:06 +0200 Subject: [PATCH 14/18] try ocp4 konflux jenkins pipeline instead of local doozer commands rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- pyartcd/pyartcd/pipelines/update_golang.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index 90bc38557a..85a41827d9 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -1483,7 +1483,14 @@ async def _refresh_ci_images( f":construction: Rebuilding CI golang builder/build-root image(s) for " f"{self.ocp_version}: {', '.join(rebuilt_image_keys)}" ) - await self._rebase_and_build_ci_images(rebuilt_image_keys) + + # await self._rebase_and_build_ci_images(rebuilt_image_keys) + build_result = jenkins.start_ocp4_konflux( + build_version=self.ocp_version, + assembly=self.assembly, + image_list=rebuilt_image_keys, + dry_run=self.dry_run + ) await self._slack_client.say_in_thread( f":white_check_mark: Rebuilt CI golang builder/build-root image(s): {', '.join(rebuilt_image_keys)}" ) From dc856b5768182961f925b69c396a4379d112a248 Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Mon, 7 Sep 2026 17:24:33 +0200 Subject: [PATCH 15/18] update sync-ci pipeline so can be used for disabled components, and test mode rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- pyartcd/pyartcd/jenkins.py | 20 +++++++++++++++-- pyartcd/pyartcd/pipelines/sync_ci_images.py | 25 ++++++++++++++++++++- pyartcd/pyartcd/pipelines/update_golang.py | 24 +++++++++++++++----- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/pyartcd/pyartcd/jenkins.py b/pyartcd/pyartcd/jenkins.py index aded16fcab..d3afa1f3ae 100644 --- a/pyartcd/pyartcd/jenkins.py +++ b/pyartcd/pyartcd/jenkins.py @@ -52,7 +52,8 @@ class Jobs(Enum): SCAN_PLASHET_RPMS = 'scanning/scanning%2Fplashet-rpms' BUILD_CONFORMA_VERIFY = 'aos-cd-builds/build%2Fbuild-conforma-verify' SCAN_OPERATOR = 'aos-cd-builds/build%2Fscan-operator' - SYNC_CI_IMAGES = 'aos-cd-builds/build%2Fsync-ci-images' + # TODO TEST + SYNC_CI_IMAGES = 'hack/bvizi/add-ci-config-to-golang-builder' OPEN_RECONCILIATION_PRS = 'aos-cd-builds/build%2Fopen-reconciliation-prs' OPEN_RECONCILIATION_PRS_LAYERED = 'aos-cd-builds/build%2Fopen-reconciliation-prs-layered-products' @@ -531,10 +532,25 @@ def start_rhcos(build_version: str, new_build: bool, job_name: str = 'build', ** ) -def start_sync_ci_images(version: str, **kwargs) -> Optional[str]: +def start_sync_ci_images( + version: str, + assembly: str = 'stream', + image_list: list = None, + dry_run: bool = False, + load_disabled: bool = False, + live_test_mode: bool = False, + **kwargs, +) -> Optional[str]: params = { 'VERSION': version, } + if image_list: + params['IMAGE_LIST'] = ','.join(image_list) + params['ART_TOOLS_COMMIT'] = 'kopero2000@ci-golang-builder-from-update-golang-ART-21958' + params['DRY_RUN'] = dry_run + params['LOAD_DISABLED'] = load_disabled + params['LIVE_TEST_MODE'] = live_test_mode + params['ASSEMBLY'] = assembly return start_build( job=Jobs.SYNC_CI_IMAGES, params=params, diff --git a/pyartcd/pyartcd/pipelines/sync_ci_images.py b/pyartcd/pyartcd/pipelines/sync_ci_images.py index 1a5c8e3de2..31dfca00e0 100644 --- a/pyartcd/pyartcd/pipelines/sync_ci_images.py +++ b/pyartcd/pyartcd/pipelines/sync_ci_images.py @@ -57,6 +57,8 @@ def __init__( skip_waits: bool = False, force_run: bool = False, update_images_only_when_missing: bool = False, + load_disabled: bool = False, + live_test_mode: bool = False, ) -> None: """ Initialize sync-ci-images pipeline. @@ -84,7 +86,8 @@ def __init__( self.skip_waits = skip_waits self.force_run = force_run self.update_images_only_when_missing = update_images_only_when_missing - + self.load_disabled = load_disabled + self.live_test_mode = live_test_mode # Validate parameters self._validate_parameters() @@ -512,6 +515,8 @@ def _build_doozer_options(self, group_dir: Path, auth_file: str) -> str: f"--build-system {self.BUILD_SYSTEM} " f"--registry-config {auth_file}" ) + if self.load_disabled: + doozer_opts += " --load-disabled" return doozer_opts @property @@ -547,6 +552,8 @@ async def _mirror_images_to_ci(self, doozer_opts: str, auth_file: str) -> None: mirror_args += "--only-if-missing " if self.runtime.dry_run: mirror_args += "--dry-run" + if self.live_test_mode: + mirror_args += "--live-test-mode" await self._run_doozer_command(doozer_opts, "images:streams mirror", mirror_args.strip()) async def _trigger_ci_builds(self, doozer_opts: str, auth_file: str) -> None: @@ -669,6 +676,18 @@ async def run(self) -> int: default=False, help='Pass --only-if-missing to doozer mirror (update only missing images)', ) +@click.option( + '--load-disabled', + is_flag=True, + default=False, + help='Pass --load-disabled to doozer mirror', +) +@click.option( + '--live-test-mode', + is_flag=True, + default=False, + help='Pass --live-test-mode to doozer mirror', +) @pass_runtime @click_coroutine async def sync_ci_images_cli( @@ -682,6 +701,8 @@ async def sync_ci_images_cli( skip_waits: bool, force_run: bool, update_images_only_when_missing: bool, + load_disabled: bool, + live_test_mode: bool, ): """ CLI entrypoint for sync-ci-images pipeline. @@ -709,6 +730,8 @@ async def sync_ci_images_cli( skip_waits=skip_waits, force_run=force_run, update_images_only_when_missing=update_images_only_when_missing, + load_disabled=load_disabled, + live_test_mode=live_test_mode, ) # Run with per-version lock diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index 85a41827d9..ae90be9eda 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -15,13 +15,13 @@ GOLANG_BUILDER_IMAGE_NAME, GOLANG_NVR_LABEL, KONFLUX_DEFAULT_FBC_REPO, + KONFLUX_DEFAULT_IMAGE_REPO, KONFLUX_DEFAULT_IMAGE_SHARE_REPO, PRODUCT_NAMESPACE_MAP, REGISTRY_CI_OPENSHIFT, REGISTRY_QUAY_OCP_RELEASE_DEV, REGISTRY_QUAY_OPENSHIFT, REGISTRY_REDHAT_IO, - KONFLUX_DEFAULT_IMAGE_REPO, ) from artcommonlib.github_auth import get_github_client_for_org from artcommonlib.konflux.konflux_build_record import ArtifactType, Engine, KonfluxBuildOutcome, KonfluxBuildRecord @@ -1489,8 +1489,12 @@ async def _refresh_ci_images( build_version=self.ocp_version, assembly=self.assembly, image_list=rebuilt_image_keys, - dry_run=self.dry_run + dry_run=self.dry_run, + block_until_complete=True, ) + if build_result != "SUCCESS": + raise RuntimeError(f"CI image build for {self.ocp_version} failed with result: {build_result}") + await self._slack_client.say_in_thread( f":white_check_mark: Rebuilt CI golang builder/build-root image(s): {', '.join(rebuilt_image_keys)}" ) @@ -1503,10 +1507,18 @@ async def _refresh_ci_images( # Sync every image considered this run, not just what was just rebuilt -- mirroring an # already-current image is a no-op cost-wise (a handful of images at most), and it keeps CI # in sync with the latest successful build even on runs where nothing needed rebuilding. - try: - await self._sync_ci_images(scan_keys) - except Exception as e: - raise RuntimeError(f"Failed to sync CI image(s) to CI: {e}") from e + + sync_result = jenkins.start_sync_ci_images( + version=self.ocp_version, + block_until_complete=True, + assembly=self.assembly, + image_list=scan_keys, + dry_run=self.dry_run, + load_disabled=True, + live_test_mode=not self.is_production_assembly, + ) + if sync_result != "SUCCESS": + raise RuntimeError(f"CI image sync for {self.ocp_version} failed with result: {sync_result}") await self._slack_client.say_in_thread(f":white_check_mark: Synced CI image(s): {', '.join(scan_keys)}") From 8ac4fb727207299979a59d5577e4afd9bafd25f5 Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Mon, 7 Sep 2026 17:32:05 +0200 Subject: [PATCH 16/18] remove unused code rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- pyartcd/pyartcd/pipelines/update_golang.py | 176 +-------- pyartcd/tests/pipelines/test_update_golang.py | 354 ++++++++---------- 2 files changed, 156 insertions(+), 374 deletions(-) diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index ae90be9eda..d6723fa165 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -14,19 +14,11 @@ BREW_HUB, GOLANG_BUILDER_IMAGE_NAME, GOLANG_NVR_LABEL, - KONFLUX_DEFAULT_FBC_REPO, - KONFLUX_DEFAULT_IMAGE_REPO, - KONFLUX_DEFAULT_IMAGE_SHARE_REPO, PRODUCT_NAMESPACE_MAP, - REGISTRY_CI_OPENSHIFT, - REGISTRY_QUAY_OCP_RELEASE_DEV, - REGISTRY_QUAY_OPENSHIFT, - REGISTRY_REDHAT_IO, ) from artcommonlib.github_auth import get_github_client_for_org from artcommonlib.konflux.konflux_build_record import ArtifactType, Engine, KonfluxBuildOutcome, KonfluxBuildRecord from artcommonlib.konflux.konflux_db import KonfluxDb -from artcommonlib.registry_config import RegistryConfig, RegistryCredential from artcommonlib.release_util import isolate_assembly_in_release, isolate_el_version_in_release from artcommonlib.rpm_utils import parse_nvr from artcommonlib.util import new_roundtrip_yaml_handler @@ -1242,155 +1234,6 @@ async def _scan_stale_ci_images(self, image_keys: List[str]) -> List[str]: changes = get_changes(report) return [image_key for image_key in changes.get('images', []) if image_key in image_keys] - async def _rebase_ci_images(self, image_keys: List[str], version: str, release: str): - _LOGGER.info("Rebasing %s for Konflux...", ", ".join(image_keys)) - group = self._get_ci_group() - cmd = [ - "doozer", - f"--working-dir={self._doozer_working_dir}-ci-rebase", - "--build-system=konflux", - ] - cmd.extend(self._get_doozer_assembly_args()) - if self.data_path: - cmd.append(f"--data-path={self.data_path}") - cmd.extend( - [ - "--group", - group, - # All CI images being rebuilt this run are rebased together in one doozer - # invocation (matching ocp4_konflux's approach), so a `from: member:` reference - # between them (e.g. ci-openshift-build-root-* -> ci-openshift-golang-builder-*) - # is resolved in-process. --latest-parent-version is kept as a fallback for a - # member that isn't part of this batch (e.g. only the build-root side went stale). - "--latest-parent-version", - "-i", - ",".join(image_keys), - "beta:images:konflux:rebase", - "--version", - version, - "--release", - release, - "--message", - f"Bump golang parent image for {', '.join(image_keys)}", - ] - ) - if self.network_mode: - cmd.extend(["--network-mode", self.network_mode]) - if not self.dry_run: - cmd.append("--push") - await exectools.cmd_assert_async(cmd, env=self._doozer_env_vars) - - async def _build_ci_images(self, image_keys: List[str]): - _LOGGER.info("Building %s on Konflux...", ", ".join(image_keys)) - group = self._get_ci_group() - konflux_namespace = PRODUCT_NAMESPACE_MAP["ocp"] - cmd = [ - "doozer", - f"--working-dir={self._doozer_working_dir}-ci-build", - "--build-system=konflux", - ] - cmd.extend(self._get_doozer_assembly_args()) - if self.data_path: - cmd.append(f"--data-path={self.data_path}") - cmd.extend( - [ - "--group", - group, - "--latest-parent-version", - "-i", - ",".join(image_keys), - "beta:images:konflux:build", - f"--konflux-namespace={konflux_namespace}", - "--skip-ec-verify", - ] - ) - if self.kubeconfig: - cmd.extend(['--konflux-kubeconfig', self.kubeconfig]) - if self.network_mode: - cmd.extend(["--network-mode", self.network_mode]) - if self.dry_run: - cmd.append("--dry-run") - await exectools.cmd_assert_async(cmd, env=self._doozer_env_vars, log_stdout=True) - - async def _rebase_and_build_ci_images(self, image_keys: List[str]): - """ - Rebase, then build, all given CI images together -- one doozer invocation per phase -- - directly via doozer, without touching streams.yml or creating an ocp-build-data PR. - """ - version = f"v{self.ocp_version}.0" - release = default_release_suffix() - await self._rebase_ci_images(image_keys, version, release) - await self._build_ci_images(image_keys) - - @staticmethod - def _get_required_env(var_name: str) -> str: - value = os.getenv(var_name) - if not value: - raise ValueError(f"Required environment variable {var_name} not set") - return value - - def _create_ci_sync_registry_config(self) -> RegistryConfig: - """ - Build registry credentials for pushing a newly built CI image straight to CI, using the - same env vars (and QCI credential setup) as the scheduled sync-ci-images job. - """ - quay_auth_file = self._get_required_env('QUAY_AUTH_FILE') - kubeconfig = self._get_required_env('KUBECONFIG') - qci_user = self._get_required_env('QCI_USER') - qci_password = self._get_required_env('QCI_PASSWORD') - - return RegistryConfig( - source_files=[quay_auth_file], - kubeconfig=kubeconfig, - registries=[ - REGISTRY_CI_OPENSHIFT, - REGISTRY_QUAY_OCP_RELEASE_DEV, - KONFLUX_DEFAULT_IMAGE_REPO, - KONFLUX_DEFAULT_IMAGE_SHARE_REPO, - KONFLUX_DEFAULT_FBC_REPO, - REGISTRY_REDHAT_IO, - ], - credentials=[ - RegistryCredential(REGISTRY_QUAY_OPENSHIFT, qci_user, qci_password), - ], - ) - - async def _sync_ci_images(self, image_keys: List[str]): - """ - Mirror newly rebuilt CI image(s) straight to CI (the same `images:streams mirror` doozer - verb the scheduled sync-ci-images job uses), so CI does not have to wait for that job's - next run to pick up the rebuild. For the test assembly, `--live-test-mode` is passed so - doozer publishes to the `.test`-suffixed CI imagestream tag instead of the real one -- - `images:streams mirror` resolves its destination entirely from each image's - ci_alignment.upstream_image config, not from anything on this command line, so there is no - other way to redirect it away from the production tag. - """ - _LOGGER.info("Syncing CI image(s) to CI: %s", ", ".join(image_keys)) - group = self._get_ci_group() - with self._create_ci_sync_registry_config() as auth_file: - cmd = [ - "doozer", - f"--working-dir={self._doozer_working_dir}-ci-sync", - "--build-system=konflux", - ] - cmd.extend(self._get_doozer_assembly_args()) - if self.data_path: - cmd.append(f"--data-path={self.data_path}") - cmd.extend(["--group", group]) - # These images are `mode: disabled` in ocp-build-data (see _scan_stale_ci_images), and - # unlike that command this one doesn't use the global `-i` image filter (its `--image` - # below is a subcommand-local option), so the runtime's default enabled-only filter - # would otherwise drop them before `images:streams mirror` can look them up. - cmd.append("--load-disabled") - cmd.extend(["images:streams", "mirror", "--registry-auth", auth_file]) - for image_key in image_keys: - cmd.extend(["--image", image_key]) - if not self.is_production_assembly: - cmd.append("--live-test-mode") - if self.dry_run: - cmd.append("--dry-run") - await exectools.cmd_assert_async(cmd, env=self._doozer_env_vars, log_stdout=True) - CI_VARIANT_BY_GROUP_VAR = { "GO_LATEST": "latest", "GO_EXTRA": "extra", @@ -1414,15 +1257,17 @@ async def _refresh_ci_images( source changes are caught directly. Loading both together also means doozer's own change propagation (a changing image marks its `member`-referencing descendants as changing too) naturally covers the case where build-root itself hasn't changed but its golang-builder - parent has. Whatever comes back stale is rebased+built together in one batch (see - `_rebase_and_build_ci_images`) -- doozer only resolves a `from: member:` reference - correctly when both images are loaded in the same run. Once done, every image considered - this run (not just what was rebuilt) is synced to CI in a single final step -- re-mirroring - an already-current image is cheap for a handful of images, and it keeps CI in sync with the + parent has. Whatever comes back stale is rebased+built together in one batch by triggering + the standing `ocp4-konflux` job (see `jenkins.start_ocp4_konflux`), scoped to just these + image(s) via `IMAGE_LIST` -- doozer only resolves a `from: member:` reference correctly + when both images are loaded in the same run. Once done, every image considered this run + (not just what was rebuilt) is synced to CI in a single final step by triggering the + standing `sync-ci-images` job (see `jenkins.start_sync_ci_images`) -- re-mirroring an + already-current image is cheap for a handful of images, and it keeps CI in sync with the latest successful build even when nothing needed rebuilding. For the test assembly, - `_sync_ci_images` passes `--live-test-mode`, which publishes to the `.test`-suffixed CI - imagestream tag instead of the real one, so test-assembly runs never overwrite what - production CI actually consumes. + `live_test_mode` is passed so that job publishes to the `.test`-suffixed CI imagestream + tag instead of the real one, so test-assembly runs never overwrite what production CI + actually consumes. """ variant = next( ( @@ -1484,7 +1329,6 @@ async def _refresh_ci_images( f"{self.ocp_version}: {', '.join(rebuilt_image_keys)}" ) - # await self._rebase_and_build_ci_images(rebuilt_image_keys) build_result = jenkins.start_ocp4_konflux( build_version=self.ocp_version, assembly=self.assembly, diff --git a/pyartcd/tests/pipelines/test_update_golang.py b/pyartcd/tests/pipelines/test_update_golang.py index 0a484fda4f..96840968b0 100644 --- a/pyartcd/tests/pipelines/test_update_golang.py +++ b/pyartcd/tests/pipelines/test_update_golang.py @@ -1,5 +1,4 @@ import asyncio -import contextlib import os import tempfile import unittest @@ -2158,112 +2157,6 @@ async def test_build_konflux_dry_run(self, mock_cmd_assert, mock_konflux_db): cmd = mock_cmd_assert.call_args[0][0] self.assertIn("--dry-run", cmd) - @patch("pyartcd.pipelines.update_golang.KonfluxDb") - @patch("artcommonlib.exectools.cmd_assert_async") - async def test_rebase_ci_images_includes_assembly(self, mock_cmd_assert, mock_konflux_db): - """Test _rebase_ci_images passes --assembly for the pipeline's assembly""" - mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) - mock_runtime.new_slack_client.return_value = Mock() - - pipeline = UpdateGolangPipeline( - runtime=mock_runtime, - ocp_version="4.18", - cves=None, - force_update_tracker=False, - go_nvrs=["golang-1.22.9-1.el9"], - art_jira="ART-1234", - tag_builds=False, - build_system="konflux", - assembly="test", - ) - - await pipeline._rebase_ci_images(["ci-openshift-golang-builder-latest.rhel9"], "v4.18.0", "1") - - cmd = mock_cmd_assert.call_args[0][0] - self.assertIn("beta:images:konflux:rebase", cmd) - self.assertEqual(cmd[cmd.index("--assembly") + 1], "test") - - @patch("pyartcd.pipelines.update_golang.KonfluxDb") - @patch("artcommonlib.exectools.cmd_assert_async") - async def test_build_ci_images_includes_assembly(self, mock_cmd_assert, mock_konflux_db): - """Test _build_ci_images passes --assembly for the pipeline's assembly""" - mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) - mock_runtime.new_slack_client.return_value = Mock() - - pipeline = UpdateGolangPipeline( - runtime=mock_runtime, - ocp_version="4.18", - cves=None, - force_update_tracker=False, - go_nvrs=["golang-1.22.9-1.el9"], - art_jira="ART-1234", - tag_builds=True, - build_system="konflux", - ) - - await pipeline._build_ci_images(["ci-openshift-golang-builder-latest.rhel9"]) - - cmd = mock_cmd_assert.call_args[0][0] - self.assertIn("beta:images:konflux:build", cmd) - self.assertEqual(cmd[cmd.index("--assembly") + 1], "stream") - - @patch("pyartcd.pipelines.update_golang.KonfluxDb") - @patch("artcommonlib.exectools.cmd_assert_async") - async def test_sync_ci_images_includes_assembly(self, mock_cmd_assert, mock_konflux_db): - """Test _sync_ci_images passes --assembly for the pipeline's assembly""" - mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) - mock_runtime.new_slack_client.return_value = Mock() - - pipeline = UpdateGolangPipeline( - runtime=mock_runtime, - ocp_version="4.18", - cves=None, - force_update_tracker=False, - go_nvrs=["golang-1.22.9-1.el9"], - art_jira="ART-1234", - tag_builds=True, - build_system="konflux", - ) - pipeline._create_ci_sync_registry_config = Mock(return_value=contextlib.nullcontext("/tmp/auth.json")) - - await pipeline._sync_ci_images(["ci-openshift-golang-builder-latest.rhel9"]) - - cmd = mock_cmd_assert.call_args[0][0] - self.assertIn("images:streams", cmd) - self.assertEqual(cmd[cmd.index("--assembly") + 1], "stream") - self.assertNotIn("--live-test-mode", cmd) - - @patch("pyartcd.pipelines.update_golang.KonfluxDb") - @patch("artcommonlib.exectools.cmd_assert_async") - async def test_sync_ci_images_uses_live_test_mode_for_test_assembly(self, mock_cmd_assert, mock_konflux_db): - """ - For the test assembly, doozer must publish to the .test-suffixed CI imagestream tag - instead of the real one -- images:streams mirror resolves its destination purely from - each image's ci_alignment.upstream_image config, so --live-test-mode is the only way to - redirect it away from what production CI actually consumes. - """ - mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) - mock_runtime.new_slack_client.return_value = Mock() - - pipeline = UpdateGolangPipeline( - runtime=mock_runtime, - ocp_version="4.18", - cves=None, - force_update_tracker=False, - go_nvrs=["golang-1.22.9-1.el9"], - art_jira="ART-1234", - tag_builds=False, - build_system="konflux", - assembly="test", - ) - pipeline._create_ci_sync_registry_config = Mock(return_value=contextlib.nullcontext("/tmp/auth.json")) - - await pipeline._sync_ci_images(["ci-openshift-golang-builder-latest.rhel9"]) - - cmd = mock_cmd_assert.call_args[0][0] - self.assertEqual(cmd[cmd.index("--assembly") + 1], "test") - self.assertIn("--live-test-mode", cmd) - @patch("pyartcd.pipelines.update_golang.KonfluxDb") @patch("artcommonlib.exectools.cmd_assert_async") async def test_test_assembly_dry_run_commands_for_konflux(self, mock_cmd_assert, mock_konflux_db): @@ -2615,58 +2508,94 @@ async def test_get_ci_image_keys_returns_empty_when_none_found(self, mock_konflu # Common args for _refresh_ci_images: a GO_LATEST bump on rhel9. RECONCILE_ARGS = ("1.22", {"GO_LATEST": "1.22"}, {9: "golang-1.22.9-1.el9"}) + @patch("pyartcd.pipelines.update_golang.jenkins") @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_reconcile_noops_when_variant_unmatched(self, mock_konflux_db): + async def test_reconcile_noops_when_variant_unmatched(self, mock_konflux_db, mock_jenkins): """No GO_LATEST/GO_EXTRA/GO_PREVIOUS var matches this build; nothing to check.""" pipeline = self._make_pipeline() pipeline._get_ci_image_keys = AsyncMock() - pipeline._rebase_and_build_ci_images = AsyncMock() await pipeline._refresh_ci_images("1.23", {"GO_LATEST": "1.22"}, {9: "golang"}) pipeline._get_ci_image_keys.assert_not_awaited() - pipeline._rebase_and_build_ci_images.assert_not_awaited() + mock_jenkins.start_ocp4_konflux.assert_not_called() + mock_jenkins.start_sync_ci_images.assert_not_called() pipeline._slack_client.say_in_thread.assert_not_awaited() + @patch("pyartcd.pipelines.update_golang.jenkins") @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_reconcile_noops_when_no_matching_images_found(self, mock_konflux_db): + async def test_reconcile_noops_when_no_matching_images_found(self, mock_konflux_db, mock_jenkins): """GO_LATEST matches, but no ci-openshift-golang-builder-latest.rhel* image exists.""" pipeline = self._make_pipeline() pipeline._get_ci_image_keys = AsyncMock(return_value=[]) pipeline._scan_stale_ci_images = AsyncMock() - pipeline._rebase_and_build_ci_images = AsyncMock() await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) pipeline._scan_stale_ci_images.assert_not_awaited() - pipeline._rebase_and_build_ci_images.assert_not_awaited() + mock_jenkins.start_ocp4_konflux.assert_not_called() + mock_jenkins.start_sync_ci_images.assert_not_called() pipeline._slack_client.say_in_thread.assert_not_awaited() + @patch("pyartcd.pipelines.update_golang.jenkins") @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_reconcile_noops_when_ci_image_already_current(self, mock_konflux_db): + async def test_reconcile_syncs_but_does_not_rebuild_when_ci_image_already_current( + self, mock_konflux_db, mock_jenkins + ): + """ + No image was stale, so the ocp4-konflux build is never triggered -- but the sync-ci-images + job still runs unconditionally, to keep CI in sync with the latest successful build even + when nothing needed rebuilding this run. + """ pipeline = self._make_pipeline() pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) pipeline._scan_stale_ci_images = AsyncMock(return_value=[]) - pipeline._rebase_and_build_ci_images = AsyncMock() + mock_jenkins.start_sync_ci_images.return_value = "SUCCESS" await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) pipeline._scan_stale_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) - pipeline._rebase_and_build_ci_images.assert_not_awaited() - pipeline._slack_client.say_in_thread.assert_not_awaited() + mock_jenkins.start_ocp4_konflux.assert_not_called() + mock_jenkins.start_sync_ci_images.assert_called_once_with( + version=pipeline.ocp_version, + block_until_complete=True, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + load_disabled=True, + live_test_mode=False, + ) + slack_messages = [call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list] + self.assertFalse(any("Rebuilding CI golang builder/build-root image" in m for m in slack_messages)) + self.assertTrue(any("Synced CI image" in m for m in slack_messages)) + @patch("pyartcd.pipelines.update_golang.jenkins") @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_reconcile_rebuilds_stale_image(self, mock_konflux_db): + async def test_reconcile_rebuilds_stale_image(self, mock_konflux_db, mock_jenkins): pipeline = self._make_pipeline() pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) - pipeline._rebase_and_build_ci_images = AsyncMock() - pipeline._sync_ci_images = AsyncMock() + mock_jenkins.start_ocp4_konflux.return_value = "SUCCESS" + mock_jenkins.start_sync_ci_images.return_value = "SUCCESS" await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) - pipeline._rebase_and_build_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) - pipeline._sync_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) + mock_jenkins.start_ocp4_konflux.assert_called_once_with( + build_version=pipeline.ocp_version, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + block_until_complete=True, + ) + mock_jenkins.start_sync_ci_images.assert_called_once_with( + version=pipeline.ocp_version, + block_until_complete=True, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + load_disabled=True, + live_test_mode=False, + ) self.assertTrue( any( "Rebuilding CI golang builder/build-root image" in call.args[0] @@ -2677,124 +2606,178 @@ async def test_reconcile_rebuilds_stale_image(self, mock_konflux_db): any("Synced CI image" in call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list) ) + @patch("pyartcd.pipelines.update_golang.jenkins") @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_reconcile_raises_when_scan_fails(self, mock_konflux_db): + async def test_reconcile_raises_when_scan_fails(self, mock_konflux_db, mock_jenkins): pipeline = self._make_pipeline() pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) pipeline._scan_stale_ci_images = AsyncMock(side_effect=RuntimeError("doozer scan-sources failed")) - pipeline._rebase_and_build_ci_images = AsyncMock() with self.assertRaisesRegex(RuntimeError, "scan-sources failed"): await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) - pipeline._rebase_and_build_ci_images.assert_not_awaited() + mock_jenkins.start_ocp4_konflux.assert_not_called() + mock_jenkins.start_sync_ci_images.assert_not_called() + @patch("pyartcd.pipelines.update_golang.jenkins") @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_reconcile_raises_when_rebuild_fails(self, mock_konflux_db): + async def test_reconcile_raises_when_rebuild_fails(self, mock_konflux_db, mock_jenkins): pipeline = self._make_pipeline() pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) - pipeline._rebase_and_build_ci_images = AsyncMock(side_effect=RuntimeError("Failed to rebuild 1/1")) + mock_jenkins.start_ocp4_konflux.return_value = "FAILURE" - with self.assertRaisesRegex(RuntimeError, "Failed to rebuild"): + with self.assertRaisesRegex(RuntimeError, "CI image build .* failed with result: FAILURE"): await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) + # Sync only happens after the build step completes, so a build failure means nothing + # gets mirrored to CI this run. + mock_jenkins.start_sync_ci_images.assert_not_called() + + @patch("pyartcd.pipelines.update_golang.jenkins") @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_reconcile_triggers_build_root_after_golang_builder_rebuild(self, mock_konflux_db): + async def test_reconcile_triggers_build_root_after_golang_builder_rebuild(self, mock_konflux_db, mock_jenkins): """ Build-root images pull their parent via a `member` reference to the golang-builder CI - image, not via the golang-builder staleness scan, so once golang builder is stale its - build-root sibling is triggered unconditionally -- no extra staleness check for build-root. + image. Both are scanned together in one `_scan_stale_ci_images` call, so doozer's own + change propagation (a changing image marks its `member`-referencing descendants as + changing too) reports build-root as stale right alongside its golang-builder parent. """ pipeline = self._make_pipeline() pipeline._get_ci_image_keys = AsyncMock( return_value=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] ) - pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) - pipeline._rebase_and_build_ci_images = AsyncMock() - pipeline._sync_ci_images = AsyncMock() + pipeline._scan_stale_ci_images = AsyncMock( + return_value=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + ) + mock_jenkins.start_ocp4_konflux.return_value = "SUCCESS" + mock_jenkins.start_sync_ci_images.return_value = "SUCCESS" await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) - # Only the golang-builder image is scanned; build-root is triggered directly, without a - # staleness check of its own. - pipeline._scan_stale_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) + # Both families are scanned together in a single call. + pipeline._scan_stale_ci_images.assert_awaited_once_with( + ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + ) # Golang builder and build-root are rebuilt together in a single batch call, so doozer # resolves the `from: member:` reference between them in-process. - pipeline._rebase_and_build_ci_images.assert_awaited_once_with( - ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + mock_jenkins.start_ocp4_konflux.assert_called_once_with( + build_version=pipeline.ocp_version, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"], + dry_run=pipeline.dry_run, + block_until_complete=True, ) # Both families are synced together in a single call, after the rebuild batch completes. - pipeline._sync_ci_images.assert_awaited_once_with( - ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] + mock_jenkins.start_sync_ci_images.assert_called_once_with( + version=pipeline.ocp_version, + block_until_complete=True, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"], + dry_run=pipeline.dry_run, + load_disabled=True, + live_test_mode=False, ) slack_messages = [call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list] self.assertTrue(any("Rebuilding CI golang builder/build-root image" in m for m in slack_messages)) self.assertTrue(any("Synced CI image" in m for m in slack_messages)) + @patch("pyartcd.pipelines.update_golang.jenkins") @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_reconcile_skips_build_root_when_golang_builder_already_fresh(self, mock_konflux_db): + async def test_reconcile_skips_build_root_when_golang_builder_already_fresh(self, mock_konflux_db, mock_jenkins): """If golang builder wasn't rebuilt, its build-root sibling is left untouched.""" pipeline = self._make_pipeline() pipeline._get_ci_image_keys = AsyncMock( return_value=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] ) pipeline._scan_stale_ci_images = AsyncMock(return_value=[]) - pipeline._rebase_and_build_ci_images = AsyncMock() + mock_jenkins.start_sync_ci_images.return_value = "SUCCESS" await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) - pipeline._rebase_and_build_ci_images.assert_not_awaited() - pipeline._slack_client.say_in_thread.assert_not_awaited() + mock_jenkins.start_ocp4_konflux.assert_not_called() + slack_messages = [call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list] + self.assertFalse(any("Rebuilding CI golang builder/build-root image" in m for m in slack_messages)) + @patch("pyartcd.pipelines.update_golang.jenkins") @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_reconcile_skips_build_root_when_none_defined_for_variant(self, mock_konflux_db): + async def test_reconcile_skips_build_root_when_none_defined_for_variant(self, mock_konflux_db, mock_jenkins): """Golang builder rebuilds fine even when no matching build-root image is defined.""" pipeline = self._make_pipeline() pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) - pipeline._rebase_and_build_ci_images = AsyncMock() - pipeline._sync_ci_images = AsyncMock() + mock_jenkins.start_ocp4_konflux.return_value = "SUCCESS" + mock_jenkins.start_sync_ci_images.return_value = "SUCCESS" await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) - pipeline._rebase_and_build_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) - pipeline._sync_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) + mock_jenkins.start_ocp4_konflux.assert_called_once_with( + build_version=pipeline.ocp_version, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + block_until_complete=True, + ) + mock_jenkins.start_sync_ci_images.assert_called_once_with( + version=pipeline.ocp_version, + block_until_complete=True, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + load_disabled=True, + live_test_mode=False, + ) + @patch("pyartcd.pipelines.update_golang.jenkins") @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_reconcile_raises_when_build_root_rebuild_fails(self, mock_konflux_db): + async def test_reconcile_raises_when_build_root_rebuild_fails(self, mock_konflux_db, mock_jenkins): pipeline = self._make_pipeline() pipeline._get_ci_image_keys = AsyncMock( return_value=["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] ) pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) - pipeline._rebase_and_build_ci_images = AsyncMock(side_effect=RuntimeError("Failed to rebuild 1/1")) - pipeline._sync_ci_images = AsyncMock() + mock_jenkins.start_ocp4_konflux.return_value = "FAILURE" - with self.assertRaisesRegex(RuntimeError, "Failed to rebuild"): + with self.assertRaisesRegex(RuntimeError, "CI image build .* failed with result: FAILURE"): await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) # Sync only happens after the rebuild batch completes, so a build-root failure means # nothing gets mirrored to CI this run -- even though the batch call is single-shot. - pipeline._sync_ci_images.assert_not_awaited() + mock_jenkins.start_sync_ci_images.assert_not_called() + @patch("pyartcd.pipelines.update_golang.jenkins") @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_reconcile_syncs_for_test_assembly(self, mock_konflux_db): + async def test_reconcile_syncs_for_test_assembly(self, mock_konflux_db, mock_jenkins): """ - Test-assembly builds get rebuilt and synced too -- _sync_ci_images itself is responsible - for redirecting the publish to the .test-suffixed CI tag via --live-test-mode, so - _refresh_ci_images doesn't need to skip the call for the test assembly. + Test-assembly builds get rebuilt and synced too -- start_sync_ci_images itself is + responsible for redirecting the publish to the .test-suffixed CI tag via + `live_test_mode`, so _refresh_ci_images doesn't need to skip the call for the test + assembly. """ pipeline = self._make_pipeline(assembly="test") pipeline._get_ci_image_keys = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) pipeline._scan_stale_ci_images = AsyncMock(return_value=["ci-openshift-golang-builder-latest.rhel9"]) - pipeline._rebase_and_build_ci_images = AsyncMock() - pipeline._sync_ci_images = AsyncMock() + mock_jenkins.start_ocp4_konflux.return_value = "SUCCESS" + mock_jenkins.start_sync_ci_images.return_value = "SUCCESS" await pipeline._refresh_ci_images(*self.RECONCILE_ARGS) - pipeline._rebase_and_build_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) - pipeline._sync_ci_images.assert_awaited_once_with(["ci-openshift-golang-builder-latest.rhel9"]) + mock_jenkins.start_ocp4_konflux.assert_called_once_with( + build_version=pipeline.ocp_version, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + block_until_complete=True, + ) + mock_jenkins.start_sync_ci_images.assert_called_once_with( + version=pipeline.ocp_version, + block_until_complete=True, + assembly=pipeline.assembly, + image_list=["ci-openshift-golang-builder-latest.rhel9"], + dry_run=pipeline.dry_run, + load_disabled=True, + live_test_mode=True, + ) self.assertTrue( any("Synced CI image" in call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list) ) @@ -2872,51 +2855,6 @@ async def test_raises_when_doozer_command_fails(self, mock_cmd_gather, mock_konf await pipeline._scan_stale_ci_images(["ci-openshift-golang-builder-latest.rhel9"]) -class TestRebaseAndBuildCiImages(IsolatedAsyncioTestCase): - """Test the low-level rebase+build batch helper used by CI image reconciliation""" - - def _make_pipeline(self): - mock_runtime = Mock(dry_run=False, working_dir=Path("/tmp/working")) - mock_runtime.new_slack_client.return_value = Mock() - return UpdateGolangPipeline( - runtime=mock_runtime, - ocp_version="4.18", - cves=None, - force_update_tracker=False, - go_nvrs=["golang-1.22.9-1.el9"], - art_jira="ART-1234", - tag_builds=True, - ) - - @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_rebases_then_builds_the_batch_together(self, mock_konflux_db): - """ - All given image keys are rebased in one doozer call, then built in one doozer call -- - not per-image -- so a `from: member:` reference between them (e.g. - ci-openshift-build-root-* -> ci-openshift-golang-builder-*) resolves in-process. - """ - pipeline = self._make_pipeline() - pipeline._rebase_ci_images = AsyncMock() - pipeline._build_ci_images = AsyncMock() - - image_keys = ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] - await pipeline._rebase_and_build_ci_images(image_keys) - - self.assertEqual(pipeline._rebase_ci_images.await_args.args[0], image_keys) - pipeline._build_ci_images.assert_awaited_once_with(image_keys) - - @patch("pyartcd.pipelines.update_golang.KonfluxDb") - async def test_raises_when_build_fails(self, mock_konflux_db): - pipeline = self._make_pipeline() - pipeline._rebase_ci_images = AsyncMock() - pipeline._build_ci_images = AsyncMock(side_effect=ChildProcessError("build failed")) - - with self.assertRaises(ChildProcessError): - await pipeline._rebase_and_build_ci_images( - ["ci-openshift-golang-builder-latest.rhel9", "ci-openshift-build-root-latest.rhel9"] - ) - - class TestMonobranchDispatch(IsolatedAsyncioTestCase): """Test that doozer group and image methods work with monobranch""" From b430ba4384cf6ac257663573f597b8b9fc9aaa56 Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Tue, 8 Sep 2026 09:40:57 +0200 Subject: [PATCH 17/18] typo rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- pyartcd/pyartcd/jenkins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyartcd/pyartcd/jenkins.py b/pyartcd/pyartcd/jenkins.py index d3afa1f3ae..bf50f9dada 100644 --- a/pyartcd/pyartcd/jenkins.py +++ b/pyartcd/pyartcd/jenkins.py @@ -53,7 +53,7 @@ class Jobs(Enum): BUILD_CONFORMA_VERIFY = 'aos-cd-builds/build%2Fbuild-conforma-verify' SCAN_OPERATOR = 'aos-cd-builds/build%2Fscan-operator' # TODO TEST - SYNC_CI_IMAGES = 'hack/bvizi/add-ci-config-to-golang-builder' + SYNC_CI_IMAGES = 'hack/bvizi/add-load-disabled' OPEN_RECONCILIATION_PRS = 'aos-cd-builds/build%2Fopen-reconciliation-prs' OPEN_RECONCILIATION_PRS_LAYERED = 'aos-cd-builds/build%2Fopen-reconciliation-prs-layered-products' From 1076113130f4bfe84d6080d6a7ae932460ad5ee6 Mon Sep 17 00:00:00 2001 From: Bela Vizi Date: Tue, 8 Sep 2026 14:57:15 +0200 Subject: [PATCH 18/18] typo rh-pre-commit.version: 2.4.0 rh-pre-commit.check-secrets: ENABLED --- pyartcd/pyartcd/jenkins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyartcd/pyartcd/jenkins.py b/pyartcd/pyartcd/jenkins.py index bf50f9dada..fdc8e0e200 100644 --- a/pyartcd/pyartcd/jenkins.py +++ b/pyartcd/pyartcd/jenkins.py @@ -545,7 +545,7 @@ def start_sync_ci_images( 'VERSION': version, } if image_list: - params['IMAGE_LIST'] = ','.join(image_list) + params['IMAGES'] = ','.join(image_list) params['ART_TOOLS_COMMIT'] = 'kopero2000@ci-golang-builder-from-update-golang-ART-21958' params['DRY_RUN'] = dry_run params['LOAD_DISABLED'] = load_disabled