diff --git a/pyartcd/pyartcd/pipelines/update_golang.py b/pyartcd/pyartcd/pipelines/update_golang.py index a6d1bd45ec..26c635223c 100644 --- a/pyartcd/pyartcd/pipelines/update_golang.py +++ b/pyartcd/pyartcd/pipelines/update_golang.py @@ -207,7 +207,6 @@ def __init__( self.force_image_build = force_image_build self.go_nvrs = go_nvrs self.art_jira = art_jira - self.build_system = build_system self.koji_session = koji.ClientSession(BREW_HUB) # Always needed for RPM builds self.tag_builds = tag_builds self.data_path = data_path @@ -218,9 +217,12 @@ def __init__( self.major_bump = major_bump if assembly not in GOLANG_ASSEMBLIES: raise ValueError(f"Unsupported golang assembly {assembly!r}; expected one of {GOLANG_ASSEMBLIES}") - if assembly == "test" and build_system in ("brew", "both"): + if assembly == "test" and build_system == "brew": _LOGGER.error(BREW_TEST_ASSEMBLY_UNSUPPORTED) raise ValueError(BREW_TEST_ASSEMBLY_UNSUPPORTED) + # Brew floating tags are incompatible with test assemblies. For a dual-system + # run, silently limit builder operations to Konflux. + self.build_system = "konflux" if assembly == "test" and build_system == "both" else build_system self.assembly = assembly self._slack_client = self.runtime.new_slack_client() self._doozer_working_dir = self.runtime.working_dir / "doozer-working" @@ -235,7 +237,7 @@ def __init__( # GitHub auth is handled by get_github_client_for_org() with App auth / PAT fallback # Initialize KonfluxDb for Konflux build system - if build_system in ('konflux', 'both'): + if self.build_system in ('konflux', 'both'): self.konflux_db = KonfluxDb() self.konflux_db.bind(KonfluxBuildRecord) @@ -309,12 +311,13 @@ def _get_allowed_go_major_minors(self) -> tuple[str, dict[str, str]]: } return branch, allowed_major_minors - def validate_go_version_matches_group_vars(self, go_version: str): + def validate_golang_assemblies_enabled(self): repo, branch = self._get_ocp_build_data_repo_and_branch(self.GOLANG_DATA_BRANCH) golang_group = self._load_yaml_from_repo(repo, "group.yml", branch) if not golang_group.get("assemblies", {}).get("enabled", False): raise ValueError(f"Assemblies are not enabled in ocp-build-data branch {branch}") + def validate_go_version_matches_group_vars(self, go_version: str): branch, allowed_major_minors = self._get_allowed_go_major_minors() build_major_minor = extract_major_minor(go_version, "golang build version") if build_major_minor not in allowed_major_minors.values(): @@ -362,15 +365,39 @@ async def run(self): go_version, el_nvr_map = extract_and_validate_golang_nvrs(self.ocp_version, self.go_nvrs) _LOGGER.info(f'Golang version detected: {go_version}') _LOGGER.info(f'NVRs by rhel version: {el_nvr_map}') - branch, allowed_major_minors, build_major_minor = self.validate_go_version_matches_group_vars(go_version) - if self.tag_builds: - self.validate_tag_builds_go_latest(branch, allowed_major_minors, build_major_minor) - process_rpm_builds = self.major_bump or build_major_minor == allowed_major_minors["GO_LATEST"] + self.validate_golang_assemblies_enabled() # el10 is only supported for build roots (RPM tagging), not for golang-builder images yet el_nvr_map_for_images = {el_v: nvr for el_v, nvr in el_nvr_map.items() if el_v != 10} if el_nvr_map.keys() - el_nvr_map_for_images.keys(): _LOGGER.info("RHEL 10 NVRs will only be used for build root tagging, not for golang-builder images") + + # Check for existing builders before running validations and setup required only for new image builds. + brew_nvrs = {} + konflux_records: dict[int, KonfluxBuildRecord] = {} + if not self.force_image_build and el_nvr_map_for_images: + if self.build_system in ['both', 'brew']: + brew_nvrs = self.get_existing_builders_brew(el_nvr_map_for_images, go_version) + if self.build_system in ['both', 'konflux']: + konflux_records = await self.get_existing_builders_konflux(el_nvr_map_for_images, go_version) + + brew_missing = ( + el_nvr_map_for_images.keys() - brew_nvrs.keys() if self.build_system in ['both', 'brew'] else set() + ) + konflux_missing = ( + el_nvr_map_for_images.keys() - konflux_records.keys() if self.build_system in ['both', 'konflux'] else set() + ) + missing_builder_els = brew_missing | konflux_missing + needs_rhel10_rpm_prep = 10 in el_nvr_map and not self.external_golang_rpms + rpm_els_to_prepare = missing_builder_els | ({10} if needs_rhel10_rpm_prep else set()) + + process_rpm_builds = False + if rpm_els_to_prepare: + branch, allowed_major_minors, build_major_minor = self.validate_go_version_matches_group_vars(go_version) + if self.tag_builds: + self.validate_tag_builds_go_latest(branch, allowed_major_minors, build_major_minor) + process_rpm_builds = self.major_bump or build_major_minor == allowed_major_minors["GO_LATEST"] + self._slack_client.bind_channel(self.ocp_version) running_in_jenkins = os.environ.get('BUILD_ID', False) if running_in_jenkins: @@ -388,59 +415,47 @@ async def run(self): f"(building {self.assembly} images on {self.build_system}{external_repos_msg}) :construction:" ) - if self.external_golang_rpms: - _LOGGER.warning( - "Using golang RPMs from external repos. Skipping tagging and availability checks. " - "Ensure external repos are enabled in golang-builder image metadata config." - ) - await self._slack_client.say_in_thread( - ":warning: Using golang RPMs from external repos. Skipping tagging and availability checks." - ) - elif process_rpm_builds: - # Process golang RPM builds (always from Brew) - cannot_proceed = not all( - await asyncio.gather(*[self.process_build(el_v, nvr) for el_v, nvr in el_nvr_map.items()]) - ) - if cannot_proceed: - raise ValueError( - 'Cannot proceed until all builds are tagged and available, did you forget check TAG_BUILD?' + if rpm_els_to_prepare: + if self.external_golang_rpms: + _LOGGER.warning( + "Using golang RPMs from external repos. Skipping tagging and availability checks. " + "Ensure external repos are enabled in golang-builder image metadata config." ) + await self._slack_client.say_in_thread( + ":warning: Using golang RPMs from external repos. Skipping tagging and availability checks." + ) + elif process_rpm_builds: + # Process golang RPM builds (always from Brew) + cannot_proceed = not all( + await asyncio.gather( + *[self.process_build(el_v, el_nvr_map[el_v]) for el_v in sorted(rpm_els_to_prepare)] + ) + ) + if cannot_proceed: + raise ValueError( + 'Cannot proceed until all builds are tagged and available, did you forget check TAG_BUILD?' + ) - _LOGGER.info('All golang RPM builds are tagged and available!') - await self._slack_client.say_in_thread("All golang RPM builds are tagged and available!") - - # Build plashets for golang RPMs before building images - await self._build_golang_plashets(go_version, el_nvr_map_for_images.keys()) - else: - matching_vars = sorted( - var_name for var_name, major_minor in allowed_major_minors.items() if major_minor == build_major_minor - ) - _LOGGER.info( - "Skipping RPM tagging, buildroot availability checks, and plashet builds for non-GO_LATEST " - "golang %s (%s). Existing builder images will be reused.", - build_major_minor, - ", ".join(matching_vars), - ) - - # Check if openshift-golang-builder image builds exist for the provided compiler builds - # Only for RHEL versions that support golang-builder images (excludes el10 for now) - brew_nvrs = {} - konflux_records: dict[int, KonfluxBuildRecord] = {} - if not self.force_image_build: - if self.build_system in ['both', 'brew']: - brew_nvrs = self.get_existing_builders_brew(el_nvr_map_for_images, go_version) - if self.build_system in ['both', 'konflux']: - konflux_records = await self.get_existing_builders_konflux(el_nvr_map_for_images, go_version) + _LOGGER.info('All golang RPM builds are tagged and available!') + await self._slack_client.say_in_thread("All golang RPM builds are tagged and available!") - # Determine which rhel versions need builds - brew_missing = ( - el_nvr_map_for_images.keys() - brew_nvrs.keys() if self.build_system in ['both', 'brew'] else set() - ) - konflux_missing = ( - el_nvr_map_for_images.keys() - konflux_records.keys() if self.build_system in ['both', 'konflux'] else set() - ) + # Build plashets only for RHEL versions whose builder images are missing. + if missing_builder_els: + await self._build_golang_plashets(go_version, sorted(missing_builder_els)) + else: + matching_vars = sorted( + var_name + for var_name, major_minor in allowed_major_minors.items() + if major_minor == build_major_minor + ) + _LOGGER.info( + "Skipping RPM tagging, buildroot availability checks, and plashet builds for non-GO_LATEST " + "golang %s (%s). Existing builder images will be reused.", + build_major_minor, + ", ".join(matching_vars), + ) - if brew_missing or konflux_missing: + if missing_builder_els: if not process_rpm_builds and not self.external_golang_rpms: missing = [] if brew_missing: @@ -1360,7 +1375,7 @@ async def update_golang( raise ValueError('CVEs must be provided with --force-update-tracker') if network_mode and build_system == 'brew': raise click.BadParameter('--network-mode only applies when --build-system is "konflux" or "both".') - if assembly == "test" and build_system in ("brew", "both"): + if assembly == "test" and build_system == "brew": _LOGGER.error(BREW_TEST_ASSEMBLY_UNSUPPORTED) raise click.BadParameter(BREW_TEST_ASSEMBLY_UNSUPPORTED, param_hint='--build-system') pipeline = UpdateGolangPipeline( diff --git a/pyartcd/tests/pipelines/test_update_golang.py b/pyartcd/tests/pipelines/test_update_golang.py index 8a5d70e4a3..48f7ae26c4 100644 --- a/pyartcd/tests/pipelines/test_update_golang.py +++ b/pyartcd/tests/pipelines/test_update_golang.py @@ -447,7 +447,7 @@ def test_existing_test_build_lookup_requires_explicit_test_assembly(self, mock_k go_nvrs=["golang-1.25.8-1.el9"], art_jira="ART-1234", tag_builds=False, - build_system="konflux", + build_system="both", assembly="test", ) @@ -470,23 +470,39 @@ def test_rejects_non_stream_type_assembly(self, mock_konflux_db): assembly="art-1234", ) - def test_test_assembly_rejects_brew_build_systems(self): - for build_system in ("brew", "both"): - with self.subTest(build_system=build_system): - with self.assertLogs("pyartcd.pipelines.update_golang", level="ERROR") as logs: - with self.assertRaisesRegex(ValueError, "Brew floating tags are updated"): - UpdateGolangPipeline( - runtime=self._make_test_runtime(), - ocp_version="5.0", - cves=None, - force_update_tracker=False, - go_nvrs=["golang-1.26.5-1.el8"], - art_jira="ART-1234", - tag_builds=False, - build_system=build_system, - assembly="test", - ) - self.assertTrue(any("successful build" in message for message in logs.output), logs.output) + def test_test_assembly_rejects_brew_build_system(self): + with self.assertLogs("pyartcd.pipelines.update_golang", level="ERROR") as logs: + with self.assertRaisesRegex(ValueError, "Brew floating tags are updated"): + UpdateGolangPipeline( + runtime=self._make_test_runtime(), + ocp_version="5.0", + cves=None, + force_update_tracker=False, + go_nvrs=["golang-1.26.5-1.el8"], + art_jira="ART-1234", + tag_builds=False, + build_system="brew", + assembly="test", + ) + self.assertTrue(any("successful build" in message for message in logs.output), logs.output) + + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + def test_test_assembly_both_silently_uses_konflux(self, mock_konflux_db): + with self.assertNoLogs("pyartcd.pipelines.update_golang", level="WARNING"): + pipeline = UpdateGolangPipeline( + runtime=self._make_test_runtime(), + ocp_version="5.0", + cves=None, + force_update_tracker=False, + go_nvrs=["golang-1.26.5-1.el8"], + art_jira="ART-1234", + tag_builds=False, + build_system="both", + assembly="test", + ) + + self.assertEqual(pipeline.build_system, "konflux") + mock_konflux_db.assert_called_once() @patch("pyartcd.pipelines.update_golang.KonfluxDb") def test_get_doozer_var_args(self, mock_konflux_db): @@ -505,7 +521,7 @@ def test_monobranch_validation_requires_assemblies_enabled(self, mock_konflux_db pipeline._get_ocp_build_data_repo_and_branch = Mock(return_value=(repo, "golang")) with self.assertRaisesRegex(ValueError, "Assemblies are not enabled.*golang"): - pipeline.validate_go_version_matches_group_vars("1.25.8") + pipeline.validate_golang_assemblies_enabled() @patch("pyartcd.pipelines.update_golang.KonfluxDb") def test_monobranch_validation_accepts_assemblies_enabled(self, mock_konflux_db): @@ -513,11 +529,7 @@ def test_monobranch_validation_accepts_assemblies_enabled(self, mock_konflux_db) repo = Mock() repo.get_contents.return_value = Mock(decoded_content=b"assemblies:\n enabled: true\n") pipeline._get_ocp_build_data_repo_and_branch = Mock(return_value=(repo, "golang")) - pipeline._get_allowed_go_major_minors = Mock(return_value=("openshift-4.16", {"GO_LATEST": "1.25"})) - - result = pipeline.validate_go_version_matches_group_vars("1.25.8") - - self.assertEqual(result, ("openshift-4.16", {"GO_LATEST": "1.25"}, "1.25")) + pipeline.validate_golang_assemblies_enabled() @patch("pyartcd.pipelines.update_golang.get_github_client_for_org") @patch("pyartcd.pipelines.update_golang.KonfluxDb") @@ -545,6 +557,7 @@ def test_validate_tag_builds_go_latest_accepts_matching_major_minor(self, mock_k tag_builds=True, ) + pipeline.validate_golang_assemblies_enabled() branch, allowed_major_minors, build_major_minor = pipeline.validate_go_version_matches_group_vars("1.22.9") pipeline.validate_tag_builds_go_latest(branch, allowed_major_minors, build_major_minor) @@ -609,6 +622,7 @@ def test_validate_go_version_matches_group_vars_accepts_unquoted_trailing_zero( tag_builds=False, ) + pipeline.validate_golang_assemblies_enabled() pipeline.validate_go_version_matches_group_vars("1.20.12") requested_paths = [call.args[0] for call in upstream_repo.get_contents.call_args_list] @@ -650,6 +664,7 @@ def get_contents(path, ref): tag_builds=False, ) + pipeline.validate_golang_assemblies_enabled() pipeline.validate_go_version_matches_group_vars("1.22.9") await pipeline.update_golang_streams("1.22.9", {}) @@ -1099,6 +1114,7 @@ async def test_ensure_builder_pullspec_available_errors_when_oc_helper_fails(sel async def test_run_brew_only_skips_updating_streams(self, mock_konflux_db, move_golang_bugs, mock_kinit): """Test brew-only runs skip streams.yml updates because streams use Konflux pullspecs""" pipeline = self._make_pipeline(build_system="brew") + pipeline.validate_golang_assemblies_enabled = Mock() pipeline.validate_go_version_matches_group_vars = Mock( return_value=("openshift-4.16", {"GO_LATEST": "1.25"}, "1.25") ) @@ -1113,6 +1129,9 @@ async def test_run_brew_only_skips_updating_streams(self, mock_konflux_db, move_ await pipeline.run() mock_kinit.assert_awaited_once() + pipeline.validate_go_version_matches_group_vars.assert_not_called() + pipeline.process_build.assert_not_awaited() + pipeline._build_golang_plashets.assert_not_awaited() pipeline.update_golang_streams.assert_not_awaited() move_golang_bugs.assert_awaited_once() slack_messages = [call.args[0] for call in pipeline._slack_client.say_in_thread.await_args_list] @@ -1121,6 +1140,88 @@ async def test_run_brew_only_skips_updating_streams(self, mock_konflux_db, move_ slack_messages, ) + @patch("pyartcd.pipelines.update_golang.kinit", new_callable=AsyncMock) + @patch("pyartcd.pipelines.update_golang.move_golang_bugs", new_callable=AsyncMock) + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_run_el10_only_prepares_rpm_without_builder_lookup( + self, mock_konflux_db, move_golang_bugs, mock_kinit + ): + pipeline = UpdateGolangPipeline( + runtime=self._make_test_runtime(), + ocp_version="5.0", + cves=None, + force_update_tracker=False, + go_nvrs=["golang-1.26.5-1.el10"], + art_jira="ART-1234", + tag_builds=False, + build_system="brew", + ) + pipeline.validate_golang_assemblies_enabled = Mock() + pipeline.validate_go_version_matches_group_vars = Mock( + return_value=("openshift-5.0", {"GO_LATEST": "1.26"}, "1.26") + ) + pipeline.process_build = AsyncMock(return_value=True) + pipeline._build_golang_plashets = AsyncMock() + pipeline.get_existing_builders_brew = Mock() + pipeline.update_golang_streams = AsyncMock() + + await pipeline.run() + + pipeline.get_existing_builders_brew.assert_not_called() + pipeline.validate_go_version_matches_group_vars.assert_called_once_with("1.26.5") + pipeline.process_build.assert_awaited_once_with(10, "golang-1.26.5-1.el10") + pipeline._build_golang_plashets.assert_not_awaited() + pipeline.update_golang_streams.assert_not_awaited() + move_golang_bugs.assert_awaited_once() + + @patch("pyartcd.pipelines.update_golang.kinit", new_callable=AsyncMock) + @patch("pyartcd.pipelines.update_golang.move_golang_bugs", new_callable=AsyncMock) + @patch("pyartcd.pipelines.update_golang.KonfluxDb") + async def test_run_existing_image_builders_only_prepares_el10_rpm( + self, mock_konflux_db, move_golang_bugs, mock_kinit + ): + pipeline = UpdateGolangPipeline( + runtime=self._make_test_runtime(), + ocp_version="5.0", + cves=None, + force_update_tracker=False, + go_nvrs=[ + "golang-1.26.5-1.el8", + "golang-1.26.5-1.el9", + "golang-1.26.5-1.el10", + ], + art_jira="ART-1234", + tag_builds=False, + build_system="brew", + ) + pipeline.validate_golang_assemblies_enabled = Mock() + pipeline.validate_go_version_matches_group_vars = Mock( + return_value=("openshift-5.0", {"GO_LATEST": "1.26"}, "1.26") + ) + pipeline.process_build = AsyncMock(return_value=True) + pipeline._build_golang_plashets = AsyncMock() + pipeline.get_existing_builders_brew = Mock( + return_value={ + 8: "openshift-golang-builder-container-v1.26.5-existing.el8", + 9: "openshift-golang-builder-container-v1.26.5-existing.el9", + } + ) + pipeline.update_golang_streams = AsyncMock() + + await pipeline.run() + + pipeline.get_existing_builders_brew.assert_called_once_with( + { + 8: "golang-1.26.5-1.el8", + 9: "golang-1.26.5-1.el9", + }, + "1.26.5", + ) + pipeline.process_build.assert_awaited_once_with(10, "golang-1.26.5-1.el10") + pipeline._build_golang_plashets.assert_not_awaited() + pipeline.update_golang_streams.assert_not_awaited() + move_golang_bugs.assert_awaited_once() + @patch("pyartcd.pipelines.update_golang.kinit", new_callable=AsyncMock) @patch("pyartcd.pipelines.update_golang.move_golang_bugs", new_callable=AsyncMock) @patch("pyartcd.pipelines.update_golang.KonfluxDb") @@ -1133,14 +1234,19 @@ async def test_run_test_assembly_skips_production_operations(self, mock_konflux_ go_nvrs=["golang-1.26.5-1.el8"], art_jira="ART-1234", tag_builds=False, - build_system="konflux", + build_system="both", assembly="test", ) + call_order = [] + pipeline.validate_golang_assemblies_enabled = Mock(side_effect=lambda: call_order.append("assemblies")) pipeline.validate_go_version_matches_group_vars = Mock(return_value=("golang", {"GO_LATEST": "1.26"}, "1.26")) pipeline.process_build = AsyncMock(return_value=True) pipeline._build_golang_plashets = AsyncMock() builder_record = Mock(nvr="openshift-golang-builder-container-v1.26.5-202608071200.p0.assembly.test.el8") - pipeline.get_existing_builders_konflux = AsyncMock(return_value={8: builder_record}) + pipeline.get_existing_builders_brew = Mock() + pipeline.get_existing_builders_konflux = AsyncMock( + side_effect=lambda *_: call_order.append("existing_builders") or {8: builder_record} + ) pipeline._get_builder_pullspec = Mock() pipeline._ensure_builder_pullspec_available = AsyncMock() pipeline.update_golang_streams = AsyncMock() @@ -1148,9 +1254,12 @@ async def test_run_test_assembly_skips_production_operations(self, mock_konflux_ await pipeline.run() mock_kinit.assert_awaited_once() - pipeline._build_golang_plashets.assert_awaited_once() - self.assertEqual(pipeline._build_golang_plashets.await_args.args[0], "1.26.5") - self.assertEqual(list(pipeline._build_golang_plashets.await_args.args[1]), [8]) + self.assertEqual(pipeline.build_system, "konflux") + self.assertEqual(call_order, ["assemblies", "existing_builders"]) + pipeline.validate_go_version_matches_group_vars.assert_not_called() + pipeline.process_build.assert_not_awaited() + pipeline._build_golang_plashets.assert_not_awaited() + pipeline.get_existing_builders_brew.assert_not_called() pipeline._get_builder_pullspec.assert_not_called() pipeline._ensure_builder_pullspec_available.assert_not_awaited() pipeline.update_golang_streams.assert_not_awaited() @@ -1175,6 +1284,7 @@ async def test_run_go_extra_reuses_builder_without_processing_rpm( tag_builds=False, build_system="konflux", ) + pipeline.validate_golang_assemblies_enabled = Mock() pipeline.validate_go_version_matches_group_vars = Mock( return_value=("openshift-5.0", {"GO_LATEST": "1.26", "GO_EXTRA": "1.25"}, "1.25") ) @@ -1190,6 +1300,7 @@ async def test_run_go_extra_reuses_builder_without_processing_rpm( pipeline.process_build.assert_not_awaited() pipeline._build_golang_plashets.assert_not_awaited() + pipeline.validate_go_version_matches_group_vars.assert_not_called() pipeline.get_existing_builders_konflux.assert_awaited_once_with( {8: "golang-1.25.11-1.el8_10"}, "1.25.11", @@ -1217,6 +1328,7 @@ async def test_run_go_extra_fails_before_building_when_builder_is_missing( tag_builds=False, build_system="konflux", ) + pipeline.validate_golang_assemblies_enabled = Mock() pipeline.validate_go_version_matches_group_vars = Mock( return_value=("openshift-5.0", {"GO_LATEST": "1.26", "GO_EXTRA": "1.25"}, "1.25") ) @@ -1253,6 +1365,7 @@ async def test_run_go_extra_external_rpms_builds_missing_builder( build_system="konflux", external_golang_rpms=True, ) + pipeline.validate_golang_assemblies_enabled = Mock() pipeline.validate_go_version_matches_group_vars = Mock( return_value=("openshift-5.0", {"GO_LATEST": "1.26", "GO_EXTRA": "1.25"}, "1.25") )