diff --git a/openqa-advanced-retrigger-jobs b/openqa-advanced-retrigger-jobs index d6424c3e..a031e8ad 100755 --- a/openqa-advanced-retrigger-jobs +++ b/openqa-advanced-retrigger-jobs @@ -1,81 +1,191 @@ -#!/bin/bash -e -host="${host:-"openqa.opensuse.org"}" -failed_since="${failed_since:-"$(date -I)"}" -instance_string="${INSTANCE+" and instance='$INSTANCE'"}" -worker_string="${WORKER+"assigned_worker_id in (select id from workers where (host='$WORKER'$instance_string)) and "}" -result="${result:-"result='incomplete'"}" -additional_filters="${additional_filters+" and $additional_filters"}" -comment="${comment:-""}" -max_jobs_per_request=25 -dry_run="${dry_run:-"0"}" - -usage() { - cat << EOF -Usage: $0 [OPTIONS] - -Retrigger openQA jobs based on database queries. - -By default retriggers openQA jobs with '$result' since '$failed_since' on -'$host'. - -Needs SSH access to the target openQA host '$host' to query the database -unless 'JOB_IDS' is provided. - -Options: - -h, --help display this help - -Environment variables: - host target openQA host (default: $host) - failed_since retrigger jobs finished since this date (default: $failed_since) - result retrigger jobs with this result (default: $result) - additional_filters additional SQL filters for the job query - comment comment to add to retriggered jobs - dry_run set to 1 to only print what would be done - WORKER restrict to jobs that ran on this worker host - INSTANCE restrict to jobs that ran on this worker instance - sql_command custom SQL command to fetch job IDs - JOB_IDS comma or space separated list of job IDs to retrigger - cli_protocol protocol for openqa-cli (e.g. https) - cli_port port for openqa-cli -EOF - exit "$1" -} - -restart-jobs() { - [[ ${#query_params[@]} -lt 1 ]] && return - [[ $comment ]] && query_params+=("comment=$comment") - $client_prefix openqa-cli api --host "$host" -X POST jobs/restart "${query_params[@]}" - query_params=() -} - -main() { - opts=$(getopt -o h -l help -n "$0" -- "$@") || usage 1 - eval set -- "$opts" - while true; do - case "$1" in - -h | --help) usage 0 ;; - --) - shift - break - ;; - *) break ;; - esac - done - - [ "$dry_run" = "1" ] && client_prefix="echo" - sql_command=${sql_command:-"select id from jobs where (${worker_string}${result} and clone_id is null and t_finished >= '$failed_since'$additional_filters);"} - # shellcheck disable=SC2029 - job_ids=${JOB_IDS:-$(ssh "$host" "sudo -u geekotest psql --no-align --tuples-only --command=\"$sql_command\" openqa")} - - query_params=() - [[ $cli_protocol ]] && host=$cli_protocol://$host - [[ $cli_port ]] && host+=:$cli_port - - for job_id in $job_ids; do - query_params+=("jobs=$job_id") - [[ ${#query_params[@]} -ge "$max_jobs_per_request" ]] && restart-jobs - done - restart-jobs -} - -caller 0 > /dev/null || main "$@" +#!/usr/bin/env python3 +# Copyright SUSE LLC +"""Retrigger openQA jobs selected by a database query over SSH.""" + +from __future__ import annotations + +import logging +import shlex +import subprocess +import sys +from datetime import UTC, datetime +from typing import Annotated + +import typer +from openqa_client.client import OpenQA_Client + +DEFAULT_HOST = "openqa.opensuse.org" +DEFAULT_RESULT = "result='incomplete'" +DEFAULT_MAX_JOBS_PER_REQUEST = 25 +PSQL_USER = "geekotest" +SQL_PREFIX = "select id from jobs where (" +SQL_WORKER_PREFIX = "assigned_worker_id in (select id from workers where (host='" +SQL_WORKER_SUFFIX = ")) and " +SQL_FILTERS = " and clone_id is null and t_finished >= " +SQL_SUFFIX = ");" + +app = typer.Typer(add_completion=False) +log = logging.getLogger(__name__) + + +def setup_logging(verbose: int) -> None: + """Configure logging on stderr according to the verbosity counter.""" + level_map = {0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG} + logging.basicConfig( + level=level_map.get(min(verbose, 2), logging.WARNING), + format="%(levelname)s: %(message)s", + stream=sys.stderr, + force=True, + ) + + +def build_sql_command( + worker: str | None, instance: str | None, result: str, failed_since: str, additional_filters: str | None +) -> str: + """Assemble the SQL query selecting the job IDs to retrigger.""" + worker_filter = "" + if worker: + instance_filter = f" and instance='{instance}'" if instance else "" + worker_filter = f"{SQL_WORKER_PREFIX}{worker}'{instance_filter}{SQL_WORKER_SUFFIX}" + extra_filters = f" and {additional_filters}" if additional_filters else "" + return f"{SQL_PREFIX}{worker_filter}{result}{SQL_FILTERS}'{failed_since}'{extra_filters}{SQL_SUFFIX}" + + +def parse_job_ids(raw: str) -> list[int]: + """Parse a comma or space separated list of job IDs.""" + return [int(token) for token in raw.replace(",", " ").split()] + + +def fetch_job_ids(host: str, sql_command: str) -> list[int]: + """Query the openQA database of the given host over SSH.""" + remote_command = shlex.join([ + "sudo", + "-u", + PSQL_USER, + "psql", + "--no-align", + "--tuples-only", + f"--command={sql_command}", + "openqa", + ]) + argv = ["ssh", host, remote_command] + completed = subprocess.run(argv, check=True, stdout=subprocess.PIPE, text=True) + return [int(token) for token in completed.stdout.split()] + + +def restart_job_ids( + client: OpenQA_Client, + job_ids: list[int], + *, + comment: str = "", + max_jobs_per_request: int = DEFAULT_MAX_JOBS_PER_REQUEST, + dry_run: bool = False, +) -> None: + """Retrigger the jobs via the openQA API, in batches of the given size.""" + for start in range(0, len(job_ids), max_jobs_per_request): + batch = job_ids[start : start + max_jobs_per_request] + params: dict[str, object] = {"jobs": batch} + if comment: + params["comment"] = comment + if dry_run: + typer.echo(f"Dry run: would POST jobs/restart with {params}") + else: + client.openqa_request("POST", "jobs/restart", params=params) + + +@app.command() +def main( # ruff: ignore[too-many-arguments] + *, + host: Annotated[ + str, + typer.Option("--host", "-H", envvar="host", help="openQA host queried over SSH."), + ] = DEFAULT_HOST, + failed_since: Annotated[ + str | None, + typer.Option("--failed-since", envvar="failed_since", help="Only retrigger jobs finished since this date."), + ] = None, + result: Annotated[ + str, + typer.Option("--result", envvar="result", help="SQL condition selecting the job result."), + ] = DEFAULT_RESULT, + additional_filters: Annotated[ + str | None, + typer.Option( + "--additional-filters", envvar="additional_filters", help="Additional SQL filters for the job query." + ), + ] = None, + comment: Annotated[ + str, + typer.Option("--comment", envvar="comment", help="Comment to add to the retriggered jobs."), + ] = "", + dry_run: Annotated[ + bool, + typer.Option("--dry-run", "-n", envvar="dry_run", help="Only print what would be done."), + ] = False, + worker: Annotated[ + str | None, + typer.Option("--worker", envvar="WORKER", help="Only retrigger jobs that ran on this worker host."), + ] = None, + instance: Annotated[ + str | None, + typer.Option("--instance", envvar="INSTANCE", help="Only retrigger jobs that ran on this worker instance."), + ] = None, + sql_command: Annotated[ + str | None, + typer.Option("--sql-command", envvar="sql_command", help="Custom SQL command fetching the job IDs."), + ] = None, + job_ids: Annotated[ + str | None, + typer.Option( + "--job-ids", envvar="JOB_IDS", help="Comma or space separated job IDs, skipping the database query." + ), + ] = None, + cli_protocol: Annotated[ + str | None, + typer.Option("--cli-protocol", envvar="cli_protocol", help="Protocol used for the openQA API."), + ] = None, + cli_port: Annotated[ + int | None, + typer.Option("--cli-port", envvar="cli_port", help="Port used for the openQA API."), + ] = None, + max_jobs_per_request: Annotated[ + int, + typer.Option( + "--max-jobs-per-request", envvar="max_jobs_per_request", help="Maximum number of jobs per restart request." + ), + ] = DEFAULT_MAX_JOBS_PER_REQUEST, + verbose: Annotated[ + int, + typer.Option("--verbose", "-v", count=True, help="Increase verbosity."), + ] = 0, +) -> None: + """Retrigger openQA jobs matching the query, in batches.""" + setup_logging(verbose) + if max_jobs_per_request < 1: + log.error("max-jobs-per-request must be at least 1") + raise typer.Exit(2) + + since = failed_since or datetime.now(tz=UTC).date().isoformat() + query = sql_command or build_sql_command(worker, instance, result, since, additional_filters) + try: + selected = parse_job_ids(job_ids) if job_ids else fetch_job_ids(host, query) + except (OSError, subprocess.CalledProcessError, ValueError) as err: + log.exception("Failed to determine the job IDs to retrigger") + raise typer.Exit(1) from err + + if not selected: + log.warning("No jobs to retrigger") + return + + server = f"{host}:{cli_port}" if cli_port else host + restart_job_ids( + OpenQA_Client(server=server, scheme=cli_protocol or ""), + selected, + comment=comment, + max_jobs_per_request=max_jobs_per_request, + dry_run=dry_run, + ) + + +if __name__ == "__main__": # pragma: no cover + app() diff --git a/tests/test_openqa_advanced_retrigger_jobs.py b/tests/test_openqa_advanced_retrigger_jobs.py new file mode 100644 index 00000000..d3f59081 --- /dev/null +++ b/tests/test_openqa_advanced_retrigger_jobs.py @@ -0,0 +1,219 @@ +# Copyright SUSE LLC +"""Unit tests for openqa-advanced-retrigger-jobs.""" + +from __future__ import annotations + +import importlib.machinery +import importlib.util +import pathlib +import shlex +import subprocess +import sys +from typing import TYPE_CHECKING +from unittest.mock import MagicMock + +import pytest +from typer.testing import CliRunner + +if TYPE_CHECKING: + from pytest_mock import MockerFixture + +MODULE_NAME = "openqa_advanced_retrigger_jobs" +rootpath = pathlib.Path(__file__).parent.parent.resolve() +script_path = rootpath / "openqa-advanced-retrigger-jobs" +spec = importlib.util.spec_from_file_location( + MODULE_NAME, + script_path, + loader=importlib.machinery.SourceFileLoader(MODULE_NAME, str(script_path)), +) +assert spec is not None +assert spec.loader is not None +retrigger = importlib.util.module_from_spec(spec) +sys.modules[MODULE_NAME] = retrigger +spec.loader.exec_module(retrigger) + +runner = CliRunner() + +SQL_PREFIX = "select id from jobs where (" +SQL_TAIL = "result='incomplete' and clone_id is null and t_finished >= '2026-10-09');" +DEFAULT_QUERY = SQL_PREFIX + SQL_TAIL + + +def test_build_sql_command_default() -> None: + query = retrigger.build_sql_command(None, None, "result='incomplete'", "2026-10-09", None) + assert query == DEFAULT_QUERY + + +def test_build_sql_command_worker_filters() -> None: + query = retrigger.build_sql_command("worker1", "42", "result='failed'", "2026-01-01", "build='1'") + assert query.startswith("select id from jobs where (assigned_worker_id in (select id from workers where ") + assert "host='worker1' and instance='42'" in query + assert query.endswith(")) and result='failed' and clone_id is null and t_finished >= '2026-01-01' and build='1');") + + +@pytest.mark.parametrize( + ("worker", "instance", "worker_clause"), + [ + (None, None, ""), + ("worker1", None, "assigned_worker_id in (select id from workers where (host='worker1')) and "), + ( + "worker1", + "7", + "assigned_worker_id in (select id from workers where (host='worker1' and instance='7')) and ", + ), + ], +) +def test_build_sql_command_worker_clause(worker: str | None, instance: str | None, worker_clause: str) -> None: + query = retrigger.build_sql_command(worker, instance, "result='incomplete'", "2026-10-09", None) + assert query == SQL_PREFIX + worker_clause + SQL_TAIL + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("1,2 3", [1, 2, 3]), + (" 5 ", [5]), + ("", []), + ], +) +def test_parse_job_ids(raw: str, expected: list[int]) -> None: + assert retrigger.parse_job_ids(raw) == expected + + +def test_parse_job_ids_invalid() -> None: + with pytest.raises(ValueError, match="boom"): + retrigger.parse_job_ids("1,boom") + + +def test_fetch_job_ids(mocker: MockerFixture) -> None: + completed = subprocess.CompletedProcess(args=[], returncode=0, stdout="11\n12\n", stderr="") + mocked = mocker.patch("openqa_advanced_retrigger_jobs.subprocess.run", return_value=completed) + assert retrigger.fetch_job_ids("openqa.opensuse.org", DEFAULT_QUERY) == [11, 12] + argv = mocked.call_args.args[0] + assert argv[:2] == ["ssh", "openqa.opensuse.org"] + assert shlex.split(argv[2]) == [ + "sudo", + "-u", + "geekotest", + "psql", + "--no-align", + "--tuples-only", + "--command=" + DEFAULT_QUERY, + "openqa", + ] + + +def test_fetch_job_ids_failure(mocker: MockerFixture) -> None: + mocker.patch( + "openqa_advanced_retrigger_jobs.subprocess.run", + side_effect=subprocess.CalledProcessError(2, "ssh"), + ) + with pytest.raises(subprocess.CalledProcessError): + retrigger.fetch_job_ids("openqa.opensuse.org", "select id from jobs where (1);") + + +def test_fetch_job_ids_leaves_stderr_on_terminal(mocker: MockerFixture) -> None: + completed = subprocess.CompletedProcess(args=[], returncode=0, stdout="1\n", stderr=None) + mocked = mocker.patch("openqa_advanced_retrigger_jobs.subprocess.run", return_value=completed) + retrigger.fetch_job_ids("openqa.opensuse.org", DEFAULT_QUERY) + assert mocked.call_args.kwargs == {"check": True, "stdout": subprocess.PIPE, "text": True} + + +def test_restart_job_ids_batching(mocker: MockerFixture) -> None: + client = mocker.MagicMock() + retrigger.restart_job_ids(client, [1, 2, 3, 4], comment="why", max_jobs_per_request=3) + assert client.openqa_request.call_count == 2 + assert client.method_calls[-1].kwargs["params"] == {"jobs": [4], "comment": "why"} + + +def test_restart_job_ids_without_comment(mocker: MockerFixture) -> None: + client = mocker.MagicMock() + retrigger.restart_job_ids(client, [1]) + assert client.method_calls[-1].kwargs["params"] == {"jobs": [1]} + + +def test_restart_job_ids_empty(mocker: MockerFixture) -> None: + client = mocker.MagicMock() + retrigger.restart_job_ids(client, []) + assert client.openqa_request.call_count == 0 + + +def test_restart_job_ids_dry_run(capsys: pytest.CaptureFixture[str]) -> None: + client = MagicMock() + retrigger.restart_job_ids(client, [1, 2], comment="why", max_jobs_per_request=1, dry_run=True) + assert client.openqa_request.call_count == 0 + output = capsys.readouterr().out + assert "would POST jobs/restart with {'jobs': [1], 'comment': 'why'}" in output + assert "would POST jobs/restart with {'jobs': [2], 'comment': 'why'}" in output + + +def test_main_dry_run(mocker: MockerFixture) -> None: + client = mocker.MagicMock() + mocker.patch("openqa_advanced_retrigger_jobs.OpenQA_Client", return_value=client) + result = runner.invoke(retrigger.app, ["-v", "--dry-run", "--job-ids", "7,8", "--max-jobs-per-request", "1"]) + assert result.exit_code == 0 + assert "would POST jobs/restart with {'jobs': [7]}" in result.output + assert "would POST jobs/restart with {'jobs': [8]}" in result.output + assert client.openqa_request.call_count == 0 + + +def test_main_retriggers_via_api(mocker: MockerFixture) -> None: + client = mocker.MagicMock() + mocker.patch("openqa_advanced_retrigger_jobs.OpenQA_Client", return_value=client) + mocker.patch("openqa_advanced_retrigger_jobs.fetch_job_ids", return_value=[9, 10]) + result = runner.invoke(retrigger.app, ["--worker", "worker1", "--instance", "3", "--comment", "again"]) + assert result.exit_code == 0 + assert client.openqa_request.call_count == 1 + assert client.method_calls[-1].kwargs["params"] == {"jobs": [9, 10], "comment": "again"} + + +def test_main_query_failure(mocker: MockerFixture) -> None: + mocker.patch("openqa_advanced_retrigger_jobs.fetch_job_ids", side_effect=subprocess.CalledProcessError(255, "ssh")) + result = runner.invoke(retrigger.app, []) + assert result.exit_code == 1 + assert "Failed to determine the job IDs to retrigger" in result.output + + +def test_main_invalid_job_ids() -> None: + result = runner.invoke(retrigger.app, ["--job-ids", "1,boom"]) + assert result.exit_code == 1 + assert "Failed to determine the job IDs to retrigger" in result.output + + +def test_main_no_jobs(mocker: MockerFixture) -> None: + mocker.patch("openqa_advanced_retrigger_jobs.fetch_job_ids", return_value=[]) + result = runner.invoke(retrigger.app, []) + assert result.exit_code == 0 + assert "No jobs to retrigger" in result.output + + +def test_main_invalid_batch_size() -> None: + result = runner.invoke(retrigger.app, ["--job-ids", "1", "--max-jobs-per-request", "0"]) + assert result.exit_code == 2 + assert "max-jobs-per-request must be at least 1" in result.output + + +def test_main_environment_defaults(mocker: MockerFixture) -> None: + client = mocker.MagicMock() + mocker.patch("openqa_advanced_retrigger_jobs.OpenQA_Client", return_value=client) + result = runner.invoke(retrigger.app, ["-v"], env={"JOB_IDS": "7,8", "dry_run": "1", "comment": "env"}) + assert result.exit_code == 0 + assert client.openqa_request.call_count == 0 + assert "would POST jobs/restart with {'jobs': [7, 8], 'comment': 'env'}" in result.output + + +def test_main_server_url_with_protocol_and_port(mocker: MockerFixture) -> None: + mocked_client = mocker.patch("openqa_advanced_retrigger_jobs.OpenQA_Client") + mocker.patch("openqa_advanced_retrigger_jobs.fetch_job_ids", return_value=[1]) + result = runner.invoke(retrigger.app, ["--cli-protocol", "http", "--cli-port", "8080"]) + assert result.exit_code == 0 + assert mocked_client.call_args.kwargs == {"server": "openqa.opensuse.org:8080", "scheme": "http"} + + +def test_main_dry_run_output_visible_without_verbosity(mocker: MockerFixture) -> None: + client = mocker.MagicMock() + mocker.patch("openqa_advanced_retrigger_jobs.OpenQA_Client", return_value=client) + result = runner.invoke(retrigger.app, ["--dry-run", "--job-ids", "7"]) + assert result.exit_code == 0 + assert "would POST jobs/restart with {'jobs': [7]}" in result.output + assert client.openqa_request.call_count == 0