diff --git a/.github/workflows/classic-python-checks.yml b/.github/workflows/classic-python-checks.yml index 399764286b76..674ba1ce5409 100644 --- a/.github/workflows/classic-python-checks.yml +++ b/.github/workflows/classic-python-checks.yml @@ -4,25 +4,19 @@ on: push: branches: [ master, dev, ci-test* ] paths: - - '.github/workflows/classic-python-checks-ci.yml' - - 'classic/original_autogpt/**' - - 'classic/forge/**' - - 'classic/direct_benchmark/**' + - '.github/workflows/classic-python-checks.yml' + - 'classic/**.py' - 'classic/pyproject.toml' - 'classic/poetry.lock' - - '**.py' - - '!classic/forge/tests/vcr_cassettes' + - '!classic/forge/tests/vcr_cassettes/**' pull_request: branches: [ master, dev, release-* ] paths: - - '.github/workflows/classic-python-checks-ci.yml' - - 'classic/original_autogpt/**' - - 'classic/forge/**' - - 'classic/direct_benchmark/**' + - '.github/workflows/classic-python-checks.yml' + - 'classic/**.py' - 'classic/pyproject.toml' - 'classic/poetry.lock' - - '**.py' - - '!classic/forge/tests/vcr_cassettes' + - '!classic/forge/tests/vcr_cassettes/**' concurrency: group: ${{ format('classic-python-checks-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }} diff --git a/.github/workflows/platform-backend-ci.yml b/.github/workflows/platform-backend-ci.yml index 5e47afb8dc32..2e188e5a1453 100644 --- a/.github/workflows/platform-backend-ci.yml +++ b/.github/workflows/platform-backend-ci.yml @@ -37,16 +37,16 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - - name: Set up Python 3.12 + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - name: Set up Python dependency cache uses: actions/cache@v5 with: path: ~/.cache/pypoetry - key: poetry-${{ runner.os }}-py3.12-${{ hashFiles('autogpt_platform/backend/poetry.lock') }} + key: poetry-${{ runner.os }}-py3.13-${{ hashFiles('autogpt_platform/backend/poetry.lock') }} - name: Install Poetry run: | @@ -68,26 +68,22 @@ jobs: permissions: contents: read timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - python-version: ["3.11", "3.12", "3.13"] runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v6 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.13" - name: Set up Python dependency cache uses: actions/cache@v5 with: path: ~/.cache/pypoetry - key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }} + key: poetry-${{ runner.os }}-py3.13-${{ hashFiles('autogpt_platform/backend/poetry.lock') }} - name: Install Poetry run: | @@ -102,7 +98,7 @@ jobs: run: poetry run prisma generate && poetry run gen-prisma-stub - name: Run Pyright - run: poetry run pyright --pythonversion ${{ matrix.python-version }} + run: poetry run pyright --pythonversion 3.13 env: CI: true @@ -112,10 +108,6 @@ jobs: permissions: contents: read timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - python-version: ["3.11", "3.12", "3.13"] runs-on: ubuntu-latest services: @@ -163,10 +155,10 @@ jobs: fetch-depth: 0 submodules: true - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.13" - name: Start Redis Cluster (3 shards) run: | @@ -243,7 +235,7 @@ jobs: uses: actions/cache@v5 with: path: ~/.cache/pypoetry - key: poetry-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }} + key: poetry-${{ runner.os }}-py3.13-${{ hashFiles('autogpt_platform/backend/poetry.lock') }} - name: Install Poetry run: | diff --git a/.github/workflows/platform-fullstack-ci.yml b/.github/workflows/platform-fullstack-ci.yml index 605c13c38b0c..73b4326aac7e 100644 --- a/.github/workflows/platform-fullstack-ci.yml +++ b/.github/workflows/platform-fullstack-ci.yml @@ -142,8 +142,39 @@ jobs: run: pnpm types if: success() || (steps.generate-api-client.outcome == 'success') + preflight-e2e: + name: Wait for upstream CI + # Don't burn a big-boi runner on E2E if backend/frontend CI is already + # broken on this SHA. Polls upstream workflow runs and gates the + # downstream e2e_test job. A workflow that didn't trigger for this SHA + # (path-filtered out) is treated as a non-blocker. + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + actions: read + outputs: + proceed: ${{ steps.gate.outputs.proceed }} + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - run: pip install requests + working-directory: . + - id: gate + working-directory: . + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }} + UPSTREAM_WORKFLOWS: | + AutoGPT Platform - Backend CI + AutoGPT Platform - Frontend CI + run: python .github/workflows/scripts/wait_for_upstream.py + e2e_test: name: end-to-end tests + needs: preflight-e2e + if: needs.preflight-e2e.outputs.proceed == 'true' runs-on: big-boi steps: diff --git a/.github/workflows/repo-pr-label.yml b/.github/workflows/repo-pr-label.yml index 97579c278464..49644a4110c9 100644 --- a/.github/workflows/repo-pr-label.yml +++ b/.github/workflows/repo-pr-label.yml @@ -1,11 +1,13 @@ name: Repo - Pull Request auto-label on: - # So that PRs touching the same files as the push are updated + # So that PRs touching the same files as the push are updated. + # Only `dev` — that's the active integration branch most PRs target. + # Push to master/release-* would re-fan-out across all open PRs for marginal value. push: - branches: [ master, dev, release-* ] + branches: [ dev ] paths-ignore: - - 'classic/forge/tests/vcr_cassettes' + - 'classic/forge/tests/vcr_cassettes/**' - 'classic/benchmark/reports/**' # So that the `dirtyLabel` is removed if conflicts are resolve # We recommend `pull_request_target` so that github secrets are available. diff --git a/.github/workflows/scripts/wait_for_upstream.py b/.github/workflows/scripts/wait_for_upstream.py new file mode 100644 index 000000000000..bb02c893dea2 --- /dev/null +++ b/.github/workflows/scripts/wait_for_upstream.py @@ -0,0 +1,185 @@ +"""Wait for upstream workflow runs (by display name) on the given head SHA. + +Used by platform-fullstack-ci's preflight job to gate the expensive +big-boi E2E job: only run E2E if every listed upstream workflow has +completed successfully on the same SHA. Outputs ``proceed=true|false`` +to ``$GITHUB_OUTPUT`` for downstream ``if:`` conditions. + +A workflow that has not appeared on the SHA after the grace period is +treated as not-triggered and blocks the gate (proceed=false), so a slow +webhook or path-filtered upstream cannot let the expensive job through. +""" + +import os +import sys +import time +from typing import Dict, List, Optional + +import requests + +REQUEST_TIMEOUT = 15 +# Wait this long before we even start polling — gives webhooks + +# workflow registration time on busy queues. +INITIAL_DELAY = 60 +POLL_INTERVAL = 30 +MAX_WAIT_SECONDS = 90 * 60 +# Grace period (measured from preflight start) during which a missing +# upstream workflow is treated as "still pending registration" rather +# than "path-filtered out / never going to run". Without this, a slow +# webhook delivery could let us emit proceed=true before backend/frontend +# CI have even appeared on the SHA — defeating the cost guard. +MISSING_GRACE_SECONDS = 10 * 60 + + +def get_env() -> Optional[Dict[str, object]]: + raw = os.environ.get("UPSTREAM_WORKFLOWS", "") + workflows = [w.strip() for w in raw.splitlines() if w.strip()] + if not workflows: + return None + return { + "api": os.environ["GITHUB_API_URL"], + "repo": os.environ["GITHUB_REPOSITORY"], + "sha": os.environ["HEAD_SHA"], + "token": os.environ["GITHUB_TOKEN"], + "workflows": workflows, + } + + +def fetch_runs(env: Dict[str, object], headers: Dict[str, str]) -> List[Dict]: + runs: List[Dict] = [] + url = ( + f"{env['api']}/repos/{env['repo']}/actions/runs" + f"?head_sha={env['sha']}&per_page=100" + ) + while url: + response = requests.get(url, headers=headers, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + runs.extend(response.json().get("workflow_runs", [])) + next_url = None + for part in response.headers.get("Link", "").split(","): + if 'rel="next"' in part: + next_url = part.split(";")[0].strip().strip("<>") + break + url = next_url + return runs + + +def latest_per_workflow(runs: List[Dict], names: List[str]) -> Dict[str, Dict]: + by_name: Dict[str, Dict] = {} + name_set = set(names) + for run in runs: + name = run.get("name") + if name not in name_set: + continue + prev = by_name.get(name) + run_key = (run["run_number"], run.get("run_attempt", 0)) + if prev is None or run_key > ( + prev["run_number"], + prev.get("run_attempt", 0), + ): + by_name[name] = run + return by_name + + +def write_output(key: str, value: str) -> None: + out = os.environ.get("GITHUB_OUTPUT") + if out: + with open(out, "a") as f: + f.write(f"{key}={value}\n") + print(f"output: {key}={value}") + + +def main() -> None: + env = get_env() + if env is None: + print("No UPSTREAM_WORKFLOWS configured; proceeding.") + write_output("proceed", "true") + return + + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {env['token']}", + "X-GitHub-Api-Version": "2022-11-28", + } + + print(f"Gating on upstream workflows for SHA {env['sha']}:") + for w in env["workflows"]: + print(f" - {w}") + start = time.monotonic() + print(f"Initial delay {INITIAL_DELAY}s to let upstream runs register...") + time.sleep(INITIAL_DELAY) + + deadline = start + MAX_WAIT_SECONDS + grace_deadline = start + MISSING_GRACE_SECONDS + + while True: + try: + runs = fetch_runs(env, headers) + except requests.RequestException as exc: + if time.monotonic() > deadline: + print( + f"Timeout after {MAX_WAIT_SECONDS}s while fetching runs: {exc}" + ) + write_output("proceed", "false") + sys.exit(1) + print(f"Transient GitHub API error while fetching runs: {exc}") + time.sleep(POLL_INTERVAL) + continue + + by_name = latest_per_workflow(runs, env["workflows"]) + + missing = [w for w in env["workflows"] if w not in by_name] + in_progress = [ + w for w, r in by_name.items() if r["status"] != "completed" + ] + failed = [ + f"{w}={by_name[w]['conclusion']}" + for w in by_name + if by_name[w]["status"] == "completed" + and by_name[w]["conclusion"] not in ("success", "skipped", "neutral") + ] + + if failed: + print(f"Upstream failed: {failed}") + write_output("proceed", "false") + return + + # Don't treat "missing" as "skipped" until the grace period + # has fully elapsed — otherwise a slow webhook / queue could + # let us pass the gate before backend/frontend CI even appears. + if missing and time.monotonic() < grace_deadline: + remaining = int(grace_deadline - time.monotonic()) + print( + f"Waiting (within {MISSING_GRACE_SECONDS}s grace, " + f"{remaining}s left): missing={missing}, " + f"in_progress={in_progress}" + ) + time.sleep(POLL_INTERVAL) + continue + + if not in_progress: + if missing: + print( + "Workflow(s) did not trigger after grace period " + f"(treating as not-triggered, blocking): {missing}" + ) + write_output("proceed", "false") + return + print(f"Upstream passed: {list(by_name.keys())}") + write_output("proceed", "true") + return + + if time.monotonic() > deadline: + print( + f"Timeout after {MAX_WAIT_SECONDS}s; still in_progress=" + f"{in_progress}, missing={missing}" + ) + write_output("proceed", "false") + sys.exit(1) + + print(f"Waiting: in_progress={in_progress}, missing={missing}") + time.sleep(POLL_INTERVAL) + + +if __name__ == "__main__": + main()