diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 124f105b492..80ab8a083b0 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,3 +1,12 @@ +# The GPU tests are *not* run automatically for every push or pull request, since they are +# not essential for the vast majority of pull requests and the GPU machines are a scarce +# resource. Therefore, it can take quite a while for them to start. They only run if +# * they were explicitly requested by commenting `/run_gpu_tests` on a pull request, +# which triggers a build via `.github/workflows/TriggerGPUTests.yml`, or +# * the build is for the default branch (`main`), i.e., after a pull request was merged, +# or +# * the build was started manually from the Buildkite web interface. + env: steps: @@ -16,7 +25,11 @@ steps: TRIXI_TEST_VERBOSE: "true" agents: queue: "cuda" - if: build.message !~ /\[skip ci\]/ + if: > + build.message !~ /\[skip ci\]/ && + (build.source == "api" || + build.branch == pipeline.default_branch || + build.source == "ui") timeout_in_minutes: 60 soft_fail: - exit_status: 3 @@ -35,7 +48,11 @@ steps: TRIXI_TEST_VERBOSE: "true" agents: queue: "rocm" - if: build.message !~ /\[skip ci\]/ + if: > + build.message !~ /\[skip ci\]/ && + (build.source == "api" || + build.branch == pipeline.default_branch || + build.source == "ui") timeout_in_minutes: 60 soft_fail: - exit_status: 3 diff --git a/.github/workflows/TriggerGPUTests.yml b/.github/workflows/TriggerGPUTests.yml new file mode 100644 index 00000000000..ef0acecf32d --- /dev/null +++ b/.github/workflows/TriggerGPUTests.yml @@ -0,0 +1,90 @@ +name: Trigger GPU Tests + +# The GPU tests run on dedicated machines with actual GPUs, which are provided via +# Buildkite (see `.buildkite/pipeline.yml`). Since they are irrelevant for the vast +# majority of pull requests and the GPU machines are a scarce resource, they are not run +# automatically. Instead, they are requested explicitly by commenting `/run_gpu_tests` on +# a pull request, which is what this workflow reacts to. + +on: + issue_comment: + types: + - created + +permissions: + # Required to look up whether the author of the comment has write access. + contents: read + # Required to look up the head commit of the pull request. + pull-requests: read + +jobs: + trigger_buildkite: + # Only react to comments on pull requests (not on issues) that *start* with the + # command. Matching it anywhere in the comment would also trigger on comments that + # merely mention it, e.g., "Do not run /run_gpu_tests before the CPU tests are green" + # or a quote of the documentation. + # The `author_association` check is only a cheap pre-filter to avoid starting a job + # for arbitrary users; the authoritative permission check happens in the first step. + if: > + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/run_gpu_tests') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), + github.event.comment.author_association) + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + # A `/run_gpu_tests` comment causes the code of the pull request to be executed on + # the GPU machines, so it must be restricted to people who can be trusted to have + # reviewed that code first. We require write access to this repository, which is the + # same bar GitHub uses for approving workflow runs of forked pull requests. + # Note that `author_association` alone is not sufficient for this: it reports + # `MEMBER` for everyone in the `trixi-framework` organization, including people who + # only have read access to this repository. + # + # We also resolve the head commit of the pull request here and pass it to Buildkite + # below. Otherwise, Buildkite would resolve `refs/pull//head` only when the + # build starts, i.e., we could end up testing a commit that was pushed after the + # GPU tests had been requested - and thus different code than the one that was + # reviewed before requesting them. + - name: Check write access and resolve the head commit + id: pr + uses: actions/github-script@v9 + with: + script: | + const user = context.payload.comment.user.login; + const { data: permission } = + await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: user + }); + // `permission` is one of "admin", "write", "read", "none", where the + // "maintain" role is mapped to "write" and "triage" is mapped to "read". + if (!["admin", "write"].includes(permission.permission)) { + core.setFailed(`@${user} does not have write access to this repository ` + + `(permission: "${permission.permission}"), so the GPU ` + + `tests are not started. Please ask a maintainer to ` + + `request them.`); + return; + } + const { data: pullRequest } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.issue.number + }); + core.setOutput("sha", pullRequest.head.sha); + core.info(`Requesting GPU tests for ${pullRequest.head.sha}.`); + + - name: Trigger Buildkite pipeline + uses: buildkite/trigger-pipeline-action@v2.5.0 + with: + buildkite_api_access_token: ${{ secrets.TRIGGER_BK_BUILD_TOKEN }} + pipeline: "julialang/trixi-dot-jl" + commit: ${{ steps.pr.outputs.sha }} + branch: "refs/pull/${{ github.event.issue.number }}/head" + # The Buildkite pipeline only builds `main` automatically, so this explicitly + # requested build has to be exempt from that branch filter. + ignore_pipeline_branch_filter: true + message: >- + GPU tests requested by @${{ github.event.comment.user.login }} + on PR #${{ github.event.issue.number }} diff --git a/docs/src/testing.md b/docs/src/testing.md index 3fd46043539..22308569945 100644 --- a/docs/src/testing.md +++ b/docs/src/testing.md @@ -79,6 +79,32 @@ and [`.github/workflows/ci.yml`](https://github.com/trixi-framework/Trixi.jl/blob/main/.github/workflows/ci.yml). +### GPU tests +The tests for the GPU backends cannot run on GitHub Actions since they require actual +hardware. They are therefore executed on [Buildkite](https://buildkite.com) on dedicated +machines with NVIDIA (`TRIXI_TEST=CUDA`) and AMD (`TRIXI_TEST=AMDGPU`) GPUs, configured in +[`.buildkite/pipeline.yml`](https://github.com/trixi-framework/Trixi.jl/blob/main/.buildkite/pipeline.yml). + +Since most pull requests do not touch any GPU-related code and the GPU machines are a +scarce resource, these tests do **not** run automatically. Instead, they are only run +* on demand, by writing a comment starting with + ``` + /run_gpu_tests + ``` + on the pull request. This runs both the CUDA and the AMDGPU tests. The comment must be written by someone + with write access to this repository, +* automatically for every push to `main`, i.e., after a pull request has been merged, and +* when a build is started manually from the Buildkite web interface. + +The comment is picked up by +[`.github/workflows/TriggerGPUTests.yml`](https://github.com/trixi-framework/Trixi.jl/blob/main/.github/workflows/TriggerGPUTests.yml), +which first verifies that the author of the comment has write access and then asks +Buildkite to build the *current* head commit of the pull request. Hence, you need to +comment again after pushing further changes. If you modify GPU code, please request a GPU +run before merging. If you do not have write access, the workflow fails with a +corresponding message; please ask a maintainer to request the GPU tests for you. + + ## Adding new tests We use [TestItems.jl](https://github.com/julia-vscode/TestItems.jl) on top of Julia's built-in [unit testing capabilities](https://docs.julialang.org/en/v1/stdlib/Test/):