Run GPU tests only on explicit trigger

.buildkite/pipeline.yml

@@ -1,3 +1,12 @@
+# The GPU tests are *not* run automatically for every push or pull request, since they are
+# not essential for the vast majority of pull requests and the GPU machines are a scarce
+# resource. Therefore, it can take quite a while for them to start. They only run if
+#   * they were explicitly requested by commenting `/run_gpu_tests` on a pull request,
+#     which triggers a build via `.github/workflows/TriggerGPUTests.yml`, or
+#   * the build is for the default branch (`main`), i.e., after a pull request was merged,
+#     or
+#   * the build was started manually from the Buildkite web interface.
+
 env:
 
 steps:
@@ -16,7 +25,11 @@ steps:
       TRIXI_TEST_VERBOSE: "true"
     agents:
       queue: "cuda"
-    if: build.message !~ /\[skip ci\]/
+    if: >
+      build.message !~ /\[skip ci\]/ &&
+      (build.source == "api" ||
+      build.branch == pipeline.default_branch ||
+      build.source == "ui")
     timeout_in_minutes: 60
     soft_fail:
       - exit_status: 3
@@ -35,7 +48,11 @@ steps:
       TRIXI_TEST_VERBOSE: "true"
     agents:
       queue: "rocm"
-    if: build.message !~ /\[skip ci\]/
+    if: >
+      build.message !~ /\[skip ci\]/ &&
+      (build.source == "api" ||
+      build.branch == pipeline.default_branch ||
+      build.source == "ui")
     timeout_in_minutes: 60
     soft_fail:
       - exit_status: 3

.github/workflows/TriggerGPUTests.yml

@@ -0,0 +1,90 @@
+name: Trigger GPU Tests
+
+# The GPU tests run on dedicated machines with actual GPUs, which are provided via
+# Buildkite (see `.buildkite/pipeline.yml`). Since they are irrelevant for the vast
+# majority of pull requests and the GPU machines are a scarce resource, they are not run
+# automatically. Instead, they are requested explicitly by commenting `/run_gpu_tests` on
+# a pull request, which is what this workflow reacts to.
+
+on:
+  issue_comment:
+    types:
+      - created
+
+permissions:
+  # Required to look up whether the author of the comment has write access.
+  contents: read
+  # Required to look up the head commit of the pull request.
+  pull-requests: read
+
+jobs:
+  trigger_buildkite:
+    # Only react to comments on pull requests (not on issues) that *start* with the
+    # command. Matching it anywhere in the comment would also trigger on comments that
+    # merely mention it, e.g., "Do not run /run_gpu_tests before the CPU tests are green"
+    # or a quote of the documentation.
+    # The `author_association` check is only a cheap pre-filter to avoid starting a job
+    # for arbitrary users; the authoritative permission check happens in the first step.
+    if: >
+      github.event.issue.pull_request &&
+      startsWith(github.event.comment.body, '/run_gpu_tests') &&
+      contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'),
+      github.event.comment.author_association)
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      # A `/run_gpu_tests` comment causes the code of the pull request to be executed on
+      # the GPU machines, so it must be restricted to people who can be trusted to have
+      # reviewed that code first. We require write access to this repository, which is the
+      # same bar GitHub uses for approving workflow runs of forked pull requests.
+      # Note that `author_association` alone is not sufficient for this: it reports
+      # `MEMBER` for everyone in the `trixi-framework` organization, including people who
+      # only have read access to this repository.
+      #
+      # We also resolve the head commit of the pull request here and pass it to Buildkite
+      # below. Otherwise, Buildkite would resolve `refs/pull/<number>/head` only when the
+      # build starts, i.e., we could end up testing a commit that was pushed after the
+      # GPU tests had been requested - and thus different code than the one that was
+      # reviewed before requesting them.
+      - name: Check write access and resolve the head commit
+        id: pr
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const user = context.payload.comment.user.login;
+            const { data: permission } =
+              await github.rest.repos.getCollaboratorPermissionLevel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                username: user
+              });
+            // `permission` is one of "admin", "write", "read", "none", where the
+            // "maintain" role is mapped to "write" and "triage" is mapped to "read".
+            if (!["admin", "write"].includes(permission.permission)) {
+              core.setFailed(`@${user} does not have write access to this repository ` +
+                             `(permission: "${permission.permission}"), so the GPU ` +
+                             `tests are not started. Please ask a maintainer to ` +
+                             `request them.`);
+              return;
+            }
+            const { data: pullRequest } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.payload.issue.number
+            });
+            core.setOutput("sha", pullRequest.head.sha);
+            core.info(`Requesting GPU tests for ${pullRequest.head.sha}.`);
+
+      - name: Trigger Buildkite pipeline
+        uses: buildkite/trigger-pipeline-action@v2.5.0
+        with:
+          buildkite_api_access_token: ${{ secrets.TRIGGER_BK_BUILD_TOKEN }}
+          pipeline: "julialang/trixi-dot-jl"
+          commit: ${{ steps.pr.outputs.sha }}
+          branch: "refs/pull/${{ github.event.issue.number }}/head"
+          # The Buildkite pipeline only builds `main` automatically, so this explicitly
+          # requested build has to be exempt from that branch filter.
+          ignore_pipeline_branch_filter: true
+          message: >-
+            GPU tests requested by @${{ github.event.comment.user.login }}
+            on PR #${{ github.event.issue.number }}

docs/src/testing.md

@@ -79,6 +79,32 @@ and
 [`.github/workflows/ci.yml`](https://github.com/trixi-framework/Trixi.jl/blob/main/.github/workflows/ci.yml).
 
 
+### GPU tests
+The tests for the GPU backends cannot run on GitHub Actions since they require actual
+hardware. They are therefore executed on [Buildkite](https://buildkite.com) on dedicated
+machines with NVIDIA (`TRIXI_TEST=CUDA`) and AMD (`TRIXI_TEST=AMDGPU`) GPUs, configured in
+[`.buildkite/pipeline.yml`](https://github.com/trixi-framework/Trixi.jl/blob/main/.buildkite/pipeline.yml).
+
+Since most pull requests do not touch any GPU-related code and the GPU machines are a
+scarce resource, these tests do **not** run automatically. Instead, they are only run
+* on demand, by writing a comment starting with
+  ```
+  /run_gpu_tests
+  ```
+  on the pull request. This runs both the CUDA and the AMDGPU tests. The comment must be written by someone
+  with write access to this repository,
+* automatically for every push to `main`, i.e., after a pull request has been merged, and
+* when a build is started manually from the Buildkite web interface.
+
+The comment is picked up by
+[`.github/workflows/TriggerGPUTests.yml`](https://github.com/trixi-framework/Trixi.jl/blob/main/.github/workflows/TriggerGPUTests.yml),
+which first verifies that the author of the comment has write access and then asks
+Buildkite to build the *current* head commit of the pull request. Hence, you need to
+comment again after pushing further changes. If you modify GPU code, please request a GPU
+run before merging. If you do not have write access, the workflow fails with a
+corresponding message; please ask a maintainer to request the GPU tests for you.
+
+
 ## Adding new tests
 We use [TestItems.jl](https://github.com/julia-vscode/TestItems.jl) on top of Julia's
 built-in [unit testing capabilities](https://docs.julialang.org/en/v1/stdlib/Test/):