Run GPU tests only on explicit trigger

.buildkite/pipeline.yml

@@ -1,3 +1,12 @@
+# The GPU tests are *not* run automatically for every push or pull request, since they are
+# not essential for the vast majority of pull requests and the GPU machines are a scarce
+# resource. Therefore, it can take quite a while for them to start. They only run if
+#   * they were explicitly requested by commenting `/run_gpu_tests` on a pull request,
+#     which triggers a build via `.github/workflows/TriggerGPUTests.yml`, or
+#   * the build is for the default branch (`main`), i.e., after a pull request was merged,
+#     or
+#   * the build was started manually from the Buildkite web interface.
+
 env:
 
 steps:
@@ -16,7 +25,11 @@ steps:
       TRIXI_TEST_VERBOSE: "true"
     agents:
       queue: "cuda"
-    if: build.message !~ /\[skip ci\]/
+    if: >
+      build.message !~ /\[skip ci\]/ &&
+      (build.source == "api" ||
+      build.branch == pipeline.default_branch ||
+      build.source == "ui")
     timeout_in_minutes: 60
     soft_fail:
       - exit_status: 3
@@ -35,7 +48,11 @@ steps:
       TRIXI_TEST_VERBOSE: "true"
     agents:
       queue: "rocm"
-    if: build.message !~ /\[skip ci\]/
+    if: >
+      build.message !~ /\[skip ci\]/ &&
+      (build.source == "api" ||
+      build.branch == pipeline.default_branch ||
+      build.source == "ui")
     timeout_in_minutes: 60
     soft_fail:
       - exit_status: 3

.github/workflows/TriggerGPUTests.yml

@@ -0,0 +1,38 @@
+name: Trigger GPU Tests
+
+# The GPU tests run on dedicated machines with actual GPUs, which are provided via
+# Buildkite (see `.buildkite/pipeline.yml`). Since they are irrelevant for the vast
+# majority of pull requests and the GPU machines are a scarce resource, they are not run
+# automatically. Instead, they are requested explicitly by commenting `/run_gpu_tests` on
+# a pull request, which is what this workflow reacts to.
+
+on:
+  issue_comment:
+    types:
+      - created
+
+# This workflow only talks to the Buildkite API, using the token below.
+permissions: {}
+
+jobs:
+  trigger_buildkite:
+    # Only react to comments on pull requests (not on issues), and only if they were
+    # written by someone who can be trusted to have reviewed the code. Without the
+    # `author_association` check, anyone could run arbitrary code from a forked pull
+    # request on the GPU machines.
+    if: >
+      github.event.issue.pull_request &&
+      contains(github.event.comment.body, '/run_gpu_tests') &&
+      contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'),
+      github.event.comment.author_association)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger Buildkite pipeline
+        uses: buildkite/trigger-pipeline-action@v2.5.0
+        with:
+          buildkite_api_access_token: ${{ secrets.TRIGGER_BK_BUILD_TOKEN }}
+          pipeline: "julialang/trixi-dot-jl"
+          branch: "refs/pull/${{ github.event.issue.number }}/head"
+          message: >-
+            GPU tests requested by @${{ github.event.comment.user.login }}
+            on PR #${{ github.event.issue.number }}

docs/src/testing.md

@@ -79,6 +79,30 @@ and
 [`.github/workflows/ci.yml`](https://github.com/trixi-framework/Trixi.jl/blob/main/.github/workflows/ci.yml).
 
 
+### GPU tests
+The tests for the GPU backends cannot run on GitHub Actions since they require actual
+hardware. They are therefore executed on [Buildkite](https://buildkite.com) on dedicated
+machines with NVIDIA (`TRIXI_TEST=CUDA`) and AMD (`TRIXI_TEST=AMDGPU`) GPUs, configured in
+[`.buildkite/pipeline.yml`](https://github.com/trixi-framework/Trixi.jl/blob/main/.buildkite/pipeline.yml).
+
+Since most pull requests do not touch any GPU-related code and the GPU machines are a
+scarce resource, these tests do **not** run automatically. Instead, they are only run
+* on demand, by writing a comment containing
+  ```
+  /run_gpu_tests
+  ```
+  on the pull request. This runs both the CUDA and the AMDGPU tests. The comment must be
+  written by an owner, member, or collaborator of the repository,
+* automatically for every push to `main`, i.e., after a pull request has been merged, and
+* when a build is started manually from the Buildkite web interface.
+
+The comment is picked up by
+[`.github/workflows/TriggerGPUTests.yml`](https://github.com/trixi-framework/Trixi.jl/blob/main/.github/workflows/TriggerGPUTests.yml),
+which asks Buildkite to build the *current* head commit of the pull request. Hence, you
+need to comment again after pushing further changes. If you modify GPU code, please
+request a GPU run before merging.
+
+
 ## Adding new tests
 We use [TestItems.jl](https://github.com/julia-vscode/TestItems.jl) on top of Julia's
 built-in [unit testing capabilities](https://docs.julialang.org/en/v1/stdlib/Test/):