feat(platform): make the cache engine selectable via REDIS_IMAGE, with a Valkey CI leg

.github/workflows/platform-backend-ci.yml

@@ -176,18 +176,39 @@ jobs:
     # merge_group that is fatal: `Check PR Status` treats a non-success check as
     # a failure and GitHub ejects the PR from the merge queue. This is a guard
     # against a hung job, not a performance budget - keep it well above p99.
+    name: test (${{ matrix.python-version }}, ${{ matrix.cache-image }})
     timeout-minutes: 35
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.11", "3.12", "3.13"]
+        # Engine for the 3-shard cluster started below. Mirrors REDIS_IMAGE in
+        # autogpt_platform/.env.default.
+        cache-image: ["redis:7"]
+        # One extra leg on Valkey, on the newest supported Python only.
+        # `single-container/` already ships a Valkey cluster, and the cache and
+        # coordination layer uses no Redis modules and no post-7.0 command
+        # semantics — this leg is what keeps that true instead of leaving it a
+        # claim in the docs. Because the include entry would have to overwrite
+        # `cache-image` on every existing combination, Actions cannot merge it
+        # into one and adds a 4th leg instead; redis:7 coverage on 3.13 stays.
+        #
+        # `optional` makes the engine-specific steps below non-blocking (see
+        # their `continue-on-error`), because redis:7 remains the supported
+        # default and a Valkey regression should be visible without gating
+        # anyone's merge. Delete the `optional` line to promote the leg to a
+        # required one; nothing else needs to change.
+        include:
+          - python-version: "3.13"
+            cache-image: "valkey/valkey:8.1"
+            optional: true
     runs-on: ubuntu-latest
 
     services:
-      # Redis is provisioned as a real 3-shard cluster below via docker
-      # run (see the "Start Redis Cluster" step). GHA services can't
-      # override the image CMD or stand up multi-container clusters, so
-      # that setup is inlined — it mirrors the topology of the local dev
+      # The cache cluster is provisioned as a real 3-shard cluster below
+      # via docker run (see the "Start Redis Cluster" step). GHA services
+      # can't override the image CMD or stand up multi-container clusters,
+      # so that setup is inlined — it mirrors the topology of the local dev
       # compose stack (autogpt_platform/docker-compose.platform.yml) and
       # prod helm chart.
       rabbitmq:
@@ -271,7 +292,14 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Start Redis Cluster (3 shards)
+      - name: Start Redis Cluster (3 shards, ${{ matrix.cache-image }})
+        # Step-level, not job-level: `continue-on-error` on the *job* still
+        # leaves the job's own check run concluding `failure`, and
+        # `Check PR Status` — the only required check — fails on any check run
+        # not concluding success/skipped/neutral, so a red advisory leg would
+        # eject PRs from the merge queue. Tolerating the failure at the step
+        # level is what actually makes the job conclude `success`.
+        continue-on-error: ${{ matrix.optional || false }}
         run: |
           # 3-master Redis Cluster matching the local compose stack
           # (autogpt_platform/docker-compose.platform.yml) and prod. Each
@@ -286,14 +314,23 @@ jobs:
           # validates the full file even when only some services are ``up``
           # — pulling it in would needlessly couple CI to the full
           # local-dev stack.
+          #
+          # ``--user 999:999`` mirrors the compose stack. Both images gate
+          # their privilege drop on being invoked as their own server binary
+          # (``redis-server`` for Redis, ``valkey-server`` for Valkey) and the
+          # command below is ``redis-server``, a symlink under Valkey — so
+          # without the pin the Valkey leg would run as root while the Redis
+          # legs do not. 999 is ``redis``/``valkey`` respectively, and /data is
+          # owned by it in both images.
           docker network create redis-cluster-ci
           for i in 0 1 2; do
             port=$((17000 + i))
             bus=$((27000 + i))
             docker run -d --name redis-$i --network redis-cluster-ci \
               --network-alias redis-$i \
+              --user 999:999 \
               -p $port:$port \
-              redis:7 \
+              "$CACHE_IMAGE" \
               redis-server --port $port \
                 --cluster-enabled yes \
                 --cluster-config-file nodes.conf \
@@ -314,23 +351,27 @@ jobs:
           done
           # Form the cluster from an init container on the same network so
           # --cluster-preferred-endpoint-type hostname resolves redis-0/1/2.
-          docker run --rm --network redis-cluster-ci redis:7 \
+          docker run --rm --network redis-cluster-ci --user 999:999 "$CACHE_IMAGE" \
             redis-cli --cluster create \
               redis-0:17000 redis-1:17001 redis-2:17002 \
               --cluster-replicas 0 --cluster-yes
           # Confirm convergence.
           for _ in $(seq 1 30); do
             state=$(docker exec redis-0 redis-cli -p 17000 cluster info | awk -F: '/^cluster_state:/ {print $2}' | tr -d '[:cntrl:]')
             if [ "$state" = "ok" ]; then
-              echo "Redis Cluster ready (3 shards, state=ok)"
+              echo "Cluster ready ($CACHE_IMAGE, 3 shards, state=ok)"
+              docker exec redis-0 redis-cli -p 17000 info server | grep -Ei '^(redis|valkey)_version:' || true
               docker exec redis-0 redis-cli -p 17000 cluster nodes
               exit 0
             fi
             sleep 1
           done
-          echo "Redis Cluster failed to reach ok state" >&2
+          echo "Cluster ($CACHE_IMAGE) failed to reach ok state" >&2
           docker exec redis-0 redis-cli -p 17000 cluster info >&2 || true
+          docker logs redis-0 >&2 || true
           exit 1
+        env:
+          CACHE_IMAGE: ${{ matrix.cache-image }}
 
       - id: get_date
         name: Get date
@@ -444,6 +485,8 @@ jobs:
           DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
 
       - name: Run pytest with coverage
+        # Advisory on the optional engine leg — see "Start Redis Cluster".
+        continue-on-error: ${{ matrix.optional || false }}
         run: |
           if [[ "${{ runner.debug }}" == "1" ]]; then
             poetry run pytest -s -vv -o log_cli=true -o log_cli_level=DEBUG \
@@ -460,14 +503,21 @@ jobs:
           REDIS_HOST: "localhost"
           REDIS_PORT: "17000"
           ENCRYPTION_KEY: "dvziYgz0KSK8FENhju0ZYi8-fRTfAdlz6YLhdB_jhNw=" # DO NOT USE IN PRODUCTION!!
+          # Without this the isolated cluster below would come up on the
+          # default engine even on a non-default leg, quietly testing Redis
+          # twice. Read only by e2e_redis_restart_test.py.
+          REDIS_IMAGE: ${{ matrix.cache-image }}
           # Opt-in: lets backend/data/e2e_redis_restart_test.py spin up its
           # own isolated 3-shard cluster (ports 27110–27112) and exercise
           # ``docker restart <shard>`` mid-stream. Off locally so a
           # contributor's ``poetry run test`` doesn't pay the ~15s cost.
           E2E_RESTART_ISOLATED: "1"
 
       - name: Upload coverage reports to Codecov
-        if: ${{ !cancelled() }}
+        # Skipped on the optional engine leg: it runs the same suite as the
+        # 3.13 default leg, so its report would only duplicate a `platform-backend`
+        # upload — and a partial one if the advisory pytest step failed.
+        if: ${{ !cancelled() && !matrix.optional }}
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}

autogpt_platform/.env.default

@@ -13,3 +13,25 @@ POSTGRES_DB=postgres
 POSTGRES_PORT=5432
 # default user is postgres
 POSTGRES_PASSWORD=your-super-secret-and-long-postgres-password
+
+############
+# Cache and coordination engine
+#
+# Image for the three Redis Cluster shards and the redis-init sidecar in
+# docker-compose.platform.yml. Note this file, unlike backend/.env(.default),
+# is read by docker compose itself for ${...} interpolation — a value set in
+# backend/.env will NOT reach this variable.
+#
+# redis:7 is the default and what backend CI covers on every leg. Any
+# Redis-compatible image works as long as it provides redis-server and
+# redis-cli and implements Redis 7.0 cluster semantics (sharded pub/sub,
+# EXPIRE NX), because the shard command lines are engine-neutral. Valkey
+# qualifies, ships both binary names as symlinks, and is already the engine
+# inside the single-container distribution.
+#
+# Left commented so the compose default stays authoritative — uncomment only
+# to pin a different engine:
+#   REDIS_IMAGE=valkey/valkey:8.1
+############
+
+# REDIS_IMAGE=redis:7

autogpt_platform/backend/backend/data/e2e_redis_restart_test.py

@@ -19,6 +19,10 @@
 ISOLATED_PROJECT = "redis-restart-test"
 ISOLATED_PORTS = (27110, 27111, 27112)
 ISOLATED_BUS_PORTS = (37110, 37111, 37112)
+# Same knob as the compose stack, so a run against a non-default engine
+# exercises restart/reconnect on that engine too rather than silently
+# falling back to Redis. Any image providing redis-server/redis-cli works.
+ISOLATED_IMAGE = os.getenv("REDIS_IMAGE") or "redis:7"
 
 
 def _docker_available() -> bool:
@@ -79,7 +83,7 @@ def _start_isolated_cluster() -> None:
                 f"redis-{i}",
                 "-p",
                 f"{port}:{port}",
-                "redis:7",
+                ISOLATED_IMAGE,
                 "redis-server",
                 "--port",
                 str(port),
@@ -112,7 +116,7 @@ def _start_isolated_cluster() -> None:
             "--rm",
             "--network",
             network,
-            "redis:7",
+            ISOLATED_IMAGE,
             "redis-cli",
             "--cluster",
             "create",

autogpt_platform/docker-compose.platform.yml

@@ -49,10 +49,32 @@ x-codex-tmpfs: &codex-tmpfs
   - /run/autogpt-codex:rw,nosuid,nodev,noexec,mode=0700,size=128m
 
 # Shared base for the three Redis Cluster shards + one-shot init sidecar.
-# The anchor absorbs image + network so each service body collapses to its
-# hostname + command + (for the seed) volume + healthcheck.
+# The anchor absorbs image + user + network so each service body collapses to
+# its hostname + command + (for the seed) healthcheck.
+#
+# ``REDIS_IMAGE`` selects the engine for all four containers at once.
+# ``redis:7`` is the default here and what backend CI covers on every leg; the
+# escape hatch exists because the shard command lines and healthchecks below
+# are engine-neutral — any Redis-compatible image that provides
+# ``redis-server``/``redis-cli`` and Redis 7.0 cluster semantics (sharded
+# pub/sub, ``EXPIRE NX``) drops in unchanged. Valkey qualifies, ships both
+# names as symlinks, and is already the engine inside the single-container
+# distribution:
+#   REDIS_IMAGE=valkey/valkey:8.1 docker compose up -d deps
+# Documented in ``.env.default`` alongside this file — note that is the file
+# compose interpolates from, not ``backend/.env(.default)``.
+#
+# ``user`` is pinned rather than left to the image entrypoint. Both images
+# gate their privilege drop on being invoked as their *own* server binary —
+# ``redis:7`` on ``redis-server``, Valkey on ``valkey-server`` — and the
+# commands below call ``redis-server``, which is a symlink under Valkey. So
+# Valkey would stay root here while ``redis:7`` does not. The numeric form
+# covers both: uid/gid 999 exists in each image (``redis`` and ``valkey``
+# respectively) and owns the ``/data`` workdir where ``nodes.conf`` is
+# written.
 x-redis-node: &redis-node
-  image: redis:7
+  image: ${REDIS_IMAGE:-redis:7}
+  user: "999:999"
   networks:
     - app-network
 
@@ -93,14 +115,14 @@ services:
       retries: 3
       start_period: 5s
 
-  # Three-container 3-master Redis Cluster for local dev. Stock ``redis:7``
-  # on each shard runs its default entrypoint; the one-shot ``redis-init``
-  # sidecar runs ``redis-cli --cluster create`` once and exits. No volumes
-  # — local dev treats the cluster as cache-only, so every ``docker compose
-  # up`` starts fresh. (Persisting only the seed volume across restarts is
-  # a trap: the other shards come back with new IDs and the seed's
-  # ``nodes.conf`` pins stale peers that cluster gossip can't heal.) Each
-  # shard announces its own compose hostname via
+  # Three-container 3-master Redis Cluster for local dev. Each shard runs the
+  # engine image's default entrypoint (see ``x-redis-node``); the one-shot
+  # ``redis-init`` sidecar runs ``redis-cli --cluster create`` once and exits.
+  # No volumes — local dev treats the cluster as cache-only, so every
+  # ``docker compose up`` starts fresh. (Persisting only the seed volume
+  # across restarts is a trap: the other shards come back with new IDs and
+  # the seed's ``nodes.conf`` pins stale peers that cluster gossip can't
+  # heal.) Each shard announces its own compose hostname via
   # ``--cluster-announce-hostname`` so both in-compose clients (Docker DNS)
   # and host-native clients (handled by the backend's ``address_remap``)
   # receive a consistent address on CLUSTER SLOTS. Ports 17000/17001/17002