diff --git a/.github/workflows/platform-backend-ci.yml b/.github/workflows/platform-backend-ci.yml index a53a5c613f17..3a749062cf52 100644 --- a/.github/workflows/platform-backend-ci.yml +++ b/.github/workflows/platform-backend-ci.yml @@ -176,18 +176,39 @@ jobs: # merge_group that is fatal: `Check PR Status` treats a non-success check as # a failure and GitHub ejects the PR from the merge queue. This is a guard # against a hung job, not a performance budget - keep it well above p99. + name: test (${{ matrix.python-version }}, ${{ matrix.cache-image }}) timeout-minutes: 35 strategy: fail-fast: false matrix: python-version: ["3.11", "3.12", "3.13"] + # Engine for the 3-shard cluster started below. Mirrors REDIS_IMAGE in + # autogpt_platform/.env.default. + cache-image: ["redis:7"] + # One extra leg on Valkey, on the newest supported Python only. + # `single-container/` already ships a Valkey cluster, and the cache and + # coordination layer uses no Redis modules and no post-7.0 command + # semantics — this leg is what keeps that true instead of leaving it a + # claim in the docs. Because the include entry would have to overwrite + # `cache-image` on every existing combination, Actions cannot merge it + # into one and adds a 4th leg instead; redis:7 coverage on 3.13 stays. + # + # `optional` makes the engine-specific steps below non-blocking (see + # their `continue-on-error`), because redis:7 remains the supported + # default and a Valkey regression should be visible without gating + # anyone's merge. Delete the `optional` line to promote the leg to a + # required one; nothing else needs to change. + include: + - python-version: "3.13" + cache-image: "valkey/valkey:8.1" + optional: true runs-on: ubuntu-latest services: - # Redis is provisioned as a real 3-shard cluster below via docker - # run (see the "Start Redis Cluster" step). GHA services can't - # override the image CMD or stand up multi-container clusters, so - # that setup is inlined — it mirrors the topology of the local dev + # The cache cluster is provisioned as a real 3-shard cluster below + # via docker run (see the "Start Redis Cluster" step). GHA services + # can't override the image CMD or stand up multi-container clusters, + # so that setup is inlined — it mirrors the topology of the local dev # compose stack (autogpt_platform/docker-compose.platform.yml) and # prod helm chart. rabbitmq: @@ -271,7 +292,14 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Start Redis Cluster (3 shards) + - name: Start Redis Cluster (3 shards, ${{ matrix.cache-image }}) + # Step-level, not job-level: `continue-on-error` on the *job* still + # leaves the job's own check run concluding `failure`, and + # `Check PR Status` — the only required check — fails on any check run + # not concluding success/skipped/neutral, so a red advisory leg would + # eject PRs from the merge queue. Tolerating the failure at the step + # level is what actually makes the job conclude `success`. + continue-on-error: ${{ matrix.optional || false }} run: | # 3-master Redis Cluster matching the local compose stack # (autogpt_platform/docker-compose.platform.yml) and prod. Each @@ -286,14 +314,23 @@ jobs: # validates the full file even when only some services are ``up`` # — pulling it in would needlessly couple CI to the full # local-dev stack. + # + # ``--user 999:999`` mirrors the compose stack. Both images gate + # their privilege drop on being invoked as their own server binary + # (``redis-server`` for Redis, ``valkey-server`` for Valkey) and the + # command below is ``redis-server``, a symlink under Valkey — so + # without the pin the Valkey leg would run as root while the Redis + # legs do not. 999 is ``redis``/``valkey`` respectively, and /data is + # owned by it in both images. docker network create redis-cluster-ci for i in 0 1 2; do port=$((17000 + i)) bus=$((27000 + i)) docker run -d --name redis-$i --network redis-cluster-ci \ --network-alias redis-$i \ + --user 999:999 \ -p $port:$port \ - redis:7 \ + "$CACHE_IMAGE" \ redis-server --port $port \ --cluster-enabled yes \ --cluster-config-file nodes.conf \ @@ -314,7 +351,7 @@ jobs: done # Form the cluster from an init container on the same network so # --cluster-preferred-endpoint-type hostname resolves redis-0/1/2. - docker run --rm --network redis-cluster-ci redis:7 \ + docker run --rm --network redis-cluster-ci --user 999:999 "$CACHE_IMAGE" \ redis-cli --cluster create \ redis-0:17000 redis-1:17001 redis-2:17002 \ --cluster-replicas 0 --cluster-yes @@ -322,15 +359,19 @@ jobs: for _ in $(seq 1 30); do state=$(docker exec redis-0 redis-cli -p 17000 cluster info | awk -F: '/^cluster_state:/ {print $2}' | tr -d '[:cntrl:]') if [ "$state" = "ok" ]; then - echo "Redis Cluster ready (3 shards, state=ok)" + echo "Cluster ready ($CACHE_IMAGE, 3 shards, state=ok)" + docker exec redis-0 redis-cli -p 17000 info server | grep -Ei '^(redis|valkey)_version:' || true docker exec redis-0 redis-cli -p 17000 cluster nodes exit 0 fi sleep 1 done - echo "Redis Cluster failed to reach ok state" >&2 + echo "Cluster ($CACHE_IMAGE) failed to reach ok state" >&2 docker exec redis-0 redis-cli -p 17000 cluster info >&2 || true + docker logs redis-0 >&2 || true exit 1 + env: + CACHE_IMAGE: ${{ matrix.cache-image }} - id: get_date name: Get date @@ -444,6 +485,8 @@ jobs: DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres - name: Run pytest with coverage + # Advisory on the optional engine leg — see "Start Redis Cluster". + continue-on-error: ${{ matrix.optional || false }} run: | if [[ "${{ runner.debug }}" == "1" ]]; then poetry run pytest -s -vv -o log_cli=true -o log_cli_level=DEBUG \ @@ -460,6 +503,10 @@ jobs: REDIS_HOST: "localhost" REDIS_PORT: "17000" ENCRYPTION_KEY: "dvziYgz0KSK8FENhju0ZYi8-fRTfAdlz6YLhdB_jhNw=" # DO NOT USE IN PRODUCTION!! + # Without this the isolated cluster below would come up on the + # default engine even on a non-default leg, quietly testing Redis + # twice. Read only by e2e_redis_restart_test.py. + REDIS_IMAGE: ${{ matrix.cache-image }} # Opt-in: lets backend/data/e2e_redis_restart_test.py spin up its # own isolated 3-shard cluster (ports 27110–27112) and exercise # ``docker restart `` mid-stream. Off locally so a @@ -467,7 +514,10 @@ jobs: E2E_RESTART_ISOLATED: "1" - name: Upload coverage reports to Codecov - if: ${{ !cancelled() }} + # Skipped on the optional engine leg: it runs the same suite as the + # 3.13 default leg, so its report would only duplicate a `platform-backend` + # upload — and a partial one if the advisory pytest step failed. + if: ${{ !cancelled() && !matrix.optional }} uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/autogpt_platform/.env.default b/autogpt_platform/.env.default index 5dc5c7a5878d..fd2e4ca5cd3f 100644 --- a/autogpt_platform/.env.default +++ b/autogpt_platform/.env.default @@ -13,3 +13,25 @@ POSTGRES_DB=postgres POSTGRES_PORT=5432 # default user is postgres POSTGRES_PASSWORD=your-super-secret-and-long-postgres-password + +############ +# Cache and coordination engine +# +# Image for the three Redis Cluster shards and the redis-init sidecar in +# docker-compose.platform.yml. Note this file, unlike backend/.env(.default), +# is read by docker compose itself for ${...} interpolation — a value set in +# backend/.env will NOT reach this variable. +# +# redis:7 is the default and what backend CI covers on every leg. Any +# Redis-compatible image works as long as it provides redis-server and +# redis-cli and implements Redis 7.0 cluster semantics (sharded pub/sub, +# EXPIRE NX), because the shard command lines are engine-neutral. Valkey +# qualifies, ships both binary names as symlinks, and is already the engine +# inside the single-container distribution. +# +# Left commented so the compose default stays authoritative — uncomment only +# to pin a different engine: +# REDIS_IMAGE=valkey/valkey:8.1 +############ + +# REDIS_IMAGE=redis:7 diff --git a/autogpt_platform/backend/backend/data/e2e_redis_restart_test.py b/autogpt_platform/backend/backend/data/e2e_redis_restart_test.py index cad8210e24b6..67dffdb16426 100644 --- a/autogpt_platform/backend/backend/data/e2e_redis_restart_test.py +++ b/autogpt_platform/backend/backend/data/e2e_redis_restart_test.py @@ -19,6 +19,10 @@ ISOLATED_PROJECT = "redis-restart-test" ISOLATED_PORTS = (27110, 27111, 27112) ISOLATED_BUS_PORTS = (37110, 37111, 37112) +# Same knob as the compose stack, so a run against a non-default engine +# exercises restart/reconnect on that engine too rather than silently +# falling back to Redis. Any image providing redis-server/redis-cli works. +ISOLATED_IMAGE = os.getenv("REDIS_IMAGE") or "redis:7" def _docker_available() -> bool: @@ -79,7 +83,7 @@ def _start_isolated_cluster() -> None: f"redis-{i}", "-p", f"{port}:{port}", - "redis:7", + ISOLATED_IMAGE, "redis-server", "--port", str(port), @@ -112,7 +116,7 @@ def _start_isolated_cluster() -> None: "--rm", "--network", network, - "redis:7", + ISOLATED_IMAGE, "redis-cli", "--cluster", "create", diff --git a/autogpt_platform/docker-compose.platform.yml b/autogpt_platform/docker-compose.platform.yml index 5eb6f587bfd4..4c3b2db5155b 100644 --- a/autogpt_platform/docker-compose.platform.yml +++ b/autogpt_platform/docker-compose.platform.yml @@ -49,10 +49,32 @@ x-codex-tmpfs: &codex-tmpfs - /run/autogpt-codex:rw,nosuid,nodev,noexec,mode=0700,size=128m # Shared base for the three Redis Cluster shards + one-shot init sidecar. -# The anchor absorbs image + network so each service body collapses to its -# hostname + command + (for the seed) volume + healthcheck. +# The anchor absorbs image + user + network so each service body collapses to +# its hostname + command + (for the seed) healthcheck. +# +# ``REDIS_IMAGE`` selects the engine for all four containers at once. +# ``redis:7`` is the default here and what backend CI covers on every leg; the +# escape hatch exists because the shard command lines and healthchecks below +# are engine-neutral — any Redis-compatible image that provides +# ``redis-server``/``redis-cli`` and Redis 7.0 cluster semantics (sharded +# pub/sub, ``EXPIRE NX``) drops in unchanged. Valkey qualifies, ships both +# names as symlinks, and is already the engine inside the single-container +# distribution: +# REDIS_IMAGE=valkey/valkey:8.1 docker compose up -d deps +# Documented in ``.env.default`` alongside this file — note that is the file +# compose interpolates from, not ``backend/.env(.default)``. +# +# ``user`` is pinned rather than left to the image entrypoint. Both images +# gate their privilege drop on being invoked as their *own* server binary — +# ``redis:7`` on ``redis-server``, Valkey on ``valkey-server`` — and the +# commands below call ``redis-server``, which is a symlink under Valkey. So +# Valkey would stay root here while ``redis:7`` does not. The numeric form +# covers both: uid/gid 999 exists in each image (``redis`` and ``valkey`` +# respectively) and owns the ``/data`` workdir where ``nodes.conf`` is +# written. x-redis-node: &redis-node - image: redis:7 + image: ${REDIS_IMAGE:-redis:7} + user: "999:999" networks: - app-network @@ -93,14 +115,14 @@ services: retries: 3 start_period: 5s - # Three-container 3-master Redis Cluster for local dev. Stock ``redis:7`` - # on each shard runs its default entrypoint; the one-shot ``redis-init`` - # sidecar runs ``redis-cli --cluster create`` once and exits. No volumes - # — local dev treats the cluster as cache-only, so every ``docker compose - # up`` starts fresh. (Persisting only the seed volume across restarts is - # a trap: the other shards come back with new IDs and the seed's - # ``nodes.conf`` pins stale peers that cluster gossip can't heal.) Each - # shard announces its own compose hostname via + # Three-container 3-master Redis Cluster for local dev. Each shard runs the + # engine image's default entrypoint (see ``x-redis-node``); the one-shot + # ``redis-init`` sidecar runs ``redis-cli --cluster create`` once and exits. + # No volumes — local dev treats the cluster as cache-only, so every + # ``docker compose up`` starts fresh. (Persisting only the seed volume + # across restarts is a trap: the other shards come back with new IDs and + # the seed's ``nodes.conf`` pins stale peers that cluster gossip can't + # heal.) Each shard announces its own compose hostname via # ``--cluster-announce-hostname`` so both in-compose clients (Docker DNS) # and host-native clients (handled by the backend's ``address_remap``) # receive a consistent address on CLUSTER SLOTS. Ports 17000/17001/17002