[WIP] switch to yhub as a collab backend

.github/workflows/impress.yml

@@ -27,10 +27,6 @@ jobs:
           fetch-depth: 0
       - name: show
         run: git log
-      - name: Enforce absence of print statements in code
-        if: always()
-        run: |
-          ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- src/backend ':(exclude)**/impress.yml' | grep "print("
       - name: Check absence of fixup commits
         if: always()
         run: |
@@ -132,6 +128,13 @@ jobs:
         # needed because the postgres container does not provide a healthcheck
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
+      # message stream for the collaboration server (see the yhub steps below)
+      valkey:
+        image: valkey/valkey:alpine
+        ports:
+          - 6379:6379
+        options: --health-cmd "valkey-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
+
     env:
       DJANGO_CONFIGURATION: Test
       DJANGO_SETTINGS_MODULE: impress.settings
@@ -146,6 +149,12 @@ jobs:
       AWS_S3_ENDPOINT_URL: http://localhost:9000
       AWS_S3_ACCESS_KEY_ID: impress
       AWS_S3_SECRET_ACCESS_KEY: password
+      # Collaboration server. The integration tests reach it over this url and
+      # skip themselves when nothing answers; yhub reads the JWKS back from the
+      # django server started alongside it, so both sides must share
+      # JWT_PRIVATE_KEY_FILE.
+      COLLABORATION_API_URL: http://localhost:3002/collaboration
+      JWT_PRIVATE_KEY_FILE: ${{ github.workspace }}/data/jwt/private.pem
 
     steps:
       - name: Checkout repository
@@ -210,5 +219,58 @@ jobs:
       - name: Generate a MO file from strings extracted from the project
         run: uv run python manage.py compilemessages
 
+      # --- collaboration server -------------------------------------------
+      # The yhub integration tests drive a real collaboration server: it reads
+      # legacy documents out of MinIO and reads this backend's JWKS back to
+      # verify the admin token the tests mint, so the two must share the
+      # signing key. Tests skip themselves when nothing answers on
+      # COLLABORATION_API_URL.
+      - name: Generate the JWT signing key
+        working-directory: .
+        run: bin/generate-jwt-private-key.sh
+
+      - name: Create the collaboration server database
+        run: |
+          PGPASSWORD=pass psql -h localhost -U dinum -d impress \
+            -c 'CREATE DATABASE yhub'
+          PGPASSWORD=pass psql -h localhost -U dinum -d yhub \
+            -f ../../docker/files/yhub/initdb/01-yhub.sql
+
+      - name: Set up Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+        with:
+          node-version: "22.x"
+
+      - name: Install the collaboration server
+        working-directory: src/yhub-server
+        run: npm ci --omit=dev
+
+      - name: Start the backend for the collaboration server to authenticate against
+        env:
+          DJANGO_ALLOWED_HOSTS: "*"
+        run: |
+          nohup uv run python manage.py runserver 0.0.0.0:8000 --noreload \
+            > /tmp/backend.log 2>&1 &
+          dockerize -wait http://localhost:8000/api/v1.0/jwks -timeout 60s
+
+      - name: Start the collaboration server
+        working-directory: src/yhub-server
+        env:
+          PORT: 3002
+          REDIS: redis://localhost:6379
+          POSTGRES: postgres://dinum:pass@localhost:5432/yhub
+          REDIS_PREFIX: yhub
+          COLLABORATION_BACKEND_BASE_URL: http://localhost:8000
+          COLLABORATION_SERVER_ORIGIN: http://localhost:3000
+          AWS_STORAGE_BUCKET_NAME: impress-media-storage
+          SOFT_MIGRATION: "true"
+        run: |
+          nohup node server.js > /tmp/yhub.log 2>&1 &
+          dockerize -wait tcp://localhost:3002 -timeout 30s
+
       - name: Run tests
         run: uv run pytest -n 2
+
+      - name: Collaboration server logs
+        if: failure()
+        run: cat /tmp/yhub.log /tmp/backend.log

.gitignore

@@ -47,6 +47,9 @@ env.d/terraform
 compose.override.yml
 docker/auth/*.local
 
+# yhub server local install
+src/yhub-server/node_modules/
+
 # npm
 node_modules
 
@@ -90,3 +93,4 @@ AGENTS.md
 .aider*
 .copilot/
 .github/copilot-instructions.md
+.playwright-mcp

CHANGELOG.md

@@ -8,6 +8,70 @@ and this project adheres to
 
 ### Added
 
+- ✅(collaboration) add integration tests covering both legacy migrations,
+  running against a real yhub: CI now starts the collaboration server and a
+  valkey alongside the backend test job. They skip themselves when nothing
+  answers on `COLLABORATION_API_URL`, so a `make test` without the dev stack
+  still passes
+- ✨(collaboration) soft-migrate legacy S3 documents into yhub on first access
+  (`SOFT_MIGRATION=true`): when yhub does not know a document yet, its legacy
+  snapshot (`{id}/file`, base64 Yjs update) is fetched from the Django S3
+  media bucket and seeded server-side (attributed to `system`, with no
+  timestamp — a lazy seed is not an editing event, and stamping one would
+  collide with the real per-version times the migrate endpoint writes) before
+  the connection is admitted. A missing S3 object means a brand-new document and
+  yields an empty room. Seeding never decides access: a legacy object that
+  cannot be migrated (undecodable or oversized) opens as a new document, logged
+  per access, since no retry could fix it and refusing would make the document
+  permanently unopenable. Every other failure — an unreachable store, but also
+  any refusal from S3 such as `AccessDenied` on a rotated key or a wrong bucket
+  name — answers a retryable `503`, so an empty document is never started on
+  top of content that exists.
+  Backend reads carrying the admin JWT are seeded too, so a server-side read of
+  an unmigrated document never answers with an empty one. Enabled in the dev
+  stack via compose.yml
+- ✨(collaboration) add a migrate endpoint on yhub:
+  `POST /collaboration/migrate/v1/docs/{id}` replays a document's **full**
+  legacy version history from the versioned S3 media bucket into a
+  `gc: false` Yjs document, crediting each S3 version with its own S3
+  timestamp — so `activity?group=false` reports the same timeline as the
+  backend's `/documents/{id}/versions/`, instead of the single
+  migration-time change the
+  lazy soft migration leaves behind. Purely additive: the result is stored as
+  one new row at clock `0`, so nothing existing is deleted and the next
+  compaction merges it like any other row. Idempotent by construction (the
+  clock-`0` insert is `ON CONFLICT DO NOTHING`, and migrated ids are recorded
+  in a valkey set), lock-free, admin JWT only, and the intended way to backfill
+  the corpus before `SOFT_MIGRATION` is turned off
+- ✨(collaboration) add a create-ydoc endpoint on yhub:
+  `POST /collaboration/create-ydoc/v1/docs/{id}` seeds a document's initial
+  Yjs state from a raw binary update posted as `application/octet-stream`,
+  so the Django backend can create documents without speaking yhub's lib0
+  wire encoding. Strict create (409 when the document already has content),
+  initial content attributed to the optional `X-User-Id` header; guarded by
+  standard document write access (the `aud: "yhub"` admin JWT, or a user
+  session with update ability)
+- ✨(collaboration) add an admin reset-connections endpoint on yhub:
+  `POST /collaboration/reset-connections/v1/docs/{id}` re-checks the
+  authorization of the document's connected clients and disconnects (close
+  code 4401) only those whose access changed. Authenticated with an admin JWT
+  verified against the backend JWKS and required to carry `aud: "yhub"`, so
+  an admin token Django issued for another service (e.g. the `y-converter`
+  one) cannot be replayed here; not yet triggered by the backend on
+  permission changes (follow-up)
+- ⬆️(collaboration) upgrade yhub to 0.5.0 and serve all its routes under the
+  `/collaboration/` prefix (`server.apiPrefix`): the websocket moves to
+  `/collaboration/ws/v1/docs`. All `/collaboration/` routes are meant to be
+  publicly exposed except `reset-connections` and `migrate`, which stay
+  backend-internal (admin JWT only). Following 0.5.0's error semantics
+  (`4xx` permanent, `5xx`/`429` retryable), the auth plugin now reports a
+  temporarily unreachable Django backend, JWKS endpoint or legacy S3 store as
+  `503` instead of denying access like a permission failure, so clients retry
+  instead of giving up. The built-in endpoints can also answer JSON on
+  `Accept: application/json`
+- ✨(backend) add a service generating cached RS256 JWT tokens
+- ✨(backend) publish the JWT public key on a JWKS endpoint
+- 🔧(dev) generate the JWT signing key when bootstrapping the dev stack
 - ♿️(frontend) restore skip to content link after header redesign #2510
 - 🌐(i18n) rename cn_CN to zh_CN, add eo_PL and zh_TW locales #2486
 
@@ -17,7 +81,21 @@ and this project adheres to
 
 ### Changed
 
+- 💥(backend) move the resource server JWKS from `/api/{version}/jwks` to
+  `/external_api/{version}/jwks`
 - ♿️(frontend) use semantic `<dl>` structure in document info card #2379
+- ♻️(collaboration) migrate the collaboration server from hocuspocus to yhub:
+  the dev stack gains dedicated valkey and postgres services for yhub. The
+  kick flow is deferred with TODO(yhub) stubs (the reset-connections
+  endpoint added above is its server-side replacement, backend wiring
+  pending). The get-connections API is dropped for good: its only consumer
+  was the removed can-edit mechanism, so it is not needed anymore
+- 🔥(backend) remove the unused `CollaborationService`
+- 💥(backend) remove the `documents/{id}/can-edit/` endpoint
+- 💥(y-provider) the published `lasuite/impress-y-provider` image becomes
+  converter-only and no longer serves `/collaboration/ws/`; deployments using
+  the existing helm values lose collaboration until the helm chart routes
+  collaboration to yhub (follow-up)
 
 ## [v5.4.1] - 2026-07-09
 

Makefile

@@ -69,6 +69,11 @@ data/media:
 data/static:
 	@mkdir -p data/static
 
+# RSA key signing the JWT tokens the backend issues. Generated locally, never
+# committed: "data/" is gitignored. Regenerate it by deleting the file.
+data/jwt/private.pem:
+	@bin/generate-jwt-private-key.sh
+
 # -- Project
 
 create-env-local-files: ## create env.local files in env.d/development
@@ -81,7 +86,8 @@ create-env-local-files:
 .PHONY: create-env-local-files
 
 generate-secret-keys:
-generate-secret-keys: ## generate secret keys to be stored in common.local
+generate-secret-keys: ## generate the secret keys needed by the dev stack
+generate-secret-keys: data/jwt/private.pem
 	@bin/generate-oidc-store-refresh-token-key.sh
 .PHONY: generate-secret-keys
 
@@ -190,6 +196,7 @@ bootstrap-e2e: \
 build: cache ?=
 build: ## build the project containers
 	@$(MAKE) build-backend cache=$(cache)
+	@$(MAKE) build-yhub cache=$(cache)
 	@$(MAKE) build-yjs-provider cache=$(cache)
 	@$(MAKE) build-frontend cache=$(cache)
 .PHONY: build
@@ -199,9 +206,14 @@ build-backend: ## build the app-dev container
 	@$(COMPOSE) build app-dev $(cache)
 .PHONY: build-backend
 
+build-yhub: cache ?=
+build-yhub: ## build the yhub collaboration server container
+	@$(COMPOSE) build yhub $(cache)
+.PHONY: build-yhub
+
 build-yjs-provider: cache ?=
 build-yjs-provider: ## build the y-provider container
-	@$(COMPOSE) build y-provider-development $(cache)
+	@$(COMPOSE) build y-provider-development-converter $(cache)
 .PHONY: build-yjs-provider
 
 build-frontend: cache ?=
@@ -212,8 +224,9 @@ build-frontend: ## build the frontend container
 build-e2e: cache ?=
 build-e2e: ## build the e2e container
 	@$(MAKE) build-backend cache=$(cache)
+	@$(MAKE) build-yhub cache=$(cache)
 	@$(COMPOSE_E2E) build frontend $(cache)
-	@$(COMPOSE_E2E) build y-provider $(cache)
+	@$(COMPOSE_E2E) build y-provider-converter $(cache)
 .PHONY: build-e2e
 
 nginx-frontend: ## build the nginx-frontend container
@@ -230,10 +243,11 @@ logs: ## display app-dev logs (follow mode)
 
 run-backend: ## Start only the backend application and all needed services
 	@$(MAKE) create-docker-network
+	@$(MAKE) data/jwt/private.pem
 	@$(COMPOSE) up --force-recreate -d docspec
 	@$(COMPOSE) up --force-recreate -d celery-dev
-	@$(COMPOSE) up --force-recreate -d y-provider-development
 	@$(COMPOSE) up --force-recreate -d y-provider-development-converter
+	@$(COMPOSE) up --force-recreate -d yhub
 	@$(COMPOSE) up --force-recreate -d nginx
 .PHONY: run-backend
 
@@ -246,9 +260,7 @@ run:
 run-e2e: ## start the e2e server
 run-e2e:
 	@$(MAKE) run-backend
-	@$(COMPOSE_E2E) stop y-provider-development
 	@$(COMPOSE_E2E) up --force-recreate -d frontend
-	@$(COMPOSE_E2E) up --force-recreate -d y-provider
 	@$(COMPOSE_E2E) up --force-recreate -d y-provider-converter
 .PHONY: run-e2e
 

UPGRADE.md

@@ -16,6 +16,13 @@ the following command inside your docker container:
 
 ## [Unreleased]
 
+- The JWKS of the resource server moved from `/api/{version}/jwks` to
+  `/external_api/{version}/jwks`, alongside the rest of the resource server
+  endpoints. `/api/{version}/jwks` now publishes the public key validating the
+  tokens Docs issues to call external services. If you enabled the resource
+  server (`OIDC_RESOURCE_SERVER_ENABLED`), update the JWKS URI declared to your
+  OIDC provider accordingly.
+
 ### [5.0.0] - 2026-04-30
 
 We made several changes around document content management leading to several breaking changes in the API.

bin/_config.sh

@@ -38,6 +38,10 @@ function _set_user() {
 # options: docker compose command options
 # ARGS   : docker compose command arguments
 function _docker_compose() {
+    # The backend settings point at this key and the containers mount it, so it
+    # has to exist before any of them starts.
+    "${REPO_DIR}/bin/generate-jwt-private-key.sh"
+
     # Set DOCKER_USER for Windows compatibility with MinIO
     if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || -n "${WSL_DISTRO_NAME:-}" ]]; then
         export DOCKER_USER="0:0"

bin/generate-jwt-private-key.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Generate the RSA private key signing the JWT tokens issued by the backend.
+#
+# Development only. The key is generated locally and never committed: it lands
+# in "data/", which is gitignored. The dev stack mounts it in the backend
+# containers, where JWT_PRIVATE_KEY_FILE points at it.
+#
+# Idempotent: an existing key is kept. Delete the file to roll the key.
+
+set -eo pipefail
+
+REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+KEY_PATH="${REPO_DIR}/data/jwt/private.pem"
+
+if [ -f "${KEY_PATH}" ]; then
+    exit 0
+fi
+
+mkdir -p "$(dirname "${KEY_PATH}")"
+openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${KEY_PATH}" 2>/dev/null
+chmod 600 "${KEY_PATH}"
+echo "✓ JWT private key generated in ${KEY_PATH}"

compose-e2e.yml

@@ -13,7 +13,7 @@ services:
     ports:
       - "3000:3000"
 
-  y-provider:
+  y-provider-converter:
     user: ${DOCKER_USER:-1000}
     build:
       context: .
@@ -24,16 +24,3 @@ services:
     env_file:
       - env.d/development/common
       - env.d/development/common.local
-    ports:
-      - "4444:4444"
-
-  y-provider-converter:
-    user: ${DOCKER_USER:-1000}
-    image: impress:y-provider-production
-    restart: unless-stopped
-    env_file:
-      - env.d/development/common
-      - env.d/development/common.local
-    depends_on:
-      y-provider:
-        condition: service_started