✨ (backend) POC: Store recordings in LaSuite Drive

CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to
 - ✨(backend) add roomkit viewset to start a room without WebRTC join
 - ✨(frontend) let users set default configuration for generated links
 - ✨(frontend) expose media state to external gateways
+- ✨(backend) push recordings to the owner's Drive (POC)
 
 ### Changed
 

Makefile

@@ -93,6 +93,7 @@ bootstrap: \
 	data/media \
 	data/static \
 	create-env-files \
+	create-docker-network \
 	build \
 	migrate \
 	demo \
@@ -126,11 +127,16 @@ down: ## stop and remove containers, networks, images, and volumes
 	@$(COMPOSE) down
 .PHONY: down
 
+create-docker-network: ## create the shared lasuite-network if it doesn't exist
+	@docker network create lasuite-network || true
+.PHONY: create-docker-network
+
 logs: ## display app-dev logs (follow mode)
 	@$(COMPOSE) logs -f app-dev
 .PHONY: logs
 
 run-backend: ## start only the backend application and all needed services
+	@$(MAKE) create-docker-network
 	@$(COMPOSE) up --force-recreate -d celery-dev --remove-orphans
 	@$(COMPOSE) up --force-recreate -d nginx
 	@$(COMPOSE) up -d livekit

compose.yml

@@ -14,6 +14,9 @@ services:
     image: sj26/mailcatcher:latest
     ports:
       - "1081:1080"
+    networks:
+      - default
+      - lasuite
 
   minio:
     user: ${DOCKER_USER:-1000}
@@ -33,6 +36,10 @@ services:
     command: minio server --console-address :9001 /data
     volumes:
       - ./data/media:/data
+    networks:
+      default:
+        aliases:
+          - meet-minio
 
   createbuckets:
     image: minio/mc
@@ -92,6 +99,7 @@ services:
     networks:
       - resource-server
       - default
+      - lasuite
 
   celery-dev:
     user: ${DOCKER_USER:-1000}
@@ -108,6 +116,9 @@ services:
       - /app/.venv
     depends_on:
       - app-dev
+    networks:
+      - default
+      - lasuite
 
   app:
     build:
@@ -195,32 +206,32 @@ services:
       - env.d/development/kc_postgresql
 
   keycloak:
-    image: quay.io/keycloak/keycloak:20.0.1
+    image: quay.io/keycloak/keycloak:26.3.2
     volumes:
       - ./docker/auth/realm.json:/opt/keycloak/data/import/realm.json
     command:
       - start-dev
       - --features=preview
       - --import-realm
-      - --proxy=edge
-      - --hostname-url=http://localhost:8083
-      - --hostname-admin-url=http://localhost:8083/
+      - --proxy-headers=xforwarded
+      - --hostname=http://localhost:8083
       - --hostname-strict=false
-      - --hostname-strict-https=false
     environment:
-      KEYCLOAK_ADMIN: admin
-      KEYCLOAK_ADMIN_PASSWORD: admin
+      KC_BOOTSTRAP_ADMIN_USERNAME: admin
+      KC_BOOTSTRAP_ADMIN_PASSWORD: admin
       KC_DB: postgres
       KC_DB_URL_HOST: kc_postgresql
       KC_DB_URL_DATABASE: keycloak
       KC_DB_PASSWORD: pass
       KC_DB_USERNAME: meet
       KC_DB_SCHEMA: public
-      PROXY_ADDRESS_FORWARDING: 'true'
     ports:
       - "8080:8080"
     depends_on:
       - kc_postgresql
+    networks:
+      - default
+      - lasuite
 
   livekit:
     image: livekit/livekit-server
@@ -337,3 +348,6 @@ services:
 networks:
   default:
   resource-server:
+  lasuite:
+    name: lasuite-network
+    external: true

docker/auth/realm.json

@@ -845,6 +845,142 @@
         "offline_access",
         "microprofile-jwt"
       ]
+    },
+    {
+      "clientId": "drive",
+      "name": "",
+      "description": "",
+      "rootUrl": "",
+      "adminUrl": "",
+      "baseUrl": "",
+      "surrogateAuthRequired": false,
+      "enabled": "true",
+      "alwaysDisplayInConsole": false,
+      "clientAuthenticatorType": "client-secret",
+      "secret": "ThisIsAnExampleKeyForDevPurposeOnly",
+      "redirectUris": [
+        "http://localhost:3100/*",
+        "http://localhost:8171/*",
+        "http://localhost:8085/*"
+      ],
+      "webOrigins": [
+        "http://localhost:3100",
+        "http://localhost:8171",
+        "http://localhost:8085"
+      ],
+      "notBefore": 0,
+      "bearerOnly": false,
+      "consentRequired": false,
+      "standardFlowEnabled": true,
+      "implicitFlowEnabled": false,
+      "directAccessGrantsEnabled": false,
+      "serviceAccountsEnabled": false,
+      "publicClient": false,
+      "frontchannelLogout": true,
+      "protocol": "openid-connect",
+      "attributes": {
+        "access.token.lifespan": "-1",
+        "client.secret.creation.time": "1707820779",
+        "user.info.response.signature.alg": "RS256",
+        "post.logout.redirect.uris": "http://localhost:3100/*##http://localhost:8171/*##http://localhost:8085/*",
+        "oauth2.device.authorization.grant.enabled": "false",
+        "use.jwks.url": "false",
+        "backchannel.logout.revoke.offline.tokens": "false",
+        "use.refresh.tokens": "true",
+        "tls-client-certificate-bound-access-tokens": "false",
+        "oidc.ciba.grant.enabled": "false",
+        "backchannel.logout.session.required": "true",
+        "client_credentials.use_refresh_token": "false",
+        "acr.loa.map": "{}",
+        "require.pushed.authorization.requests": "false",
+        "display.on.consent.screen": "false",
+        "client.session.idle.timeout": "-1",
+        "token.response.type.bearer.lower-case": "false"
+      },
+      "authenticationFlowBindingOverrides": {},
+      "fullScopeAllowed": true,
+      "nodeReRegistrationTimeout": -1,
+      "defaultClientScopes": [
+        "web-origins",
+        "acr",
+        "roles",
+        "profile",
+        "email"
+      ],
+      "optionalClientScopes": [
+        "address",
+        "phone",
+        "offline_access",
+        "microprofile-jwt"
+      ]
+    },
+    {
+      "clientId": "deploycenter",
+      "name": "",
+      "description": "",
+      "rootUrl": "",
+      "adminUrl": "",
+      "baseUrl": "",
+      "surrogateAuthRequired": false,
+      "enabled": "true",
+      "alwaysDisplayInConsole": false,
+      "clientAuthenticatorType": "client-secret",
+      "secret": "ThisIsAnExampleKeyForDevPurposeOnly",
+      "redirectUris": [
+        "http://localhost:3100/*",
+        "http://localhost:8171/*",
+        "http://localhost:8085/*"
+      ],
+      "webOrigins": [
+        "http://localhost:3100",
+        "http://localhost:8171",
+        "http://localhost:8085"
+      ],
+      "notBefore": 0,
+      "bearerOnly": false,
+      "consentRequired": false,
+      "standardFlowEnabled": true,
+      "implicitFlowEnabled": false,
+      "directAccessGrantsEnabled": false,
+      "serviceAccountsEnabled": false,
+      "publicClient": false,
+      "frontchannelLogout": true,
+      "protocol": "openid-connect",
+      "attributes": {
+        "access.token.lifespan": "-1",
+        "client.secret.creation.time": "1707820779",
+        "user.info.response.signature.alg": "RS256",
+        "post.logout.redirect.uris": "http://localhost:3100/*##http://localhost:8171/*##http://localhost:8085/*",
+        "oauth2.device.authorization.grant.enabled": "false",
+        "use.jwks.url": "false",
+        "backchannel.logout.revoke.offline.tokens": "false",
+        "use.refresh.tokens": "true",
+        "tls-client-certificate-bound-access-tokens": "false",
+        "oidc.ciba.grant.enabled": "false",
+        "backchannel.logout.session.required": "true",
+        "client_credentials.use_refresh_token": "false",
+        "acr.loa.map": "{}",
+        "require.pushed.authorization.requests": "false",
+        "display.on.consent.screen": "false",
+        "client.session.idle.timeout": "-1",
+        "token.response.type.bearer.lower-case": "false"
+      },
+      "authenticationFlowBindingOverrides": {},
+      "fullScopeAllowed": true,
+      "nodeReRegistrationTimeout": -1,
+      "defaultClientScopes": [
+        "web-origins",
+        "acr",
+        "roles",
+        "profile",
+        "email"
+      ],
+      "optionalClientScopes": [
+        "address",
+        "phone",
+        "offline_access",
+        "microprofile-jwt"
+      ]
     }
   ],
   "clientScopes": [

docs/features/recording.md

@@ -126,6 +126,61 @@ RECORDING_STORAGE_EVENT_TOKEN = <token>
 > Questions? Open an issue on [GitHub](https://github.com/suitenumerique/meet/issues/new?assignees=&labels=bug&template=Bug_report.md) or join our [Matrix community](https://matrix.to/#/#meet-official:matrix.org).
 
 
+## Push recordings to Drive
+
+Once a recording is over, it can be pushed (streamed) to the user's [Drive](https://github.com/suitenumerique/drive), in addition to staying in the object storage.
+
+Drive is called as a resource server, following its
+[resource server documentation](https://github.com/suitenumerique/drive/blob/main/docs/resource_server.md):
+
+
+### Special requirements
+
+- Drive configured as an OIDC resource server, accepting Meet's audience
+  (`OIDC_RS_ALLOWED_AUDIENCES` must contain Meet's client id), with the `items`
+  endpoint allowing the `list`, `children` and `upload_ended` actions.
+- `OIDC_STORE_ACCESS_TOKEN` enabled on Meet, along with
+  `OIDC_STORE_REFRESH_TOKEN_KEY`, the Fernet key encrypting the stored token.
+
+> [!CAUTION]
+> This is a proof of concept: the access token is captured when the recording
+> starts and assumed to still be valid when the recording ends. Long recordings
+> may therefore fail to be pushed. Exchanging it for a long-lived, narrowly
+> scoped token is the intended follow-up.
+
+### Configuration options
+
+| Option                                                | Type        | Default | Description                                                                                                                                        |
+| ----------------------------------------------------- | ----------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------- |
+| **RECORDING_PUSH_TO_DRIVE_ENABLED**                   | Boolean     | `False` | Enable pushing recordings to the owner's Drive.                                                                                                     |
+| **DRIVE_API_BASE_URL**                                | String      | `None`  | Base URL of Drive's external API, e.g. `https://fichiers.numerique.gouv.fr/external_api/v1.0`.                                                                |
+| **RECORDING_PUSH_TO_DRIVE_SIGNED_URL_EXPIRY_SECONDS** | Integer     | `3600`  | Lifetime of the signed URL the worker downloads the recording from.                                                                                 |
+| **OIDC_STORE_ACCESS_TOKEN**                           | Boolean     | `False` | Keep the user's access token in the session, required to call Drive on their behalf.                                                                 |
+| **OIDC_STORE_REFRESH_TOKEN_KEY**                      | Secret/File | `None`  | Fernet key encrypting OIDC tokens at rest. Generate one with `Fernet.generate_key()`.                                                                |
+| **DRIVE_UPLOAD_STORAGE_NETLOC**                       | String      | `None`  | Development only: `host:port` to reach Drive's object storage at, when the domain Drive signs its upload URLs with only resolves from a browser.     |
+
+### Local development
+
+Meet and Drive run as two separate compose projects, joined by the external
+`lasuite-network` (`make create-docker-network`). Meet's backend containers reach
+Drive's nginx at `drive-nginx:8083` and its object storage at `drive-minio:9000`.
+
+On the Drive side:
+
+```bash
+OIDC_RESOURCE_SERVER_ENABLED=True
+OIDC_RS_CLIENT_ID=drive
+OIDC_RS_CLIENT_SECRET=ThisIsAnExampleKeyForDevPurposeOnly
+OIDC_RS_AUDIENCE_CLAIM=client_id
+OIDC_RS_ALLOWED_AUDIENCES=meet
+```
+
+`DRIVE_UPLOAD_STORAGE_NETLOC` is needed because Drive signs its upload URLs
+with `localhost:9100`, which does not resolve from Meet's containers. The
+presigned signature covers the `Host` header, so the backend keeps announcing the
+signed host and only swaps the address it connects to.
+
+
 ## LiveKit Egress
 
 La Suite Meet uses LiveKit Egress to record room sessions. For reference, see the [LiveKit Egress repository](https://github.com/livekit/egress) and the [official documentation](https://docs.livekit.io/home/egress/overview/).

env.d/development/common.dist

@@ -24,7 +24,7 @@ MEET_BASE_URL="http://localhost:8072"
 # Media
 STORAGES_STATICFILES_BACKEND=django.contrib.staticfiles.storage.StaticFilesStorage
 AWS_S3_DOMAIN_REPLACE=http://localhost:9000
-AWS_S3_ENDPOINT_URL=http://minio:9000
+AWS_S3_ENDPOINT_URL=http://meet-minio:9000
 AWS_S3_ACCESS_KEY_ID=meet
 AWS_S3_SECRET_ACCESS_KEY=password
 MEDIA_BASE_URL=http://localhost:3000
@@ -50,6 +50,10 @@ LOGOUT_REDIRECT_URL=http://localhost:3000
 OIDC_REDIRECT_ALLOWED_HOSTS=localhost:8083,localhost:3000
 OIDC_AUTH_REQUEST_EXTRA_PARAMS={"acr_values": "eidas1"}
 
+# The key below encrypts OIDC tokens at rest.
+OIDC_STORE_ACCESS_TOKEN=True
+OIDC_STORE_REFRESH_TOKEN_KEY=
+
 OIDC_RS_CLIENT_ID=meet
 OIDC_RS_CLIENT_SECRET=ThisIsAnExampleKeyForDevPurposeOnly
 
@@ -70,6 +74,12 @@ SUMMARY_SERVICE_API_TOKEN=password
 SUMMARY_SERVICE_WEBHOOK_API_TOKEN=webhook-password
 RECORDING_DOWNLOAD_BASE_URL=http://localhost:3000/recording
 
+# Push recordings to Drive
+# DRIVE_UPLOAD_STORAGE_NETLOC is a development-only workaround.
+RECORDING_PUSH_TO_DRIVE_ENABLED=True
+DRIVE_API_BASE_URL=http://drive-app-dev:8000/external_api/v1.0
+DRIVE_UPLOAD_STORAGE_NETLOC=drive-minio:9000
+
 # Recording encoding (LiveKit Egress advanced options).
 # When RECORDING_ENCODING_ENABLED is False (default), LiveKit uses its built-in
 # H264_720P_30 preset (1280x720, 30fps, 3000 kbps). Enable and tune to reduce

env.d/development/metadata_collector.dist

@@ -2,7 +2,7 @@ LIVEKIT_URL=ws://livekit:7880
 LIVEKIT_API_KEY=devkey
 LIVEKIT_API_SECRET=secret
 
-AWS_S3_ENDPOINT_URL=minio:9000
+AWS_S3_ENDPOINT_URL=meet-minio:9000
 AWS_S3_ACCESS_KEY_ID=meet
 AWS_S3_SECRET_ACCESS_KEY=password
 AWS_STORAGE_BUCKET_NAME=meet-media-storage

env.d/development/summary.dist

@@ -2,7 +2,7 @@ APP_NAME="meet-app-summary-dev"
 APP_API_TOKEN="password"
 
 AWS_STORAGE_BUCKET_NAME="meet-media-storage"
-AWS_S3_ENDPOINT_URL="minio:9000"
+AWS_S3_ENDPOINT_URL="meet-minio:9000"
 AWS_S3_SECURE_ACCESS=false
 
 AWS_S3_ACCESS_KEY_ID="meet"