diff --git a/README.md b/README.md index 93a666d..bee5ea1 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,12 @@ From a fresh clone: docker compose up --build ``` -The Compose entrypoint generates and persists a random creator-authority encryption +The Compose entrypoint generates and persists a random runtime master key in the private `lock-home` volume. Set -`PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY` before startup only when you need to supply -your own 32-byte base64url key. +`PUBKY_LOCK_RUNTIME_MASTER_KEY` before startup only when you need to supply +your own 32-byte unpadded-base64url key. A supplied override is atomically +persisted to that volume, so a later startup without the environment variable +continues using the same key rather than silently reverting to an older key. Verified browser-facing defaults are: @@ -425,7 +427,8 @@ Example: "params": { "recipient_pubky": "pubky", "amount": "50000", - "asset": "BTC" + "asset": "BTC", + "payment_in": 24 } } ], @@ -458,7 +461,7 @@ For example: A submitted proof bundle is sent by the viewer before verification. It is not stored as an entitlement unless verification succeeds. -For `paykit-payment`, the content lock criterion params are exactly `recipient_pubky`, positive base-unit string `amount`, and non-empty `asset`. `recipient_pubky` must equal the content-lock creator. In v1 it must be the lock's only criterion, referenced exactly once by the lock logic. The submitted proof carries no payment details in its proof payload; it uses top-level `reader_public_key` plus the canonical `pubky_lock_resource` so the Lock Server can create the Paykit invoice. +For `paykit-payment`, the content lock criterion params are exactly `recipient_pubky`, positive base-unit string `amount`, non-empty `asset`, and positive whole-hour JSON `u64` `payment_in`. `recipient_pubky` must equal the content-lock creator. In v1 it must be the lock's only criterion, referenced exactly once by the lock logic. The submitted proof carries no payment details in its proof payload; it uses top-level `reader_public_key` plus the canonical `pubky_lock_resource` so the Lock Server can create the Paykit invoice. Example: diff --git a/docker-compose.yml b/docker-compose.yml index f63d6ba..b5f2b23 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,7 +49,7 @@ services: network_mode: service:pubky-testnet environment: PUBKY_LOCK_DATABASE_URL: postgres://locks:locks@postgres:5432/locks_test - PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY: ${PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY:-} + PUBKY_LOCK_RUNTIME_MASTER_KEY: ${PUBKY_LOCK_RUNTIME_MASTER_KEY:-} volumes: - lock-home:/var/lib/pubky-lock command: ["locks-server-compose-entrypoint.sh"] diff --git a/docker/locks-server-compose-entrypoint.sh b/docker/locks-server-compose-entrypoint.sh index ed28851..ed33851 100644 --- a/docker/locks-server-compose-entrypoint.sh +++ b/docker/locks-server-compose-entrypoint.sh @@ -5,25 +5,74 @@ service_home="${LOCKS_SERVICE_HOME:-/var/lib/pubky-lock/.pubky-lock}" generated_config="$service_home/config.toml" compose_config="${LOCKS_COMPOSE_CONFIG:-/var/lib/pubky-lock/config.compose.toml}" secret_path="$service_home/secret.sess" -creator_authority_key_path="$service_home/creator-authority-encryption-key" +runtime_master_key_path="$service_home/runtime-master-key" +retired_creator_authority_key_path="$service_home/creator-authority-encryption-key" mkdir -p "$service_home" -if [ -z "${PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY:-}" ]; then - if [ ! -f "$creator_authority_key_path" ]; then - echo "[locks-compose] generating creator-authority encryption key" +if [ -f "$retired_creator_authority_key_path" ]; then + echo "[locks-compose] retired creator-authority key detected: $retired_creator_authority_key_path" >&2 + echo "[locks-compose] stop the stack, discard and reacquire creator authority rows or recreate the local database, remove the retired key file, then restart" >&2 + exit 1 +fi + +if [ -n "${PUBKY_LOCK_RUNTIME_MASTER_KEY:-}" ]; then + if ! printf '%s' "$PUBKY_LOCK_RUNTIME_MASTER_KEY" | grep -Eq '^[A-Za-z0-9_-]{43}$'; then + echo "[locks-compose] PUBKY_LOCK_RUNTIME_MASTER_KEY must be an unpadded base64url-encoded 32-byte key" >&2 + exit 1 + fi + umask 077 + decoded_key_path="$runtime_master_key_path.decoded.$$" + cleanup_decoded_key() { + rm -f "$decoded_key_path" + } + trap cleanup_decoded_key EXIT HUP INT TERM + if ! printf '%s=' "$PUBKY_LOCK_RUNTIME_MASTER_KEY" \ + | tr '_-' '/+' \ + | base64 -d > "$decoded_key_path" 2>/dev/null \ + || [ "$(wc -c < "$decoded_key_path" | tr -d ' ')" -ne 32 ]; then + echo "[locks-compose] PUBKY_LOCK_RUNTIME_MASTER_KEY must be an unpadded base64url-encoded 32-byte key" >&2 + exit 1 + fi + canonical_runtime_master_key="$( + base64 < "$decoded_key_path" \ + | tr '+/' '-_' \ + | tr -d '=\n' + )" + if [ "$canonical_runtime_master_key" != "$PUBKY_LOCK_RUNTIME_MASTER_KEY" ]; then + echo "[locks-compose] PUBKY_LOCK_RUNTIME_MASTER_KEY must be an unpadded base64url-encoded 32-byte key" >&2 + exit 1 + fi + cleanup_decoded_key + trap - EXIT HUP INT TERM + if [ -f "$runtime_master_key_path" ]; then + persisted_runtime_master_key="$(cat "$runtime_master_key_path")" + if [ "$persisted_runtime_master_key" != "$PUBKY_LOCK_RUNTIME_MASTER_KEY" ]; then + echo "[locks-compose] PUBKY_LOCK_RUNTIME_MASTER_KEY does not match the persisted runtime master key" >&2 + echo "[locks-compose] rotate only through an explicit data migration or reset that handles encrypted state" >&2 + exit 1 + fi + else + temporary_key_path="$runtime_master_key_path.tmp.$$" + printf '%s' "$PUBKY_LOCK_RUNTIME_MASTER_KEY" > "$temporary_key_path" + chmod 600 "$temporary_key_path" + mv "$temporary_key_path" "$runtime_master_key_path" + fi +else + if [ ! -f "$runtime_master_key_path" ]; then + echo "[locks-compose] generating runtime master key" umask 077 - temporary_key_path="$creator_authority_key_path.tmp.$$" + temporary_key_path="$runtime_master_key_path.tmp.$$" head -c 32 /dev/urandom \ | base64 \ | tr '+/' '-_' \ | tr -d '=\n' > "$temporary_key_path" chmod 600 "$temporary_key_path" - mv "$temporary_key_path" "$creator_authority_key_path" + mv "$temporary_key_path" "$runtime_master_key_path" fi - PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY="$(cat "$creator_authority_key_path")" - export PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY + PUBKY_LOCK_RUNTIME_MASTER_KEY="$(cat "$runtime_master_key_path")" + export PUBKY_LOCK_RUNTIME_MASTER_KEY fi if [ ! -f "$generated_config" ] || [ ! -f "$secret_path" ]; then @@ -78,7 +127,14 @@ frontend_session_code_ttl_seconds = 120 allowed_return_origins = ["http://localhost:8080"] [secrets] -creator_authority_key_env = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" +runtime_master_key_env = "PUBKY_LOCK_RUNTIME_MASTER_KEY" + +[deletion] +retry_max_attempts = 10 +retry_initial_backoff_seconds = 1 +retry_max_backoff_seconds = 300 +final_credential_issuance_window_seconds = 900 +final_read_window_seconds = 900 [logging] level = "info" diff --git a/docs/ADRs/0020-locks-paykit-v1-integration-boundary.md b/docs/ADRs/0020-locks-paykit-v1-integration-boundary.md index 9c999fd..b3da988 100644 --- a/docs/ADRs/0020-locks-paykit-v1-integration-boundary.md +++ b/docs/ADRs/0020-locks-paykit-v1-integration-boundary.md @@ -27,13 +27,15 @@ The v1 content-lock criterion has verifier wire value `paykit-payment` and param { "recipient_pubky": "pubky", "amount": "50000", - "asset": "BTC" + "asset": "BTC", + "payment_in": 24 } ``` - `recipient_pubky` must equal the canonical content-lock creator. - `amount` is a positive decimal integer string in the asset's base unit. - `asset` is an opaque, non-empty string to Locks. Paykit Server owns deployment-specific asset support and base-unit interpretation. +- `payment_in` is a required, nonzero JSON `u64` number of whole hours in Locks policy. - V1 permits exactly one payment criterion, referenced exactly once by the lock logic, and exactly one submitted payment proof. - The submitted payment proof payload is `{}`. `reader_public_key` is top-level submission data. - Content-lock authoring does not require runtime Paykit configuration or availability. @@ -44,14 +46,14 @@ The v1 content-lock criterion has verifier wire value `paykit-payment` and param 1. apply rate limiting; 2. validate proof shape; -3. load and validate the current canonical Lock Resource and payment policy; -4. resolve the current reader through Pubky discovery; -5. compare any persisted lifecycle under `{ creator, bundle_id }`; -6. return an exact persisted replay or reject changed submitted proof material with `409 task_state_conflict`; -7. require configured Paykit and create an invoice only for a new identity; and -8. insert the verification task with post-invoice race reconciliation. +3. compare any durable lifecycle or admission reservation under `{ creator, bundle_id }` before mutable Lock Resource lookup or reader discovery; +4. return an exact ready replay, resume an exact unready reservation from its persisted submission fields, or reject changed submitted proof material with `409 task_state_conflict`; +5. only for a genuinely new identity, load and validate the current canonical Lock Resource and payment policy and resolve the current reader through Pubky discovery; +6. require configured Paykit and atomically persist a hidden, unclaimable admission reservation under the per-lock deletion/admission fence; +7. create or idempotently replay the Paykit invoice; and +8. mark the reservation ready so the verification task becomes publicly visible and worker-claimable. -Exact and changed persisted replays do not call Paykit. Terminal lifecycle state is not restarted under the same identity; clients needing another attempt must generate a new Bundle ID. +Exact ready replay does not call Paykit. Exact unready replay requires configured Paykit and repeats the same persisted idempotent invoice request before marking the reservation ready. Changed replay conflicts without calling Paykit. Terminal lifecycle state is not restarted under the same identity; clients needing another attempt must generate a new Bundle ID. ### Invoice request @@ -67,7 +69,7 @@ For a new lifecycle identity, Locks sends RFC 8785 canonical JSON to `POST /invo Locks signs the exact canonical body bytes with its existing Ed25519 keypair and sends the unpadded-base64url signature in `X-Paykit-Signature`. -The durable Paykit invoice identity is `(creator, bundle_id)`, where Paykit derives `creator` from `lock_resource`. Exact replay must return the original generic success without repeating mutable lookups, allocation, address creation, or delivery side effects. A different binding under the same identity returns Paykit `409 Conflict`, which Locks maps to `409 task_state_conflict`. Locks accepts any Paykit 2xx response and ignores its body; other invoice failures return `502 paykit_invoice_creation_failed` without creating a new verification task. +The durable Paykit invoice identity is `(creator, bundle_id)`, where Paykit derives `creator` from `lock_resource`. Exact replay must return the original generic success without repeating mutable lookups, allocation, address creation, or delivery side effects. A different binding under the same identity returns Paykit `409 Conflict`, which Locks maps to `409 task_state_conflict`. Locks accepts any Paykit 2xx response and ignores its body; other invoice failures return `502 paykit_invoice_creation_failed` while the internal Locks reservation remains hidden and unclaimable for exact retry. ### Status request and access policy @@ -92,7 +94,7 @@ V1 has no invoice expiry, TTL, `expires_at`, or terminal Paykit payment-failure ### Runtime boundary -- A new payment lifecycle requires `[paykit]`; exact persisted replay does not. +- A new payment lifecycle and exact unready reconciliation require `[paykit]`; exact ready replay does not. - Paykit HTTP connect timeout is 5 seconds and whole-request timeout is 20 seconds. - An enabled in-process Paykit worker requires `claim_timeout_seconds > 20`. - `worker.poll_interval_ms` must be greater than zero whether the worker is enabled or disabled. @@ -112,7 +114,7 @@ V1 has no invoice expiry, TTL, `expires_at`, or terminal Paykit payment-failure - Both services must implement the same canonical-body signing contract. - Paykit must parse the public Locks payment criterion and therefore depends on its versioned shape. -- Exact submission replay intentionally performs current canonical lock and reader preflight before returning persisted lifecycle state. +- Incomplete admission reservations require durable reconciliation before deletion may start the Paykit drain. - Unpaid invoices and pending Locks tasks have no protocol expiry in v1 and therefore require operational retention policy outside the payment-status contract. ## Rejected alternatives diff --git a/docs/API.md b/docs/API.md index 516ebd3..dceabf3 100644 --- a/docs/API.md +++ b/docs/API.md @@ -19,11 +19,12 @@ The Lock Server has one non-production route family and one authenticated creato - `POST /verification-task-completions` - Requires `runtime.environment = "development"`. - `staging` and `production` never mount it. -- Authenticated creator publishing routes: `PUT /creator/priv-resources/content/`, `DELETE /creator/priv-resources/content/`, `POST /creator/content-locks`, `POST /creator/lock-service-config` +- Authenticated creator publishing routes: `PUT /creator/priv-resources/content/`, `DELETE /creator/priv-resources/content/`, `POST /creator/content-locks`, `DELETE /creator/content-locks/{lock_id}`, `GET /creator/content-locks/{lock_id}/deletion`, `POST /creator/lock-service-config` - Always Pubky homeserver-backed. - Can run in `development`, `staging`, or `production`. - Require `Authorization: Bearer `. - Derive creator identity from the frontend session. Request-body `creator` is rejected for authenticated routes. + - A guarded path can be owned by only one managed Content Lock for that creator. Creating a different Lock ID for an owned path returns `409 content_lock_path_conflict`. - Missing/unknown/expired frontend sessions use the JSON error envelope (`401 frontend_session_unavailable` or `401 frontend_session_expired`). - Missing/revoked creator-granted homeserver authority remains a separate operational error (`503 creator_authority_unavailable`). - Creator authority status route: `GET /creator/authority-status` @@ -46,7 +47,9 @@ Gated-off routes are plain Axum `404 Not Found` responses because the route is i | --- | --- | --- | --- | --- | | `PUT /creator/priv-resources/content/` | `200` JSON guarded-resource descriptor | Requires `Authorization: Bearer `. Raw bytes body; MIME from `Content-Type`. | No bearer secrets or raw bytes in response. | `400 invalid_request`, `401 frontend_session_unavailable`, `401 frontend_session_expired`, `413 payload_too_large`, `503 creator_authority_unavailable` | | `DELETE /creator/priv-resources/content/` | `204` empty response | Requires `Authorization: Bearer `. | No bearer secrets or raw bytes in response. | `401 frontend_session_unavailable`, `401 frontend_session_expired`, `404 guarded_resource_not_found`, `503 creator_authority_unavailable` | -| `POST /creator/content-locks` | `200` JSON content lock | Requires `Authorization: Bearer `. | No bearer secrets in response. | `400 invalid_request`, `404 guarded_resource_not_found`, `401 frontend_session_unavailable`, `401 frontend_session_expired`, `503 creator_authority_unavailable` | +| `POST /creator/content-locks` | `200` JSON content lock | Requires `Authorization: Bearer `. | No bearer secrets in response. | `400 invalid_request`, `401 frontend_session_unavailable`, `401 frontend_session_expired`, `404 guarded_resource_not_found`, `409 content_lock_path_conflict`, `409 content_lock_deletion_in_progress`, `503 creator_authority_unavailable` | +| `DELETE /creator/content-locks/{lock_id}` | Graceful: `202` redacted lifecycle, or `200` completed absent postcondition. Replaying a failed graceful job requeues the same frozen manifest. `force=true`: synchronous `200` force summary when no active graceful job exists; an active graceful job is marked for worker escalation and returns `202`. | Requires `Authorization: Bearer `. Default and `graceful=true` are graceful; `force=true` is explicit and mutually exclusive. | No snapshots, task IDs, attempts, dependency errors, or force marker in lifecycle responses. Force summary contains only Lock ID, lock-deleted boolean, and failed guarded paths. | `400 invalid_request`, `400 invalid_identifier`, `401 frontend_session_unavailable`, `401 frontend_session_expired`, `503 creator_authority_unavailable` | +| `GET /creator/content-locks/{lock_id}/deletion` | `200` redacted lifecycle JSON | Requires `Authorization: Bearer `. | Contains only Lock ID, stable status, and an optional closed failure code. Permanent force receipts project as completed without exposing force mode. | `400 invalid_identifier`, `401 frontend_session_unavailable`, `401 frontend_session_expired`, `404 content_lock_deletion_not_found` | | `POST /creator/lock-service-config` | `200` JSON lock-service pointer | Requires `Authorization: Bearer `. | No bearer secrets in response. | `400 invalid_request`, `401 frontend_session_unavailable`, `401 frontend_session_expired`, `503 creator_authority_unavailable` | | `GET /connect` | `200` HTML Lock-Server-hosted connect shell | No bearer auth. Mounted when `[creator_authority_acquisition].enabled = true`; `return_to` must match `allowed_return_origins` or explicit wildcard policy. | HTML intentionally contains the secret-bearing Pubky authorization URL on Lock Server origin; response must not contain frontend session token, one-time code, or creator authority secret. | `400 invalid_request`, `503 creator_authority_unavailable`, `404` when route gated off | | `POST /connect/{flow_id}/complete` | `303` redirect to stored `return_to` | No bearer auth. Mounted when `[creator_authority_acquisition].enabled = true`; stored `return_to` is revalidated before redirect. | `Location` contains only callback `state` and one-time `code`; no authorization URL, frontend session token, or creator authority secret. | `400 invalid_request`, `404 creator_connect_flow_unavailable`, `410 creator_connect_flow_expired`, `503 creator_authority_unavailable`, `404` when route gated off | @@ -54,7 +57,7 @@ Gated-off routes are plain Axum `404 Not Found` responses because the route is i | `DELETE /frontend-sessions/current` | `204` empty response | Requires `Authorization: Bearer `. Mounted with creator authority acquisition. | Token is request-only and is deleted from the frontend session store. | `401 frontend_session_unavailable`, `401 frontend_session_expired`, `404` when route gated off | | `GET /.well-known/locks-server` | `200` JSON service identity | Public. Always mounted. CORS-enabled. | No secrets. Used by browser SDK to verify service, API version, and Lock Server Pubky identity. | n/a | | `GET /creator/authority-status` | `200` JSON secret-free authority status | Requires `Authorization: Bearer `. Creator is derived from the frontend session. | Response contains only creator, boolean status, auth kind, scopes, and optional expiry; no tokens, codes, authorization URLs, secrets, or DB/config values. | `401 frontend_session_unavailable`, `401 frontend_session_expired`, `404` only if route absent in older deployments | -| `POST /proof-bundles` | `200` JSON lifecycle | Public viewer route. A new `paykit-payment` lifecycle identity requires `[paykit]` runtime config; an exact persisted replay does not. | No bearer secrets, invoice data, or raw proof material in response. | `400 invalid_request`, `409 task_state_conflict`, `422 unsupported_verifier_type`, `422 paykit_not_configured`, `422 reader_pubky_unresolvable`, `429 rate_limited`, `502 paykit_invoice_creation_failed` | +| `POST /proof-bundles` | `200` JSON lifecycle | Public viewer route. A new or unready `paykit-payment` lifecycle identity requires `[paykit]` runtime config; an exact ready replay does not. | No bearer secrets, invoice data, or raw proof material in response. | `400 invalid_request`, `409 task_state_conflict`, `409 content_lock_deletion_in_progress`, `422 unsupported_verifier_type`, `422 paykit_not_configured`, `422 reader_pubky_unresolvable`, `429 rate_limited`, `502 paykit_invoice_creation_failed` | | `POST /verification-task-lookups` | `200` JSON lifecycle | Public viewer route. | No bearer secrets in response. | `400 invalid_request`, `404 verification_task_not_found` | | `POST /verification-task-completions` | `200` JSON lifecycle | Dev-only completion gate. | No bearer secrets in response. | `400 invalid_request`, `404 verification_task_not_found`, `409 task_state_conflict`, `404` when route gated off | | `POST /access-credentials` | `200` JSON credential | Public viewer route after entitlement. | Response intentionally contains raw viewer access credential exactly once. | `400 invalid_request`, `403 entitlement_not_authorized`, `404 verification_task_not_found` | @@ -98,13 +101,15 @@ Stable error codes and statuses mirror `locks-server/src/api/errors.rs` tests: | `frontend_session_expired` | 401 | Frontend session token existed but expired. | | `frontend_session_state_mismatch` | 400 | One-time code exchange state did not match. | | `creator_authority_unavailable` | 503 | Creator-granted homeserver authority is unavailable or could not be revalidated. | +| `content_lock_path_conflict` | 409 | A creator-scoped guarded path already has an in-flight/published owner, or the canonical Content Lock publication itself is still in flight. | | `task_state_conflict` | 409 | Submission or completion conflicts with existing task state. | +| `content_lock_deletion_in_progress` | 409 | A deletion cutoff committed before this new proof Bundle could be admitted. | | `unsupported_verifier_type` | 422 | Proof references a verifier unavailable in the current runtime. | | `paykit_not_configured` | 422 | A `paykit-payment` proof was submitted to a Lock Server without a `[paykit]` runtime section. | | `reader_pubky_unresolvable` | 422 | A `paykit-payment` proof had a syntactically valid `reader_public_key` that could not be resolved to a Pubky homeserver/PKARR record before invoice creation. | | `rate_limited` | 429 | Submission exceeded configured rate limits. | | `payload_too_large` | 413 | Raw guarded-resource upload exceeded `[content_locks].max_resource_bytes`. | -| `paykit_invoice_creation_failed` | 502 | Lock Server could not create the Paykit invoice; no verification task is created. | +| `paykit_invoice_creation_failed` | 502 | Lock Server could not create or replay the Paykit invoice; no verification task is publicly admitted or worker-claimable. | | `internal_error` | 500 | Unexpected server-side failure. | ## Service discovery @@ -324,6 +329,22 @@ Authorization: Bearer Success returns `204 No Content`. Missing resources return `404 guarded_resource_not_found`. +### `DELETE /creator/content-locks/{lock_id}` + +Requires the authenticated creator frontend session. With no query, or with `graceful=true`, it starts or replays the durable graceful deletion job. Queued and running work returns `202` with only `lock_id` and `status`; a completed-and-forgotten absent lock returns `200 { "lock_id": "...", "status": "completed" }`. + +Graceful withdrawal compares the current public lock bytes with the frozen canonical lock before publishing the tombstone. Pubky 0.9.3 does not provide an atomic conditional write, so this check and the tombstone `PUT` are not one operation. A replacement already visible to the check is preserved and fails the deletion closed, but an out-of-band creator replacement written after that check and before the `PUT` can be overwritten by the tombstone. This is an explicitly accepted limitation until Pubky supports conditional writes. Graceful cleanup does not delete replacement bytes; unconditional public deletion remains exclusive to `force=true`. + +`force=true` is explicit and cannot be combined with `graceful=true`. With no active graceful job, force deletion synchronously stores the permanent blocking receipt, removes the public lock/tombstone, and then best-effort deletes guarded resources. A terminal graceful job supplies its frozen manifest for this synchronous cleanup. It returns exactly `{ "lock_id": "...", "lock_deleted": true, "failed_resource_paths": ["..."] }`. With a queued or running graceful job, it revokes any current worker claim, durably requeues the job for force escalation, and returns that redacted job at `202`. An in-flight canonical publication returns redacted `409 content_lock_path_conflict`; force has not started and no receipt exists, so the creator may retry after publication reconciles. + +Unknown query keys, malformed or false booleans, and ambiguous modes return `400 invalid_request`. The accepted wire forms are exactly no query, `graceful=true`, or `force=true`. + +Rust SDK callers use `CreatorLocks::delete_content_lock(DeleteContentLockRequest { lock_id, mode })`, where `DeleteContentLockMode` is closed to `DefaultGraceful`, `ExplicitGraceful`, and `Force`. JS/WASM callers use `creator.deleteContentLock(lockId, options?)`; omitting `options` selects default graceful, while `new DeleteContentLockOptions(DeleteContentLockMode.ExplicitGraceful)` and `new DeleteContentLockOptions(DeleteContentLockMode.Force)` select the two explicit query modes. + +### `GET /creator/content-locks/{lock_id}/deletion` + +Returns only `{ "lock_id": "...", "status": "queued|running|completed|failed", "failure_code"?: "..." }`. The optional failure vocabulary is closed to `tombstone_missing`, `tombstone_replaced`, `resource_replaced`, `retry_exhausted`, and `state_corrupt`. If neither a job nor permanent force receipt exists, it returns `404 content_lock_deletion_not_found`. + ### `POST /creator/content-locks` Fixtures: @@ -338,7 +359,7 @@ Creates or replaces a content lock from a resource set. A content lock may conta At least one resource is required. If a primary resource is present, its path must not also appear in `secondary_resources`. `secondary_resources` keys are full canonical private paths such as `/priv/locks.app/content/attachments/a.txt`. -With Pubky-backed repositories, this writes the public content lock JSON to the creator homeserver under its derived `content_lock_path`. Test-support composition may use in-memory repositories behind the same authenticated route contract. +With Pubky-backed repositories, this writes the public content lock JSON to the creator homeserver under its derived `content_lock_path`. Before external publication, Locks persists an opaque per-lock publication intent under the same PostgreSQL fence used by graceful deletion and force-receipt establishment. Deletion cannot start while that intent exists; force therefore cannot report permanent deletion and then lose a race to a late Pubky write. After an upsert error, Locks reads the canonical path: an exact expected lock is reconciled as published, proven absence permits reservation compensation, and a mismatched payload or failed read retains both ownership and intent for fail-closed operator reconciliation. The intent is removed only after path ownership is durably marked published or safely compensated. Test-support composition may use in-memory repositories behind the same authenticated route contract. Every referenced guarded resource must currently exist for the same creator/path and must match hash, content type, and size. If the creator has overwritten or deleted a guarded resource path, content lock creation rejects the stale descriptor. @@ -348,11 +369,12 @@ Every referenced guarded resource must currently exist for the same creator/path { "recipient_pubky": "pubky", "amount": "50000", - "asset": "BTC" + "asset": "BTC", + "payment_in": 24 } ``` -`recipient_pubky` must be a valid Pubky public key string equal to the content-lock creator, `amount` must be a positive base-unit integer encoded as a string, and `asset` must be a non-empty string. The lock params do not include Paykit server URLs, account IDs, memos, expiry, payment references, or reader identity. A v1 content lock that uses `paykit-payment` must contain exactly that one criterion, and its `all` or `any` lock logic must reference that criterion exactly once. Mixed criteria, multiple payment criteria, recipient/creator mismatch, and duplicate or mismatched logic references return `400 invalid_request`. +`recipient_pubky` must be a valid Pubky public key string equal to the content-lock creator, `amount` must be a positive base-unit integer encoded as a string, `asset` must be a non-empty string, and `payment_in` must be a positive whole-hour JSON `u64`. The lock params do not include Paykit server URLs, account IDs, memos, expiry, payment references, or reader identity. A v1 content lock that uses `paykit-payment` must contain exactly that one criterion, and its `all` or `any` lock logic must reference that criterion exactly once. Mixed criteria, multiple payment criteria, recipient/creator mismatch, and duplicate or mismatched logic references return `400 invalid_request`. #### Request @@ -523,7 +545,7 @@ Success response returns lifecycle metadata only. It does not return internal `t For non-payment verifier types, `reader_public_key` may be omitted. For `paykit-payment`, `reader_public_key` is required as a top-level field on `submitted_proof_bundle`; the payment proof payload itself must be `{}`. Payment submissions are v1 single-proof only: a bundle with more than one `paykit-payment` proof, or a mix of `paykit-payment` and any other proof type, is rejected with `400 invalid_request`. -Submission processing applies rate limiting, validates proof shape, loads the current canonical content lock referenced by `pubky_lock_resource`, verifies its lock identity and payment policy (including recipient/creator equality), and resolves `reader_public_key` through Pubky/PKARR/homeserver discovery. It then checks the permanent lifecycle identity `{ creator, bundle_id }`. An exact persisted replay returns the existing lifecycle; changed submitted proof material returns `409 task_state_conflict`. Neither case calls Paykit again. Only a new identity requires `[paykit]` configuration and calls `POST /invoices`. Task insertion retains race reconciliation after invoice creation. The signed Paykit invoice body is exactly: +Submission processing applies rate limiting and validates the closed proof shape before checking the permanent lifecycle identity `{ creator, bundle_id }`. With PostgreSQL runtime storage, exact persisted binding is classified before mutable lock lookup or reader discovery. An exact ready replay returns the existing lifecycle without calling Paykit; an exact unready replay requires `[paykit]` and repeats the idempotent invoice request from the persisted submission fields; changed submitted proof material returns `409 task_state_conflict`. Only a genuinely new identity loads and validates the current canonical content lock, verifies its lock identity and payment policy (including recipient/creator equality), resolves `reader_public_key` through Pubky/PKARR/homeserver discovery, and requires `[paykit]`. Locks then atomically persists a hidden, unclaimable admission reservation before calling `POST /invoices`; Paykit success makes that task publicly visible and worker-claimable. The signed Paykit invoice body is exactly: ```json { @@ -533,7 +555,7 @@ Submission processing applies rate limiting, validates proof shape, loads the cu } ``` -Any 2xx Paykit invoice response is accepted and its body is ignored. Paykit invoice `409 Conflict` maps to `409 task_state_conflict`. Other invoice failures return `502 paykit_invoice_creation_failed`; no verification task is created unless invoice creation was accepted. +Any 2xx Paykit invoice response is accepted and its body is ignored. With PostgreSQL runtime storage, Locks first commits an internal, unclaimable admission reservation serialized against deletion start. It calls Paykit only after that commit, then makes the task publicly visible and worker-claimable after Paykit accepts the invoice. Durable handle replay is classified before consulting the mutable public lock or reader discovery: exact ready replay returns the persisted lifecycle, exact unready replay resumes the same idempotent Paykit request from persisted fields, and changed replay conflicts. Graceful deletion snapshots these reservations and cannot start the Paykit drain until every snapshotted reservation is ready, so Paykit has durably created every pre-cutoff invoice before its drain fence activates. Paykit invoice `409 Conflict` maps to `409 task_state_conflict`. Other invoice failures return `502 paykit_invoice_creation_failed`; the internal reservation remains hidden and unclaimable for exact retry. Paykit status verification is worker-owned. The Lock Server sends canonical JSON `{ "creator": "pubky...", "bundle_id": "..." }` to `POST /transactions/status` with `X-Paykit-Signature` over those exact canonical body bytes. Valid response statuses are `undetected`, `detected`, and `confirmed`. Transport failures, timeouts, every non-2xx response (including `404` and authentication/authorization failures), and malformed success bodies are durably rescheduled as pending and are not retried again before the worker poll interval elapses. V1 has no terminal Paykit payment-failure status. diff --git a/docs/DOMAIN_MODEL.md b/docs/DOMAIN_MODEL.md index b544c7c..72c806c 100644 --- a/docs/DOMAIN_MODEL.md +++ b/docs/DOMAIN_MODEL.md @@ -62,7 +62,7 @@ Completion is worker-owned in production-shaped runtime. The server runs an in-p The dev HTTP completion route, `POST /verification-task-completions`, is not a production route. It accepts `{ creator, bundle_id }`, resolves the internal task, and is mounted only when `runtime.environment = "development"`; `staging` and `production` never mount it. -Creator publishing routes are authenticated and Pubky homeserver-backed. `PUT /creator/priv-resources/content/`, `DELETE /creator/priv-resources/content/`, `POST /creator/content-locks`, and `POST /creator/lock-service-config` require `Authorization: Bearer `, derive creator from the Locks-local frontend session, and reject request-body creator spoofing. The raw guarded-resource upload body is bytes, not JSON/base64. +Creator publishing and deletion routes are authenticated and Pubky homeserver-backed. `PUT /creator/priv-resources/content/`, `DELETE /creator/priv-resources/content/`, `POST /creator/content-locks`, `DELETE /creator/content-locks/{lock_id}`, `GET /creator/content-locks/{lock_id}/deletion`, and `POST /creator/lock-service-config` require `Authorization: Bearer ` and derive creator from the Locks-local frontend session. The raw guarded-resource upload body is bytes, not JSON/base64. Deletion validates that any fetched Content Lock hashes to the requested Lock ID and belongs to that authenticated creator before freezing or deleting its manifest. Graceful-job creation/resume and permanent force-receipt creation share one canonical per-lock PostgreSQL fence, so active graceful work and a force receipt cannot coexist. Failed graceful replay requeues the same frozen job; synchronous force replaces terminal operational job state with the permanent receipt. Force against an active graceful job remains a durable worker escalation. Runtime health and readiness are Lock Server operator concerns. `GET /healthz` reports process liveness once the HTTP router is serving. `GET /readyz` reports whether runtime dependencies are usable: `ephemeral` readiness currently means in-memory process composition, while `persisted` readiness currently means the Postgres-backed runtime can ping its configured pool. Health/readiness responses must be small and secret-free; they must not expose database URLs, secret paths, Lock Server identities, worker IDs, task counts, Task IDs, claim metadata, submitted proof material, access credentials, or rate-limit counters. diff --git a/docs/LOCAL_OPERATOR_DEMO.md b/docs/LOCAL_OPERATOR_DEMO.md index e0b214c..91c5fb0 100644 --- a/docs/LOCAL_OPERATOR_DEMO.md +++ b/docs/LOCAL_OPERATOR_DEMO.md @@ -52,7 +52,7 @@ If `/creator/lock-service-config` or `/creator/priv-resources/content/` re ## Prerequisites - A Postgres database reachable through `PUBKY_LOCK_DATABASE_URL`. -- A 32-byte base64url creator-authority encryption key in `PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY`. +- A 32-byte base64url runtime master key in `PUBKY_LOCK_RUNTIME_MASTER_KEY`. - `curl`, `jq`, and `python3` available in your shell. - A generated/default Lock Server config and secret under `~/.pubky-lock/`. @@ -62,10 +62,10 @@ The database URL below is a local development example. Real credentials must com export PUBKY_LOCK_DATABASE_URL='postgres://locks:locks@localhost:55433/locks_test' ``` -Generate a local creator-authority encryption key for this shell before starting the server: +Generate a local runtime master key for this shell before starting the server: ```bash -export PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY="$(python3 - <<'PY' +export PUBKY_LOCK_RUNTIME_MASTER_KEY="$(python3 - <<'PY' import base64, os print(base64.urlsafe_b64encode(os.urandom(32)).decode().rstrip('=')) PY diff --git a/docs/RUNTIME.md b/docs/RUNTIME.md index 5acb082..f14eb37 100644 --- a/docs/RUNTIME.md +++ b/docs/RUNTIME.md @@ -117,7 +117,7 @@ key_republisher_interval_seconds = 3600 `public_pubky_tls_port` and `public_icann_http_port` advertise externally reachable ports. `icann_domain` is browser/ICANN fallback target. Local testnet operators should set `pkarr_relays = ["http://localhost:15411"]`. -PKARR publishing starts when environment is `staging`/`production` or creator-authority acquisition is enabled, and republishes every `key_republisher_interval_seconds` seconds. +PKARR publishing starts when environment is `staging`/`production` or creator-authority acquisition is enabled, and republishes every `key_republisher_interval_seconds` seconds. Initial publication begins only after the HTTP listener binds successfully; it is cancellation-aware and bounded by `deletion_worker.shutdown_timeout_seconds`, the existing lifecycle deadline. `credentials.lock_server_secret_key` must contain: @@ -137,7 +137,7 @@ server_url = "http://127.0.0.1:3001" minimum_confirmations = 0 ``` -`server_url` is the standalone Paykit Server base URL. Any configured path prefix is preserved when appending `invoices` and `transactions/status`, with or without a trailing slash. For a new `{ creator, bundle_id }` lifecycle identity, the Lock Server calls `POST /invoices` during `POST /proof-bundles` before creating a verification task; exact persisted submission replay does not call Paykit. Workers call `POST /transactions/status` while completing pending payment verification tasks. Both request bodies are canonical JSON signed through `X-Paykit-Signature` with the existing Lock Server keypair; therefore `credentials.lock_server_secret_key` must use the `keypair-seed:` format when `[paykit]` is configured. +`server_url` is the standalone Paykit Server base URL. Any configured path prefix is preserved when appending `invoices` and `transactions/status`, with or without a trailing slash. With PostgreSQL runtime storage, a new `{ creator, bundle_id }` first commits an internal, unclaimable admission reservation under the same per-lock database fence used by deletion start. The Lock Server then calls `POST /invoices`; only success makes the task publicly visible and worker-claimable. Durable handle replay is checked before mutable public-lock lookup and reader discovery. Exact replay of a ready task does not call Paykit; exact retry of an incomplete reservation uses its persisted canonical request fields to replay the same idempotent invoice request, even after the public lock is tombstoned. A deletion job cannot advance from withdrawal to Paykit drain start while any snapshotted reservation remains unready; this ensures Paykit's drain sees every pre-cutoff invoice. Workers call `POST /transactions/status` while completing ready payment verification tasks. Both request bodies are canonical JSON signed through `X-Paykit-Signature` with the existing Lock Server keypair; therefore `credentials.lock_server_secret_key` must use the `keypair-seed:` format when `[paykit]` is configured. Paykit HTTP connections have a 5-second connect timeout and every request has a 20-second whole-request timeout. Invoice timeouts fail submission with `paykit_invoice_creation_failed`; status-query timeouts remain pending/retryable. When `[paykit]` and the in-process worker are both enabled, `worker.claim_timeout_seconds` must be greater than 20 so a Paykit request cannot outlive the worker claim lease. External worker deployments must preserve the same timeout/lease relationship operationally. @@ -145,7 +145,7 @@ Every claimed verification task receives a fresh opaque claim token. Retry, comp `minimum_confirmations = 0` accepts a Paykit status of `detected` or `confirmed` when `amount_matched = true`. Values above zero require `status = "confirmed"` and at least that many confirmations. `undetected`, insufficient confirmations, or `amount_matched = false` keep the task pending/retryable. -Omitting `[paykit]` prevents creation of new payment lifecycle identities. In that state, non-payment verifier flows continue to run, an exact persisted `paykit-payment` submission replay can still return its lifecycle after current canonical preflight, and a new `paykit-payment` submission returns `422 paykit_not_configured`. Staging deployments should omit `[paykit]` until a Paykit Server is deployed and reachable for that environment. +Omitting `[paykit]` prevents creation or reconciliation of payment lifecycle identities. In that state, non-payment verifier flows continue to run, an exact ready `paykit-payment` replay can still return its persisted lifecycle, an exact unready replay returns `422 paykit_not_configured`, and a new `paykit-payment` submission returns the same `422`. Staging deployments should omit `[paykit]` until a Paykit Server is deployed and reachable for that environment. ## Runtime storage @@ -156,14 +156,33 @@ Operator-facing readiness uses semantic storage labels: Postgres is private runtime storage for verification tasks, task claiming, access credentials, frontend sessions, and creator-granted homeserver session material. It is not storage for Pubky-owned content locks, guarded resources, Lock Service Pointers, or verified proof bundles. -Creator-granted session material is encrypted before storage. The server-side encryption key comes from an env var named by config: +Sensitive runtime material is encrypted before storage. A root key comes from an env var named by config: ```toml [secrets] -creator_authority_key_env = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" +runtime_master_key_env = "PUBKY_LOCK_RUNTIME_MASTER_KEY" ``` -The named env var must contain a 32-byte key encoded as base64url without padding: +The named env var must contain a 32-byte key encoded as base64url without padding. Locks derives separate fixed-domain keys for creator-authority material and replayable final deletion credentials; the root key is never used directly as an AEAD key. Rotating it requires an explicit data migration. In the local Compose flow, an explicitly supplied `PUBKY_LOCK_RUNTIME_MASTER_KEY` is validated and persisted only when `.pubky-lock/runtime-master-key` does not yet exist. Once persisted, an override must match those exact bytes or startup fails closed; later starts without the override reuse the persisted key. Changing the key therefore requires an explicit encrypted-data migration or an intentional reset that discards the dependent ciphertext and its old key together. + +Migration `0016_content_lock_access_drains` fails closed when an older database contains a resumable `queued`, `running`, or `failed` deletion job, because Task 7 rows do not contain enough information to reconstruct cutoff credential classification safely. Failed jobs are resumable by graceful-deletion replay, so they cannot be treated as terminal for this upgrade. Before upgrading, stop new writes and check: + +```sql +SELECT job_id, state, phase +FROM content_lock_deletion_jobs +WHERE state IN ('queued', 'running', 'failed'); +``` + +If rows are returned, resume/retry them on the pre-0016 release until every deletion reaches `completed` and then retry the upgrade, or explicitly reset the pre-production environment: stop the stack, back up anything needed, recreate the Locks PostgreSQL database/volume, reconcile or republish any public tombstones/content locks, and reacquire creator authority. Do not bypass the guard by deleting only the job rows; that can strand external Pubky state and accepted obligations. + +The runtime-master-key cutover intentionally cannot decrypt creator-authority rows written with the retired Compose `creator-authority-encryption-key`. An existing Compose volume containing that file fails startup instead of silently stranding encrypted authority. To upgrade a local stack: + +1. Stop the stack and back up any data that must be retained. +2. Either discard the existing creator-authority rows and reacquire authority after restart, or recreate the local PostgreSQL database/volume. +3. Remove `.pubky-lock/creator-authority-encryption-key` from the Locks service volume. +4. Restart. Compose creates `.pubky-lock/runtime-master-key`; keep that file stable with the database. + +Removing only the retired key file while retaining its encrypted creator-authority rows is unsupported. ```bash python3 - <<'PY' @@ -172,6 +191,19 @@ print(base64.urlsafe_b64encode(os.urandom(32)).decode().rstrip('=')) PY ``` +Deletion retry and final-access bounds are a closed configuration section: + +```toml +[deletion] +retry_max_attempts = 10 +retry_initial_backoff_seconds = 1 +retry_max_backoff_seconds = 300 +final_credential_issuance_window_seconds = 900 +final_read_window_seconds = 900 +``` + +All values must be positive, initial backoff cannot exceed maximum backoff, and both final-access windows must be at most 3600 seconds. + ## Development integration shape ```toml @@ -223,6 +255,19 @@ For `paykit-payment`, every Paykit status-call failure schedules a normal pendin Scheduled retries and crash recovery are separate mechanisms. Expected retryable results explicitly release the current claim and set `next_attempt_at`. If a worker crashes while a task is `in_progress`, another worker may reclaim it only after `claim_expires_at`; only the worker that still owns an active claim may schedule its retry. +The deletion worker has an independent closed runtime section: + +```toml +[deletion_worker] +enabled = true +poll_interval_ms = 250 +claim_timeout_seconds = 60 +shutdown_timeout_seconds = 30 +worker_id = "deletion-worker" +``` + +All three numeric values must be positive and `worker_id` must be nonblank. Queue polling cadence is independent from durable deletion retry backoff. `shutdown_timeout_seconds` bounds coordinated worker and HTTP shutdown; once shutdown begins, the deletion worker must stop acquiring new claims. + ## Health and readiness `GET /healthz` reports process liveness: @@ -231,7 +276,7 @@ Scheduled retries and crash recovery are separate mechanisms. Expected retryable { "status": "ok" } ``` -`GET /readyz` reports runtime dependency readiness: +`GET /readyz` reports runtime dependency and enabled-worker readiness: ```json { @@ -242,3 +287,5 @@ Scheduled retries and crash recovery are separate mechanisms. Expected retryable ``` Health/readiness responses must remain secret-free. They must not include database URLs, secret paths, worker IDs, task counts, public keys, credentials, raw errors, or submitted proof material. + +The response status is `not_ready` while an enabled worker is starting, stopping, or unexpectedly stopped. A transient deletion dependency failure reports `degraded` until successful dependency evidence. Ordinary pending deletion work, advisory-lock contention, and a correctly terminalized failed deletion job do not degrade readiness. diff --git a/docs/plans/2026-08-10-graceful-content-lock-deletion.md b/docs/plans/2026-08-10-graceful-content-lock-deletion.md new file mode 100644 index 0000000..066035d --- /dev/null +++ b/docs/plans/2026-08-10-graceful-content-lock-deletion.md @@ -0,0 +1,590 @@ +# Graceful Content-Lock Deletion and Payment Deadline Implementation Plan + +> **For Hermes:** Use subagent-driven-development to implement this plan one review-gated commit slice at a time. Stop after each slice; the user commits before the next slice. + +**Goal:** Add a bounded `paykit-payment` deadline and creator-authorized graceful content-lock deletion that withdraws the public lock immediately and drains accepted payment and access obligations durably. Graceful cleanup never deletes concurrent Pubky replacements, but tombstone publication has the explicitly accepted non-atomic overwrite limitation recorded below. + +**Architecture:** Locks owns the public tombstone, admission cutoff, verification tasks, credentials, guarded content, path ownership, and overall deletion job. Paykit Server owns invoice timestamps, Payment Request lifecycle classification, cancellation, Bitcoin observation, and a durable lock-wide payment drain. PostgreSQL stores retryable Locks workflow state; Pubky remains authoritative for public lock/tombstone and private guarded bytes. + +**Tech Stack:** Rust 2024, Axum, Tokio, SQLx/PostgreSQL, Pubky homeserver storage, `time`, AEAD via `chacha20poly1305`, existing Locks SDK and JS/WASM bindings. + +**Sibling plan:** Paykit Server `docs/plans/2026-08-10-lock-payment-draining.md`. Both plans repeat the shared wire contract deliberately. + +--- + +## Status and provenance + +- Plan status: **accepted product design; Tasks 1–7 committed; Tasks 8–11 remain**. +- Repository inspected: `/home/u/Projects/Synonym/Pubky/locks-public`. +- Planning base when written: clean `master` at `ba49a77`. +- There has been no production deployment. New persistence may require a clean pre-production database; no historical backfill is required. +- No Paykit Rust protocol change is planned. `proposal_expires_at` retains its existing pre-acceptance meaning. + +### Explicit requirements and confirmed decisions + +1. `paykit-payment` criterion params gain required `payment_in`. +2. `payment_in` is a nonzero JSON `u64` integer measured in whole hours. Zero, fractional, negative, string, and out-of-range values are invalid. There is no product maximum beyond checked duration/timestamp representation. +3. Locks includes `payment_in` in the signed invoice request. Paykit independently reads the canonical lock and rejects a mismatch before side effects. +4. Paykit commits `invoice_created_at` and `payment_deadline = checked(invoice_created_at + payment_in hours)` atomically with invoice creation. Exact replay returns the original timestamps. +5. Locks persists the returned timestamps before admitting the verification task. Retry never restarts the payment window. +6. Paykit sets Payment Request `proposal_expires_at` to `payment_deadline`, but the field remains proposal-only. Locks and Paykit Server enforce the post-acceptance deadline as application state. +7. Payment is timely when Paykit’s durable `first_amount_matched_observed_at <= payment_deadline`. An earlier underpayment does not lend its timestamp to a later qualifying output. Polling latency is accepted. +8. At the deadline, undetected and underpaid invoices expire and stop active observation. A timely amount-matched payment may continue confirmation observation after the deadline without a second timeout. +9. Locks alone applies configured `minimum_confirmations`; it is not sent to Paykit’s drain endpoint. +10. Payment after application expiry never opens the lock. Reader UI blocks/removes payment instructions at the deadline and warns that late payment receives no access or automatic refund. +11. Graceful deletion is the default creator DELETE mode. `graceful=true` is an alias; `force=true` is mutually exclusive and explicit. +12. Graceful deletion is irreversible once its durable job is persisted. There is no cancellation API. +13. Public withdrawal replaces `/pub/locks.app/{lock_id}.json` with an exact tombstone after durably storing the original canonical lock: + +```json +{ + "version": 1, + "type": "content_lock_deletion", + "lock_id": "", + "deletion_started_at": "" +} +``` + +14. Persisting the deletion job is the proof-admission cutoff. New Bundle IDs are rejected; exact replay/status for previously persisted tasks remains available. +15. Paykit atomically classifies Payment Requests at drain start: accepted/rejected persisted before the cutoff retain that state; unanswered requests are durably canceled; later acceptance loses. +16. Durable cancellation enqueue is enough to stop blocking; delivery/acknowledgment is not awaited. +17. Rejected and canceled requests do not block. Accepted requests block until payment expires or satisfies Locks’ frozen rule. Timely amount-matched payment continues through required confirmations. +18. Existing access credentials remain reusable until their original expiry. +19. Existing and final drain credentials resolve authorization and resource descriptors from the deletion job’s frozen canonical manifest while the public path contains a tombstone. The tombstone is never treated as a valid Content Lock, and callers outside the persisted drain receive no new access. +20. Every already-paid entitlement lacking an active credential at tombstoning, plus every payment completed during draining, may obtain exactly one final drain credential. +21. Default final-credential issuance window is 15 minutes; configured maximum is one hour. Default read window is 15 minutes; configured maximum is one hour. Retry does not extend either. +22. Final credential permits one successful GET per frozen resource. Each path uses an atomic claim. Consumption occurs after upstream bytes are fetched/validated and a `200` response is constructed; a later disconnect does not restore it. Pre-response fetch failure releases the claim. +23. Exact credential issuance replay returns the same random bearer. Persist a versioned encrypted envelope using a domain-separated key derived from the existing runtime master key; bind creator, Bundle ID, deletion job, and envelope version as AEAD context. +24. Deletion worker retries transient failures with durable exponential backoff: one second initial, five-minute cap, full jitter, ten attempts per phase by default. Attempts reset on phase advance. +25. Creator-visible job status is only `queued|running|completed|failed`; failed responses include a stable secret-free `failure_code` only. +26. Missing or replaced tombstone before final verification halts as failed. Creator restores the exact tombstone and repeats graceful DELETE to resume the same job. +27. Guarded paths are exclusive to one managed lock. Enforce unique `(creator, guarded_path)` ownership in PostgreSQL. There is no historical backfill. +28. Lock publication uses best-effort ownership compensation and a durable opaque per-lock publication intent under the same PostgreSQL fence as deletion admission. Graceful/force deletion cannot start while publication is in flight. The intent is cleared only after ownership is durably published or failed publication is safely compensated. Process death can leave operator-reconciled intent/ownership state; do not claim cross-system atomicity. +29. Graceful finalization non-destructively verifies every frozen guarded-resource generation and the exact tombstone, then purges Locks authorization/task/job state and asks Paykit to remove operational drain state. The tombstone and guarded bytes remain on Pubky because its unconditional DELETE API cannot safely remove an expected generation in the presence of out-of-band creator writes. Later republication requires an explicit follow-up design rather than an unsafe graceful delete. +30. Paykit retains terminal financial invoice/payment history; delayed old lifecycle events cannot reactivate a fresh publication. +31. New force deletion is synchronous: persist a permanent minimal blocking receipt, delete lock/tombstone first, then best-effort guarded resources. Do not drain Paykit/tasks/credentials. Return failed paths. A force-deleted Lock ID can never be republished. +32. `force=true` against an active graceful job persists `force_requested`, revokes the current claim token/lease, requeues the same frozen job, and returns `202`; a fresh worker claim escalates asynchronously under exclusive action ownership, skips drains, deletes tombstone then content, and finishes forced. +33. Graceful job insertion/resume and permanent force-receipt establishment acquire the same canonical per-lock PostgreSQL fence. The durable result is either an active graceful job or a permanent force receipt, never both. Failed graceful replay requeues the same job and frozen manifest. Force against a terminal job atomically replaces that operational row with the permanent receipt before synchronous external deletion. +34. Any Content Lock fetched from Pubky for deletion must hash to the requested Lock ID and name the authenticated creator before its manifest is frozen or used for resource verification or force deletion. +35. Runtime encryption uses one environment-only 32-byte unpadded-base64url master key selected by `secrets.runtime_master_key_env`. Creator-authority and final-credential encryption keys are derived from it with distinct fixed domain labels. The retired `creator_authority_key_env` key is rejected as unknown configuration; no compatibility alias is retained. +36. The closed `[deletion]` configuration contract is `retry_max_attempts = 10`, `retry_initial_backoff_seconds = 1`, `retry_max_backoff_seconds = 300`, `final_credential_issuance_window_seconds = 900`, and `final_read_window_seconds = 900` by default. All values are positive; initial backoff cannot exceed maximum backoff; both credential windows are bounded to at most 3600 seconds. Retry jitter remains an implementation policy rather than a configurable field. +37. Deletion admission immutably records whether each paid snapshot Bundle had any active credential at cutoff and enrolls every such ordinary credential with its original expiry. Enrolled ordinary credentials remain reusable against the frozen manifest until that expiry; they do not acquire one-shot resource-read rows. When the claimed job first enters `issue_final_credentials`, it persists `final_issuance_started_at`, `final_credential_issuance_deadline = final_issuance_started_at + final_credential_issuance_window`, and `final_read_deadline = final_credential_issuance_deadline + final_read_window` once; replay and later config changes never extend them. A paid snapshot resolved completed without an active ordinary credential at cutoff becomes durably final-credential eligible and receives exactly one encrypted replayable final credential expiring at `final_read_deadline`. Every final credential receives one claimable row per frozen manifest path. Final-read claims precede Pubky fetch, are released on pre-response failure, expire for crash recovery, and are consumed only after the complete HTTP response is constructed; consumption is permanent. Phase advancement waits until every enrolled ordinary credential is expired and every final resource is consumed or its credential/read window is expired. +38. Ordinary credential insertion and deletion admission acquire the same canonical per-lock fence. Deletion-first rejects the insert; insertion-first is attached and classified at cutoff. Database lock order is canonical per-lock fence, deletion job row, snapshot/credential row, then resource-read row. No transaction spans Pubky I/O. Final read claims use fixed 30-second leases clamped to credential expiry; stale claim tokens cannot consume or release a reclaimed row. +39. This pre-production migration intentionally has no creator-authority ciphertext compatibility path. Moving the same bytes to `runtime_master_key_env` changes the derived creator-authority key; existing local encrypted authority rows must be discarded and reacquired or the local database recreated. +40. Deletion-worker runtime configuration is a separate closed `[deletion_worker]` section with `enabled`, `poll_interval_ms`, `claim_timeout_seconds`, `shutdown_timeout_seconds`, and `worker_id`. Defaults are `true`, `250`, `60`, `30`, and `"deletion-worker"`. Enabled verification and deletion workers are independently tracked: starting, stopping, or unexpected exit is `not_ready`; a transient deletion dependency failure is `degraded` until successful dependency evidence; ordinary pending work, advisory-lock contention, and a correctly terminalized failed job do not degrade readiness. +41. **Accepted Pubky tombstone TOCTOU limitation:** the pinned Pubky 0.9.3 SDK exposes ETag metadata but no conditional write API, and the matching homeserver enforces `If-None-Match` for reads but not `If-Match` for writes. Graceful withdrawal therefore reads and compares the frozen canonical lock before an unconditional tombstone `PUT`. A replacement already visible at that read fails closed and is preserved; crash/reclaim replay also preserves any replacement it observes. However, an out-of-band creator replacement written after the comparison and before the `PUT` can be overwritten by the tombstone. Product explicitly accepts this race until Pubky provides atomic conditional writes. This exception does not authorize graceful deletion of replacement bytes; active force remains the only unconditional delete path. + +### Source-derived constraints + +- Lock ID is BLAKE3 over complete canonical lock JSON. A mutable `deleting` field cannot be added under the same ID. +- Readers fetch the public lock directly from the creator homeserver; a Locks Server GET gate cannot withdraw it. +- Guarded content and public lock JSON are separate Pubky records; no delete cascade exists. +- Current creation validates resource descriptors but does not enforce cross-lock path exclusivity (`locks-service/src/application/use_cases/create_content_lock.rs`). +- Current verification tasks use PostgreSQL leases and fresh claim tokens; deletion needs a separate queue but the same fenced-transition discipline. +- Current access-credential storage keeps only a bearer lookup hash; exact replay requires new encrypted bearer persistence. +- `proposal_expires_at` expires only `Proposed` Paykit SDK state and has no accepted-payment effect. +- Pubky, Locks PostgreSQL, and Paykit PostgreSQL cannot participate in one atomic transaction. + +### Explicitly accepted risks + +- Late Bitcoin payment may receive no content and no refund. +- Paykit polling latency can make a pre-deadline broadcast late. +- Timely amount-matched payment can block deletion indefinitely while confirmations/reorg state remains unresolved. +- Durable cancellation enqueue may precede actual counterparty delivery. +- Best-effort lock-publication reservation compensation can leave operator-cleaned orphan ownership after process death. +- Force deletion deliberately abandons active payment/access obligations and may orphan content after a crash. + +## Repository ownership matrix + +| Contract/state | Owner | +| --- | --- | +| `payment_in` criterion schema and validation | Locks Core | +| Signed invoice request producer and response persistence | Locks Server/Service | +| `invoice_created_at`, `payment_deadline`, proposal expiry | Paykit Server | +| Payment Request acceptance/rejection/cancellation projection | Paykit Server | +| Bitcoin first-observation and confirmations | Paykit Server | +| `minimum_confirmations` entitlement decision | Locks | +| Public tombstone and frozen lock manifest | Locks | +| Proof admission cutoff and task transitions | Locks | +| Credentials, per-path consumption, content serving | Locks | +| Lock-wide payment drain and aggregate status | Paykit Server | +| Overall deletion orchestration and final cleanup | Locks | +| Terminal financial history | Paykit Server | + +## Shared service-to-service contract + +All requests use existing `X-Paykit-Signature` over canonical JSON. Secret/correlation identifiers stay in POST bodies and must not be logged. + +### Invoice creation + +```http +POST /invoices + +{ + "bundle_id": "...", + "lock_resource": "pubky.../pub/locks.app/.json", + "reader": "pubky...", + "payment_in": 24 +} +``` + +Success changes from ignored-body 2xx to closed JSON: + +```json +{ + "invoice_created_at": "", + "payment_deadline": "" +} +``` + +Paykit compares request `payment_in` with canonical criterion `payment_in`. Exact replay returns the original response. + +### Lock-wide drain + +```http +POST /payment-request-drains +{ "lock_resource": "..." } +``` + +Starts or exactly replays an atomic persisted classification. No `minimum_confirmations` field. + +```http +POST /payment-request-drain-lookups +{ "lock_resource": "..." } +``` + +Both drain endpoints return `200` with the same closed aggregate body: + +```json +{ + "status": "active", + "accepted_count": 0, + "terminal_count": 0, + "cancellation_enqueued_count": 0, + "cleanup_token": "<43-character-unpadded-base64url>" +} +``` + +`status` is exactly `active` or `completed`. The response contains no drain ID, replay flag, Bundle ID, reader, Payment Request ID, address, payment reference, or raw error. Replay preserves the frozen drain identity, cancellation count, and cleanup token while returning its latest monotonic aggregate progress. + +### Per-Bundle status + +```http +POST /payment-requests/status +{ "creator": "pubky...", "bundle_id": "..." } +``` + +Returns this exact closed body with orthogonal lifecycle and payment facts: + +```json +{ + "request_state": "proposed", + "payment_state": "undetected", + "invoice_created_at": "", + "payment_deadline": "", + "confirmations": 0, + "amount_matched": false +} +``` + +The canonical persisted `request_state` is one of these exact closed snake-case values, mapped one-to-one from Paykit SDK lifecycle state: + +- `proposed` +- `proposal_expired` +- `accepted` +- `rejected` +- `canceled` +- `proof_submitted` +- `active_recurring` +- `recovery_required` +- `invalid_conflict` + +`payment_state` is exactly one of: + +- `undetected` +- `detected` +- `confirmed` +- `expired` + +`expired` is returned when the invoice has a durable `payment_expired_at`; otherwise the persisted observation state maps one-to-one to `undetected`, `detected`, or `confirmed`. `confirmations` and `amount_matched` remain orthogonal factual fields. + +Locks maps `rejected`, `canceled`, and `proposal_expired` requests to `VerificationTaskStatus::Expired`. An `accepted` request whose `payment_state` is `expired` also maps to `Expired`. These terminal outcomes carry no failure message and never map to `Failed`. + +Drain classification uses the persisted state without inference from invoice delivery or Bitcoin observation: + +- `accepted` is accepted and blocking; +- `rejected`, `canceled`, and `proposal_expired` are terminal and non-blocking; +- `proposed` is unanswered and requires durable cancellation enqueue; +- `recovery_required`, `invalid_conflict`, `proof_submitted`, and `active_recurring` fail drain classification rather than being collapsed into another lifecycle. + +For the later HTTP slice, `recovery_required` maps to `503 unavailable`; `invalid_conflict`, `proof_submitted`, and `active_recurring` map to `409 conflict`. These mappings do not alter the canonical lifecycle persisted by this projection. + +The stable drain-classification error envelopes are: + +- `409 {"error":{"code":"conflict","message":"request conflicts with persisted payment state"}}` +- `503 {"error":{"code":"unavailable","message":"payment request state is unavailable"}}` + +Absent drain lookups and absent per-Bundle statuses reuse `404 {"error":{"code":"not_found","message":"requested resource was not found"}}`. + +### Operational drain cleanup + +Drain creation and lookup responses include an opaque `cleanup_token`: the canonical unpadded base64url encoding of 32 server-keyed, domain-separated bytes bound to the immutable drain identity. The token is not an internal drain ID, is not reversible, is stable across restart/exact replay, and must never be logged. Add `POST /payment-request-drain-cleanups`, authenticated by the existing canonical Locks signature boundary. It accepts the exact query-free body: + +```json +{"cleanup_token":"<43-character-unpadded-base64url>","lock_resource":"pubky/pub/locks.app/.json"} +``` + +On success it returns the exact closed response `200 {"status":"removed"}`. Cleanup is cycle-bound and idempotent: deleting the matching completed drain advances the publication generation once and durably retains the consumed token as the generation boundary's cleanup receipt; replay of that token while no newer drain exists returns the same response without advancing again. A token that cannot be verified against either the current drain or its retained cleanup receipt—including an arbitrary token for a never-known lock or a delayed old token after a newer drain exists—returns the existing coarse `409 conflict` envelope and cannot delete or advance the newer cycle. An active matching drain also returns `409 conflict`. Authenticated envelope mismatch, corrupt receipt/generation state, or unavailable persistence returns the existing coarse `503 unavailable` envelope. The operation rejects query strings, unknown body fields, padding, non-canonical base64url, and token lengths other than exactly 32 decoded bytes. It deletes only a completed operational drain after Locks external cleanup succeeds and must never delete invoices, Bitcoin observations, Payment Request events, cancellation intents, or financial audit history. No lock, Bundle, internal drain ID, invoice, reader, or Payment Request identifier appears in the response or error envelope. + +## HTTP creator contract + +```http +DELETE /creator/content-locks/{lock_id} +DELETE /creator/content-locks/{lock_id}?graceful=true +``` + +Starts/replays/resumes graceful deletion and returns `202` for queued/running work. A completed-and-forgotten absent lock is an idempotent absent postcondition. + +Queued/running deletion and deletion status use the closed body `{ "lock_id": "...", "status": "queued|running|completed|failed", "failure_code"?: "..." }`. If both the canonical lock and deletion job are absent, graceful DELETE returns `200` with `{ "lock_id": "...", "status": "completed" }`. + +```http +DELETE /creator/content-locks/{lock_id}?force=true +``` + +- No graceful job: synchronous `200` force summary. +- Existing graceful job: persist `force_requested`, return `202` job status. + +The synchronous force summary is exactly `{ "lock_id": "...", "lock_deleted": true, "failed_resource_paths": ["..."] }`. It does not expose a force mode or internal receipt. + +Reject `force=true&graceful=true`, unknown fields, malformed booleans, and duplicate conflicting query values. + +```http +GET /creator/content-locks/{lock_id}/deletion +``` + +Authenticated response contains Lock ID and `status`; include `failure_code` only for failed jobs. The closed stable vocabulary is exactly `tombstone_missing`, `tombstone_replaced`, `resource_replaced`, `retry_exhausted`, and `state_corrupt`. `resource_replaced` means a frozen guarded-resource path no longer contains the admitted generation and therefore graceful finalization failed closed without deleting the replacement. Do not expose phases, leases, retries, Bundle IDs, readers, credentials, paths, Paykit IDs, or dependency errors. + +If no job or force receipt exists, status returns `404 content_lock_deletion_not_found`. A permanent force receipt projects as `{ "lock_id": "...", "status": "completed" }` without exposing force mode. + +## Internal state model + +Internal phase names are not public API. The implementation should represent at least: + +1. `withdraw`: persist frozen payload/job/admission cutoff, write tombstone, read back exact bytes. +2. `start_payment_drain`: exact Paykit drain creation. +3. `drain_payments`: poll aggregate drain and per-Bundle statuses; transition frozen tasks. +4. `drain_existing_credentials`: wait for credentials active at cutoff to expire. +5. `issue_final_credentials`: allow bounded issuance for eligible entitlements. +6. `drain_final_reads`: enforce per-path claims/consumption and read deadlines. +7. `delete_content`: non-destructively verify every frozen resource generation while the tombstone remains exact. +8. `delete_tombstone`: non-destructively verify that the exact tombstone remains published before the purge handoff. +9. `purge_operational_state`: remove Paykit operational drain, then atomically purge Locks lock-scoped authorization/task/job state and release path ownership. + +Use separate durable `state`, `phase`, `attempt_count`, `next_attempt_at`, claim owner/token/expiry, and force-request fields. Use a per-job PostgreSQL advisory action lock where lease expiry must not permit overlapping external effects. SQLx advisory-lock connections must be close-on-drop and explicitly unlocked/closed. + +## Implementation sequence + +Each task is a separate review/commit checkpoint. Do not commit automatically. + +### Task 1: Lock the `payment_in` core contract + +**Objective:** Make the content-addressed lock schema reject every non-approved timing shape. + +**Files:** +- Modify: `locks-core/src/lock_policy.rs` +- Modify: `locks-core/src/creator_publishing.rs` +- Modify: `locks-sdk/bindings/js/src/creator.rs` +- Test: neighboring unit/public API tests in those files and `locks-sdk/tests/public_api.rs` + +**RED:** Add serialization/validation tests for required nonzero JSON `u64`, unknown/missing field rejection, zero/fraction/string/overflow rejection, and canonical Lock ID sensitivity. + +**GREEN:** Extend the closed `paykit-payment` params parser/typed accessors and JS creator builder. + +**Verify:** + +```bash +cargo test -p locks-core +cargo test -p locks-sdk +cargo test -p locks-sdk-wasm +cargo test --workspace --no-run +``` + +**Suggested commit:** `feat(core): add paykit payment deadline hours` + +### Task 2: Persist exclusive guarded-path ownership + +**Objective:** Enforce one managed Content Lock per creator/path and retain ownership safely across deletion failures. + +**Files:** +- Modify: `locks-service/src/infrastructure/postgres/migrations.rs` +- Create: `locks-service/src/application/models/content_lock_ownership.rs` +- Create: `locks-service/src/application/ports/content_lock_ownership.rs` +- Create: `locks-service/src/infrastructure/postgres/content_lock_ownership.rs` +- Modify: relevant `mod.rs` exports +- Modify: `locks-service/src/application/use_cases/create_content_lock.rs` +- Modify: in-memory test adapters +- Test: `locks-e2e/tests/postgres_runtime.rs` +- Test: `locks-e2e/tests/production_creator_publishing_http.rs` + +**RED:** Prove duplicate `(creator,path)` rejection, atomic all-path reservation, ordinary-error compensation, retained ownership after failed deletion, and clean-database rollout. + +**GREEN:** Add unique ownership rows carrying creator, full path, intended Lock ID, and status. Reserve before Pubky publication; best-effort compensate ordinary publication failure. Do not invent historical backfill. + +**Verify:** + +```bash +TEST_DATABASE_URL="$TEST_DATABASE_URL" cargo test -p locks-e2e --test postgres_runtime +TEST_DATABASE_URL="$TEST_DATABASE_URL" cargo test -p locks-e2e --test production_creator_publishing_http +cargo test --workspace --no-run +``` + +**Suggested commit:** `feat(service): enforce guarded path ownership` + +### Task 3: Upgrade the Locks-to-Paykit invoice boundary + +**Objective:** Send `payment_in`, require the closed timestamp response, and durably bind it to the verification task before admission. + +**Files:** +- Modify: `locks-server/src/paykit_http_client.rs` +- Modify: `locks-service/src/application/models/verification.rs` +- Modify: `locks-service/src/application/ports/verification.rs` +- Modify: verification task PostgreSQL/memory adapters and migration +- Modify: `locks-service/src/application/use_cases/submit_proof_bundle.rs` +- Test: `locks-server/src/api/routes/tests.rs` +- Test: `locks-e2e/tests/postgres_runtime.rs` + +**Dependency gate:** Implement only after the Paykit Server invoice-response slice is reviewed and committed. + +**RED:** Test canonical signed request body, strict timestamp response decoding, checked ordering (`created <= deadline`), exact task replay preserving timestamps, and rollback/no-task on invoice rejection. + +**GREEN:** Persist immutable invoice timestamps with the task in the same local transaction that admits it. Do not recompute on retry. + +**Verify:** focused unit tests, PostgreSQL E2E, then `cargo test --workspace --no-run`. + +**Suggested commit:** `feat(paykit): persist invoice payment deadlines` + +### Task 4: Add deletion/tombstone domain and persistence + +**Objective:** Persist frozen manifests, cutoff state, leases, retry scheduling, force receipts, and minimal public DTOs. + +**Files:** +- Create: `locks-core/src/content_lock_deletion.rs` +- Modify: `locks-core/src/lib.rs` +- Create: `locks-service/src/application/models/content_lock_deletion.rs` +- Create: `locks-service/src/application/ports/content_lock_deletion.rs` +- Create: `locks-service/src/infrastructure/postgres/content_lock_deletions.rs` +- Create: `locks-service/src/infrastructure/memory/content_lock_deletions.rs` +- Modify: PostgreSQL migration/module exports +- Modify: `locks-service/src/application/errors.rs` + +**RED:** Test exact tombstone JSON, strict unknown-field rejection, frozen payload integrity, unique creator/Lock ID job identity, due claims, lease reclaim/fresh tokens, stale-token rejection, per-phase attempt reset, and permanent force receipt. + +**GREEN:** Implement the minimal state model. Keep public status conversion separate from internal phases. + +**Suggested commit:** `feat(service): persist content lock deletion jobs` + +### Task 5: Serialize deletion start against proof admission + +**Objective:** Make database commit order the authoritative cutoff for new Bundle IDs. + +**Files:** +- Modify: `locks-service/src/application/use_cases/submit_proof_bundle.rs` +- Create: `locks-service/src/application/use_cases/start_content_lock_deletion.rs` +- Modify: relevant repositories/PostgreSQL transaction helpers +- Test: `locks-e2e/tests/postgres_runtime.rs` +- Test: `locks-server/src/api/routes/tests.rs` + +**RED:** Concurrent tests prove task-first commit joins snapshot, deletion-first commit rejects a new Bundle, exact old replay succeeds, and conflicting replay remains rejected. + +**GREEN:** Use per-lock database serialization and one transaction for job persistence/task snapshot. For Paykit-backed submissions, atomically persist a hidden, unclaimable admission reservation before the external invoice call; classify durable exact replay/conflict before mutable lock lookup or reader resolution, and resume an unready reservation from its persisted canonical fields. Paykit success makes it ready. This guarantees that deletion either snapshots the durable obligation or commits first and prevents any Paykit call. Do not permit transition to `start_payment_drain` while a snapshotted reservation is unready: Paykit's active drain accepts exact replay only for invoices already created before drain start. Do not hold a database transaction across HTTP, and do not use viewer timestamps or tombstone publication as the cutoff. + +**Suggested commit:** `feat(service): enforce deletion admission cutoff` + +### Task 6: Add creator deletion/status APIs and SDKs + +**Objective:** Expose authenticated graceful default, explicit force, and minimal status consistently across Rust and JS. + +**Files:** +- Modify: `locks-server/src/api/creator_publishing.rs` +- Modify: `locks-server/src/api/dtos.rs` +- Modify: `locks-server/src/api/errors.rs` +- Modify: `locks-server/src/api/routes.rs` +- Modify: `locks-sdk/src/creator.rs` +- Modify: `locks-sdk/src/transport.rs` +- Modify: `locks-sdk/bindings/js/src/creator.rs` +- Test: `locks-server/src/api/routes/tests.rs` +- Test: `locks-sdk/tests/public_api.rs` +- Test: `locks-e2e/tests/production_creator_publishing_http.rs` + +**RED:** Cover query matrix, auth creator binding, 202 replay/resume/escalation, synchronous 200 force, permanent force receipt, absent postcondition, and redacted status. + +**GREEN:** Implement the closed routes exactly as documented. No immediate force through an omitted query option. + +**Suggested commit:** `feat(api): add creator content lock deletion` + +### Task 7: Integrate Paykit drain/status client + +**Objective:** Start/poll Paykit’s lock-wide drain and resolve each existing verification task from factual status. + +**Files:** +- Modify: `locks-server/src/paykit_http_client.rs` +- Modify: `locks-server/src/app_state/mod.rs` +- Create: `locks-service/src/application/ports/payment_drain.rs` +- Create: `locks-service/src/application/ports/payment_drain_repository.rs` +- Create: `locks-service/src/application/use_cases/drain_lock_payments.rs` +- Create: `locks-service/src/infrastructure/postgres/payment_drains.rs` +- Create: `locks-service/migrations/0015_content_lock_payment_drains.sql` +- Modify: `locks-service/src/infrastructure/postgres/content_lock_deletions.rs` +- Modify: `locks-service/src/infrastructure/postgres/verification_task_claims.rs` +- Test: `locks-server/src/paykit_http_client.rs` +- Test: deletion use-case tests and HTTP integration fixtures + +**Dependency gate:** Patch both plans with exact per-Bundle enums, error mappings, and drain-cleanup route before RED tests. Then implement Paykit Server routes first. + +**RED:** Test exact signed JSON, no `minimum_confirmations` leak, aggregate redaction, local application of confirmations, canceled/rejected/expired transitions, timely matched confirmation continuation, and retryable transport errors. + +**GREEN:** Freeze Paykit obligation identity, criterion, cutoff status, and authoritative invoice window in the deletion snapshot; persist the opaque drain token and aggregate under the deletion claim fence. Paykit aggregate progress is monotonic: `accepted_count` may only decrease to zero, `terminal_count` may only increase by the same amount, and `cancellation_enqueued_count` plus the opaque cleanup token remain immutable. `completed` requires `accepted_count == 0` and cannot regress to `active`. Locks persists each newer aggregate under the live deletion lease, but still requires every frozen local obligation to become terminal before phase advancement; aggregate completion never bypasses Locks-local confirmation or reorg reconciliation. Exclude ordinary verification workers for every surviving deletion snapshot. Immediately before external entitlement storage, an ordinary claimed worker durably marks entitlement publication under its exact lease. Deletion admission locks and owns every matching task row before snapshot/reset: a committed publication marker blocks deletion, while committed deletion ownership blocks publication and every ordinary claim/retry/terminal write. Ambiguous entitlement publication retains the marker until an ordinary retry reconciles an equivalent entitlement and terminalizes the task. Publish entitlements before terminalizing their task and reconcile only an equivalent existing entitlement. Compose the client and repository in `AppState`. Task 9 supplies the queue polling/supervision that invokes this phase use case. + +**Suggested commit:** `feat(paykit): drain deleting lock payments` + +### Task 8: Implement final credential/read draining + +**Objective:** Preserve existing credential TTL behavior while giving eligible paid entitlements one bounded per-resource final read. + +**Files:** +- Modify: `locks-service/src/application/models/access.rs` +- Modify: `locks-service/src/application/ports/access.rs` +- Modify: `locks-service/src/infrastructure/postgres/access_credentials.rs` +- Modify: `locks-service/src/infrastructure/postgres/migrations.rs` +- Modify: `locks-service/src/application/use_cases/issue_access_credential.rs` +- Modify: `locks-service/src/application/use_cases/proxy_read_guarded_resource.rs` +- Modify: `locks-server/src/storage.rs` and secret composition as needed +- Test: `locks-service/src/application/use_cases/credential_flow_tests.rs` +- Test: `locks-service/src/application/use_cases/retrieval_access_flow_tests.rs` +- Test: `locks-e2e/tests/retrieval_access_http.rs` + +**RED:** Cover exact encrypted replay, wrong-key/corrupt/version rejection, no secret Debug/log output, issuance/read deadlines, no deadline extension, existing/final access through the frozen manifest while the public path is a tombstone, denial outside the persisted drain, one concurrent success per path, claim release before response construction, consumption after construction, and automatic revocation when complete/expired. + +**GREEN:** Snapshot and enroll active credentials atomically at deletion admission, and initialize final-window timestamps once when entering final issuance under the deletion lease. Fence ordinary insertion against deletion admission. Use versioned AEAD and domain-separated key derivation; retain lookup hashes. Enroll cutoff-active credentials at their original expiry and create one final credential only for an eligible completed snapshot without one. Resolve draining reads from the frozen manifest rather than parsing the tombstone. Claim each credential/path before fetch, release on pre-response failure, consume only after the server constructs the complete response, and allow only expired claims to be reclaimed. Do not store plaintext bearer. + +**Suggested commit:** `feat(access): drain final deletion credentials` + +### Task 9: Implement and supervise the deletion worker + +**Objective:** Execute external phases retryably without overlapping external actions or breaking shutdown. + +**Files:** +- Create: `locks-server/src/deletion_worker.rs` +- Modify: `locks-server/src/main.rs` +- Modify: `locks-server/src/config/schema.rs` +- Modify: `locks-server/src/config/defaults.rs` +- Modify: `locks-server/src/config/validation.rs` +- Modify: `locks-server/src/app_state/readiness.rs` +- Modify: `locks-server/src/api/runtime.rs` +- Modify: deletion application ports/use cases and PostgreSQL/Pubky/in-memory adapters required for exact tombstone I/O, per-job advisory action ownership, non-failure deferral, and worker materialization of final credentials +- Test: worker unit tests and `locks-e2e/tests/postgres_runtime.rs` + +**RED:** Crash/reclaim tests after every external side effect; PostgreSQL advisory ownership exclusion; exact tombstone publication/read-back and failure on replacements observed before publication or during replay; retry exhaustion/resume; force escalation; non-destructive sorted/deduplicated verification of every frozen guarded-resource generation followed by exact retained-tombstone verification; missing or replaced frozen resources and missing or replaced tombstones fail closed; active force deletes the canonical public path before best-effort private cleanup; readiness degradation; shutdown stops claims before HTTP drain and bounds the complete worker/HTTP join. These tests do not claim atomic replacement safety across the accepted read-to-unconditional-`PUT` window in decision 41. + +**GREEN:** Reuse existing worker configuration conventions but keep queue cadence and retry due time separate. Never log manifest, resource paths, Bundle IDs, credentials, readers, or Paykit payloads. + +**Task 9 PostgreSQL crash/reclaim acceptance coverage map:** + +- Graceful public tombstone publication/read-back: `locks-e2e/tests/postgres_runtime.rs::postgres_graceful_withdraw_crash_reclaims_without_republishing_or_stale_advance` executes the production phase executor over the PostgreSQL job/lease repository, simulates process loss after publication, proves a fresh claim and advisory owner resume from exact read-back without a second publication, and fences the stale phase write. Exact missing/replaced byte classification remains covered by `locks-service/tests/content_lock_tombstones.rs` and the phase-executor failure tests. +- Payment-drain start and reconciliation: `payment_drain_reclaim_reconciles_external_start_before_local_persistence` covers remote start before local persistence and fresh-claim lookup reconciliation; `start_phase_replay_persists_monotonic_progress_after_crash_before_phase_advance`, `reclaim_first_fences_stale_initial_payment_drain_store`, `concurrent_force_winner_fences_stale_payment_drain_reconciliation`, and `concurrent_reclaim_winner_fences_stale_terminal_obligation_persistence` cover persisted aggregate replay plus stale start/reconcile/task writes against real PostgreSQL. +- Final credential generation/persistence/replay: `final_credentials_to_materialize_revalidates_exact_live_issue_claim_and_deadline` and `worker_final_issuance_is_exact_claim_fenced_in_winner_transaction` cover live-claim enumeration, fresh reclaimed ownership, stale/forced/deadline fencing, encrypted winner persistence, exact replay, and one-row cardinality against real PostgreSQL; `concurrent_final_issuers_replay_one_winner` independently covers concurrent winner replay. +- Frozen guarded-resource generation and retained-tombstone verification: `locks-e2e/tests/postgres_runtime.rs::postgres_guarded_generation_verification_crash_reclaims_and_replays_without_deletion` demonstrates two distinct loss points: after a frozen generation read but before its phase advance, and after exact retained-tombstone read-back but before the purge-handoff advance. Each boundary drops advisory ownership, recreates the PostgreSQL repository/runtime executor, reclaims with a fresh token, fences the stale advance, and idempotently replays without deleting private bytes. +- Active-force public-first/private cleanup: `locks-e2e/tests/postgres_runtime.rs::postgres_active_force_public_delete_crash_reclaims_before_private_cleanup` loses ownership immediately after canonical public deletion, before any private cleanup, then recreates the PostgreSQL repository/runtime executor, fences stale completion, replays public absence, performs private cleanup, and persists the terminal force receipt. `locks-e2e/tests/postgres_runtime.rs::postgres_active_force_private_delete_crash_reclaims_to_terminal_receipt` separately loses ownership after private deletion, then proves fresh-token reclaim, stale completion fencing, idempotent public/private replay, permanent receipt persistence, job removal, and no further claimable work. `execute_forced_content_lock_deletion::tests` supplies the sorted/deduplicated multi-resource and best-effort error matrix at the same production use-case seam. +- Every composed E2E case above drops the detached PostgreSQL advisory guard as the crash boundary and proves a later independent acquisition succeeds; `postgres_deletion_action_ownership_excludes_overlap_and_reacquires_after_release` separately proves overlap exclusion. + +**Suggested commit:** `feat(server): run graceful deletion worker` + +### Task 10: Purge graceful state and preserve force blocks + +**Objective:** Purge graceful operational state without removing the durable external tombstone, while permanently blocking force-deleted Lock IDs. + +**Files:** +- Create/modify: lock-scoped purge repository/use case in `locks-service/src/` +- Modify: deletion worker +- Modify: content-lock creation ownership/force-receipt checks +- Test: PostgreSQL E2E and creator publishing HTTP E2E + +**RED:** Prove all Locks task/proof/entitlement/credential/job rows are gone after graceful completion, ownership is released only after external cleanup, fresh same-ID publication accepts only new Bundle IDs, late old task replay cannot reactivate, force receipt blocks same-ID publication forever, and failed force paths retain ownership. + +**Suggested commit:** `feat(service): finalize lock deletion lifecycle` + +### Task 11: Reader UX and documentation + +**Objective:** Make the application deadline visible and prevent accidental late manual payment. + +**Files:** +- Modify only currently active reader/demo files discovered at implementation time; audit `examples/js-sdk/`, `README.md`, and `docs/LOCAL_OPERATOR_DEMO.md` before naming exact files. +- Modify: protocol/API documentation for criterion and deletion routes. + +**RED:** Browser/demo test with injected clock proves payment action disabled at equality boundary only after the inclusive deadline has passed, warning is visible, and no automatic payment is initiated. + +**GREEN:** Display Paykit-returned absolute deadline; do not derive from browser clock plus duration. + +**Suggested commit:** `docs: document payment deadlines and lock deletion` + +## Cross-repository implementation/review order + +1. Commit synchronized plan-only changes separately in Locks and Paykit Server. +2. Locks Task 1 (`payment_in`) and publish/review the exact Locks Core revision Paykit will consume. +3. Paykit Server invoice persistence/response and deadline observation slices. +4. Resolve and patch the exact per-Bundle enums and operational-drain cleanup route in both plans. +5. Paykit Server drain/status API slices. +6. Locks invoice persistence and payment-drain client slices. +7. Locks deletion persistence/API/worker/credential slices. +8. Cross-service E2E and docs. + +No repository may claim the sibling contract implemented until pinned dependency/revision and live tests prove it. + +## Verification + +Repository-local final verification: + +```bash +cargo fmt --all +cargo test -p locks-core +cargo test -p locks-service +cargo test -p locks-server +cargo test -p locks-sdk +cargo test -p locks-sdk-wasm +TEST_DATABASE_URL="$TEST_DATABASE_URL" cargo test -p locks-e2e --test postgres_runtime +TEST_DATABASE_URL="$TEST_DATABASE_URL" cargo test -p locks-e2e --test production_creator_publishing_http +TEST_DATABASE_URL="$TEST_DATABASE_URL" cargo test -p locks-e2e --test retrieval_access_http +cargo test --workspace +cargo clippy --workspace --all-targets -- -D warnings +cargo fmt --all --check +git diff --check +``` + +Cross-service acceptance must additionally prove: + +- invoice timestamp exact replay; +- canonical lock/request `payment_in` mismatch rejection with no side effects; +- inclusive first amount-matched-observation deadline; +- underpayment expiry and matched-payment confirmation continuation; +- atomic acceptance/cancellation drain cutoff; +- cancellation enqueue without delivery wait; +- Locks-only minimum-confirmation decision; +- deletion crash recovery after every remote effect; +- exact tombstone replacement halt/resume; +- existing/final credential drain and concurrent per-path consumption; +- graceful tombstone preservation with no old authorization revival; +- permanent force same-ID block. + +## Remaining implementation-contract gates + +None. The exact Locks-only deletion configuration and runtime-master-key contracts are fixed above. Paykit Server has no corresponding credential or deletion-worker configuration. + +## Out of scope + +- Paykit protocol `payment_due_at` field or accepted-expiry event. +- Automatic refunds or late-payment access. +- Manual/automatic Bitcoin payment from the reader. +- Cross-system transactions or exactly-once external effects. +- Historical production-data migration/backfill. +- Republishing force-deleted Lock IDs. +- Deleting reader-downloaded copies. diff --git a/examples/js-sdk/README.md b/examples/js-sdk/README.md index 99d36ec..f59fd1d 100644 --- a/examples/js-sdk/README.md +++ b/examples/js-sdk/README.md @@ -87,7 +87,7 @@ docker compose up --build On first startup, the Lock Server entrypoint generates a random creator-authority encryption key and persists it in the private `lock-home` volume. Later starts reuse that key. To supply your own 32-byte base64url key instead, export -`PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY` before running Compose. +`PUBKY_LOCK_RUNTIME_MASTER_KEY` before running Compose. The compose stack starts: @@ -159,10 +159,10 @@ The examples do **not** generate or mutate Lock Server TOML. They read the Lock ~/.pubky-lock/config.toml ``` -Generate a local creator-authority encryption key for the same shell that starts the Lock Server: +Generate a local runtime master key for the same shell that starts the Lock Server: ```bash -export PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY="$( +export PUBKY_LOCK_RUNTIME_MASTER_KEY="$( python3 - <<'PY' import base64, os print(base64.urlsafe_b64encode(os.urandom(32)).decode().rstrip('=')) @@ -170,7 +170,7 @@ PY )" ``` -To generate the default config and Lock Server secret, start the server once after setting `PUBKY_LOCK_DATABASE_URL` and `PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY`: +To generate the default config and Lock Server secret, start the server once after setting `PUBKY_LOCK_DATABASE_URL` and `PUBKY_LOCK_RUNTIME_MASTER_KEY`: ```bash cargo run -p locks-server diff --git a/locks-core/src/content_lock_deletion.rs b/locks-core/src/content_lock_deletion.rs new file mode 100644 index 0000000..fddafef --- /dev/null +++ b/locks-core/src/content_lock_deletion.rs @@ -0,0 +1,83 @@ +use serde::{Deserialize, Deserializer, Serialize}; +use time::{OffsetDateTime, UtcOffset}; + +use crate::ids::LockId; + +/// Supported public content-lock deletion tombstone version. +pub const CONTENT_LOCK_DELETION_TOMBSTONE_VERSION: u16 = 1; +const CONTENT_LOCK_DELETION_TOMBSTONE_TYPE: &str = "content_lock_deletion"; + +/// Exact public replacement for a content lock while graceful deletion runs. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ContentLockDeletionTombstone { + #[serde(deserialize_with = "deserialize_version")] + version: u16, + #[serde(rename = "type", deserialize_with = "deserialize_type")] + kind: String, + /// Identifier of the withdrawn canonical content lock. + pub lock_id: LockId, + /// Durable proof-admission cutoff, encoded as RFC3339 UTC. + #[serde( + serialize_with = "time::serde::rfc3339::serialize", + deserialize_with = "deserialize_utc_timestamp" + )] + pub deletion_started_at: OffsetDateTime, +} + +impl ContentLockDeletionTombstone { + /// Creates the exact supported tombstone payload. + pub fn new(lock_id: LockId, deletion_started_at: OffsetDateTime) -> Self { + Self { + version: CONTENT_LOCK_DELETION_TOMBSTONE_VERSION, + kind: CONTENT_LOCK_DELETION_TOMBSTONE_TYPE.to_owned(), + lock_id, + deletion_started_at: deletion_started_at.to_offset(UtcOffset::UTC), + } + } + + /// Returns the supported tombstone version. + pub fn version(&self) -> u16 { + self.version + } +} + +fn deserialize_version<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let version = u16::deserialize(deserializer)?; + if version == CONTENT_LOCK_DELETION_TOMBSTONE_VERSION { + Ok(version) + } else { + Err(serde::de::Error::custom( + "unsupported content lock deletion tombstone version", + )) + } +} + +fn deserialize_type<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let kind = String::deserialize(deserializer)?; + if kind == CONTENT_LOCK_DELETION_TOMBSTONE_TYPE { + Ok(kind) + } else { + Err(serde::de::Error::custom( + "unsupported content lock deletion tombstone type", + )) + } +} + +fn deserialize_utc_timestamp<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let timestamp = time::serde::rfc3339::deserialize(deserializer)?; + if timestamp.offset() == UtcOffset::UTC { + Ok(timestamp) + } else { + Err(serde::de::Error::custom("timestamp must use UTC offset Z")) + } +} diff --git a/locks-core/src/lib.rs b/locks-core/src/lib.rs index fca1f54..fc37d7f 100644 --- a/locks-core/src/lib.rs +++ b/locks-core/src/lib.rs @@ -1,3 +1,4 @@ +pub mod content_lock_deletion; pub mod creator_publishing; pub mod ids; pub mod lock_policy; diff --git a/locks-core/src/lock_policy.rs b/locks-core/src/lock_policy.rs index 2c6d57c..6fb2e13 100644 --- a/locks-core/src/lock_policy.rs +++ b/locks-core/src/lock_policy.rs @@ -336,6 +336,35 @@ pub enum PaykitPaymentParamsValidationError { InvalidAmount, #[error("paykit-payment asset must be a non-empty string")] InvalidAsset, + #[error("paykit-payment payment_in must be a positive whole-hour JSON u64")] + InvalidPaymentIn, +} + +/// Validated public parameters for a `paykit-payment` criterion. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PaykitPaymentParams { + recipient_pubky: CreatorPubky, + amount: String, + asset: String, + payment_in: u64, +} + +impl PaykitPaymentParams { + pub fn recipient_pubky(&self) -> &CreatorPubky { + &self.recipient_pubky + } + + pub fn amount(&self) -> &str { + &self.amount + } + + pub fn asset(&self) -> &str { + &self.asset + } + + pub fn payment_in(&self) -> u64 { + self.payment_in + } } /// Invalid v1 content-lock policy containing a `paykit-payment` criterion. @@ -370,22 +399,32 @@ pub struct Criterion { impl Criterion { /// Validates verifier-specific public criterion params. pub fn validate_params(&self) -> Result<(), PaykitPaymentParamsValidationError> { + self.paykit_payment_params().map(|_| ()) + } + + /// Returns typed parameters when this is a `paykit-payment` criterion. + pub fn paykit_payment_params( + &self, + ) -> Result, PaykitPaymentParamsValidationError> { match self.verifier_type { - VerifierType::DevStatic => Ok(()), - VerifierType::PaykitPayment => validate_paykit_payment_params(&self.params), + VerifierType::DevStatic => Ok(None), + VerifierType::PaykitPayment => validate_paykit_payment_params(&self.params).map(Some), } } } fn validate_paykit_payment_params( params: &Value, -) -> Result<(), PaykitPaymentParamsValidationError> { +) -> Result { let object = params .as_object() .ok_or(PaykitPaymentParamsValidationError::NotObject)?; for key in object.keys() { - if !matches!(key.as_str(), "recipient_pubky" | "amount" | "asset") { + if !matches!( + key.as_str(), + "recipient_pubky" | "amount" | "asset" | "payment_in" + ) { return Err(PaykitPaymentParamsValidationError::UnknownField( key.clone(), )); @@ -398,7 +437,7 @@ fn validate_paykit_payment_params( .ok_or(PaykitPaymentParamsValidationError::MissingField( "recipient_pubky", ))?; - CreatorPubky::from_str(recipient_pubky) + let recipient_pubky = CreatorPubky::from_str(recipient_pubky) .map_err(|_| PaykitPaymentParamsValidationError::InvalidRecipientPubky)?; let amount = object @@ -422,7 +461,21 @@ fn validate_paykit_payment_params( return Err(PaykitPaymentParamsValidationError::InvalidAsset); } - Ok(()) + let payment_in = object + .get("payment_in") + .ok_or(PaykitPaymentParamsValidationError::MissingField( + "payment_in", + ))? + .as_u64() + .filter(|payment_in| *payment_in > 0) + .ok_or(PaykitPaymentParamsValidationError::InvalidPaymentIn)?; + + Ok(PaykitPaymentParams { + recipient_pubky, + amount: amount.to_owned(), + asset: asset.to_owned(), + payment_in, + }) } /// Logic expression over criterion identifiers. @@ -473,7 +526,7 @@ mod tests { AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, ContentLockValidationError, Criterion, GuardedResource, GuardedResourceValidationError, LockLogic, LockServerConfig, PRIVATE_PROOF_BUNDLE_PATH_PREFIX, PRIVATE_RESOURCE_CONTENT_PATH_PREFIX, - PUBLIC_LOCKS_APP_PATH_PREFIX, PaykitPaymentParamsValidationError, + PUBLIC_LOCKS_APP_PATH_PREFIX, PaykitPaymentParams, PaykitPaymentParamsValidationError, PaykitPaymentPolicyValidationError, SecondaryGuardedResource, VerifierType, verified_proof_bundle_path, }; @@ -550,7 +603,8 @@ mod tests { params: json!({ "recipient_pubky": recipient_pubky.to_string(), "amount": "50000", - "asset": "BTC" + "asset": "BTC", + "payment_in": 24 }), } } @@ -868,77 +922,91 @@ mod tests { "recipient_pubky": test_pubky_identity(), "amount": "50000", "asset": "BTC", + "payment_in": 24, }), }; assert_eq!(criterion.validate_params(), Ok(())); + let params = criterion.paykit_payment_params().unwrap().unwrap(); + assert_eq!(params.amount(), "50000"); + assert_eq!(params.asset(), "BTC"); + assert_eq!(params.payment_in(), 24); + assert_eq!( + params.recipient_pubky().to_string(), + criterion.params["recipient_pubky"] + ); + let _: PaykitPaymentParams = params; } #[test] fn paykit_payment_params_reject_invalid_shapes() { + let recipient = test_pubky_identity(); + let overflow = serde_json::from_str(&format!( + r#"{{"recipient_pubky":"{recipient}","amount":"50000","asset":"BTC","payment_in":18446744073709551616}}"# + )) + .unwrap(); for (params, expected) in [ (json!(null), PaykitPaymentParamsValidationError::NotObject), ( - json!({ - "recipient_pubky": test_pubky_identity(), - "amount": "50000", - "asset": "BTC", - "memo": "extra", - }), + json!({ "recipient_pubky": recipient, "amount": "50000", "asset": "BTC", "payment_in": 24, "memo": "extra" }), PaykitPaymentParamsValidationError::UnknownField("memo".to_owned()), ), ( - json!({ "amount": "50000", "asset": "BTC" }), + json!({ "amount": "50000", "asset": "BTC", "payment_in": 24 }), PaykitPaymentParamsValidationError::MissingField("recipient_pubky"), ), ( - json!({ "recipient_pubky": test_pubky_identity(), "asset": "BTC" }), + json!({ "recipient_pubky": recipient, "asset": "BTC", "payment_in": 24 }), PaykitPaymentParamsValidationError::MissingField("amount"), ), ( - json!({ "recipient_pubky": test_pubky_identity(), "amount": "50000" }), + json!({ "recipient_pubky": recipient, "amount": "50000", "payment_in": 24 }), PaykitPaymentParamsValidationError::MissingField("asset"), ), ( - json!({ - "recipient_pubky": "not-a-pubky", - "amount": "50000", - "asset": "BTC", - }), + json!({ "recipient_pubky": recipient, "amount": "50000", "asset": "BTC" }), + PaykitPaymentParamsValidationError::MissingField("payment_in"), + ), + ( + json!({ "recipient_pubky": "not-a-pubky", "amount": "50000", "asset": "BTC", "payment_in": 24 }), PaykitPaymentParamsValidationError::InvalidRecipientPubky, ), ( - json!({ - "recipient_pubky": test_pubky_identity(), - "amount": "0", - "asset": "BTC", - }), + json!({ "recipient_pubky": recipient, "amount": "0", "asset": "BTC", "payment_in": 24 }), PaykitPaymentParamsValidationError::InvalidAmount, ), ( - json!({ - "recipient_pubky": test_pubky_identity(), - "amount": "0.5", - "asset": "BTC", - }), + json!({ "recipient_pubky": recipient, "amount": "0.5", "asset": "BTC", "payment_in": 24 }), PaykitPaymentParamsValidationError::InvalidAmount, ), ( - json!({ - "recipient_pubky": test_pubky_identity(), - "amount": 50000, - "asset": "BTC", - }), + json!({ "recipient_pubky": recipient, "amount": 50000, "asset": "BTC", "payment_in": 24 }), PaykitPaymentParamsValidationError::InvalidAmount, ), ( - json!({ - "recipient_pubky": test_pubky_identity(), - "amount": "50000", - "asset": "", - }), + json!({ "recipient_pubky": recipient, "amount": "50000", "asset": "", "payment_in": 24 }), PaykitPaymentParamsValidationError::InvalidAsset, ), + ( + json!({ "recipient_pubky": recipient, "amount": "50000", "asset": "BTC", "payment_in": 0 }), + PaykitPaymentParamsValidationError::InvalidPaymentIn, + ), + ( + json!({ "recipient_pubky": recipient, "amount": "50000", "asset": "BTC", "payment_in": -1 }), + PaykitPaymentParamsValidationError::InvalidPaymentIn, + ), + ( + json!({ "recipient_pubky": recipient, "amount": "50000", "asset": "BTC", "payment_in": 1.5 }), + PaykitPaymentParamsValidationError::InvalidPaymentIn, + ), + ( + json!({ "recipient_pubky": recipient, "amount": "50000", "asset": "BTC", "payment_in": "24" }), + PaykitPaymentParamsValidationError::InvalidPaymentIn, + ), + ( + overflow, + PaykitPaymentParamsValidationError::InvalidPaymentIn, + ), ] { let criterion = Criterion { criterion_id: "criterion-1".to_owned(), @@ -1214,4 +1282,17 @@ mod tests { without_override.lock_id().unwrap() ); } + + #[test] + fn changing_paykit_payment_in_changes_lock_id() { + let mut shorter = content_lock_fixture(); + shorter.criteria = vec![paykit_criterion("payment", &shorter.creator)]; + shorter.lock_logic = LockLogic::All { + criteria: vec!["payment".to_owned()], + }; + let mut longer = shorter.clone(); + longer.criteria[0].params["payment_in"] = json!(25); + + assert_ne!(shorter.lock_id().unwrap(), longer.lock_id().unwrap()); + } } diff --git a/locks-core/tests/content_lock_deletion.rs b/locks-core/tests/content_lock_deletion.rs new file mode 100644 index 0000000..50399ce --- /dev/null +++ b/locks-core/tests/content_lock_deletion.rs @@ -0,0 +1,77 @@ +use std::str::FromStr; + +use locks_core::content_lock_deletion::{ + CONTENT_LOCK_DELETION_TOMBSTONE_VERSION, ContentLockDeletionTombstone, +}; +use locks_core::ids::LockId; +use serde_json::json; +use time::macros::datetime; + +const LOCK_ID: &str = "000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG"; + +#[test] +fn tombstone_serializes_the_exact_closed_protocol_shape() { + let tombstone = ContentLockDeletionTombstone::new( + LockId::from_str(LOCK_ID).unwrap(), + datetime!(2026-08-12 05:00:00 UTC), + ); + + assert_eq!( + serde_json::to_value(&tombstone).unwrap(), + json!({ + "version": CONTENT_LOCK_DELETION_TOMBSTONE_VERSION, + "type": "content_lock_deletion", + "lock_id": LOCK_ID, + "deletion_started_at": "2026-08-12T05:00:00Z", + }) + ); +} + +#[test] +fn tombstone_rejects_unknown_version_type_fields_and_non_utc_time() { + let valid = json!({ + "version": 1, + "type": "content_lock_deletion", + "lock_id": LOCK_ID, + "deletion_started_at": "2026-08-12T05:00:00Z", + }); + assert!(serde_json::from_value::(valid.clone()).is_ok()); + + for invalid in [ + { + let mut value = valid.clone(); + value["version"] = json!(2); + value + }, + { + let mut value = valid.clone(); + value["type"] = json!("content_lock"); + value + }, + { + let mut value = valid.clone(); + value["extra"] = json!(true); + value + }, + { + let mut value = valid; + value["deletion_started_at"] = json!("2026-08-12T06:00:00+01:00"); + value + }, + ] { + assert!(serde_json::from_value::(invalid).is_err()); + } +} + +#[test] +fn tombstone_constructor_normalizes_offsets_to_utc() { + let tombstone = ContentLockDeletionTombstone::new( + LockId::from_str(LOCK_ID).unwrap(), + datetime!(2026-08-12 06:00:00 +01:00), + ); + + assert_eq!( + serde_json::to_value(tombstone).unwrap()["deletion_started_at"], + "2026-08-12T05:00:00Z" + ); +} diff --git a/locks-e2e/tests/creator_publishing_http.rs b/locks-e2e/tests/creator_publishing_http.rs index 89456c1..6734a28 100644 --- a/locks-e2e/tests/creator_publishing_http.rs +++ b/locks-e2e/tests/creator_publishing_http.rs @@ -20,6 +20,7 @@ use locks_server::testing::TestServerApp; use locks_server::worker::{VerificationWorker, WorkerTick}; use locks_service::application::models::FrontendSessionToken; use locks_service::application::ports::CriterionVerifier; +use locks_service::infrastructure::memory::content_lock_tombstones::InMemoryContentLockTombstoneRepository; use locks_service::infrastructure::memory::content_locks::InMemoryContentLockRepository; use locks_service::infrastructure::memory::entitlements::InMemoryEntitlementRepository; use locks_service::infrastructure::memory::guarded_resources::InMemoryGuardedResourceRepository; @@ -41,6 +42,7 @@ async fn creator_publishing_http_flow_registers_locks_verifies_and_proxy_reads_g TestServerApp::from_state(AppState::new_empty_in_memory_with_creator_repositories( config, Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryContentLockTombstoneRepository::new()), Arc::new(InMemoryGuardedResourceRepository::new()), Arc::new(InMemoryLockServicePointerRepository::new()), Arc::new(InMemoryEntitlementRepository::new()), @@ -335,6 +337,7 @@ async fn creator_publishing_http_rejects_invalid_guarded_path_before_lock_creati TestServerApp::from_state(AppState::new_empty_in_memory_with_creator_repositories( config, Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryContentLockTombstoneRepository::new()), Arc::new(InMemoryGuardedResourceRepository::new()), Arc::new(InMemoryLockServicePointerRepository::new()), Arc::new(InMemoryEntitlementRepository::new()), @@ -377,7 +380,8 @@ async fn creator_publishing_http_rejects_invalid_paykit_payment_params() { "params": { "recipient_pubky": creator().to_string(), "amount": "0", - "asset": "BTC" + "asset": "BTC", + "payment_in": 24 } }]), standard_lock_logic(), @@ -399,7 +403,8 @@ async fn creator_publishing_http_rejects_invalid_paykit_payment_params() { "params": { "recipient_pubky": "pubky7ir1ttte48bcp4zjychjyscicrwi1j34mtt91ptsafdbjmr8g9eo", "amount": "50000", - "asset": "BTC" + "asset": "BTC", + "payment_in": 24 } }]), standard_lock_logic(), @@ -484,6 +489,7 @@ async fn creator_publishing_http_paykit_payment_flow_creates_invoice_verifies_an let state = AppState::new_empty_in_memory_with_creator_repositories( config, Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryContentLockTombstoneRepository::new()), Arc::new(InMemoryGuardedResourceRepository::new()), Arc::new(InMemoryLockServicePointerRepository::new()), Arc::new(InMemoryEntitlementRepository::new()), @@ -515,7 +521,8 @@ async fn creator_publishing_http_paykit_payment_flow_creates_invoice_verifies_an "params": { "recipient_pubky": creator().to_string(), "amount": "50000", - "asset": "BTC" + "asset": "BTC", + "payment_in": 24 } }]), standard_lock_logic(), @@ -625,6 +632,7 @@ async fn creator_publishing_client() -> (TestServerApp, LocalCreatorPublishingCl TestServerApp::from_state(AppState::new_empty_in_memory_with_creator_repositories( config, Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryContentLockTombstoneRepository::new()), Arc::new(InMemoryGuardedResourceRepository::new()), Arc::new(InMemoryLockServicePointerRepository::new()), Arc::new(InMemoryEntitlementRepository::new()), @@ -757,7 +765,8 @@ fn paykit_criterion_json(criterion_id: &str) -> serde_json::Value { "params": { "recipient_pubky": creator().to_string(), "amount": "50000", - "asset": "BTC" + "asset": "BTC", + "payment_in": 24 } }) } @@ -823,6 +832,7 @@ impl FakePaykitServer { Some(json!({ "bundle_id": BUNDLE_ID, "lock_resource": lock_resource, + "payment_in": 24, "reader": creator().to_string(), })) ); @@ -868,14 +878,20 @@ async fn fake_invoice_handler( State(state): State>>, headers: HeaderMap, body: Bytes, -) -> StatusCode { +) -> (StatusCode, Json) { let mut state = state.lock().await; state.invoice_count += 1; state.invoice_body = Some(serde_json::from_slice(&body).unwrap()); state.invoice_signature = headers .get("X-Paykit-Signature") .map(|value| value.to_str().unwrap().to_owned()); - StatusCode::CREATED + ( + StatusCode::CREATED, + Json(json!({ + "invoice_created_at": "2026-08-12T10:00:00Z", + "payment_deadline": "2026-08-13T10:00:00Z", + })), + ) } async fn fake_status_handler( diff --git a/locks-e2e/tests/postgres_runtime.rs b/locks-e2e/tests/postgres_runtime.rs index 976dab8..ef132af 100644 --- a/locks-e2e/tests/postgres_runtime.rs +++ b/locks-e2e/tests/postgres_runtime.rs @@ -1,12 +1,19 @@ +use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::PathBuf; use std::str::FromStr; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use async_trait::async_trait; use axum::body::{Body, to_bytes}; use axum::extract::ConnectInfo; use axum::http::{Request, StatusCode, header}; +use axum::routing::post; +use locks_core::content_lock_deletion::ContentLockDeletionTombstone; use locks_core::ids::{ - BundleId, CreatorPubky, GuardedResourceHash, LockServerPubky, PubkyLockResource, + BundleId, ContentLockPath, CreatorPubky, GuardedResourceHash, LockServerPubky, + PubkyLockResource, TaskId, }; use locks_core::lock_policy::{ AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, Criterion, GuardedResource, LockLogic, @@ -14,39 +21,705 @@ use locks_core::lock_policy::{ }; use locks_core::verification::{Proof, SUBMITTED_PROOF_BUNDLE_VERSION, SubmittedProofBundle}; use locks_server::api::routes::router; -use locks_server::app_state::AppState; +use locks_server::app_state::{AppState, ReaderPubkyResolver, RuntimeSecretCiphers}; use locks_server::config::{ - ContentLocksConfig, CreatorAuthorityAcquisitionConfig, DatabaseConfig, - LockServerCredentialsConfig, LockServerRuntimeConfig, LoggingConfig, PkdnsConfig, PubkyConfig, + ContentLocksConfig, CreatorAuthorityAcquisitionConfig, DatabaseConfig, DeletionConfig, + FilesystemLockServerIdentityProvider, LockServerCredentialsConfig, LockServerIdentityProvider, + LockServerRuntimeConfig, LoggingConfig, PaykitConfig, PkdnsConfig, PubkyConfig, RateLimitsConfig, RuntimeConfig, RuntimeEnvironment, SecretsConfig, WorkerConfig, }; +use locks_server::deletion_worker::{ClaimedDeletionExecutor, RuntimeClaimedDeletionExecutor}; +use locks_server::testing::TestServerApp; use locks_server::worker::{VerificationWorker, WorkerTick}; use locks_service::application::models::{ - AccessCredential, AccessCredentialLookupKey, CreatorAuthorityAuthKind, CreatorAuthorityRecord, - CreatorAuthoritySecret, VerificationTaskStatus, + AccessCredential, AccessCredentialLookupKey, ClaimedContentLockDeletionJob, + ContentLockDeletionJob, ContentLockDeletionPhase, ContentLockDeletionState, + ContentLockOwnershipStatus, CreatorAuthorityAuthKind, CreatorAuthorityRecord, + CreatorAuthoritySecret, FrontendSessionToken, GuardedResourceRecord, + PrepareForceDeletionResult, VerificationTaskRecord, VerificationTaskStatus, }; +use locks_service::application::ports::{ + ContentLockDeletionActionAcquireResult, ContentLockDeletionActionClaim, + ContentLockDeletionActionGuard, ContentLockDeletionActionOwnership, + ContentLockDeletionRepository, ContentLockTombstoneRepository, GuardedResourceReadback, + GuardedResourceRepository, TombstoneReadback, VerificationTaskRepository, +}; +use locks_service::infrastructure::final_credentials::FinalCredentialCipher; use locks_service::infrastructure::memory::{ + content_lock_tombstones::InMemoryContentLockTombstoneRepository, content_locks::InMemoryContentLockRepository, entitlements::InMemoryEntitlementRepository, guarded_resources::InMemoryGuardedResourceRepository, lock_service_pointers::InMemoryLockServicePointerRepository, }; -use locks_service::infrastructure::postgres::{CreatorAuthoritySecretCipher, run_migrations}; +use locks_service::infrastructure::postgres::{ + CreatorAuthoritySecretCipher, PostgresContentLockDeletionActionOwnership, + PostgresContentLockDeletionRepository, PostgresVerificationTaskRepository, run_migrations, +}; use serde_json::{Value, json}; use sqlx::postgres::PgPoolOptions; use sqlx::{Connection, Executor, PgConnection, PgPool}; -use time::macros::datetime; +use time::{OffsetDateTime, macros::datetime}; use tower::ServiceExt; const BUNDLE_ID: &str = "000G40R40M30E209185GR38E1W"; +#[tokio::test] +async fn postgres_deletion_action_ownership_excludes_overlap_and_reacquires_after_release() { + let Some(database) = TestDatabase::create().await else { + return; + }; + let first_owner = PostgresContentLockDeletionActionOwnership::new(database.pool().clone()); + let second_owner = PostgresContentLockDeletionActionOwnership::new(database.pool().clone()); + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new( + uuid::Uuid::new_v4(), + content_lock(), + OffsetDateTime::now_utc(), + ) + .unwrap(); + deletions.insert_job(job).await.unwrap(); + let claimed = deletions + .claim_next("ownership-worker", time::Duration::minutes(1)) + .await + .unwrap() + .unwrap(); + + let first_guard = expect_action_acquired( + first_owner + .try_acquire(deletion_action_claim(&claimed, "ownership-worker", false)) + .await + .unwrap(), + ); + assert!(matches!( + second_owner + .try_acquire(deletion_action_claim(&claimed, "ownership-worker", false,)) + .await + .unwrap(), + ContentLockDeletionActionAcquireResult::Busy + )); + + first_guard.release().await.unwrap(); + let replacement_guard = expect_action_acquired( + second_owner + .try_acquire(deletion_action_claim(&claimed, "ownership-worker", false)) + .await + .unwrap(), + ); + replacement_guard.release().await.unwrap(); + + database.cleanup().await; +} + +#[tokio::test] +async fn postgres_missed_final_issuance_terminalizes_with_closed_creator_failure_without_external_action() + { + let Some(database) = TestDatabase::create().await else { + return; + }; + let lock = content_lock(); + let external = Arc::new(CrashExternalRepository::with_tombstone_and_resources(&lock)); + let state = deletion_app_state(database.pool().clone(), Arc::clone(&external)); + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let now = OffsetDateTime::now_utc(); + let mut task = VerificationTaskRecord { + task_id: TaskId::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(), + creator: lock.creator.clone(), + submitted_proof_bundle: submitted_proof_bundle_for(&lock), + status: VerificationTaskStatus::Completed, + submitted_at: now - time::Duration::hours(1), + started_at: Some(now - time::Duration::hours(1)), + completed_at: Some(now - time::Duration::minutes(30)), + failure_message: None, + }; + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + tasks.insert_verification_task(task).await.unwrap(); + + let job = ContentLockDeletionJob::new(uuid::Uuid::new_v4(), lock, now).unwrap(); + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + deletions.insert_job(job.clone()).await.unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET resolved_status = 'completed', resolved_at = $2, + final_credential_eligible_at = $2 + WHERE deletion_job_id = $1", + ) + .bind(job.job_id) + .bind(now - time::Duration::minutes(30)) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET phase = 'issue_final_credentials', final_issuance_started_at = $2, + final_credential_issuance_deadline = $3, final_read_deadline = $4 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(now - time::Duration::minutes(20)) + .bind(now - time::Duration::minutes(5)) + .bind(now + time::Duration::minutes(10)) + .execute(database.pool()) + .await + .unwrap(); + let claim = deletions + .claim_next( + "missed-issuance-worker", + (now + time::Duration::minutes(5)) - (now), + ) + .await + .unwrap() + .unwrap(); + + assert_eq!( + RuntimeClaimedDeletionExecutor::new(state.clone()) + .execute_claimed(claim, "missed-issuance-worker") + .await + .outcome, + locks_service::application::use_cases::execute_content_lock_deletion_phase::DeletionPhaseExecutionOutcome::TerminalFailed + ); + let failed = deletions + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .unwrap(); + assert_eq!(failed.state, ContentLockDeletionState::Failed); + assert_eq!( + failed.failure_code.map(|code| code.as_str()), + Some("state_corrupt") + ); + assert!(external.operations().is_empty()); + assert_eq!(external.resource_read_count(), 0); + assert_eq!(external.resource_delete_count(), 0); + + let app = TestServerApp::from_state(state); + let token = "missed-issuance-creator-session"; + app.insert_frontend_session_for_test( + FrontendSessionToken::new(token), + job.creator.clone(), + now + time::Duration::hours(1), + ) + .await + .unwrap(); + let response = app + .router() + .oneshot( + Request::builder() + .method("GET") + .uri(format!("/creator/content-locks/{}/deletion", job.lock_id)) + .header(header::AUTHORIZATION, format!("Bearer {token}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_json(response).await, + json!({ + "lock_id": job.lock_id, + "status": "failed", + "failure_code": "state_corrupt" + }) + ); + + database.cleanup().await; +} + +#[tokio::test] +async fn postgres_graceful_withdraw_crash_reclaims_without_republishing_or_stale_advance() { + let Some(database) = TestDatabase::create().await else { + return; + }; + let external = Arc::new(CrashExternalRepository::with_original(content_lock())); + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let ownership = PostgresContentLockDeletionActionOwnership::new(database.pool().clone()); + let now = OffsetDateTime::now_utc(); + let job = ContentLockDeletionJob::new(uuid::Uuid::new_v4(), content_lock(), now).unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + + let stale = deletions + .claim_next( + "withdraw-crashed", + (now + time::Duration::seconds(1)) - (now), + ) + .await + .unwrap() + .unwrap(); + let crash_guard = expect_action_acquired( + ownership + .try_acquire(deletion_action_claim(&stale, "withdraw-crashed", false)) + .await + .unwrap(), + ); + let tombstone = ContentLockDeletionTombstone::new(job.lock_id.clone(), now); + assert_eq!( + external + .withdraw_content_lock( + job.creator.clone(), + ContentLockPath::from_lock_id(job.lock_id.clone()), + &job.frozen_content_lock, + &tombstone, + ) + .await + .unwrap(), + TombstoneReadback::Exact + ); + drop(crash_guard); + + expire_deletion_claim(database.pool(), job.job_id).await; + let fresh = deletions + .claim_next("withdraw-reclaimed", time::Duration::minutes(1)) + .await + .unwrap() + .unwrap(); + assert_ne!(fresh.claim_token, stale.claim_token); + assert!(matches!( + deletions + .advance_phase( + job.job_id, + "withdraw-crashed", + stale.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap(), + locks_service::application::models::AdvanceContentLockDeletionPhaseResult::ClaimLost + )); + + let recreated = deletion_app_state(database.pool().clone(), Arc::clone(&external)); + assert_eq!( + RuntimeClaimedDeletionExecutor::new(recreated) + .execute_claimed(fresh, "withdraw-reclaimed") + .await + .outcome, + locks_service::application::use_cases::execute_content_lock_deletion_phase::DeletionPhaseExecutionOutcome::Progressed + ); + assert_eq!(external.tombstone_write_count(), 1); + assert_eq!(external.withdraw_call_count(), 2); + assert_eq!( + deletions + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .unwrap() + .phase, + ContentLockDeletionPhase::StartPaymentDrain + ); + let release_check = deletions + .claim_next("withdraw-release-check", time::Duration::minutes(1)) + .await + .unwrap() + .unwrap(); + expect_action_acquired( + ownership + .try_acquire(deletion_action_claim( + &release_check, + "withdraw-release-check", + false, + )) + .await + .unwrap(), + ) + .release() + .await + .unwrap(); + + database.cleanup().await; +} + +#[tokio::test] +async fn postgres_guarded_generation_verification_crash_reclaims_and_replays_without_deletion() { + let Some(database) = TestDatabase::create().await else { + return; + }; + let lock = content_lock(); + let external = Arc::new(CrashExternalRepository::with_tombstone_and_resources(&lock)); + let state = deletion_app_state(database.pool().clone(), Arc::clone(&external)); + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let ownership = PostgresContentLockDeletionActionOwnership::new(database.pool().clone()); + let now = OffsetDateTime::now_utc(); + let job = ContentLockDeletionJob::new(uuid::Uuid::new_v4(), lock.clone(), now).unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + set_deletion_phase(database.pool(), job.job_id, "delete_content").await; + + let stale = deletions + .claim_next("verify-crashed", (now + time::Duration::seconds(1)) - (now)) + .await + .unwrap() + .unwrap(); + let crash_guard = expect_action_acquired( + ownership + .try_acquire(deletion_action_claim(&stale, "verify-crashed", false)) + .await + .unwrap(), + ); + let primary = lock.primary_resource.as_ref().unwrap(); + assert_eq!( + external + .read_guarded_resource_generation(&job.creator, &primary.path, &primary.hash) + .await + .unwrap(), + GuardedResourceReadback::Exact + ); + drop(crash_guard); + + expire_deletion_claim(database.pool(), job.job_id).await; + let fresh = deletions + .claim_next("verify-reclaimed", time::Duration::minutes(1)) + .await + .unwrap() + .unwrap(); + assert!(matches!( + deletions + .advance_phase( + job.job_id, + "verify-crashed", + stale.claim_token, + ContentLockDeletionPhase::DeleteTombstone, + ) + .await + .unwrap(), + locks_service::application::models::AdvanceContentLockDeletionPhaseResult::ClaimLost + )); + assert_eq!( + RuntimeClaimedDeletionExecutor::new(state) + .execute_claimed(fresh, "verify-reclaimed") + .await + .outcome, + locks_service::application::use_cases::execute_content_lock_deletion_phase::DeletionPhaseExecutionOutcome::Progressed + ); + assert_eq!(external.resource_delete_count(), 0); + assert_eq!(external.resource_read_count(), 2); + assert_eq!( + deletions + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .unwrap() + .phase, + ContentLockDeletionPhase::DeleteTombstone + ); + + let tombstone_crashed = deletions + .claim_next("tombstone-crashed", time::Duration::seconds(1)) + .await + .unwrap() + .unwrap(); + let tombstone_crash_guard = expect_action_acquired( + ownership + .try_acquire(deletion_action_claim( + &tombstone_crashed, + "tombstone-crashed", + false, + )) + .await + .unwrap(), + ); + let tombstone = ContentLockDeletionTombstone::new(job.lock_id.clone(), job.deletion_started_at); + assert_eq!( + external + .read_tombstone( + &job.creator, + &ContentLockPath::from_lock_id(job.lock_id.clone()), + &tombstone, + ) + .await + .unwrap(), + TombstoneReadback::Exact + ); + drop(tombstone_crash_guard); + + expire_deletion_claim(database.pool(), job.job_id).await; + let recreated_deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let retained_tombstone_claim = recreated_deletions + .claim_next("tombstone-reclaimed", time::Duration::minutes(1)) + .await + .unwrap() + .unwrap(); + assert_ne!( + retained_tombstone_claim.claim_token, + tombstone_crashed.claim_token + ); + assert!(matches!( + recreated_deletions + .advance_phase( + job.job_id, + "tombstone-crashed", + tombstone_crashed.claim_token, + ContentLockDeletionPhase::PurgeOperationalState, + ) + .await + .unwrap(), + locks_service::application::models::AdvanceContentLockDeletionPhaseResult::ClaimLost + )); + let recreated = deletion_app_state(database.pool().clone(), Arc::clone(&external)); + assert_eq!( + RuntimeClaimedDeletionExecutor::new(recreated) + .execute_claimed(retained_tombstone_claim, "tombstone-reclaimed") + .await + .outcome, + locks_service::application::use_cases::execute_content_lock_deletion_phase::DeletionPhaseExecutionOutcome::Progressed + ); + assert_eq!(external.tombstone_read_count(), 4); + assert_eq!(external.resource_delete_count(), 0); + assert_eq!( + recreated_deletions + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .unwrap() + .phase, + ContentLockDeletionPhase::PurgeOperationalState + ); + let release_check = recreated_deletions + .claim_next("tombstone-release-check", time::Duration::minutes(1)) + .await + .unwrap() + .unwrap(); + expect_action_acquired( + ownership + .try_acquire(deletion_action_claim( + &release_check, + "tombstone-release-check", + false, + )) + .await + .unwrap(), + ) + .release() + .await + .unwrap(); + + database.cleanup().await; +} + +#[tokio::test] +async fn postgres_active_force_public_delete_crash_reclaims_before_private_cleanup() { + let Some(database) = TestDatabase::create().await else { + return; + }; + let lock = content_lock(); + let external = Arc::new(CrashExternalRepository::with_original_and_resources(&lock)); + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let ownership = PostgresContentLockDeletionActionOwnership::new(database.pool().clone()); + let now = OffsetDateTime::now_utc(); + let job = ContentLockDeletionJob::new(uuid::Uuid::new_v4(), lock, now).unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + assert!(matches!( + deletions + .prepare_force_deletion(&job.creator, &job.lock_id) + .await + .unwrap(), + PrepareForceDeletionResult::Active(_) + )); + + let stale = deletions + .claim_next("public-crashed", (now + time::Duration::seconds(1)) - (now)) + .await + .unwrap() + .unwrap(); + let crash_guard = expect_action_acquired( + ownership + .try_acquire(deletion_action_claim(&stale, "public-crashed", true)) + .await + .unwrap(), + ); + external + .force_delete_content_lock_and_verify_absent( + &job.creator, + &ContentLockPath::from_lock_id(job.lock_id.clone()), + ) + .await + .unwrap(); + assert_eq!(external.operations(), vec!["public"]); + drop(crash_guard); + + expire_deletion_claim(database.pool(), job.job_id).await; + let recreated_deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let fresh = recreated_deletions + .claim_next("public-reclaimed", time::Duration::minutes(1)) + .await + .unwrap() + .unwrap(); + let fresh_action_claim = fresh.clone(); + assert_ne!(fresh.claim_token, stale.claim_token); + assert!( + !deletions + .complete_force_deletion(job.job_id, "public-crashed", stale.claim_token,) + .await + .unwrap() + ); + + let recreated = deletion_app_state(database.pool().clone(), Arc::clone(&external)); + assert_eq!( + RuntimeClaimedDeletionExecutor::new(recreated) + .execute_claimed(fresh, "public-reclaimed") + .await + .outcome, + locks_service::application::use_cases::execute_content_lock_deletion_phase::DeletionPhaseExecutionOutcome::Progressed + ); + assert!( + recreated_deletions + .has_force_receipt(&job.creator, &job.lock_id) + .await + .unwrap() + ); + assert!( + recreated_deletions + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .is_none() + ); + assert_eq!(external.operations(), vec!["public", "public", "private"]); + assert_eq!(external.resource_delete_count(), 1); + assert!(matches!( + ownership + .try_acquire(deletion_action_claim( + &fresh_action_claim, + "public-reclaimed", + true, + )) + .await + .unwrap(), + ContentLockDeletionActionAcquireResult::ClaimLost + )); + + database.cleanup().await; +} + +#[tokio::test] +async fn postgres_active_force_private_delete_crash_reclaims_to_terminal_receipt() { + let Some(database) = TestDatabase::create().await else { + return; + }; + let lock = content_lock(); + let external = Arc::new(CrashExternalRepository::with_original_and_resources(&lock)); + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let ownership = PostgresContentLockDeletionActionOwnership::new(database.pool().clone()); + let now = OffsetDateTime::now_utc(); + let job = ContentLockDeletionJob::new(uuid::Uuid::new_v4(), lock.clone(), now).unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + assert!(matches!( + deletions + .prepare_force_deletion(&job.creator, &job.lock_id) + .await + .unwrap(), + PrepareForceDeletionResult::Active(_) + )); + + let stale = deletions + .claim_next( + "private-crashed", + (now + time::Duration::seconds(1)) - (now), + ) + .await + .unwrap() + .unwrap(); + let crash_guard = expect_action_acquired( + ownership + .try_acquire(deletion_action_claim(&stale, "private-crashed", true)) + .await + .unwrap(), + ); + external + .force_delete_content_lock_and_verify_absent( + &job.creator, + &ContentLockPath::from_lock_id(job.lock_id.clone()), + ) + .await + .unwrap(); + external + .delete_guarded_resource(&job.creator, &lock.primary_resource.as_ref().unwrap().path) + .await + .unwrap(); + assert_eq!(external.operations(), vec!["public", "private"]); + drop(crash_guard); + + expire_deletion_claim(database.pool(), job.job_id).await; + let recreated_deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let fresh = recreated_deletions + .claim_next("private-reclaimed", time::Duration::minutes(1)) + .await + .unwrap() + .unwrap(); + let fresh_action_claim = fresh.clone(); + assert_ne!(fresh.claim_token, stale.claim_token); + assert!( + !deletions + .complete_force_deletion(job.job_id, "private-crashed", stale.claim_token,) + .await + .unwrap() + ); + + let recreated = deletion_app_state(database.pool().clone(), Arc::clone(&external)); + assert_eq!( + RuntimeClaimedDeletionExecutor::new(recreated) + .execute_claimed(fresh, "private-reclaimed") + .await + .outcome, + locks_service::application::use_cases::execute_content_lock_deletion_phase::DeletionPhaseExecutionOutcome::Progressed + ); + assert!( + recreated_deletions + .has_force_receipt(&job.creator, &job.lock_id) + .await + .unwrap() + ); + assert!( + recreated_deletions + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .is_none() + ); + assert!( + recreated_deletions + .claim_next("after-terminal", time::Duration::minutes(1)) + .await + .unwrap() + .is_none() + ); + assert_eq!( + external.operations(), + vec!["public", "private", "public", "private"] + ); + assert_eq!(external.resource_delete_count(), 2); + assert!(matches!( + ownership + .try_acquire(deletion_action_claim( + &fresh_action_claim, + "private-reclaimed", + true, + )) + .await + .unwrap(), + ContentLockDeletionActionAcquireResult::ClaimLost + )); + + database.cleanup().await; +} + #[tokio::test] async fn postgres_runtime_state_survives_app_state_recreation() { let Some(database) = TestDatabase::create().await else { return; }; let content_lock = content_lock(); + let lock_id = content_lock.lock_id().unwrap(); + let guarded_paths = vec![content_lock.primary_resource.as_ref().unwrap().path.clone()]; let first_state = app_state(database.pool().clone()); + first_state + .content_lock_ownership() + .reserve_paths(&creator(), &guarded_paths, &lock_id) + .await + .unwrap(); + first_state + .content_lock_ownership() + .mark_paths_published(&creator(), &guarded_paths, &lock_id) + .await + .unwrap(); seed_content_lock(&first_state, content_lock.clone()).await; let first_router = router(first_state.clone()); submit_task(&first_router, submitted_proof_bundle_for(&content_lock)).await; @@ -59,6 +732,14 @@ async fn postgres_runtime_state_survives_app_state_recreation() { .unwrap() .unwrap(); assert_eq!(recreated_task.status, VerificationTaskStatus::Pending); + let ownership = recreated_state + .content_lock_ownership() + .get_path_ownership(&creator(), &guarded_paths[0]) + .await + .unwrap() + .unwrap(); + assert_eq!(ownership.lock_id, lock_id); + assert_eq!(ownership.status, ContentLockOwnershipStatus::Published); seed_content_lock(&recreated_state, content_lock.clone()).await; let worker = VerificationWorker::from_state(&recreated_state); @@ -82,12 +763,75 @@ async fn postgres_runtime_state_survives_app_state_recreation() { database.cleanup().await; } +#[tokio::test] +async fn manual_completion_hides_legacy_paykit_admission_without_authoritative_window() { + let Some(database) = TestDatabase::create().await else { + return; + }; + let submitted = paykit_submission_for(&paykit_content_lock()); + let task = VerificationTaskRecord { + task_id: TaskId::from_str(&uuid::Uuid::new_v4().to_string()).unwrap(), + creator: submitted.pubky_lock_resource.creator().clone(), + submitted_proof_bundle: submitted.clone(), + status: VerificationTaskStatus::Pending, + submitted_at: datetime!(2026-08-12 06:00:00 UTC), + started_at: None, + completed_at: None, + failure_message: None, + }; + PostgresVerificationTaskRepository::new(database.pool().clone()) + .insert_verification_task(task.clone()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO paykit_task_admissions + (verification_task_id, ready, ready_at) + VALUES ($1::uuid, TRUE, now())", + ) + .bind(task.task_id.to_string()) + .execute(database.pool()) + .await + .unwrap(); + + let response = router(app_state(database.pool().clone())) + .oneshot(json_request( + "POST", + "/verification-task-completions", + json!({ + "creator": submitted.pubky_lock_resource.creator(), + "bundle_id": submitted.bundle_id, + }), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert_eq!( + response_json(response).await, + json!({ + "error": { + "code": "verification_task_not_found", + "message": "verification task not found" + } + }) + ); + + database.cleanup().await; +} + #[tokio::test] async fn postgres_runtime_readyz_returns_ready_without_leaking_runtime_details() { let Some(database) = TestDatabase::create().await else { return; }; let state = app_state(database.pool().clone()); + state.record_worker_readiness( + locks_server::app_state::WorkerKind::Verification, + locks_server::app_state::WorkerReadinessEvidence::Ready, + ); + state.record_worker_readiness( + locks_server::app_state::WorkerKind::Deletion, + locks_server::app_state::WorkerReadinessEvidence::Ready, + ); let response = router(state) .oneshot(empty_request("GET", "/readyz")) .await @@ -153,6 +897,477 @@ async fn postgres_runtime_encrypts_creator_authority_secrets_at_rest() { database.cleanup().await; } +#[tokio::test] +async fn deletion_first_proof_submission_returns_409_without_calling_paykit() { + let Some(database) = TestDatabase::create().await else { + return; + }; + let invoice_calls = Arc::new(AtomicUsize::new(0)); + let paykit_state = Arc::clone(&invoice_calls); + let paykit_app = axum::Router::new().route( + "/invoices", + post(move || { + let paykit_state = Arc::clone(&paykit_state); + async move { + paykit_state.fetch_add(1, Ordering::SeqCst); + StatusCode::OK + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let paykit_url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, paykit_app).await.unwrap() }); + + let temp_dir = tempfile::tempdir().unwrap(); + let secret_path = temp_dir.path().join("lock-server.keypair-seed"); + let public_key = FilesystemLockServerIdentityProvider + .generate_secret(&secret_path) + .unwrap(); + let mut config = test_config(); + config.credentials.lock_server_secret_key = secret_path; + config.credentials.lock_server_public_key = public_key; + config.paykit = Some(PaykitConfig { + server_url: paykit_url, + minimum_confirmations: 0, + }); + let state = AppState::new_with_postgres_runtime_and_creator_repositories( + config, + database.pool().clone(), + RuntimeSecretCiphers::new( + CreatorAuthoritySecretCipher::new([7; 32]), + FinalCredentialCipher::new([8; 32]), + ), + Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryContentLockTombstoneRepository::new()), + Arc::new(InMemoryGuardedResourceRepository::new()), + Arc::new(InMemoryLockServicePointerRepository::new()), + Arc::new(InMemoryEntitlementRepository::new()), + ) + .with_reader_pubky_resolver(Arc::new(AlwaysResolvesReader)); + let lock = paykit_content_lock(); + seed_content_lock(&state, lock.clone()).await; + PostgresContentLockDeletionRepository::new(database.pool().clone()) + .insert_job( + ContentLockDeletionJob::new( + uuid::Uuid::new_v4(), + lock.clone(), + datetime!(2026-08-12 06:00:00 UTC), + ) + .unwrap(), + ) + .await + .unwrap(); + + let response = router(state) + .oneshot(json_request( + "POST", + "/proof-bundles", + json!({ "submitted_proof_bundle": paykit_submission_for(&lock) }), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CONFLICT); + assert_eq!( + response_json(response).await, + json!({ + "error": { + "code": "content_lock_deletion_in_progress", + "message": "content lock deletion is in progress" + } + }) + ); + assert_eq!(invoice_calls.load(Ordering::SeqCst), 0); + + database.cleanup().await; +} + +#[tokio::test] +async fn snapshotted_unready_paykit_replay_ignores_tombstoned_lock_and_reader_resolution() { + let Some(database) = TestDatabase::create().await else { + return; + }; + let invoice_calls = Arc::new(AtomicUsize::new(0)); + let paykit_state = Arc::clone(&invoice_calls); + let paykit_app = axum::Router::new().route( + "/invoices", + post(move || { + let call = paykit_state.fetch_add(1, Ordering::SeqCst); + async move { + if call == 0 { + ( + StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(json!({ "error": "injected" })), + ) + } else { + ( + StatusCode::OK, + axum::Json(json!({ + "invoice_created_at": "2026-08-12T10:00:00Z", + "payment_deadline": "2026-08-13T10:00:00Z", + })), + ) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let paykit_url = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, paykit_app).await.unwrap() }); + + let temp_dir = tempfile::tempdir().unwrap(); + let secret_path = temp_dir.path().join("lock-server.keypair-seed"); + let public_key = FilesystemLockServerIdentityProvider + .generate_secret(&secret_path) + .unwrap(); + let mut config = test_config(); + config.credentials.lock_server_secret_key = secret_path; + config.credentials.lock_server_public_key = public_key; + config.paykit = Some(PaykitConfig { + server_url: paykit_url, + minimum_confirmations: 0, + }); + let initial_state = AppState::new_with_postgres_runtime_and_creator_repositories( + config.clone(), + database.pool().clone(), + RuntimeSecretCiphers::new( + CreatorAuthoritySecretCipher::new([7; 32]), + FinalCredentialCipher::new([8; 32]), + ), + Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryContentLockTombstoneRepository::new()), + Arc::new(InMemoryGuardedResourceRepository::new()), + Arc::new(InMemoryLockServicePointerRepository::new()), + Arc::new(InMemoryEntitlementRepository::new()), + ) + .with_reader_pubky_resolver(Arc::new(AlwaysResolvesReader)); + let lock = paykit_content_lock(); + let submitted = paykit_submission_for(&lock); + seed_content_lock(&initial_state, lock.clone()).await; + + let first = router(initial_state) + .oneshot(json_request( + "POST", + "/proof-bundles", + json!({ "submitted_proof_bundle": submitted.clone() }), + )) + .await + .unwrap(); + assert_eq!(first.status(), StatusCode::BAD_GATEWAY); + PostgresContentLockDeletionRepository::new(database.pool().clone()) + .insert_job( + ContentLockDeletionJob::new( + uuid::Uuid::new_v4(), + lock, + datetime!(2026-08-12 06:00:00 UTC), + ) + .unwrap(), + ) + .await + .unwrap(); + + let tombstoned_state = AppState::new_with_postgres_runtime_and_creator_repositories( + config, + database.pool().clone(), + RuntimeSecretCiphers::new( + CreatorAuthoritySecretCipher::new([7; 32]), + FinalCredentialCipher::new([8; 32]), + ), + Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryContentLockTombstoneRepository::new()), + Arc::new(InMemoryGuardedResourceRepository::new()), + Arc::new(InMemoryLockServicePointerRepository::new()), + Arc::new(InMemoryEntitlementRepository::new()), + ) + .with_reader_pubky_resolver(Arc::new(NeverResolvesReader)); + let replay_router = router(tombstoned_state); + let replay = replay_router + .clone() + .oneshot(json_request( + "POST", + "/proof-bundles", + json!({ "submitted_proof_bundle": submitted.clone() }), + )) + .await + .unwrap(); + assert_eq!(replay.status(), StatusCode::OK); + assert_eq!(response_json(replay).await["status"], "pending"); + assert_eq!(invoice_calls.load(Ordering::SeqCst), 2); + + let ready_replay = replay_router + .clone() + .oneshot(json_request( + "POST", + "/proof-bundles", + json!({ "submitted_proof_bundle": submitted.clone() }), + )) + .await + .unwrap(); + assert_eq!(ready_replay.status(), StatusCode::OK); + assert_eq!(invoice_calls.load(Ordering::SeqCst), 2); + + let mut changed = submitted; + changed.reader_public_key = Some( + CreatorPubky::from_str("pubky7ir1ttte48bcp4zjychjyscicrwi1j34mtt91ptsafdbjmr8g9eo") + .unwrap(), + ); + let conflict = replay_router + .oneshot(json_request( + "POST", + "/proof-bundles", + json!({ "submitted_proof_bundle": changed }), + )) + .await + .unwrap(); + assert_eq!(conflict.status(), StatusCode::CONFLICT); + assert_eq!(invoice_calls.load(Ordering::SeqCst), 2); + + database.cleanup().await; +} + +#[derive(Debug)] +struct CrashExternalRepository { + // 0 = frozen original, 1 = exact tombstone, 2 = absent, 3 = replacement. + public_state: AtomicUsize, + tombstone_writes: AtomicUsize, + withdraw_calls: AtomicUsize, + tombstone_reads: AtomicUsize, + resource_reads: AtomicUsize, + resource_deletes: AtomicUsize, + resources: Mutex>, + operations: Mutex>, +} + +impl CrashExternalRepository { + fn with_original(_lock: ContentLock) -> Self { + Self::new(0, None) + } + + fn with_tombstone_and_resources(lock: &ContentLock) -> Self { + Self::new(1, Some(lock)) + } + + fn with_original_and_resources(lock: &ContentLock) -> Self { + Self::new(0, Some(lock)) + } + + fn new(public_state: usize, lock: Option<&ContentLock>) -> Self { + let mut resources = HashMap::new(); + if let Some(lock) = lock + && let Some(primary) = &lock.primary_resource + { + resources.insert( + primary.path.clone(), + GuardedResourceRecord { + creator: lock.creator.clone(), + path: primary.path.clone(), + hash: primary.hash, + content_type: primary.content_type.clone(), + size: primary.size, + bytes: vec![7; primary.size as usize], + }, + ); + } + Self { + public_state: AtomicUsize::new(public_state), + tombstone_writes: AtomicUsize::new(0), + withdraw_calls: AtomicUsize::new(0), + tombstone_reads: AtomicUsize::new(0), + resource_reads: AtomicUsize::new(0), + resource_deletes: AtomicUsize::new(0), + resources: Mutex::new(resources), + operations: Mutex::new(Vec::new()), + } + } + + fn tombstone_write_count(&self) -> usize { + self.tombstone_writes.load(Ordering::SeqCst) + } + + fn withdraw_call_count(&self) -> usize { + self.withdraw_calls.load(Ordering::SeqCst) + } + + fn tombstone_read_count(&self) -> usize { + self.tombstone_reads.load(Ordering::SeqCst) + } + + fn resource_read_count(&self) -> usize { + self.resource_reads.load(Ordering::SeqCst) + } + + fn resource_delete_count(&self) -> usize { + self.resource_deletes.load(Ordering::SeqCst) + } + + fn operations(&self) -> Vec<&'static str> { + self.operations.lock().unwrap().clone() + } + + fn tombstone_readback(&self) -> TombstoneReadback { + match self.public_state.load(Ordering::SeqCst) { + 1 => TombstoneReadback::Exact, + 2 => TombstoneReadback::Missing, + _ => TombstoneReadback::Replaced, + } + } +} + +#[async_trait] +impl ContentLockTombstoneRepository for CrashExternalRepository { + async fn withdraw_content_lock( + &self, + _creator: CreatorPubky, + _content_lock_path: ContentLockPath, + _frozen_original: &ContentLock, + _tombstone: &ContentLockDeletionTombstone, + ) -> Result { + self.withdraw_calls.fetch_add(1, Ordering::SeqCst); + if self + .public_state + .compare_exchange(0, 1, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + self.tombstone_writes.fetch_add(1, Ordering::SeqCst); + } + Ok(self.tombstone_readback()) + } + + async fn read_tombstone( + &self, + _creator: &CreatorPubky, + _content_lock_path: &ContentLockPath, + _expected: &ContentLockDeletionTombstone, + ) -> Result { + self.tombstone_reads.fetch_add(1, Ordering::SeqCst); + Ok(self.tombstone_readback()) + } + + async fn force_delete_content_lock_and_verify_absent( + &self, + _creator: &CreatorPubky, + _content_lock_path: &ContentLockPath, + ) -> Result<(), locks_service::application::errors::ApplicationError> { + self.operations.lock().unwrap().push("public"); + self.public_state.store(2, Ordering::SeqCst); + Ok(()) + } +} + +#[async_trait] +impl GuardedResourceRepository for CrashExternalRepository { + async fn upsert_guarded_resource( + &self, + resource: GuardedResourceRecord, + ) -> Result<(), locks_service::application::errors::ApplicationError> { + self.resources + .lock() + .unwrap() + .insert(resource.path.clone(), resource); + Ok(()) + } + + async fn get_guarded_resource( + &self, + _creator: &CreatorPubky, + path: &str, + hash: &GuardedResourceHash, + ) -> Result, locks_service::application::errors::ApplicationError> + { + Ok(self + .resources + .lock() + .unwrap() + .get(path) + .filter(|record| record.hash == *hash) + .cloned()) + } + + async fn get_current_guarded_resource( + &self, + _creator: &CreatorPubky, + path: &str, + ) -> Result, locks_service::application::errors::ApplicationError> + { + self.resource_reads.fetch_add(1, Ordering::SeqCst); + Ok(self.resources.lock().unwrap().get(path).cloned()) + } + + async fn delete_guarded_resource( + &self, + _creator: &CreatorPubky, + path: &str, + ) -> Result { + self.operations.lock().unwrap().push("private"); + self.resource_deletes.fetch_add(1, Ordering::SeqCst); + Ok(self.resources.lock().unwrap().remove(path).is_some()) + } +} + +fn deletion_action_claim<'a>( + claimed: &ClaimedContentLockDeletionJob, + worker_id: &'a str, + force: bool, +) -> ContentLockDeletionActionClaim<'a> { + ContentLockDeletionActionClaim { + job_id: claimed.job.job_id, + worker_id, + claim_token: claimed.claim_token, + expected_phase: claimed.job.phase, + force, + } +} + +fn expect_action_acquired( + result: ContentLockDeletionActionAcquireResult, +) -> Box { + match result { + ContentLockDeletionActionAcquireResult::Acquired(guard) => guard, + ContentLockDeletionActionAcquireResult::Busy => { + panic!("live deletion action claim was unexpectedly contended") + } + ContentLockDeletionActionAcquireResult::ClaimLost => { + panic!("live deletion action claim was unexpectedly lost") + } + } +} + +async fn set_deletion_phase(pool: &PgPool, job_id: uuid::Uuid, phase: &str) { + sqlx::query("UPDATE content_lock_deletion_jobs SET phase = $2 WHERE job_id = $1") + .bind(job_id) + .bind(phase) + .execute(pool) + .await + .unwrap(); +} + +async fn expire_deletion_claim(pool: &PgPool, job_id: uuid::Uuid) { + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET claim_expires_at = clock_timestamp() - interval '1 second' + WHERE job_id = $1", + ) + .bind(job_id) + .execute(pool) + .await + .unwrap(); +} + +fn deletion_app_state(pool: PgPool, external: Arc) -> AppState { + AppState::new_with_postgres_runtime_and_creator_repositories( + test_config(), + pool, + RuntimeSecretCiphers::new( + CreatorAuthoritySecretCipher::new([7; 32]), + FinalCredentialCipher::new([8; 32]), + ), + Arc::new(InMemoryContentLockRepository::new()), + external.clone(), + external, + Arc::new(InMemoryLockServicePointerRepository::new()), + Arc::new(InMemoryEntitlementRepository::new()), + ) +} + struct TestDatabase { pool: PgPool, schema_name: String, @@ -220,8 +1435,12 @@ fn app_state(pool: PgPool) -> AppState { AppState::new_with_postgres_runtime_and_creator_repositories( test_config(), pool, - CreatorAuthoritySecretCipher::new([7; 32]), + RuntimeSecretCiphers::new( + CreatorAuthoritySecretCipher::new([7; 32]), + FinalCredentialCipher::new([8; 32]), + ), std::sync::Arc::new(InMemoryContentLockRepository::new()), + std::sync::Arc::new(InMemoryContentLockTombstoneRepository::new()), std::sync::Arc::new(InMemoryGuardedResourceRepository::new()), std::sync::Arc::new(InMemoryLockServicePointerRepository::new()), std::sync::Arc::new(InMemoryEntitlementRepository::new()), @@ -258,6 +1477,8 @@ fn test_config() -> LockServerRuntimeConfig { max_connections: 10, run_migrations_on_startup: true, }, + deletion: DeletionConfig::default(), + deletion_worker: locks_server::config::DeletionWorkerConfig::default(), worker: WorkerConfig { enabled: true, poll_interval_ms: 250, @@ -376,6 +1597,58 @@ fn content_lock() -> ContentLock { } } +fn paykit_content_lock() -> ContentLock { + let mut lock = content_lock(); + lock.criteria = vec![Criterion { + criterion_id: "criterion-1".to_owned(), + verifier_type: VerifierType::PaykitPayment, + params: json!({ + "recipient_pubky": creator().to_string(), + "amount": "50000", + "asset": "BTC", + "payment_in": 24 + }), + }]; + lock +} + +fn paykit_submission_for(content_lock: &ContentLock) -> SubmittedProofBundle { + SubmittedProofBundle { + version: SUBMITTED_PROOF_BUNDLE_VERSION, + bundle_id: bundle_id(), + pubky_lock_resource: PubkyLockResource::new( + content_lock.creator.clone(), + content_lock.content_lock_path().unwrap(), + ), + reader_public_key: Some(creator()), + proofs: vec![Proof { + criterion_id: "criterion-1".to_owned(), + verifier_type: VerifierType::PaykitPayment, + payload: json!({}), + }], + } +} + +#[derive(Debug)] +struct AlwaysResolvesReader; + +#[async_trait] +impl ReaderPubkyResolver for AlwaysResolvesReader { + async fn reader_has_homeserver(&self, _reader: &CreatorPubky) -> bool { + true + } +} + +#[derive(Debug)] +struct NeverResolvesReader; + +#[async_trait] +impl ReaderPubkyResolver for NeverResolvesReader { + async fn reader_has_homeserver(&self, _reader: &CreatorPubky) -> bool { + false + } +} + fn creator() -> CreatorPubky { CreatorPubky::from_str("pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy").unwrap() } diff --git a/locks-e2e/tests/production_creator_publishing_http.rs b/locks-e2e/tests/production_creator_publishing_http.rs index de38083..c1e2f88 100644 --- a/locks-e2e/tests/production_creator_publishing_http.rs +++ b/locks-e2e/tests/production_creator_publishing_http.rs @@ -14,8 +14,8 @@ use locks_service::application::models::{CreatorAuthorityAuthKind, FrontendSessi use locks_service::application::ports::{CreatorAuthorityManager, CreatorAuthorityStatus}; use locks_service::infrastructure::pubky::{ AuthorizingPubkyHomeserverStorageClient, PubkyBytesResource, PubkyContentLockRepository, - PubkyEntitlementRepository, PubkyHomeserverStorageClient, PubkyLockServicePointerRepository, - PubkyPrivResourceRepository, + PubkyContentLockTombstoneRepository, PubkyEntitlementRepository, PubkyHomeserverStorageClient, + PubkyLockServicePointerRepository, PubkyPrivResourceRepository, }; use serde_json::json; use support::creator_publishing_client::LocalCreatorPublishingClient; @@ -39,6 +39,9 @@ async fn production_creator_publishing_http_flow_writes_to_pubky_storage_when_fr storage.clone(), manager.clone(), ))), + Arc::new(PubkyContentLockTombstoneRepository::new( + authorizing_storage(storage.clone(), manager.clone()), + )), Arc::new(PubkyPrivResourceRepository::new(authorizing_storage( storage.clone(), manager.clone(), @@ -96,7 +99,7 @@ async fn production_creator_publishing_http_flow_writes_to_pubky_storage_when_fr let content_lock_json = client .create_content_lock( - guarded_resource, + guarded_resource.clone(), json!([{ "criterion_id": "criterion-1", "verifier_type": "dev-static", @@ -108,6 +111,23 @@ async fn production_creator_publishing_http_flow_writes_to_pubky_storage_when_fr ) .await .unwrap(); + let conflict = client + .create_content_lock( + guarded_resource, + json!([{ + "criterion_id": "criterion-1", + "verifier_type": "dev-static", + "params": { "satisfied": false } + }]), + json!({ "type": "all", "criteria": ["criterion-1"] }), + json!({ "requested_credential_ttl_seconds": 900 }), + json!({ "override": "pubky7ir1ttte48bcp4zjychjyscicrwi1j34mtt91ptsafdbjmr8g9eo" }), + ) + .await + .unwrap_err(); + assert_eq!(conflict.status, StatusCode::CONFLICT); + assert_eq!(conflict.body["error"]["code"], "content_lock_path_conflict"); + assert_secret_free(&conflict.body); let content_lock_path = ContentLockPath::from_str( content_lock_json["content_lock_path"] .as_str() @@ -145,6 +165,9 @@ async fn production_creator_publishing_http_returns_creator_authority_unavailabl storage.clone(), manager.clone(), ))), + Arc::new(PubkyContentLockTombstoneRepository::new( + authorizing_storage(storage.clone(), manager.clone()), + )), Arc::new(PubkyPrivResourceRepository::new(authorizing_storage( storage.clone(), manager.clone(), diff --git a/locks-e2e/tests/pubky_homeserver_repositories.rs b/locks-e2e/tests/pubky_homeserver_repositories.rs index ad73c3a..e0b6fbb 100644 --- a/locks-e2e/tests/pubky_homeserver_repositories.rs +++ b/locks-e2e/tests/pubky_homeserver_repositories.rs @@ -12,8 +12,9 @@ use locks_server::testing::TestServerApp; use locks_service::application::errors::ApplicationError; use locks_service::application::models::FrontendSessionToken; use locks_service::infrastructure::pubky::{ - PubkyBytesResource, PubkyContentLockRepository, PubkyEntitlementRepository, - PubkyHomeserverStorageClient, PubkyLockServicePointerRepository, PubkyPrivResourceRepository, + PubkyBytesResource, PubkyContentLockRepository, PubkyContentLockTombstoneRepository, + PubkyEntitlementRepository, PubkyHomeserverStorageClient, PubkyLockServicePointerRepository, + PubkyPrivResourceRepository, }; use serde_json::json; use support::creator_publishing_client::{LocalCreatorPublishingClient, response_bytes}; @@ -31,6 +32,7 @@ async fn pubky_homeserver_repository_flow_writes_to_fake_homeserver_storage() { let state = AppState::new_empty_in_memory_with_creator_repositories( config, Arc::new(PubkyContentLockRepository::new(storage.clone())), + Arc::new(PubkyContentLockTombstoneRepository::new(storage.clone())), Arc::new(PubkyPrivResourceRepository::new(storage.clone())), Arc::new(PubkyLockServicePointerRepository::new(storage.clone())), Arc::new(PubkyEntitlementRepository::new(storage.clone())), diff --git a/locks-sdk/bindings/js/src/creator.rs b/locks-sdk/bindings/js/src/creator.rs index be56bae..1031446 100644 --- a/locks-sdk/bindings/js/src/creator.rs +++ b/locks-sdk/bindings/js/src/creator.rs @@ -8,7 +8,7 @@ use crate::session::{BrowserPkarrResolver, fetch_authorized_empty, fetch_authori #[cfg(any(test, target_arch = "wasm32"))] use crate::session::{JsAuthorizedRequestPlan, JsRequestBody}; #[cfg(any(test, target_arch = "wasm32"))] -use locks_core::ids::LockServerPubky; +use locks_core::ids::{LockId, LockServerPubky}; #[cfg(any(test, target_arch = "wasm32"))] use std::str::FromStr; use wasm_bindgen::prelude::*; @@ -65,6 +65,32 @@ impl DeleteGuardedResourceOptions { } } +#[wasm_bindgen] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeleteContentLockMode { + DefaultGraceful, + ExplicitGraceful, + Force, +} + +#[wasm_bindgen] +pub struct DeleteContentLockOptions { + mode: DeleteContentLockMode, +} + +#[wasm_bindgen] +impl DeleteContentLockOptions { + #[wasm_bindgen(constructor)] + pub fn new(mode: DeleteContentLockMode) -> Self { + Self { mode } + } + + #[wasm_bindgen(getter)] + pub fn mode(&self) -> DeleteContentLockMode { + self.mode + } +} + #[derive(Debug, Clone, Default)] struct CreateContentLockRequestBuilderState { primary_resource: Option, @@ -213,6 +239,14 @@ impl CreateContentLockRequestBuilder { .criteria .as_ref() .ok_or_else(|| "content lock request requires criteria".to_owned())?; + let typed_criteria: Vec = + serde_json::from_value(criteria.clone()) + .map_err(|err| format!("invalid content lock criteria: {err}"))?; + for criterion in &typed_criteria { + criterion + .validate_params() + .map_err(|err| format!("invalid content lock criterion: {err}"))?; + } body.insert("criteria".to_owned(), criteria.clone()); let lock_logic = state .lock_logic @@ -357,6 +391,41 @@ impl Creator { fetch_authorized_empty(&request).await } + #[cfg(target_arch = "wasm32")] + #[wasm_bindgen(js_name = deleteContentLock)] + pub async fn delete_content_lock( + &self, + lock_id: String, + options: Option, + ) -> crate::js_error::JsResult { + let resolver = BrowserPkarrResolver::new_with_options(self.session.options()) + .map_err(|err| crate::js_error::invalid_input(err.to_string()))?; + let request = self + .build_delete_content_lock_request(&lock_id, options.as_ref()) + .map_err(crate::js_error::invalid_input)? + .prepare_with_pkarr_resolver(&resolver, None) + .await + .map_err(|err| crate::js_error::invalid_input(err.to_string()))?; + fetch_authorized_json(&request).await + } + + #[cfg(target_arch = "wasm32")] + #[wasm_bindgen(js_name = contentLockDeletionStatus)] + pub async fn content_lock_deletion_status( + &self, + lock_id: String, + ) -> crate::js_error::JsResult { + let resolver = BrowserPkarrResolver::new_with_options(self.session.options()) + .map_err(|err| crate::js_error::invalid_input(err.to_string()))?; + let request = self + .build_content_lock_deletion_status_request(&lock_id) + .map_err(crate::js_error::invalid_input)? + .prepare_with_pkarr_resolver(&resolver, None) + .await + .map_err(|err| crate::js_error::invalid_input(err.to_string()))?; + fetch_authorized_json(&request).await + } + #[cfg(target_arch = "wasm32")] #[wasm_bindgen(js_name = setLockServicePointer)] pub async fn set_lock_service_pointer( @@ -410,6 +479,44 @@ impl Creator { self.authorized_request_plan(request) } + #[cfg(any(test, target_arch = "wasm32"))] + pub(crate) fn build_delete_content_lock_request( + &self, + lock_id: &str, + options: Option<&DeleteContentLockOptions>, + ) -> Result { + let lock_id = LockId::from_str(lock_id).map_err(|err| format!("invalid lock id: {err}"))?; + let mode = match options.map(|options| options.mode) { + None | Some(DeleteContentLockMode::DefaultGraceful) => { + locks_sdk::DeleteContentLockMode::DefaultGraceful + } + Some(DeleteContentLockMode::ExplicitGraceful) => { + locks_sdk::DeleteContentLockMode::ExplicitGraceful + } + Some(DeleteContentLockMode::Force) => locks_sdk::DeleteContentLockMode::Force, + }; + Ok(self.authorized_request_plan( + self.session + .inner() + .creator() + .delete_content_lock(locks_sdk::DeleteContentLockRequest { lock_id, mode }), + )) + } + + #[cfg(any(test, target_arch = "wasm32"))] + pub(crate) fn build_content_lock_deletion_status_request( + &self, + lock_id: &str, + ) -> Result { + let lock_id = LockId::from_str(lock_id).map_err(|err| format!("invalid lock id: {err}"))?; + Ok(self.authorized_request_plan( + self.session + .inner() + .creator() + .get_content_lock_deletion(lock_id), + )) + } + #[cfg(any(test, target_arch = "wasm32"))] #[cfg_attr(target_arch = "wasm32", allow(dead_code))] pub(crate) fn build_set_lock_service_pointer_request( @@ -540,6 +647,52 @@ mod tests { assert_eq!(request.content_type, None); } + #[test] + fn content_lock_deletion_requests_delegate_to_closed_rust_sdk_routes() { + let creator = Creator::new(test_session()); + let lock_id = LockId::from_hash(locks_core::ids::LockHash::from_bytes([9; 32])); + + let graceful = creator + .build_delete_content_lock_request(&lock_id.to_string(), None) + .unwrap(); + assert_eq!(graceful.method, "DELETE"); + assert_eq!(graceful.path, format!("/creator/content-locks/{lock_id}")); + assert_eq!(graceful.authorization, "Bearer frontend-session-secret"); + + let explicit_graceful = creator + .build_delete_content_lock_request( + &lock_id.to_string(), + Some(&DeleteContentLockOptions::new( + DeleteContentLockMode::ExplicitGraceful, + )), + ) + .unwrap(); + assert_eq!( + explicit_graceful.path, + format!("/creator/content-locks/{lock_id}?graceful=true") + ); + + let force = creator + .build_delete_content_lock_request( + &lock_id.to_string(), + Some(&DeleteContentLockOptions::new(DeleteContentLockMode::Force)), + ) + .unwrap(); + assert_eq!( + force.path, + format!("/creator/content-locks/{lock_id}?force=true") + ); + + let status = creator + .build_content_lock_deletion_status_request(&lock_id.to_string()) + .unwrap(); + assert_eq!(status.method, "GET"); + assert_eq!( + status.path, + format!("/creator/content-locks/{lock_id}/deletion") + ); + } + #[test] fn create_content_lock_request_builder_primary_only_build_succeeds() { let builder = complete_builder(); @@ -619,6 +772,31 @@ mod tests { assert!(format!("{err:?}").contains("criteria")); } + #[test] + fn create_content_lock_request_builder_rejects_invalid_paykit_payment_in() { + let builder = complete_builder(); + builder.state.borrow_mut().primary_resource = + Some(resource("/priv/locks.app/content/example.txt", "hash", 13)); + builder.state.borrow_mut().criteria = Some(serde_json::json!([{ + "criterion_id": "payment", + "verifier_type": "paykit-payment", + "params": { + "recipient_pubky": "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy", + "amount": "50000", + "asset": "BTC", + "payment_in": 0 + } + }])); + builder.state.borrow_mut().lock_logic = Some(serde_json::json!({ + "type": "all", + "criteria": ["payment"] + })); + + let err = builder.build_value().unwrap_err(); + + assert!(err.contains("payment_in")); + } + #[test] fn create_content_lock_request_builder_rejects_duplicate_secondary_path() { let builder = complete_builder(); diff --git a/locks-sdk/bindings/js/src/lib.rs b/locks-sdk/bindings/js/src/lib.rs index 6b0c589..a7b41b8 100644 --- a/locks-sdk/bindings/js/src/lib.rs +++ b/locks-sdk/bindings/js/src/lib.rs @@ -6,8 +6,8 @@ mod session; mod viewer; pub use creator::{ - CreateContentLockRequestBuilder, Creator, DeleteGuardedResourceOptions, - RegisterGuardedResourceOptions, SetLockServicePointerOptions, + CreateContentLockRequestBuilder, Creator, DeleteContentLockMode, DeleteContentLockOptions, + DeleteGuardedResourceOptions, RegisterGuardedResourceOptions, SetLockServicePointerOptions, }; pub use locks::{ ConnectCallback, ConnectUrlOptions, ExchangeFrontendSessionCodeOptions, Locks, LocksOptions, diff --git a/locks-sdk/src/creator.rs b/locks-sdk/src/creator.rs index 6edf4cc..104ee06 100644 --- a/locks-sdk/src/creator.rs +++ b/locks-sdk/src/creator.rs @@ -1,4 +1,5 @@ pub use locks_core::creator_publishing::{CreateContentLockRequest, SetLockServicePointerRequest}; +use locks_core::ids::LockId; use percent_encoding::{AsciiSet, CONTROLS, utf8_percent_encode}; use serde::Serialize; use serde_json::Value; @@ -55,6 +56,19 @@ pub struct DeleteGuardedResourceRequest { pub path: String, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeleteContentLockMode { + DefaultGraceful, + ExplicitGraceful, + Force, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeleteContentLockRequest { + pub lock_id: LockId, + pub mode: DeleteContentLockMode, +} + impl CreatorLocks { pub fn new(session: LocksSession) -> Self { Self { session } @@ -100,6 +114,22 @@ impl CreatorLocks { self.delete_guarded_resource_request(request) } + pub fn delete_content_lock(&self, request: DeleteContentLockRequest) -> SdkRequest { + let suffix = match request.mode { + DeleteContentLockMode::DefaultGraceful => "", + DeleteContentLockMode::ExplicitGraceful => "?graceful=true", + DeleteContentLockMode::Force => "?force=true", + }; + self.empty_request( + "DELETE", + format!("/creator/content-locks/{}{suffix}", request.lock_id), + ) + } + + pub fn get_content_lock_deletion(&self, lock_id: LockId) -> SdkRequest { + self.empty_request("GET", format!("/creator/content-locks/{lock_id}/deletion")) + } + pub fn create_content_lock_request(&self, request: CreateContentLockRequest) -> SdkRequest { self.post_json("/creator/content-locks", request) } @@ -130,6 +160,16 @@ impl CreatorLocks { ), } } + + fn empty_request(&self, method: &'static str, path: String) -> SdkRequest { + SdkRequest { + method, + path, + authorization: self.session.authorization_header_value(), + content_type: String::new(), + body: SdkRequestBody::Bytes(Vec::new()), + } + } } pub(crate) fn encode_content_path(path: &str) -> String { diff --git a/locks-sdk/src/lib.rs b/locks-sdk/src/lib.rs index fe931a9..e387390 100644 --- a/locks-sdk/src/lib.rs +++ b/locks-sdk/src/lib.rs @@ -8,8 +8,9 @@ pub mod viewer; pub use client::LocksClient; pub use creator::{ - CreateContentLockRequest, CreatorLocks, DeleteGuardedResourceRequest, - RegisterGuardedResourceRequest, SdkRequest, SdkRequestBody, SetLockServicePointerRequest, + CreateContentLockRequest, CreatorLocks, DeleteContentLockMode, DeleteContentLockRequest, + DeleteGuardedResourceRequest, RegisterGuardedResourceRequest, SdkRequest, SdkRequestBody, + SetLockServicePointerRequest, }; pub use discovery::{ CreatorLockServicePointer, WellKnownLocksServer, content_lock_resource_url, diff --git a/locks-sdk/tests/public_api.rs b/locks-sdk/tests/public_api.rs index 5b9c580..9761f35 100644 --- a/locks-sdk/tests/public_api.rs +++ b/locks-sdk/tests/public_api.rs @@ -2,11 +2,11 @@ use std::str::FromStr; use locks_core::ids::{BundleId, CreatorPubky, LockServerPubky, PubkyLockResource}; use locks_sdk::{ - AccessCredentialResponse, CreatorLockServicePointer, DeleteGuardedResourceRequest, LocksClient, - LocksSession, ReadLockedResourceRequest, RegisterGuardedResourceRequest, - VerificationTaskHandleRequest, VerificationTaskLifecycleResponse, VerificationTaskStatus, - ViewerLocks, content_lock_resource_url, creator_lock_service_pointer_url, - lock_server_for_content_lock, + AccessCredentialResponse, CreatorLockServicePointer, DeleteContentLockMode, + DeleteContentLockRequest, DeleteGuardedResourceRequest, LocksClient, LocksSession, + ReadLockedResourceRequest, RegisterGuardedResourceRequest, VerificationTaskHandleRequest, + VerificationTaskLifecycleResponse, VerificationTaskStatus, ViewerLocks, + content_lock_resource_url, creator_lock_service_pointer_url, lock_server_for_content_lock, }; #[test] @@ -113,3 +113,50 @@ fn crate_root_exports_foundation_sdk_types() { assert_eq!(LocksSession::new("another").export_secret(), "another"); } + +#[test] +fn creator_content_lock_deletion_requests_use_closed_routes_and_modes() { + let creator = LocksSession::new("frontend-session-secret").creator(); + let lock_id = + locks_core::ids::LockId::from_str("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG") + .unwrap(); + + let default_graceful = creator.delete_content_lock(DeleteContentLockRequest { + lock_id: lock_id.clone(), + mode: DeleteContentLockMode::DefaultGraceful, + }); + assert_eq!(default_graceful.method, "DELETE"); + assert_eq!( + default_graceful.path, + format!("/creator/content-locks/{lock_id}") + ); + assert_eq!( + default_graceful.authorization, + "Bearer frontend-session-secret" + ); + + let explicit_graceful = creator.delete_content_lock(DeleteContentLockRequest { + lock_id: lock_id.clone(), + mode: DeleteContentLockMode::ExplicitGraceful, + }); + assert_eq!( + explicit_graceful.path, + format!("/creator/content-locks/{lock_id}?graceful=true") + ); + + let force = creator.delete_content_lock(DeleteContentLockRequest { + lock_id: lock_id.clone(), + mode: DeleteContentLockMode::Force, + }); + assert_eq!( + force.path, + format!("/creator/content-locks/{lock_id}?force=true") + ); + + let status = creator.get_content_lock_deletion(lock_id.clone()); + assert_eq!(status.method, "GET"); + assert_eq!( + status.path, + format!("/creator/content-locks/{lock_id}/deletion") + ); +} diff --git a/locks-server/config/example.dev.postgres.toml b/locks-server/config/example.dev.postgres.toml index 41b0b00..ef9fb45 100644 --- a/locks-server/config/example.dev.postgres.toml +++ b/locks-server/config/example.dev.postgres.toml @@ -16,6 +16,13 @@ poll_interval_ms = 250 claim_timeout_seconds = 60 worker_id = "default-worker" +[deletion_worker] +enabled = true +poll_interval_ms = 250 +claim_timeout_seconds = 60 +shutdown_timeout_seconds = 30 +worker_id = "deletion-worker" + [runtime] environment = "development" @@ -33,7 +40,14 @@ frontend_session_code_ttl_seconds = 120 allowed_return_origins = ["http://localhost:3000"] [secrets] -creator_authority_key_env = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" +runtime_master_key_env = "PUBKY_LOCK_RUNTIME_MASTER_KEY" + +[deletion] +retry_max_attempts = 10 +retry_initial_backoff_seconds = 1 +retry_max_backoff_seconds = 300 +final_credential_issuance_window_seconds = 900 +final_read_window_seconds = 900 [logging] level = "info" diff --git a/locks-server/src/api/access.rs b/locks-server/src/api/access.rs index 8c7615a..b091397 100644 --- a/locks-server/src/api/access.rs +++ b/locks-server/src/api/access.rs @@ -8,7 +8,7 @@ use locks_service::application::use_cases::issue_access_credential::{ IssueAccessCredentialRequest, IssueAccessCredentialUseCase, }; use locks_service::application::use_cases::proxy_read_guarded_resource::{ - ProxyReadGuardedResourceRequest, ProxyReadGuardedResourceUseCase, + ProxiedGuardedResource, ProxyReadGuardedResourceRequest, ProxyReadGuardedResourceUseCase, }; use crate::api::dtos::{IssueAccessCredentialHttpRequest, IssueAccessCredentialHttpResponse}; @@ -58,6 +58,18 @@ pub(super) async fn proxy_read_guarded_resource( let proxied = use_case .execute(ProxyReadGuardedResourceRequest { credential, path }) .await?; + let response = match build_proxy_read_response(&proxied) { + Ok(response) => response, + Err(error) => { + use_case.release_prepared_deletion_read(&proxied).await?; + return Err(error); + } + }; + use_case.consume_prepared_deletion_read(&proxied).await?; + Ok(response) +} + +fn build_proxy_read_response(proxied: &ProxiedGuardedResource) -> Result { let content_type = HeaderValue::from_str(&proxied.content_type).map_err(|_| { ApiError::new( ApiErrorCode::InternalError, @@ -88,7 +100,7 @@ pub(super) async fn proxy_read_guarded_resource( .map_err(|_| ApiError::new(ApiErrorCode::InternalError, "invalid etag"))?, ), ], - Body::from(proxied.bytes), + Body::from(proxied.bytes.clone()), ) .into_response()) } diff --git a/locks-server/src/api/creator_publishing.rs b/locks-server/src/api/creator_publishing.rs index 8b37395..66d2f87 100644 --- a/locks-server/src/api/creator_publishing.rs +++ b/locks-server/src/api/creator_publishing.rs @@ -1,8 +1,13 @@ use axum::Json; use axum::body::{Body, to_bytes}; -use axum::extract::rejection::JsonRejection; -use axum::extract::{Path, State}; +use axum::extract::rejection::{JsonRejection, QueryRejection}; +use axum::extract::{Path, Query, State}; use axum::http::{HeaderMap, StatusCode, header}; +use locks_core::ids::{ContentLockPath, CreatorPubky, LockId}; +use locks_service::application::models::{ + ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionState, + PrepareForceDeletionResult, +}; use locks_service::application::use_cases::create_content_lock::{ CreateContentLockRequest, CreateContentLockUseCase, }; @@ -15,17 +20,358 @@ use locks_service::application::use_cases::register_guarded_resource::{ use locks_service::application::use_cases::set_lock_service_pointer::{ SetLockServicePointerRequest, SetLockServicePointerUseCase, }; +use serde::Deserialize; +use serde_json::{Value, json}; +use std::str::FromStr; +use uuid::Uuid; use crate::api::auth::authenticated_creator_from_headers; use crate::api::dtos::{ AuthenticatedCreateContentLockHttpRequest, AuthenticatedSetLockServicePointerHttpRequest, - CreateContentLockHttpResponse, RegisterGuardedResourceHttpResponse, - SetLockServicePointerHttpResponse, + ContentLockDeletionStatusHttpResponse, CreateContentLockHttpResponse, + RegisterGuardedResourceHttpResponse, SetLockServicePointerHttpResponse, }; use crate::api::errors::{ApiError, ApiErrorCode}; use crate::api::extractors::{guarded_resource_path_from_tail, parse_json}; use crate::app_state::AppState; +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct DeleteContentLockQuery { + force: Option, + graceful: Option, +} + +pub(super) async fn delete_content_lock_for_authenticated_creator( + State(state): State, + Path(lock_id): Path, + headers: HeaderMap, + query: Result, QueryRejection>, +) -> Result<(StatusCode, Json), ApiError> { + let creator = authenticated_creator_from_headers(&state, &headers).await?; + let Query(query) = + query.map_err(|_| ApiError::new(ApiErrorCode::InvalidRequest, "invalid request"))?; + let force = match (query.force, query.graceful) { + (None, None | Some(true)) => false, + (Some(true), None) => true, + _ => { + return Err(ApiError::new( + ApiErrorCode::InvalidRequest, + "invalid request", + )); + } + }; + let lock_id = LockId::from_str(&lock_id) + .map_err(|_| ApiError::new(ApiErrorCode::InvalidIdentifier, "invalid lock id"))?; + + if force { + return force_delete_content_lock(&state, creator, lock_id).await; + } + + if state + .content_lock_deletions() + .has_force_receipt(&creator, &lock_id) + .await? + { + return Ok(( + StatusCode::OK, + Json(json!({ "lock_id": lock_id, "status": "completed" })), + )); + } + + if let Some(job) = state + .content_lock_deletions() + .get_job(&creator, &lock_id) + .await? + { + let job = if job.state == ContentLockDeletionState::Failed { + match state + .content_lock_deletions() + .resume_failed_job(&creator, &lock_id, state.clock().now()) + .await? + { + Some(resumed) => resumed, + None if state + .content_lock_deletions() + .has_force_receipt(&creator, &lock_id) + .await? => + { + return Ok(( + StatusCode::OK, + Json(json!({ "lock_id": lock_id, "status": "completed" })), + )); + } + None => job, + } + } else { + job + }; + let status = match job.state { + ContentLockDeletionState::Queued | ContentLockDeletionState::Running => { + StatusCode::ACCEPTED + } + ContentLockDeletionState::Completed | ContentLockDeletionState::Failed => { + StatusCode::OK + } + }; + return Ok(( + status, + Json(deletion_status_json(lock_id, job.state, job.failure_code)), + )); + } + + let path = ContentLockPath::from_lock_id(lock_id.clone()); + let Some(content_lock) = state + .content_locks() + .get_content_lock(&creator, &path) + .await? + else { + if state + .content_lock_deletions() + .publication_in_progress(&creator, &lock_id) + .await? + { + return Err(ApiError::new( + ApiErrorCode::ContentLockPathConflict, + "content lock publication is in progress", + )); + } + return Ok(( + StatusCode::OK, + Json(json!({ "lock_id": lock_id, "status": "completed" })), + )); + }; + + validate_content_lock_identity(&content_lock, &creator, &lock_id)?; + + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock, state.clock().now())?; + match state.content_lock_deletions().insert_job(job.clone()).await { + Ok(()) => Ok(( + StatusCode::ACCEPTED, + Json(deletion_status_json(lock_id, job.state, job.failure_code)), + )), + Err(locks_service::application::errors::ApplicationError::DuplicateRecord { + record: "content_lock_deletion_job", + }) => { + let persisted = state + .content_lock_deletions() + .get_job(&creator, &lock_id) + .await? + .ok_or_else(|| { + ApiError::new( + ApiErrorCode::InternalError, + "content lock deletion unavailable", + ) + })?; + let status = match persisted.state { + ContentLockDeletionState::Queued | ContentLockDeletionState::Running => { + StatusCode::ACCEPTED + } + ContentLockDeletionState::Completed | ContentLockDeletionState::Failed => { + StatusCode::OK + } + }; + Ok(( + status, + Json(deletion_status_json( + lock_id, + persisted.state, + persisted.failure_code, + )), + )) + } + Err( + locks_service::application::errors::ApplicationError::ContentLockDeletionInProgress, + ) if state + .content_lock_deletions() + .has_force_receipt(&creator, &lock_id) + .await? => + { + Ok(( + StatusCode::OK, + Json(json!({ "lock_id": lock_id, "status": "completed" })), + )) + } + Err(error) => Err(error.into()), + } +} + +async fn force_delete_content_lock( + state: &AppState, + creator: CreatorPubky, + lock_id: LockId, +) -> Result<(StatusCode, Json), ApiError> { + let path = ContentLockPath::from_lock_id(lock_id.clone()); + let existing_job = state + .content_lock_deletions() + .get_job(&creator, &lock_id) + .await?; + let published_content_lock = if existing_job.is_none() { + state + .content_locks() + .get_content_lock(&creator, &path) + .await? + } else { + None + }; + if let Some(content_lock) = published_content_lock.as_ref() { + validate_content_lock_identity(content_lock, &creator, &lock_id)?; + } + let content_lock = match state + .content_lock_deletions() + .prepare_force_deletion(&creator, &lock_id) + .await? + { + PrepareForceDeletionResult::PublicationInProgress => { + return Err(ApiError::new( + ApiErrorCode::ContentLockPathConflict, + "content lock publication is in progress", + )); + } + PrepareForceDeletionResult::Active(job) => { + return Ok(( + StatusCode::ACCEPTED, + Json(deletion_status_json(lock_id, job.state, job.failure_code)), + )); + } + PrepareForceDeletionResult::Synchronous(Some(job)) => Some(job.frozen_content_lock), + PrepareForceDeletionResult::Synchronous(None) => published_content_lock, + }; + + if let Some(content_lock) = content_lock.as_ref() { + validate_content_lock_identity(content_lock, &creator, &lock_id)?; + } + + state + .content_locks() + .delete_content_lock(&creator, &path) + .await?; + if state + .content_locks() + .get_content_lock(&creator, &path) + .await? + .is_some() + { + return Err(ApiError::new( + ApiErrorCode::InternalError, + "content lock deletion postcondition failed", + )); + } + + let mut failed_resource_paths = Vec::new(); + if let Some(content_lock) = content_lock { + let mut resource_paths = content_lock + .secondary_resources + .keys() + .cloned() + .collect::>(); + if let Some(primary) = content_lock.primary_resource { + resource_paths.push(primary.path); + } + resource_paths.sort(); + resource_paths.dedup(); + for resource_path in resource_paths { + if state + .guarded_resources() + .delete_guarded_resource(&creator, &resource_path) + .await + .is_err() + { + failed_resource_paths.push(resource_path); + } + } + } + + Ok(( + StatusCode::OK, + Json(json!({ + "lock_id": lock_id, + "lock_deleted": true, + "failed_resource_paths": failed_resource_paths + })), + )) +} + +fn validate_content_lock_identity( + content_lock: &locks_core::lock_policy::ContentLock, + creator: &CreatorPubky, + expected_lock_id: &LockId, +) -> Result<(), ApiError> { + let actual_lock_id = content_lock + .lock_id() + .map_err(|_| ApiError::new(ApiErrorCode::InternalError, "content lock unavailable"))?; + if &content_lock.creator != creator || &actual_lock_id != expected_lock_id { + return Err(ApiError::new( + ApiErrorCode::InternalError, + "content lock unavailable", + )); + } + Ok(()) +} + +fn deletion_status_json( + lock_id: LockId, + state: ContentLockDeletionState, + failure_code: Option, +) -> Value { + serde_json::to_value(deletion_status_response(lock_id, state, failure_code)) + .expect("deletion status response must serialize") +} + +pub(super) async fn get_content_lock_deletion_status_for_authenticated_creator( + State(state): State, + Path(lock_id): Path, + headers: HeaderMap, +) -> Result, ApiError> { + let creator = authenticated_creator_from_headers(&state, &headers).await?; + let lock_id = LockId::from_str(&lock_id) + .map_err(|_| ApiError::new(ApiErrorCode::InvalidIdentifier, "invalid lock id"))?; + if state + .content_lock_deletions() + .has_force_receipt(&creator, &lock_id) + .await? + { + return Ok(Json(ContentLockDeletionStatusHttpResponse { + lock_id, + status: "completed", + failure_code: None, + })); + } + if let Some(job) = state + .content_lock_deletions() + .get_job(&creator, &lock_id) + .await? + { + return Ok(Json(deletion_status_response( + lock_id, + job.state, + job.failure_code, + ))); + } + Err(ApiError::new( + ApiErrorCode::ContentLockDeletionNotFound, + "content lock deletion not found", + )) +} + +fn deletion_status_response( + lock_id: LockId, + state: ContentLockDeletionState, + failure_code: Option, +) -> ContentLockDeletionStatusHttpResponse { + let status = match state { + ContentLockDeletionState::Queued => "queued", + ContentLockDeletionState::Running => "running", + ContentLockDeletionState::Completed => "completed", + ContentLockDeletionState::Failed => "failed", + }; + ContentLockDeletionStatusHttpResponse { + lock_id, + status, + failure_code: failure_code.map(|code| code.as_str().to_owned()), + } +} + pub(super) async fn register_guarded_resource_empty_tail_for_authenticated_creator( State(state): State, headers: HeaderMap, @@ -98,6 +444,8 @@ pub(super) async fn create_content_lock_for_authenticated_creator( validate_content_lock_limits(&request, state.config().content_locks.clone())?; let use_case = CreateContentLockUseCase::new( state.content_locks().as_ref(), + state.content_lock_deletions().as_ref(), + state.content_lock_ownership().as_ref(), state.guarded_resources().as_ref(), state.clock().as_ref(), ); diff --git a/locks-server/src/api/dtos.rs b/locks-server/src/api/dtos.rs index 16a0127..0fbd1a9 100644 --- a/locks-server/src/api/dtos.rs +++ b/locks-server/src/api/dtos.rs @@ -51,6 +51,14 @@ pub struct CreateContentLockHttpResponse { pub content_lock: ContentLock, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ContentLockDeletionStatusHttpResponse { + pub lock_id: LockId, + pub status: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub failure_code: Option, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct SetLockServicePointerHttpResponse { pub creator: CreatorPubky, diff --git a/locks-server/src/api/errors.rs b/locks-server/src/api/errors.rs index 13cc5a8..ecddddb 100644 --- a/locks-server/src/api/errors.rs +++ b/locks-server/src/api/errors.rs @@ -23,6 +23,9 @@ pub enum ApiErrorCode { FrontendSessionUnavailable, FrontendSessionExpired, FrontendSessionStateMismatch, + ContentLockPathConflict, + ContentLockDeletionInProgress, + ContentLockDeletionNotFound, TaskStateConflict, UnsupportedVerifierType, PaykitNotConfigured, @@ -53,6 +56,9 @@ impl ApiErrorCode { Self::FrontendSessionUnavailable => "frontend_session_unavailable", Self::FrontendSessionExpired => "frontend_session_expired", Self::FrontendSessionStateMismatch => "frontend_session_state_mismatch", + Self::ContentLockPathConflict => "content_lock_path_conflict", + Self::ContentLockDeletionInProgress => "content_lock_deletion_in_progress", + Self::ContentLockDeletionNotFound => "content_lock_deletion_not_found", Self::TaskStateConflict => "task_state_conflict", Self::UnsupportedVerifierType => "unsupported_verifier_type", Self::PaykitNotConfigured => "paykit_not_configured", @@ -74,6 +80,7 @@ impl ApiErrorCode { Self::VerificationTaskNotFound | Self::GuardedResourceNotFound | Self::ContentLockNotFound + | Self::ContentLockDeletionNotFound | Self::CreatorConnectFlowUnavailable | Self::FrontendSessionCodeUnavailable => StatusCode::NOT_FOUND, Self::CreatorAuthorityUnavailable => StatusCode::SERVICE_UNAVAILABLE, @@ -84,7 +91,9 @@ impl ApiErrorCode { StatusCode::UNAUTHORIZED } Self::FrontendSessionStateMismatch => StatusCode::BAD_REQUEST, - Self::TaskStateConflict => StatusCode::CONFLICT, + Self::ContentLockPathConflict + | Self::ContentLockDeletionInProgress + | Self::TaskStateConflict => StatusCode::CONFLICT, Self::UnsupportedVerifierType | Self::PaykitNotConfigured | Self::ReaderPubkyUnresolvable => StatusCode::UNPROCESSABLE_ENTITY, @@ -193,6 +202,14 @@ impl From for ApiError { ApiErrorCode::FrontendSessionStateMismatch, "frontend session state mismatch", ), + ApplicationError::ContentLockPathConflict { .. } => Self::new( + ApiErrorCode::ContentLockPathConflict, + "content lock path is already owned", + ), + ApplicationError::ContentLockDeletionInProgress => Self::new( + ApiErrorCode::ContentLockDeletionInProgress, + "content lock deletion is in progress", + ), ApplicationError::InvalidGuardedResource { .. } => { Self::new(ApiErrorCode::InvalidRequest, "invalid guarded resource") } @@ -223,6 +240,9 @@ impl From for ApiError { Self::new(ApiErrorCode::RateLimited, "rate limit exceeded") } ApplicationError::Storage { .. } + | ApplicationError::VerificationDependencyUnavailable + | ApplicationError::FinalCredentialSecret { .. } + | ApplicationError::InvalidContentLockDeletionState { .. } | ApplicationError::Verifier { .. } | ApplicationError::CredentialGeneration { .. } | ApplicationError::ContentLockCanonicalization { .. } @@ -410,6 +430,11 @@ mod tests { StatusCode::CONFLICT, "task_state_conflict", ), + ( + ApiErrorCode::ContentLockPathConflict, + StatusCode::CONFLICT, + "content_lock_path_conflict", + ), ( ApiErrorCode::UnsupportedVerifierType, StatusCode::UNPROCESSABLE_ENTITY, @@ -478,6 +503,42 @@ mod tests { ); } + #[test] + fn content_lock_path_conflict_maps_to_409_stable_envelope() { + let api_error = ApiError::from(ApplicationError::ContentLockPathConflict { + guarded_path: "/priv/locks.app/content/already-owned.txt".to_owned(), + }); + + assert_eq!(api_error.status_code(), StatusCode::CONFLICT); + let json = serde_json::to_value(api_error.error_response()).unwrap(); + assert_eq!( + json, + json!({ + "error": { + "code": "content_lock_path_conflict", + "message": "content lock path is already owned" + } + }) + ); + assert!(!json.to_string().contains("already-owned.txt")); + } + + #[test] + fn content_lock_deletion_cutoff_maps_to_409_stable_envelope() { + let api_error = ApiError::from(ApplicationError::ContentLockDeletionInProgress); + + assert_eq!(api_error.status_code(), StatusCode::CONFLICT); + assert_eq!( + serde_json::to_value(api_error.error_response()).unwrap(), + json!({ + "error": { + "code": "content_lock_deletion_in_progress", + "message": "content lock deletion is in progress" + } + }) + ); + } + #[test] fn invalid_guarded_resource_maps_to_400_stable_envelope() { let api_error = ApiError::from(ApplicationError::InvalidGuardedResource { diff --git a/locks-server/src/api/routes.rs b/locks-server/src/api/routes.rs index af7fce7..acbbe4b 100644 --- a/locks-server/src/api/routes.rs +++ b/locks-server/src/api/routes.rs @@ -8,8 +8,9 @@ use crate::api::creator_authority::{ exchange_frontend_session_code_route, frontend_session_signout_route, }; use crate::api::creator_publishing::{ - create_content_lock_for_authenticated_creator, + create_content_lock_for_authenticated_creator, delete_content_lock_for_authenticated_creator, delete_guarded_resource_for_authenticated_creator, + get_content_lock_deletion_status_for_authenticated_creator, register_guarded_resource_empty_tail_for_authenticated_creator, register_guarded_resource_for_authenticated_creator, set_lock_service_pointer_for_authenticated_creator, @@ -65,6 +66,14 @@ pub fn router(state: AppState) -> Router { "/creator/content-locks", post(create_content_lock_for_authenticated_creator), ) + .route( + "/creator/content-locks/{lock_id}", + delete(delete_content_lock_for_authenticated_creator), + ) + .route( + "/creator/content-locks/{lock_id}/deletion", + get(get_content_lock_deletion_status_for_authenticated_creator), + ) .route( "/creator/lock-service-config", post(set_lock_service_pointer_for_authenticated_creator), diff --git a/locks-server/src/api/routes/tests.rs b/locks-server/src/api/routes/tests.rs index 03b6cac..16803d1 100644 --- a/locks-server/src/api/routes/tests.rs +++ b/locks-server/src/api/routes/tests.rs @@ -10,7 +10,8 @@ use axum::body::{Body, to_bytes}; use axum::extract::ConnectInfo; use axum::http::{HeaderMap, Request, StatusCode, header}; use locks_core::ids::{ - BundleId, CreatorPubky, GuardedResourceHash, LockServerPubky, PubkyLockResource, + BundleId, ContentLockPath, CreatorPubky, GuardedResourceHash, LockId, LockServerPubky, + PubkyLockResource, }; use locks_core::lock_policy::{ AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, Criterion, GuardedResource, LockLogic, @@ -19,16 +20,20 @@ use locks_core::lock_policy::{ use locks_core::verification::{Proof, SUBMITTED_PROOF_BUNDLE_VERSION, SubmittedProofBundle}; use locks_service::application::errors::ApplicationError; use locks_service::application::models::{ - CreatorAuthorityAuthKind, CreatorAuthorityRecord, CreatorAuthoritySecret, - CreatorConnectAuthorizationUrl, CreatorConnectFlowId, FrontendSessionRecord, - FrontendSessionToken, GuardedResourceRecord, LegacyCreatorConnectFlowApproval, - PendingCreatorConnectFlowRecord, + AccessCredentialLookupKey, AccessCredentialRecord, ContentLockDeletionFailureCode, + ContentLockDeletionState, CreatorAuthorityAuthKind, CreatorAuthorityRecord, + CreatorAuthoritySecret, CreatorConnectAuthorizationUrl, CreatorConnectFlowId, + DeletionReadAuthorization, FrontendSessionRecord, FrontendSessionToken, GuardedResourceRecord, + LegacyCreatorConnectFlowApproval, PendingCreatorConnectFlowRecord, +}; +use locks_service::application::ports::{ + AccessCredentialStore, Clock, LegacyCreatorConnectFlowClient, }; -use locks_service::application::ports::{Clock, LegacyCreatorConnectFlowClient}; use serde_json::{Value, json}; use sqlx::postgres::PgPoolOptions; use time::macros::datetime; use tower::ServiceExt; +use uuid::Uuid; use super::router; use crate::api::auth::parse_frontend_session_token; @@ -46,6 +51,7 @@ use crate::config::{ VerificationSubmissionRateLimitConfig, WorkerConfig, }; +use locks_service::infrastructure::memory::content_lock_tombstones::InMemoryContentLockTombstoneRepository; use locks_service::infrastructure::memory::content_locks::InMemoryContentLockRepository; use locks_service::infrastructure::memory::entitlements::InMemoryEntitlementRepository; use locks_service::infrastructure::memory::guarded_resources::InMemoryGuardedResourceRepository; @@ -69,6 +75,107 @@ impl ReaderPubkyResolver for AlwaysResolvesReader { } } +struct ResponseBoundaryAccessCredentialStore { + content_type: String, + claim_token: Uuid, + consume_succeeds: bool, + releases: AtomicUsize, + consumes: AtomicUsize, +} + +impl ResponseBoundaryAccessCredentialStore { + fn new(content_type: &str) -> Self { + Self { + content_type: content_type.to_owned(), + claim_token: Uuid::new_v4(), + consume_succeeds: true, + releases: AtomicUsize::new(0), + consumes: AtomicUsize::new(0), + } + } + + fn losing_consume(content_type: &str) -> Self { + Self { + consume_succeeds: false, + ..Self::new(content_type) + } + } +} + +#[async_trait] +impl AccessCredentialStore for ResponseBoundaryAccessCredentialStore { + async fn insert_access_credential( + &self, + _lock_id: &LockId, + _lookup_key: AccessCredentialLookupKey, + _record: AccessCredentialRecord, + ) -> Result<(), ApplicationError> { + unreachable!("response-boundary fake does not issue credentials") + } + + async fn get_access_credential( + &self, + _lookup_key: &AccessCredentialLookupKey, + ) -> Result, ApplicationError> { + Ok(None) + } + + async fn delete_access_credential( + &self, + _lookup_key: &AccessCredentialLookupKey, + ) -> Result<(), ApplicationError> { + Ok(()) + } + + async fn prepare_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + path: &str, + _claim_duration: time::Duration, + ) -> Result, ApplicationError> { + Ok(Some(DeletionReadAuthorization { + claim_token: Some(self.claim_token), + creator: creator(), + resource: GuardedResource { + path: path.to_owned(), + hash: GuardedResourceHash::from_bytes([7; 32]), + content_type: self.content_type.clone(), + size: 13, + }, + })) + } + + async fn deletion_credential_enrolled( + &self, + _lookup_key: &AccessCredentialLookupKey, + ) -> Result { + Ok(true) + } + + async fn release_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + claim_token: Uuid, + _now: time::OffsetDateTime, + ) -> Result { + assert_eq!(claim_token, self.claim_token); + self.releases.fetch_add(1, Ordering::SeqCst); + Ok(true) + } + + async fn consume_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + claim_token: Uuid, + ) -> Result { + assert_eq!(claim_token, self.claim_token); + self.consumes.fetch_add(1, Ordering::SeqCst); + Ok(self.consume_succeeds) + } +} + #[tokio::test] async fn healthz_returns_process_liveness_without_runtime_details() { let response = router(test_state()) @@ -156,8 +263,653 @@ async fn cors_preflight_allows_browser_sdk_requests() { } #[tokio::test] -async fn readyz_returns_ready_for_ephemeral_runtime_without_secrets() { +async fn creator_content_lock_delete_requires_frontend_session() { + let lock_id = content_lock(true).lock_id().unwrap(); let response = router(test_state()) + .oneshot(empty_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!( + response_json(response).await["error"]["code"], + "frontend_session_unavailable" + ); +} + +#[tokio::test] +async fn creator_content_lock_delete_rejects_ambiguous_or_unknown_modes() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let lock_id = content_lock(true).lock_id().unwrap(); + + for query in [ + "force=true&graceful=true", + "force=maybe", + "force=false", + "graceful=false", + "force=false&graceful=false", + "unknown=true", + "force=true&force=false", + ] { + let response = router(state.clone()) + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}?{query}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{query}"); + assert_eq!( + response_json(response).await["error"]["code"], + "invalid_request", + "{query}" + ); + } +} + +#[tokio::test] +async fn authenticated_other_creator_cannot_mutate_or_discover_target_deletion() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + seed_frontend_session(&state, "other-session", other_creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + seed_content_lock(&state, content_lock).await; + let app = router(state.clone()); + + let other_delete = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "other-session", + )) + .await + .unwrap(); + assert_eq!(other_delete.status(), StatusCode::OK); + assert_eq!( + response_json(other_delete).await, + json!({ "lock_id": lock_id, "status": "completed" }) + ); + assert!( + state + .content_lock_deletions() + .get_job(&creator(), &lock_id) + .await + .unwrap() + .is_none() + ); + assert!( + state + .content_locks() + .get_content_lock(&creator(), &ContentLockPath::from_lock_id(lock_id.clone())) + .await + .unwrap() + .is_some() + ); + + let other_status = app + .oneshot(authenticated_json_request( + "GET", + &format!("/creator/content-locks/{lock_id}/deletion"), + json!(null), + "other-session", + )) + .await + .unwrap(); + assert_error_response( + other_status, + StatusCode::NOT_FOUND, + "content_lock_deletion_not_found", + ) + .await; +} + +#[tokio::test] +async fn graceful_delete_of_absent_lock_returns_completed_postcondition() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let lock_id = content_lock(true).lock_id().unwrap(); + + let response = router(state) + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_json(response).await, + json!({ "lock_id": lock_id, "status": "completed" }) + ); +} + +#[tokio::test] +async fn graceful_delete_of_existing_lock_queues_one_frozen_job_and_replays() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + seed_content_lock(&state, content_lock.clone()).await; + let app = router(state.clone()); + + for _ in 0..2 { + let response = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::ACCEPTED); + assert_eq!( + response_json(response).await, + json!({ "lock_id": lock_id, "status": "queued" }) + ); + } + + let job = state + .content_lock_deletions() + .get_job(&creator(), &lock_id) + .await + .unwrap() + .unwrap(); + assert_eq!(job.frozen_content_lock, content_lock); + assert_eq!(job.state, ContentLockDeletionState::Queued); +} + +#[tokio::test] +async fn force_delete_removes_resources_and_lock_records_receipt_and_replays() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + let path = content_lock.content_lock_path().unwrap(); + let resource = content_lock.primary_resource.clone().unwrap(); + seed_content_lock(&state, content_lock.clone()).await; + seed_guarded_resource(&state, &content_lock, b"guarded".to_vec()).await; + let app = router(state.clone()); + + for _ in 0..2 { + let response = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}?force=true"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_json(response).await, + json!({ + "lock_id": lock_id, + "lock_deleted": true, + "failed_resource_paths": [] + }) + ); + } + + assert!( + state + .content_locks() + .get_content_lock(&creator(), &path) + .await + .unwrap() + .is_none() + ); + assert!( + state + .guarded_resources() + .get_current_guarded_resource(&creator(), &resource.path) + .await + .unwrap() + .is_none() + ); + assert!( + state + .content_lock_deletions() + .has_force_receipt(&creator(), &lock_id) + .await + .unwrap() + ); + + let status = app + .oneshot(authenticated_json_request( + "GET", + &format!("/creator/content-locks/{lock_id}/deletion"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(status.status(), StatusCode::OK); + assert_eq!( + response_json(status).await, + json!({ "lock_id": lock_id, "status": "completed" }) + ); +} + +#[tokio::test] +async fn force_during_publication_intent_returns_redacted_conflict_without_receipt() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + let publication_token = Uuid::new_v4(); + state + .content_lock_deletions() + .begin_publication(&creator(), &lock_id, publication_token) + .await + .unwrap(); + + for query in ["?force=true", ""] { + let response = router(state.clone()) + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}{query}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::CONFLICT); + assert_eq!( + response_json(response).await, + json!({ + "error": { + "code": "content_lock_path_conflict", + "message": "content lock publication is in progress" + } + }) + ); + } + assert!( + !state + .content_lock_deletions() + .has_force_receipt(&creator(), &lock_id) + .await + .unwrap() + ); + assert!( + state + .content_lock_deletions() + .abandon_publication(&creator(), &lock_id, publication_token) + .await + .unwrap() + ); +} + +#[tokio::test] +async fn force_escalation_marks_existing_job_for_worker_owned_async_force() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + seed_content_lock(&state, content_lock.clone()).await; + seed_guarded_resource(&state, &content_lock, b"guarded".to_vec()).await; + let app = router(state.clone()); + + let graceful = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(graceful.status(), StatusCode::ACCEPTED); + + let forced = app + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}?force=true"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(forced.status(), StatusCode::ACCEPTED); + assert_eq!( + response_json(forced).await, + json!({ "lock_id": lock_id, "status": "queued" }) + ); + + let job = state + .content_lock_deletions() + .get_job(&creator(), &lock_id) + .await + .unwrap() + .unwrap(); + assert_eq!(job.frozen_content_lock, content_lock); + assert!(job.force_requested_at.is_some()); + assert!( + !state + .content_lock_deletions() + .has_force_receipt(&creator(), &lock_id) + .await + .unwrap() + ); + assert!( + state + .content_locks() + .get_content_lock(&creator(), &ContentLockPath::from_lock_id(lock_id)) + .await + .unwrap() + .is_some() + ); +} + +#[tokio::test] +async fn force_after_terminal_graceful_failure_runs_synchronously_from_frozen_manifest() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + let path = content_lock.content_lock_path().unwrap(); + let resource = content_lock.primary_resource.clone().unwrap(); + seed_content_lock(&state, content_lock.clone()).await; + seed_guarded_resource(&state, &content_lock, b"guarded".to_vec()).await; + let app = router(state.clone()); + + let graceful = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(graceful.status(), StatusCode::ACCEPTED); + + let now = state.clock().now(); + let claimed = state + .content_lock_deletions() + .claim_next("test-worker", (now + time::Duration::minutes(1)) - (now)) + .await + .unwrap() + .unwrap(); + state + .content_lock_deletions() + .finish( + claimed.job.job_id, + "test-worker", + claimed.claim_token, + Some(ContentLockDeletionFailureCode::TombstoneMissing), + ) + .await + .unwrap() + .unwrap(); + + let forced = app + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}?force=true"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(forced.status(), StatusCode::OK); + assert_eq!( + response_json(forced).await, + json!({ + "lock_id": lock_id, + "lock_deleted": true, + "failed_resource_paths": [] + }) + ); + assert!( + state + .content_locks() + .get_content_lock(&creator(), &path) + .await + .unwrap() + .is_none() + ); + assert!( + state + .guarded_resources() + .get_current_guarded_resource(&creator(), &resource.path) + .await + .unwrap() + .is_none() + ); + assert!( + state + .content_lock_deletions() + .has_force_receipt(&creator(), &lock_id) + .await + .unwrap() + ); +} + +#[tokio::test] +async fn graceful_replay_of_failed_job_requeues_same_frozen_manifest() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + seed_content_lock(&state, content_lock.clone()).await; + let app = router(state.clone()); + + let started = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(started.status(), StatusCode::ACCEPTED); + let original = state + .content_lock_deletions() + .get_job(&creator(), &lock_id) + .await + .unwrap() + .unwrap(); + let now = state.clock().now(); + let claimed = state + .content_lock_deletions() + .claim_next("test-worker", (now + time::Duration::minutes(1)) - (now)) + .await + .unwrap() + .unwrap(); + state + .content_lock_deletions() + .finish( + claimed.job.job_id, + "test-worker", + claimed.claim_token, + Some(ContentLockDeletionFailureCode::TombstoneMissing), + ) + .await + .unwrap() + .unwrap(); + + let replay = app + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(replay.status(), StatusCode::ACCEPTED); + assert_eq!( + response_json(replay).await, + json!({ "lock_id": lock_id, "status": "queued" }) + ); + let resumed = state + .content_lock_deletions() + .get_job(&creator(), &lock_id) + .await + .unwrap() + .unwrap(); + assert_eq!(resumed.job_id, original.job_id); + assert_eq!(resumed.frozen_content_lock, content_lock); + assert_eq!(resumed.state, ContentLockDeletionState::Queued); + assert_eq!(resumed.failure_code, None); +} + +#[tokio::test] +async fn graceful_and_force_reject_content_lock_stored_under_wrong_canonical_path() { + for force_query in ["", "?force=true"] { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let requested = content_lock(true); + let requested_lock_id = requested.lock_id().unwrap(); + let wrong_lock = content_lock(false); + let wrong_resource = wrong_lock.primary_resource.clone().unwrap(); + seed_guarded_resource(&state, &wrong_lock, b"guarded".to_vec()).await; + state + .content_locks() + .upsert_content_lock( + creator(), + ContentLockPath::from_lock_id(requested_lock_id.clone()), + wrong_lock, + ) + .await + .unwrap(); + + let response = router(state.clone()) + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{requested_lock_id}{force_query}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert!( + state + .content_lock_deletions() + .get_job(&creator(), &requested_lock_id) + .await + .unwrap() + .is_none() + ); + assert!( + !state + .content_lock_deletions() + .has_force_receipt(&creator(), &requested_lock_id) + .await + .unwrap() + ); + assert!( + state + .guarded_resources() + .get_current_guarded_resource(&creator(), &wrong_resource.path) + .await + .unwrap() + .is_some() + ); + } +} + +#[tokio::test] +async fn creator_content_lock_deletion_status_reports_job_and_absence() { + let state = test_state(); + seed_frontend_session(&state, "creator-session", creator()).await; + let content_lock = content_lock(true); + let lock_id = content_lock.lock_id().unwrap(); + seed_content_lock(&state, content_lock).await; + let app = router(state.clone()); + + let started = app + .clone() + .oneshot(authenticated_json_request( + "DELETE", + &format!("/creator/content-locks/{lock_id}"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(started.status(), StatusCode::ACCEPTED); + + let response = app + .oneshot(authenticated_json_request( + "GET", + &format!("/creator/content-locks/{lock_id}/deletion"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response_json(response).await, + json!({ "lock_id": lock_id, "status": "queued" }) + ); + + let missing_lock_id = LockId::from_hash(locks_core::ids::LockHash::from_bytes([99; 32])); + let missing = router(state) + .oneshot(authenticated_json_request( + "GET", + &format!("/creator/content-locks/{missing_lock_id}/deletion"), + json!(null), + "creator-session", + )) + .await + .unwrap(); + assert_eq!(missing.status(), StatusCode::NOT_FOUND); + assert_eq!( + response_json(missing).await["error"]["code"], + "content_lock_deletion_not_found" + ); +} + +#[tokio::test] +async fn readyz_requires_independent_ready_evidence_for_enabled_workers() { + let state = test_state(); + let starting = router(state.clone()) + .oneshot(empty_request("GET", "/readyz")) + .await + .unwrap(); + assert_eq!(starting.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(response_json(starting).await["status"], "not_ready"); + + state.record_worker_readiness( + crate::app_state::WorkerKind::Verification, + crate::app_state::WorkerReadinessEvidence::Ready, + ); + let verification_only = router(state.clone()) + .oneshot(empty_request("GET", "/readyz")) + .await + .unwrap(); + assert_eq!(verification_only.status(), StatusCode::SERVICE_UNAVAILABLE); + + state.record_worker_readiness( + crate::app_state::WorkerKind::Deletion, + crate::app_state::WorkerReadinessEvidence::Ready, + ); + let response = router(state) .oneshot(empty_request("GET", "/readyz")) .await .unwrap(); @@ -190,6 +942,7 @@ async fn readyz_returns_ready_for_ephemeral_runtime_without_secrets() { async fn readyz_reports_worker_disabled_for_ephemeral_runtime() { let mut config = test_config(RuntimeEnvironment::Development, true); config.worker.enabled = false; + config.deletion_worker.enabled = false; let response = router(AppState::new_empty_in_memory(config)) .oneshot(empty_request("GET", "/readyz")) .await @@ -203,6 +956,31 @@ async fn readyz_reports_worker_disabled_for_ephemeral_runtime() { assert_eq!(body.as_object().unwrap().len(), 3); } +#[tokio::test] +async fn readyz_reports_degraded_when_a_worker_dependency_is_degraded() { + let state = test_state(); + state.record_worker_readiness( + crate::app_state::WorkerKind::Verification, + crate::app_state::WorkerReadinessEvidence::Ready, + ); + state.record_worker_readiness( + crate::app_state::WorkerKind::Deletion, + crate::app_state::WorkerReadinessEvidence::TransientDependencyFailure, + ); + + let response = router(state) + .oneshot(empty_request("GET", "/readyz")) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let body = response_json(response).await; + assert_eq!(body["status"], "degraded"); + assert_eq!(body["runtime_storage"], "ephemeral"); + assert_eq!(body["worker_enabled"], true); + assert_eq!(body.as_object().unwrap().len(), 3); +} + #[tokio::test] async fn readyz_returns_not_ready_for_persisted_runtime_when_pool_ping_fails() { let pool = PgPoolOptions::new() @@ -214,6 +992,15 @@ async fn readyz_returns_not_ready_for_persisted_runtime_when_pool_ping_fails() { test_config(RuntimeEnvironment::Development, true), pool, CreatorAuthoritySecretCipher::new([7; 32]), + locks_service::infrastructure::final_credentials::FinalCredentialCipher::new([8; 32]), + ); + state.record_worker_readiness( + crate::app_state::WorkerKind::Verification, + crate::app_state::WorkerReadinessEvidence::Ready, + ); + state.record_worker_readiness( + crate::app_state::WorkerKind::Deletion, + crate::app_state::WorkerReadinessEvidence::TransientDependencyFailure, ); let response = router(state) @@ -359,7 +1146,8 @@ async fn post_proof_bundles_rejects_paykit_payment_when_paykit_is_not_configured content_lock.criteria[0].params = json!({ "recipient_pubky": creator().to_string(), "amount": "50000", - "asset": "BTC" + "asset": "BTC", + "payment_in": 24 }); let mut bundle = submitted_proof_bundle_for(&content_lock); bundle.reader_public_key = Some(creator()); @@ -818,6 +1606,7 @@ async fn dev_pubky_homeserver_routes_mount_authenticated_creator_routes_and_manu let app = router(AppState::new_empty_in_memory_with_creator_repositories( config, Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryContentLockTombstoneRepository::new()), Arc::new(InMemoryGuardedResourceRepository::new()), Arc::new(InMemoryLockServicePointerRepository::new()), Arc::new(InMemoryEntitlementRepository::new()), @@ -1346,6 +2135,69 @@ async fn proxy_read_with_valid_bearer_credential_returns_raw_guarded_resource_by assert_eq!(response_bytes(response).await, b"guarded bytes".to_vec()); } +#[tokio::test] +async fn deletion_proxy_read_releases_claim_when_http_response_construction_fails() { + let store = Arc::new(ResponseBoundaryAccessCredentialStore::new( + "invalid\ncontent-type", + )); + let state = test_state().with_access_credentials(store.clone()); + seed_response_boundary_resource(&state, "invalid\ncontent-type").await; + + let response = router(state) + .oneshot(auth_request( + "GET", + "/priv-resources/content/hello.txt", + "Bearer deletion-credential", + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(store.releases.load(Ordering::SeqCst), 1); + assert_eq!(store.consumes.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn deletion_proxy_read_consumes_claim_before_returning_http_200() { + let store = Arc::new(ResponseBoundaryAccessCredentialStore::new("text/plain")); + let state = test_state().with_access_credentials(store.clone()); + seed_response_boundary_resource(&state, "text/plain").await; + + let response = router(state) + .oneshot(auth_request( + "GET", + "/priv-resources/content/hello.txt", + "Bearer deletion-credential", + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(store.consumes.load(Ordering::SeqCst), 1); + assert_eq!(store.releases.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn deletion_proxy_read_does_not_return_constructed_response_when_consume_loses() { + let store = Arc::new(ResponseBoundaryAccessCredentialStore::losing_consume( + "text/plain", + )); + let state = test_state().with_access_credentials(store.clone()); + seed_response_boundary_resource(&state, "text/plain").await; + + let response = router(state) + .oneshot(auth_request( + "GET", + "/priv-resources/content/hello.txt", + "Bearer deletion-credential", + )) + .await + .unwrap(); + + assert_ne!(response.status(), StatusCode::OK); + assert_eq!(store.consumes.load(Ordering::SeqCst), 1); +} + #[tokio::test] async fn proxy_read_accepts_bearer_scheme_case_insensitively() { let state = test_state(); @@ -2677,6 +3529,7 @@ fn test_state_with_runtime( AppState::new_empty_in_memory_with_creator_repositories( config, Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryContentLockTombstoneRepository::new()), Arc::new(InMemoryGuardedResourceRepository::new()), Arc::new(InMemoryLockServicePointerRepository::new()), Arc::new(InMemoryEntitlementRepository::new()), @@ -2711,6 +3564,7 @@ fn test_state_with_content_lock_limits(content_locks: ContentLocksConfig) -> App AppState::new_empty_in_memory_with_creator_repositories( config, Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryContentLockTombstoneRepository::new()), Arc::new(InMemoryGuardedResourceRepository::new()), Arc::new(InMemoryLockServicePointerRepository::new()), Arc::new(InMemoryEntitlementRepository::new()), @@ -2725,6 +3579,7 @@ fn test_state_with_creator_repository_backend( AppState::new_empty_in_memory_with_creator_repositories( config, Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryContentLockTombstoneRepository::new()), Arc::new(InMemoryGuardedResourceRepository::new()), Arc::new(InMemoryLockServicePointerRepository::new()), Arc::new(InMemoryEntitlementRepository::new()), @@ -2764,6 +3619,8 @@ fn test_config( pkdns: crate::config::PkdnsConfig::default(), rate_limits: RateLimitsConfig::default(), content_locks: ContentLocksConfig::default(), + deletion: crate::config::DeletionConfig::default(), + deletion_worker: crate::config::DeletionWorkerConfig::default(), paykit: None, } } @@ -2826,6 +3683,21 @@ async fn seed_content_lock(state: &AppState, content_lock: ContentLock) { .unwrap(); } +async fn seed_response_boundary_resource(state: &AppState, content_type: &str) { + state + .guarded_resources() + .upsert_guarded_resource(GuardedResourceRecord { + creator: creator(), + path: "/priv/locks.app/content/hello.txt".to_owned(), + hash: GuardedResourceHash::from_bytes([7; 32]), + content_type: content_type.to_owned(), + size: 13, + bytes: b"guarded bytes".to_vec(), + }) + .await + .unwrap(); +} + async fn seed_guarded_resource(state: &AppState, content_lock: &ContentLock, bytes: Vec) { let guarded_resource = content_lock.primary_resource.as_ref().unwrap(); state diff --git a/locks-server/src/api/runtime.rs b/locks-server/src/api/runtime.rs index 3a6d56c..881076b 100644 --- a/locks-server/src/api/runtime.rs +++ b/locks-server/src/api/runtime.rs @@ -6,48 +6,46 @@ use axum::response::{IntoResponse, Response}; use crate::api::dtos::{ HealthHttpResponse, ReadinessHttpResponse, WellKnownLocksServerHttpResponse, }; -use crate::app_state::{AppState, RuntimeStorageKind}; +use crate::app_state::{AppState, ReadinessStatus, RuntimeStorageKind}; pub(super) async fn healthz() -> Json { Json(HealthHttpResponse { status: "ok" }) } pub(super) async fn readyz(State(state): State) -> Response { - let worker_enabled = state.config().worker.enabled; - match state.private_runtime_storage_kind() { - RuntimeStorageKind::InMemory => Json(ReadinessHttpResponse { - status: "ready", - runtime_storage: "ephemeral", - worker_enabled, - }) - .into_response(), + let worker_enabled = state.config().worker.enabled || state.config().deletion_worker.enabled; + let (runtime_storage, database_ready) = match state.private_runtime_storage_kind() { + RuntimeStorageKind::InMemory => ("ephemeral", true), RuntimeStorageKind::Postgres => { - let is_ready = match state.postgres_pool() { + let database_ready = match state.postgres_pool() { Some(pool) => sqlx::query_scalar::<_, i32>("SELECT 1") .fetch_one(pool) .await .is_ok(), None => false, }; - if is_ready { - Json(ReadinessHttpResponse { - status: "ready", - runtime_storage: "persisted", - worker_enabled, - }) - .into_response() - } else { - ( - StatusCode::SERVICE_UNAVAILABLE, - Json(ReadinessHttpResponse { - status: "not_ready", - runtime_storage: "persisted", - worker_enabled, - }), - ) - .into_response() - } + ("persisted", database_ready) } + }; + + let status = if !database_ready { + ReadinessStatus::NotReady + } else { + state.worker_readiness_status() + }; + let response = Json(ReadinessHttpResponse { + status: match status { + ReadinessStatus::Ready => "ready", + ReadinessStatus::Degraded => "degraded", + ReadinessStatus::NotReady => "not_ready", + }, + runtime_storage, + worker_enabled, + }); + + match status { + ReadinessStatus::Ready | ReadinessStatus::Degraded => response.into_response(), + ReadinessStatus::NotReady => (StatusCode::SERVICE_UNAVAILABLE, response).into_response(), } } diff --git a/locks-server/src/api/verification.rs b/locks-server/src/api/verification.rs index d13d11a..d1c7d1e 100644 --- a/locks-server/src/api/verification.rs +++ b/locks-server/src/api/verification.rs @@ -20,6 +20,9 @@ use locks_service::application::use_cases::submit_proof_bundle::{ use locks_service::application::use_cases::validate_paykit_payment_submission::{ ValidatePaykitPaymentSubmissionRequest, ValidatePaykitPaymentSubmissionUseCase, }; +use locks_service::infrastructure::postgres::{ + PaykitInvoiceWindow, PostgresPaykitTaskAdmissionRepository, +}; use locks_service::infrastructure::verifiers::registry::StaticCriterionVerifierRegistry; use crate::api::dtos::{ @@ -111,7 +114,24 @@ async fn maybe_prepare_paykit_submission( "paykit-payment requires reader_public_key", ) })?; - ValidatePaykitPaymentSubmissionUseCase::new(state.content_locks().as_ref()) + if let Some(pool) = state.postgres_pool() { + let admissions = PostgresPaykitTaskAdmissionRepository::new(pool.clone()); + if let Some(admission) = admissions.find_existing(submitted).await? { + if admission.requires_paykit { + let invoice_window = create_paykit_invoice( + state, + &admission.task.submitted_proof_bundle, + admission.payment_in, + ) + .await?; + admissions + .mark_ready(&admission.task, invoice_window) + .await?; + } + return Ok(Some(admission.task.into())); + } + } + let validated = ValidatePaykitPaymentSubmissionUseCase::new(state.content_locks().as_ref()) .execute(ValidatePaykitPaymentSubmissionRequest { submitted_proof_bundle: submitted.clone(), }) @@ -126,6 +146,23 @@ async fn maybe_prepare_paykit_submission( "reader pubky is unresolvable", )); } + if let Some(pool) = state.postgres_pool() { + let task = submit_use_case.prepare_task(submitted.clone()).await?; + let admissions = PostgresPaykitTaskAdmissionRepository::new(pool.clone()); + let admission = admissions.reserve(task, validated.payment_in).await?; + if admission.requires_paykit { + let invoice_window = create_paykit_invoice( + state, + &admission.task.submitted_proof_bundle, + admission.payment_in, + ) + .await?; + admissions + .mark_ready(&admission.task, invoice_window) + .await?; + } + return Ok(Some(admission.task.into())); + } if let Some(existing) = submit_use_case.find_existing(submitted).await? { return Ok(Some(existing)); } @@ -139,6 +176,7 @@ async fn maybe_prepare_paykit_submission( .create_invoice(&PaykitInvoiceRequest { bundle_id: submitted.bundle_id.to_string(), lock_resource: submitted.pubky_lock_resource.to_string(), + payment_in: validated.payment_in, reader: reader.to_string(), }) .await @@ -146,6 +184,39 @@ async fn maybe_prepare_paykit_submission( Ok(None) } +async fn create_paykit_invoice( + state: &AppState, + submitted: &SubmittedProofBundle, + payment_in: u64, +) -> Result { + let reader = submitted.reader_public_key.as_ref().ok_or_else(|| { + ApiError::new( + ApiErrorCode::InvalidRequest, + "paykit-payment requires reader_public_key", + ) + })?; + let response = state + .paykit_http_client() + .ok_or_else(|| { + ApiError::new( + ApiErrorCode::PaykitNotConfigured, + "paykit is not configured", + ) + })? + .create_invoice(&PaykitInvoiceRequest { + bundle_id: submitted.bundle_id.to_string(), + lock_resource: submitted.pubky_lock_resource.to_string(), + payment_in, + reader: reader.to_string(), + }) + .await + .map_err(map_paykit_invoice_error)?; + Ok(PaykitInvoiceWindow { + invoice_created_at: response.invoice_created_at, + payment_deadline: response.payment_deadline, + }) +} + fn map_paykit_invoice_error(error: PaykitClientError) -> ApiError { if matches!( error, diff --git a/locks-server/src/app_state/creator_repositories.rs b/locks-server/src/app_state/creator_repositories.rs index 74613b4..f2389b1 100644 --- a/locks-server/src/app_state/creator_repositories.rs +++ b/locks-server/src/app_state/creator_repositories.rs @@ -2,17 +2,17 @@ use std::sync::Arc; use locks_service::{ application::ports::{ - ContentLockRepository, EntitlementRepository, GuardedResourceRepository, - LockServicePointerRepository, + ContentLockRepository, ContentLockTombstoneRepository, EntitlementRepository, + GuardedResourceRepository, LockServicePointerRepository, }, infrastructure::{ postgres::PostgresCreatorAuthorityStore, pubky::{ LegacyCookieCreatorScopedPubkyStorageProvider, ProviderBackedPubkyHomeserverStorageClient, PubkyContentLockRepository, - PubkyEntitlementRepository, PubkyHomeserverStorageClient, - PubkyLegacyCookieSessionImporter, PubkyLockServicePointerRepository, - PubkyPrivResourceRepository, + PubkyContentLockTombstoneRepository, PubkyEntitlementRepository, + PubkyHomeserverStorageClient, PubkyLegacyCookieSessionImporter, + PubkyLockServicePointerRepository, PubkyPrivResourceRepository, }, }, }; @@ -20,6 +20,7 @@ use locks_service::{ #[derive(Clone)] pub(super) struct CreatorRepositoryAdapters { pub(super) content_locks: Arc, + pub(super) content_lock_tombstones: Arc, pub(super) guarded_resources: Arc, pub(super) lock_service_pointers: Arc, pub(super) entitlements: Arc, @@ -28,12 +29,14 @@ pub(super) struct CreatorRepositoryAdapters { impl CreatorRepositoryAdapters { pub(super) fn new( content_locks: Arc, + content_lock_tombstones: Arc, guarded_resources: Arc, lock_service_pointers: Arc, entitlements: Arc, ) -> Self { Self { content_locks, + content_lock_tombstones, guarded_resources, lock_service_pointers, entitlements, @@ -52,6 +55,7 @@ impl CreatorRepositoryAdapters { Self::new( Arc::new(PubkyContentLockRepository::new(client.clone())), + Arc::new(PubkyContentLockTombstoneRepository::new(client.clone())), Arc::new(PubkyPrivResourceRepository::new(client.clone())), Arc::new(PubkyLockServicePointerRepository::new(client.clone())), Arc::new(PubkyEntitlementRepository::new(client)), diff --git a/locks-server/src/app_state/mod.rs b/locks-server/src/app_state/mod.rs index 3f497f1..4361420 100644 --- a/locks-server/src/app_state/mod.rs +++ b/locks-server/src/app_state/mod.rs @@ -17,31 +17,43 @@ use locks_service::{ errors::ApplicationError, models::AccessCredentialPolicy, ports::{ - AccessCredentialStore, Clock, ContentLockRepository, CreatorAuthorityManager, - CreatorAuthorityStore, CreatorConnectFlowStore, EntitlementRepository, - FrontendSessionCodeStore, FrontendSessionStore, GuardedResourceRepository, - LegacyCreatorConnectFlowClient, LockServicePointerRepository, VerificationTaskClaimer, - VerificationTaskRepository, + AccessCredentialStore, Clock, ContentLockDeletionActionOwnership, + ContentLockDeletionRepository, ContentLockOwnershipRepository, ContentLockRepository, + ContentLockTombstoneRepository, CreatorAuthorityManager, CreatorAuthorityStore, + CreatorConnectFlowStore, EntitlementRepository, FrontendSessionCodeStore, + FrontendSessionStore, GuardedResourceRepository, LegacyCreatorConnectFlowClient, + LockServicePointerRepository, PaymentDrainClient, PaymentDrainRepository, + VerificationTaskClaimer, VerificationTaskRepository, }, }, infrastructure::{ + final_credentials::FinalCredentialCipher, memory::{ access_credentials::InMemoryAccessCredentialStore, + content_lock_deletion_action_ownership::InMemoryContentLockDeletionActionOwnership, + content_lock_deletions::InMemoryContentLockDeletionRepository, + content_lock_ownership::InMemoryContentLockOwnershipRepository, + content_lock_tombstones::InMemoryContentLockTombstoneRepository, + content_locks::InMemoryContentLockRepository, + public_content_locks::InMemoryPublicContentLockStore, verification_task_claims::InMemoryVerificationTaskClaimer, + verification_task_deletion_fence::InMemoryVerificationTaskDeletionFence, verification_tasks::InMemoryVerificationTaskRepository, }, postgres::{ CreatorAuthoritySecretCipher, PostgresAccessCredentialStore, - PostgresCreatorAuthorityStore, PostgresCreatorConnectFlowStore, - PostgresFrontendSessionCodeStore, PostgresFrontendSessionStore, + PostgresContentLockDeletionActionOwnership, PostgresContentLockDeletionRepository, + PostgresContentLockOwnershipRepository, PostgresCreatorAuthorityStore, + PostgresCreatorConnectFlowStore, PostgresFrontendSessionCodeStore, + PostgresFrontendSessionStore, PostgresPaymentDrainRepository, PostgresVerificationTaskClaimer, PostgresVerificationTaskRepository, }, pubky::{ AuthorizingPubkyHomeserverStorageClient, LegacyCookieCreatorAuthorityManager, - PubkyBytesResource, PubkyContentLockRepository, PubkyEntitlementRepository, - PubkyHomeserverStorageClient, PubkyLegacyCookieSessionRevalidator, - PubkyLegacyCreatorConnectFlowClient, PubkyLockServicePointerRepository, - PubkyPrivResourceRepository, + PubkyBytesResource, PubkyContentLockRepository, PubkyContentLockTombstoneRepository, + PubkyEntitlementRepository, PubkyHomeserverStorageClient, + PubkyLegacyCookieSessionRevalidator, PubkyLegacyCreatorConnectFlowClient, + PubkyLockServicePointerRepository, PubkyPrivResourceRepository, }, verifiers::dev_static::DevStaticVerifier, verifiers::paykit_payment::PaykitPaymentVerifier, @@ -65,7 +77,10 @@ use crate::app_state::private_runtime::{ use crate::app_state::pubky_clients::{ build_pubky_client, build_pubky_http_client, pubky_auth_relay_for_network, }; -pub use crate::app_state::readiness::RuntimeStorageKind; +pub use crate::app_state::readiness::{ + ReadinessStatus, RuntimeStorageKind, WorkerKind, WorkerReadiness, WorkerReadinessEvidence, + WorkerReadinessState, +}; use crate::config::LockServerRuntimeConfig; use crate::paykit_http_client::PaykitHttpClient; use crate::rate_limit::InMemoryVerificationSubmissionRateLimiter; @@ -144,9 +159,15 @@ pub struct AppState { config: LockServerRuntimeConfig, private_runtime_storage_kind: RuntimeStorageKind, postgres_pool: Option, + worker_readiness: WorkerReadiness, content_locks: Arc, + content_lock_tombstones: Arc, guarded_resources: Arc, lock_service_pointers: Arc, + content_lock_ownership: Arc, + content_lock_deletions: Arc, + content_lock_deletion_action_ownership: Arc, + payment_drains: Option>, verification_tasks: Arc, verification_task_claimer: Arc, entitlements: Arc, @@ -169,6 +190,25 @@ pub struct AppState { verification_submission_rate_limiter: Arc, reader_pubky_resolver: Arc, paykit_http_client: Option>, + payment_drain_client: Option>, +} + +/// Purpose-separated runtime ciphers derived from the configured master key. +pub struct RuntimeSecretCiphers { + creator_authority: CreatorAuthoritySecretCipher, + final_credential: FinalCredentialCipher, +} + +impl RuntimeSecretCiphers { + pub fn new( + creator_authority: CreatorAuthoritySecretCipher, + final_credential: FinalCredentialCipher, + ) -> Self { + Self { + creator_authority, + final_credential, + } + } } impl std::fmt::Debug for AppState { @@ -202,28 +242,61 @@ impl std::fmt::Debug for AppState { impl AppState { pub fn new_empty_in_memory(config: LockServerRuntimeConfig) -> Self { - let verification_tasks = Arc::new(InMemoryVerificationTaskRepository::new()); - let verification_task_claimer = - Arc::new(InMemoryVerificationTaskClaimer::with_task_repository( + let verification_task_deletion_fence = + Arc::new(InMemoryVerificationTaskDeletionFence::new()); + let verification_tasks = Arc::new(InMemoryVerificationTaskRepository::with_deletion_fence( + Arc::clone(&verification_task_deletion_fence), + )); + let verification_task_claimer = Arc::new( + InMemoryVerificationTaskClaimer::with_task_repository_and_deletion_fence( vec![], verification_tasks.clone(), - )); - let access_credentials = Arc::new(InMemoryAccessCredentialStore::new()); + Arc::clone(&verification_task_deletion_fence), + ), + ); + let access_credentials = Arc::new( + InMemoryAccessCredentialStore::with_verification_task_repository_and_deletion_fence( + verification_tasks.clone(), + Arc::clone(&verification_task_deletion_fence), + ), + ); let creator_authority_store = InMemoryCreatorAuthorityStore::new(); let creator_authorities = Arc::new(creator_authority_store.clone()); let creator_authority_manager = Arc::new(LegacyCookieCreatorAuthorityManager::new( creator_authority_store, NoopLegacyCookieSessionRevalidator, )); - let unavailable_storage = UnavailablePubkyHomeserverStorageClient; + let unavailable_storage: Arc = + Arc::new(UnavailablePubkyHomeserverStorageClient); + let public_content_locks = InMemoryPublicContentLockStore::new(); let creator_repositories = CreatorRepositoryAdapters::new( - Arc::new(PubkyContentLockRepository::new(unavailable_storage)), - Arc::new(PubkyPrivResourceRepository::new(unavailable_storage)), - Arc::new(PubkyLockServicePointerRepository::new(unavailable_storage)), + Arc::new(InMemoryContentLockRepository::with_public_store( + public_content_locks.clone(), + )), + Arc::new(InMemoryContentLockTombstoneRepository::with_public_store( + public_content_locks, + )), + Arc::new(PubkyPrivResourceRepository::new( + unavailable_storage.clone(), + )), + Arc::new(PubkyLockServicePointerRepository::new( + unavailable_storage.clone(), + )), Arc::new(PubkyEntitlementRepository::new(unavailable_storage)), ); + let content_lock_deletions = Arc::new( + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + Arc::clone(&access_credentials), + verification_task_deletion_fence, + ), + ); let private_runtime = PrivateRuntimeAdapters { + content_lock_ownership: Arc::new(InMemoryContentLockOwnershipRepository::new()), + content_lock_deletions: content_lock_deletions.clone(), + content_lock_deletion_action_ownership: Arc::new( + InMemoryContentLockDeletionActionOwnership::new(content_lock_deletions), + ), verification_tasks, verification_task_claimer, access_credentials, @@ -247,17 +320,29 @@ impl AppState { pub fn new_empty_in_memory_with_creator_repositories( config: LockServerRuntimeConfig, content_locks: Arc, + content_lock_tombstones: Arc, guarded_resources: Arc, lock_service_pointers: Arc, entitlements: Arc, ) -> Self { - let verification_tasks = Arc::new(InMemoryVerificationTaskRepository::new()); - let verification_task_claimer = - Arc::new(InMemoryVerificationTaskClaimer::with_task_repository( + let verification_task_deletion_fence = + Arc::new(InMemoryVerificationTaskDeletionFence::new()); + let verification_tasks = Arc::new(InMemoryVerificationTaskRepository::with_deletion_fence( + Arc::clone(&verification_task_deletion_fence), + )); + let verification_task_claimer = Arc::new( + InMemoryVerificationTaskClaimer::with_task_repository_and_deletion_fence( vec![], verification_tasks.clone(), - )); - let access_credentials = Arc::new(InMemoryAccessCredentialStore::new()); + Arc::clone(&verification_task_deletion_fence), + ), + ); + let access_credentials = Arc::new( + InMemoryAccessCredentialStore::with_verification_task_repository_and_deletion_fence( + verification_tasks.clone(), + Arc::clone(&verification_task_deletion_fence), + ), + ); let creator_authority_store = InMemoryCreatorAuthorityStore::new(); let creator_authorities = Arc::new(creator_authority_store.clone()); let creator_authority_manager = Arc::new(LegacyCookieCreatorAuthorityManager::new( @@ -266,12 +351,24 @@ impl AppState { )); let creator_repositories = CreatorRepositoryAdapters::new( content_locks, + content_lock_tombstones, guarded_resources, lock_service_pointers, entitlements, ); + let content_lock_deletions = Arc::new( + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + Arc::clone(&access_credentials), + verification_task_deletion_fence, + ), + ); let private_runtime = PrivateRuntimeAdapters { + content_lock_ownership: Arc::new(InMemoryContentLockOwnershipRepository::new()), + content_lock_deletions: content_lock_deletions.clone(), + content_lock_deletion_action_ownership: Arc::new( + InMemoryContentLockDeletionActionOwnership::new(content_lock_deletions), + ), verification_tasks, verification_task_claimer, access_credentials, @@ -299,44 +396,64 @@ impl AppState { where S: PubkyHomeserverStorageClient + Clone + 'static, { - let verification_tasks = Arc::new(InMemoryVerificationTaskRepository::new()); - let verification_task_claimer = - Arc::new(InMemoryVerificationTaskClaimer::with_task_repository( + let verification_task_deletion_fence = + Arc::new(InMemoryVerificationTaskDeletionFence::new()); + let verification_tasks = Arc::new(InMemoryVerificationTaskRepository::with_deletion_fence( + Arc::clone(&verification_task_deletion_fence), + )); + let verification_task_claimer = Arc::new( + InMemoryVerificationTaskClaimer::with_task_repository_and_deletion_fence( vec![], verification_tasks.clone(), - )); - let access_credentials = Arc::new(InMemoryAccessCredentialStore::new()); + Arc::clone(&verification_task_deletion_fence), + ), + ); + let access_credentials = Arc::new( + InMemoryAccessCredentialStore::with_verification_task_repository_and_deletion_fence( + verification_tasks.clone(), + Arc::clone(&verification_task_deletion_fence), + ), + ); let creator_authority_store = InMemoryCreatorAuthorityStore::new(); let creator_authorities = Arc::new(creator_authority_store.clone()); let creator_authority_manager = Arc::new(LegacyCookieCreatorAuthorityManager::new( creator_authority_store.clone(), NoopLegacyCookieSessionRevalidator, )); - let authorizing_storage = |storage: S| { - AuthorizingPubkyHomeserverStorageClient::new( + let authorizing_storage: Arc = + Arc::new(AuthorizingPubkyHomeserverStorageClient::new( storage, LegacyCookieCreatorAuthorityManager::new( creator_authority_store.clone(), NoopLegacyCookieSessionRevalidator, ), - ) - }; + )); let creator_repositories = CreatorRepositoryAdapters::new( - Arc::new(PubkyContentLockRepository::new(authorizing_storage( - storage.clone(), - ))), - Arc::new(PubkyPrivResourceRepository::new(authorizing_storage( - storage.clone(), - ))), - Arc::new(PubkyLockServicePointerRepository::new(authorizing_storage( - storage.clone(), - ))), - Arc::new(PubkyEntitlementRepository::new(authorizing_storage( - storage, - ))), + Arc::new(PubkyContentLockRepository::new(authorizing_storage.clone())), + Arc::new(PubkyContentLockTombstoneRepository::new( + authorizing_storage.clone(), + )), + Arc::new(PubkyPrivResourceRepository::new( + authorizing_storage.clone(), + )), + Arc::new(PubkyLockServicePointerRepository::new( + authorizing_storage.clone(), + )), + Arc::new(PubkyEntitlementRepository::new(authorizing_storage)), ); + let content_lock_deletions = Arc::new( + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + Arc::clone(&access_credentials), + verification_task_deletion_fence, + ), + ); let private_runtime = PrivateRuntimeAdapters { + content_lock_ownership: Arc::new(InMemoryContentLockOwnershipRepository::new()), + content_lock_deletions: content_lock_deletions.clone(), + content_lock_deletion_action_ownership: Arc::new( + InMemoryContentLockDeletionActionOwnership::new(content_lock_deletions), + ), verification_tasks, verification_task_claimer, access_credentials, @@ -361,11 +478,16 @@ impl AppState { config: LockServerRuntimeConfig, pool: PgPool, creator_authority_cipher: CreatorAuthoritySecretCipher, + final_credential_cipher: FinalCredentialCipher, ) -> Self { let verification_tasks = Arc::new(PostgresVerificationTaskRepository::new(pool.clone())); let verification_task_claimer = Arc::new(PostgresVerificationTaskClaimer::new(pool.clone())); - let access_credentials = Arc::new(PostgresAccessCredentialStore::new(pool.clone())); + let access_credentials = + Arc::new(PostgresAccessCredentialStore::with_final_credential_cipher( + pool.clone(), + final_credential_cipher, + )); let creator_authority_store = PostgresCreatorAuthorityStore::new_encrypted(pool.clone(), creator_authority_cipher); let creator_authorities = Arc::new(creator_authority_store.clone()); @@ -391,6 +513,15 @@ impl AppState { }; let private_runtime = PrivateRuntimeAdapters { + content_lock_ownership: Arc::new(PostgresContentLockOwnershipRepository::new( + pool.clone(), + )), + content_lock_deletions: Arc::new(PostgresContentLockDeletionRepository::new( + pool.clone(), + )), + content_lock_deletion_action_ownership: Arc::new( + PostgresContentLockDeletionActionOwnership::new(pool.clone()), + ), verification_tasks, verification_task_claimer, access_credentials, @@ -411,11 +542,13 @@ impl AppState { ) } + #[allow(clippy::too_many_arguments)] pub fn new_with_postgres_runtime_and_creator_repositories( config: LockServerRuntimeConfig, pool: PgPool, - creator_authority_cipher: CreatorAuthoritySecretCipher, + ciphers: RuntimeSecretCiphers, content_locks: Arc, + content_lock_tombstones: Arc, guarded_resources: Arc, lock_service_pointers: Arc, entitlements: Arc, @@ -423,9 +556,13 @@ impl AppState { let verification_tasks = Arc::new(PostgresVerificationTaskRepository::new(pool.clone())); let verification_task_claimer = Arc::new(PostgresVerificationTaskClaimer::new(pool.clone())); - let access_credentials = Arc::new(PostgresAccessCredentialStore::new(pool.clone())); + let access_credentials = + Arc::new(PostgresAccessCredentialStore::with_final_credential_cipher( + pool.clone(), + ciphers.final_credential, + )); let creator_authority_store = - PostgresCreatorAuthorityStore::new_encrypted(pool.clone(), creator_authority_cipher); + PostgresCreatorAuthorityStore::new_encrypted(pool.clone(), ciphers.creator_authority); let creator_authorities = Arc::new(creator_authority_store.clone()); let pubky_http_client = build_pubky_http_client(config.pubky.network); let creator_authority_manager: Arc = @@ -447,11 +584,21 @@ impl AppState { }; let creator_repositories = CreatorRepositoryAdapters::new( content_locks, + content_lock_tombstones, guarded_resources, lock_service_pointers, entitlements, ); let private_runtime = PrivateRuntimeAdapters { + content_lock_ownership: Arc::new(PostgresContentLockOwnershipRepository::new( + pool.clone(), + )), + content_lock_deletions: Arc::new(PostgresContentLockDeletionRepository::new( + pool.clone(), + )), + content_lock_deletion_action_ownership: Arc::new( + PostgresContentLockDeletionActionOwnership::new(pool.clone()), + ), verification_tasks, verification_task_claimer, access_credentials, @@ -479,6 +626,8 @@ impl AppState { creator_repositories: CreatorRepositoryAdapters, private_runtime: PrivateRuntimeAdapters, ) -> Self { + let worker_readiness = + WorkerReadiness::new(config.worker.enabled, config.deletion_worker.enabled); let access_credential_policy = AccessCredentialPolicy::new(config.credentials.max_ttl_seconds); let verification_submission_rate_limiter = @@ -502,14 +651,27 @@ impl AppState { )) }) }); + let payment_drains: Option> = postgres_pool + .as_ref() + .map(|pool| Arc::new(PostgresPaymentDrainRepository::new(pool.clone())) as Arc<_>); + let payment_drain_client = paykit_http_client + .as_ref() + .map(|client| Arc::clone(client) as Arc); Self { config, private_runtime_storage_kind, postgres_pool, + worker_readiness, content_locks: creator_repositories.content_locks, + content_lock_tombstones: creator_repositories.content_lock_tombstones, guarded_resources: creator_repositories.guarded_resources, lock_service_pointers: creator_repositories.lock_service_pointers, + content_lock_ownership: private_runtime.content_lock_ownership, + content_lock_deletions: private_runtime.content_lock_deletions, + content_lock_deletion_action_ownership: private_runtime + .content_lock_deletion_action_ownership, + payment_drains, verification_tasks: private_runtime.verification_tasks, verification_task_claimer: private_runtime.verification_task_claimer, entitlements: creator_repositories.entitlements, @@ -532,6 +694,7 @@ impl AppState { verification_submission_rate_limiter, reader_pubky_resolver, paykit_http_client, + payment_drain_client, } } @@ -547,14 +710,52 @@ impl AppState { self.postgres_pool.as_ref() } + pub fn worker_readiness(&self) -> &WorkerReadiness { + &self.worker_readiness + } + + pub fn worker_readiness_status(&self) -> ReadinessStatus { + self.worker_readiness.status() + } + + pub fn record_worker_readiness(&self, worker: WorkerKind, evidence: WorkerReadinessEvidence) { + self.worker_readiness.record(worker, evidence); + } + pub fn content_locks(&self) -> &Arc { &self.content_locks } + pub fn content_lock_tombstones(&self) -> &Arc { + &self.content_lock_tombstones + } + pub fn guarded_resources(&self) -> &Arc { &self.guarded_resources } + pub fn content_lock_ownership(&self) -> &Arc { + &self.content_lock_ownership + } + + pub fn content_lock_deletions(&self) -> &Arc { + &self.content_lock_deletions + } + + pub fn content_lock_deletion_action_ownership( + &self, + ) -> &Arc { + &self.content_lock_deletion_action_ownership + } + + pub fn payment_drains(&self) -> Option<&Arc> { + self.payment_drains.as_ref() + } + + pub fn payment_drain_client(&self) -> Option<&Arc> { + self.payment_drain_client.as_ref() + } + pub fn lock_service_pointers(&self) -> &Arc { &self.lock_service_pointers } @@ -639,6 +840,15 @@ impl AppState { self } + #[cfg(test)] + pub fn with_access_credentials( + mut self, + access_credentials: Arc, + ) -> Self { + self.access_credentials = access_credentials; + self + } + #[cfg(any(test, feature = "test-support"))] pub fn with_legacy_creator_connect_flow_client( mut self, diff --git a/locks-server/src/app_state/private_runtime.rs b/locks-server/src/app_state/private_runtime.rs index 93d83c7..58d3196 100644 --- a/locks-server/src/app_state/private_runtime.rs +++ b/locks-server/src/app_state/private_runtime.rs @@ -11,7 +11,8 @@ use locks_service::application::{ PendingCreatorConnectFlowRecord, }, ports::{ - AccessCredentialStore, CreatorAuthorityManager, CreatorAuthorityStore, + AccessCredentialStore, ContentLockDeletionActionOwnership, ContentLockDeletionRepository, + ContentLockOwnershipRepository, CreatorAuthorityManager, CreatorAuthorityStore, CreatorConnectFlowStore, FrontendSessionCodeStore, FrontendSessionStore, LegacyCreatorConnectFlowClient, VerificationTaskClaimer, VerificationTaskRepository, }, @@ -21,6 +22,9 @@ use tokio::sync::RwLock; #[derive(Clone)] pub(super) struct PrivateRuntimeAdapters { + pub(super) content_lock_ownership: Arc, + pub(super) content_lock_deletions: Arc, + pub(super) content_lock_deletion_action_ownership: Arc, pub(super) verification_tasks: Arc, pub(super) verification_task_claimer: Arc, pub(super) access_credentials: Arc, diff --git a/locks-server/src/app_state/readiness.rs b/locks-server/src/app_state/readiness.rs index 96ac469..203dd95 100644 --- a/locks-server/src/app_state/readiness.rs +++ b/locks-server/src/app_state/readiness.rs @@ -1,6 +1,192 @@ +use std::sync::{Arc, RwLock}; + /// Runtime composition state for the Lock Server process. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RuntimeStorageKind { InMemory, Postgres, } + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WorkerKind { + Verification, + Deletion, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WorkerReadinessState { + Disabled, + Starting, + Ready, + Degraded, + NotReady, + Stopped, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WorkerReadinessEvidence { + Starting, + Ready, + DependencySucceeded, + TransientDependencyFailure, + PendingWork, + LockContention, + TerminalBusinessFailure, + Stopping, + Stopped, + UnexpectedExit, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReadinessStatus { + Ready, + Degraded, + NotReady, +} + +#[derive(Debug, Clone)] +pub struct WorkerReadiness { + states: Arc>, +} + +impl WorkerReadiness { + pub fn new(verification_enabled: bool, deletion_enabled: bool) -> Self { + let initial = |enabled| { + if enabled { + WorkerReadinessState::Starting + } else { + WorkerReadinessState::Disabled + } + }; + Self { + states: Arc::new(RwLock::new([ + initial(verification_enabled), + initial(deletion_enabled), + ])), + } + } + + pub fn status(&self) -> ReadinessStatus { + let states = self.states.read().expect("worker readiness lock poisoned"); + if states.iter().any(|state| { + matches!( + state, + WorkerReadinessState::Starting + | WorkerReadinessState::NotReady + | WorkerReadinessState::Stopped + ) + }) { + ReadinessStatus::NotReady + } else if states.contains(&WorkerReadinessState::Degraded) { + ReadinessStatus::Degraded + } else { + ReadinessStatus::Ready + } + } + + pub fn worker_state(&self, worker: WorkerKind) -> WorkerReadinessState { + self.states.read().expect("worker readiness lock poisoned")[worker.index()] + } + + pub fn record(&self, worker: WorkerKind, evidence: WorkerReadinessEvidence) { + let state = match evidence { + WorkerReadinessEvidence::Starting => Some(WorkerReadinessState::Starting), + WorkerReadinessEvidence::Ready | WorkerReadinessEvidence::DependencySucceeded => { + Some(WorkerReadinessState::Ready) + } + WorkerReadinessEvidence::TransientDependencyFailure => { + Some(WorkerReadinessState::Degraded) + } + WorkerReadinessEvidence::Stopping | WorkerReadinessEvidence::UnexpectedExit => { + Some(WorkerReadinessState::NotReady) + } + WorkerReadinessEvidence::Stopped => Some(WorkerReadinessState::Stopped), + WorkerReadinessEvidence::PendingWork + | WorkerReadinessEvidence::LockContention + | WorkerReadinessEvidence::TerminalBusinessFailure => None, + }; + if let Some(state) = state { + self.states.write().expect("worker readiness lock poisoned")[worker.index()] = state; + } + } +} + +impl WorkerKind { + fn index(self) -> usize { + match self { + Self::Verification => 0, + Self::Deletion => 1, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn enabled_workers_start_not_ready_and_require_independent_ready_evidence() { + let readiness = WorkerReadiness::new(true, true); + + assert_eq!(readiness.status(), ReadinessStatus::NotReady); + assert_eq!( + readiness.worker_state(WorkerKind::Verification), + WorkerReadinessState::Starting + ); + assert_eq!( + readiness.worker_state(WorkerKind::Deletion), + WorkerReadinessState::Starting + ); + + readiness.record(WorkerKind::Verification, WorkerReadinessEvidence::Ready); + assert_eq!(readiness.status(), ReadinessStatus::NotReady); + + readiness.record(WorkerKind::Deletion, WorkerReadinessEvidence::Ready); + assert_eq!(readiness.status(), ReadinessStatus::Ready); + } + + #[test] + fn deletion_dependency_failure_degrades_until_success_without_business_outcome_noise() { + let readiness = WorkerReadiness::new(false, true); + readiness.record(WorkerKind::Deletion, WorkerReadinessEvidence::Ready); + + readiness.record( + WorkerKind::Deletion, + WorkerReadinessEvidence::TransientDependencyFailure, + ); + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + assert_eq!( + readiness.worker_state(WorkerKind::Deletion), + WorkerReadinessState::Degraded + ); + + for evidence in [ + WorkerReadinessEvidence::PendingWork, + WorkerReadinessEvidence::LockContention, + WorkerReadinessEvidence::TerminalBusinessFailure, + ] { + readiness.record(WorkerKind::Deletion, evidence); + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + } + + readiness.record( + WorkerKind::Deletion, + WorkerReadinessEvidence::DependencySucceeded, + ); + assert_eq!(readiness.status(), ReadinessStatus::Ready); + } + + #[test] + fn stopping_stopped_and_unexpected_exit_are_not_ready() { + for evidence in [ + WorkerReadinessEvidence::Stopping, + WorkerReadinessEvidence::Stopped, + WorkerReadinessEvidence::UnexpectedExit, + ] { + let readiness = WorkerReadiness::new(false, true); + readiness.record(WorkerKind::Deletion, WorkerReadinessEvidence::Ready); + readiness.record(WorkerKind::Deletion, evidence); + assert_eq!(readiness.status(), ReadinessStatus::NotReady); + } + } +} diff --git a/locks-server/src/app_state/test_support.rs b/locks-server/src/app_state/test_support.rs index 94aa3c4..5f4a1fc 100644 --- a/locks-server/src/app_state/test_support.rs +++ b/locks-server/src/app_state/test_support.rs @@ -3,7 +3,8 @@ use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; -use locks_core::ids::{CreatorPubky, LockServerPubky}; +use locks_core::content_lock_deletion::ContentLockDeletionTombstone; +use locks_core::ids::{ContentLockPath, CreatorPubky, LockId, LockServerPubky}; use sqlx::postgres::PgPoolOptions; use time::macros::datetime; @@ -19,10 +20,14 @@ use crate::config::{ }; use crate::rate_limit::VerificationSubmissionRateLimitKey; use locks_service::application::errors::ApplicationError; +use locks_service::application::models::ContentLockDeletionPhase; use locks_service::application::ports::{ + ContentLockDeletionActionAcquireResult, ContentLockDeletionActionClaim, + ContentLockDeletionActionOwnership, ContentLockTombstoneRepository, CreatorConnectFlowIdGenerator, FrontendSessionCodeGenerator, FrontendSessionTokenGenerator, - VerificationTaskIdGenerator, + TombstoneReadback, VerificationTaskIdGenerator, }; +use locks_service::infrastructure::final_credentials::FinalCredentialCipher; use locks_service::infrastructure::postgres::CreatorAuthoritySecretCipher; #[test] @@ -41,23 +46,109 @@ async fn postgres_state_uses_postgres_for_private_runtime_adapters() { .connect_lazy("postgres://locks:locks@localhost/locks_test") .unwrap(); - let state = - AppState::new_with_postgres_runtime(test_config(), pool, test_creator_authority_cipher()); + let state = AppState::new_with_postgres_runtime( + test_config(), + pool, + test_creator_authority_cipher(), + test_final_credential_cipher(), + ); assert_eq!( state.private_runtime_storage_kind(), RuntimeStorageKind::Postgres ); + assert_action_ownership_adapter(state.content_lock_deletion_action_ownership().as_ref()); + assert_tombstone_adapter(state.content_lock_tombstones().as_ref()); } +#[tokio::test] +async fn in_memory_state_exposes_callable_deletion_action_and_tombstone_adapters() { + let state = AppState::new_empty_in_memory(test_config()); + let result = state + .content_lock_deletion_action_ownership() + .try_acquire(ContentLockDeletionActionClaim { + job_id: uuid::Uuid::new_v4(), + worker_id: "test-worker", + claim_token: uuid::Uuid::new_v4(), + expected_phase: ContentLockDeletionPhase::Withdraw, + force: false, + }) + .await + .unwrap(); + assert!(matches!( + result, + ContentLockDeletionActionAcquireResult::ClaimLost + )); + + let lock_id = LockId::from_str("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG").unwrap(); + let path = ContentLockPath::from_lock_id(lock_id.clone()); + let tombstone = ContentLockDeletionTombstone::new(lock_id, datetime!(2026-08-12 05:00:00 UTC)); + assert_eq!( + state + .content_lock_tombstones() + .read_tombstone(&rate_limit_key().creator, &path, &tombstone) + .await + .unwrap(), + TombstoneReadback::Missing + ); + + let creator = rate_limit_key().creator; + let content_lock: locks_core::lock_policy::ContentLock = + serde_json::from_value(serde_json::json!({ + "version": 1, + "creator": creator, + "primary_resource": null, + "secondary_resources": {}, + "criteria": [], + "lock_logic": { "type": "all", "criteria": [] }, + "access_policy": { "requested_credential_ttl_seconds": 900 }, + "lock_server": { "override": null }, + "created_at": "2026-08-12T04:00:00Z" + })) + .unwrap(); + state + .content_locks() + .upsert_content_lock(creator.clone(), path.clone(), content_lock.clone()) + .await + .unwrap(); + state + .content_lock_tombstones() + .withdraw_content_lock(creator.clone(), path.clone(), &content_lock, &tombstone) + .await + .unwrap(); + assert!( + state + .content_locks() + .get_content_lock(&creator, &path) + .await + .is_err() + ); + assert_eq!( + state + .content_lock_tombstones() + .read_tombstone(&creator, &path, &tombstone) + .await + .unwrap(), + TombstoneReadback::Exact + ); +} + +fn assert_action_ownership_adapter(_: &dyn ContentLockDeletionActionOwnership) {} + +fn assert_tombstone_adapter(_: &dyn ContentLockTombstoneRepository) {} + #[tokio::test] async fn postgres_state_wires_legacy_connect_flow_runtime_state() { let pool = PgPoolOptions::new() .connect_lazy("postgres://locks:locks@localhost/locks_test") .unwrap(); - let state = - AppState::new_with_postgres_runtime(test_config(), pool, test_creator_authority_cipher()); + let state = AppState::new_with_postgres_runtime( + test_config(), + pool, + test_creator_authority_cipher(), + test_final_credential_cipher(), + ); assert!(Arc::strong_count(state.creator_connect_flows()) >= 1); assert!(Arc::strong_count(state.frontend_session_codes()) >= 1); @@ -74,7 +165,12 @@ async fn postgres_state_uses_acquisition_gate_to_wire_legacy_connect_client() { let mut config = test_config(); config.creator_authority_acquisition.enabled = true; - let state = AppState::new_with_postgres_runtime(config, pool, test_creator_authority_cipher()); + let state = AppState::new_with_postgres_runtime( + config, + pool, + test_creator_authority_cipher(), + test_final_credential_cipher(), + ); let result = state .legacy_creator_connect_flow_client() @@ -125,8 +221,12 @@ async fn persisted_state_keeps_postgres_pool_for_readiness() { .connect_lazy("postgres://locks:locks@localhost/locks_test") .unwrap(); - let state = - AppState::new_with_postgres_runtime(test_config(), pool, test_creator_authority_cipher()); + let state = AppState::new_with_postgres_runtime( + test_config(), + pool, + test_creator_authority_cipher(), + test_final_credential_cipher(), + ); assert!(state.postgres_pool().is_some()); } @@ -137,8 +237,12 @@ async fn persisted_state_composes_pubky_homeserver_creator_repositories() { .connect_lazy("postgres://locks:locks@localhost/locks_test") .unwrap(); - let state = - AppState::new_with_postgres_runtime(test_config(), pool, test_creator_authority_cipher()); + let state = AppState::new_with_postgres_runtime( + test_config(), + pool, + test_creator_authority_cipher(), + test_final_credential_cipher(), + ); assert!(Arc::strong_count(state.content_locks()) >= 1); assert!(Arc::strong_count(state.guarded_resources()) >= 1); @@ -187,6 +291,7 @@ async fn postgres_state_has_rate_limiter_configured_from_runtime_config() { test_config_with_rate_limit(true, 1, 60), pool, test_creator_authority_cipher(), + test_final_credential_cipher(), ); let key = rate_limit_key(); let now = datetime!(2026-06-03 12:00:00 UTC); @@ -256,6 +361,10 @@ fn disabled_runtime_rate_limiter_in_state_always_allows() { } } +fn test_final_credential_cipher() -> FinalCredentialCipher { + FinalCredentialCipher::new([8; 32]) +} + fn test_creator_authority_cipher() -> CreatorAuthoritySecretCipher { CreatorAuthoritySecretCipher::new([7; 32]) } @@ -292,6 +401,8 @@ fn test_config() -> LockServerRuntimeConfig { pkdns: crate::config::PkdnsConfig::default(), rate_limits: RateLimitsConfig::default(), content_locks: ContentLocksConfig::default(), + deletion: crate::config::DeletionConfig::default(), + deletion_worker: crate::config::DeletionWorkerConfig::default(), paykit: None, } } diff --git a/locks-server/src/config/defaults.rs b/locks-server/src/config/defaults.rs index fd030ec..f246ad7 100644 --- a/locks-server/src/config/defaults.rs +++ b/locks-server/src/config/defaults.rs @@ -2,4 +2,13 @@ pub(super) const DEFAULT_SERVICE_HOME: &str = ".pubky-lock"; pub(super) const DEFAULT_CONFIG_FILE: &str = "config.toml"; pub(super) const DEFAULT_SECRET_FILE: &str = "secret.sess"; pub(super) const PUBLIC_KEY_PLACEHOLDER: &str = ""; -pub(super) const DEFAULT_CREATOR_AUTHORITY_KEY_ENV: &str = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY"; +pub(super) const DEFAULT_RUNTIME_MASTER_KEY_ENV: &str = "PUBKY_LOCK_RUNTIME_MASTER_KEY"; +pub(super) const DEFAULT_DELETION_RETRY_MAX_ATTEMPTS: u32 = 10; +pub(super) const DEFAULT_DELETION_RETRY_INITIAL_BACKOFF_SECONDS: u64 = 1; +pub(super) const DEFAULT_DELETION_RETRY_MAX_BACKOFF_SECONDS: u64 = 300; +pub(super) const DEFAULT_FINAL_CREDENTIAL_ISSUANCE_WINDOW_SECONDS: u64 = 900; +pub(super) const DEFAULT_FINAL_READ_WINDOW_SECONDS: u64 = 900; +pub(super) const DEFAULT_DELETION_WORKER_POLL_INTERVAL_MS: u64 = 250; +pub(super) const DEFAULT_DELETION_WORKER_CLAIM_TIMEOUT_SECONDS: u64 = 60; +pub(super) const DEFAULT_DELETION_WORKER_SHUTDOWN_TIMEOUT_SECONDS: u64 = 30; +pub(super) const DEFAULT_DELETION_WORKER_ID: &str = "deletion-worker"; diff --git a/locks-server/src/config/examples.rs b/locks-server/src/config/examples.rs index 6182eb2..a8a817b 100644 --- a/locks-server/src/config/examples.rs +++ b/locks-server/src/config/examples.rs @@ -48,7 +48,14 @@ frontend_session_code_ttl_seconds = 120 allowed_return_origins = ["http://localhost:3000"] [secrets] -creator_authority_key_env = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" +runtime_master_key_env = "PUBKY_LOCK_RUNTIME_MASTER_KEY" + +[deletion] +retry_max_attempts = 10 +retry_initial_backoff_seconds = 1 +retry_max_backoff_seconds = 300 +final_credential_issuance_window_seconds = 900 +final_read_window_seconds = 900 [logging] level = "info" @@ -226,6 +233,26 @@ fn rejects_zero_worker_poll_interval() { } } +#[test] +fn rejects_zero_pkarr_republisher_interval_before_runtime_startup() { + let temp_dir = tempdir().unwrap(); + let secret_path = temp_dir.path().join("secret.sess"); + let public_key = test_identity(&secret_path); + let config_path = temp_dir.path().join("config.toml"); + let config = minimal_config(&secret_path, &public_key, "development").replace( + "key_republisher_interval_seconds = 3600", + "key_republisher_interval_seconds = 0", + ); + std::fs::write(&config_path, config).unwrap(); + + let error = load_existing_config_from_path(&config_path).unwrap_err(); + + assert!(matches!( + error, + ConfigError::InvalidPkarrRepublisherInterval + )); +} + #[test] fn rejects_removed_creator_repositories_section() { let temp_dir = tempdir().unwrap(); @@ -308,6 +335,182 @@ fn allows_wildcard_return_origin_outside_production() { ); } +#[test] +fn accepts_closed_deletion_defaults_and_runtime_master_key_contract() { + let temp_dir = tempdir().unwrap(); + let secret_path = temp_dir.path().join("secret.sess"); + let public_key = test_identity(&secret_path); + let config_path = temp_dir.path().join("config.toml"); + std::fs::write( + &config_path, + minimal_config(&secret_path, &public_key, "development"), + ) + .unwrap(); + + let config = load_existing_config_from_path(&config_path).unwrap(); + assert_eq!( + config.secrets.runtime_master_key_env, + "PUBKY_LOCK_RUNTIME_MASTER_KEY" + ); + assert_eq!(config.deletion.retry_max_attempts, 10); + assert_eq!(config.deletion.retry_initial_backoff_seconds, 1); + assert_eq!(config.deletion.retry_max_backoff_seconds, 300); + assert_eq!( + config.deletion.final_credential_issuance_window_seconds, + 900 + ); + assert_eq!(config.deletion.final_read_window_seconds, 900); + assert!(config.deletion_worker.enabled); + assert_eq!(config.deletion_worker.poll_interval_ms, 250); + assert_eq!(config.deletion_worker.claim_timeout_seconds, 60); + assert_eq!(config.deletion_worker.shutdown_timeout_seconds, 30); + assert_eq!(config.deletion_worker.worker_id, "deletion-worker"); +} + +#[test] +fn rejects_non_positive_or_blank_deletion_worker_settings() { + let temp_dir = tempdir().unwrap(); + let secret_path = temp_dir.path().join("secret.sess"); + let public_key = test_identity(&secret_path); + let base = minimal_config(&secret_path, &public_key, "development").replace( + "[runtime]", + "[deletion_worker]\nenabled = true\npoll_interval_ms = 250\nclaim_timeout_seconds = 60\nshutdown_timeout_seconds = 30\nworker_id = \"deletion-worker\"\n\n[runtime]", + ); + + for (name, from, to) in [ + ("poll", "poll_interval_ms = 250", "poll_interval_ms = 0"), + ( + "claim", + "claim_timeout_seconds = 60", + "claim_timeout_seconds = 0", + ), + ( + "shutdown", + "shutdown_timeout_seconds = 30", + "shutdown_timeout_seconds = 0", + ), + ( + "worker-id", + "worker_id = \"deletion-worker\"", + "worker_id = \" \"", + ), + ] { + let config_path = temp_dir.path().join(format!("{name}.toml")); + std::fs::write(&config_path, base.replace(from, to)).unwrap(); + let error = load_existing_config_from_path(&config_path).unwrap_err(); + assert!( + matches!(error, ConfigError::InvalidDeletionWorkerConfig), + "unexpected error for {name}: {error}" + ); + } +} + +#[test] +fn rejects_retired_creator_authority_key_env() { + let temp_dir = tempdir().unwrap(); + let secret_path = temp_dir.path().join("secret.sess"); + let public_key = test_identity(&secret_path); + let config_path = temp_dir.path().join("config.toml"); + let config = minimal_config(&secret_path, &public_key, "development").replace( + "runtime_master_key_env = \"PUBKY_LOCK_RUNTIME_MASTER_KEY\"", + "creator_authority_key_env = \"PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY\"", + ); + std::fs::write(&config_path, config).unwrap(); + + let error = load_existing_config_from_path(&config_path).unwrap_err(); + assert!(matches!(error, ConfigError::ParseConfig { .. })); + assert!(error.to_string().contains("creator_authority_key_env")); +} + +#[test] +fn rejects_zero_or_inverted_deletion_retry_contract() { + let temp_dir = tempdir().unwrap(); + let secret_path = temp_dir.path().join("secret.sess"); + let public_key = test_identity(&secret_path); + + for (name, from, to, expected) in [ + ( + "zero-attempts", + "retry_max_attempts = 10", + "retry_max_attempts = 0", + ConfigError::InvalidDeletionRetry, + ), + ( + "zero-initial", + "retry_initial_backoff_seconds = 1", + "retry_initial_backoff_seconds = 0", + ConfigError::InvalidDeletionRetry, + ), + ( + "inverted", + "retry_initial_backoff_seconds = 1", + "retry_initial_backoff_seconds = 301", + ConfigError::InvalidDeletionRetryBackoffOrder, + ), + ] { + let config_path = temp_dir.path().join(format!("{name}.toml")); + let config = minimal_config(&secret_path, &public_key, "development").replace(from, to); + std::fs::write(&config_path, config).unwrap(); + + let error = load_existing_config_from_path(&config_path).unwrap_err(); + assert_eq!( + std::mem::discriminant(&error), + std::mem::discriminant(&expected) + ); + } +} + +#[test] +fn accepts_maximum_deletion_windows_and_rejects_out_of_range_values() { + let temp_dir = tempdir().unwrap(); + let secret_path = temp_dir.path().join("secret.sess"); + let public_key = test_identity(&secret_path); + let base = minimal_config(&secret_path, &public_key, "development"); + + let max_path = temp_dir.path().join("max.toml"); + let max = base + .replace( + "final_credential_issuance_window_seconds = 900", + "final_credential_issuance_window_seconds = 3600", + ) + .replace( + "final_read_window_seconds = 900", + "final_read_window_seconds = 3600", + ); + std::fs::write(&max_path, max).unwrap(); + assert!(load_existing_config_from_path(&max_path).is_ok()); + + for (name, from, to) in [ + ( + "zero-issuance", + "final_credential_issuance_window_seconds = 900", + "final_credential_issuance_window_seconds = 0", + ), + ( + "long-issuance", + "final_credential_issuance_window_seconds = 900", + "final_credential_issuance_window_seconds = 3601", + ), + ( + "zero-read", + "final_read_window_seconds = 900", + "final_read_window_seconds = 0", + ), + ( + "long-read", + "final_read_window_seconds = 900", + "final_read_window_seconds = 3601", + ), + ] { + let config_path = temp_dir.path().join(format!("{name}.toml")); + std::fs::write(&config_path, base.replace(from, to)).unwrap(); + assert!(matches!( + load_existing_config_from_path(&config_path).unwrap_err(), + ConfigError::InvalidDeletionCredentialWindow + )); + } +} + fn test_identity(secret_path: &std::path::Path) -> LockServerPubky { let keypair = pubky_common::crypto::Keypair::from_secret(&[9; 32]); let public_key = LockServerPubky::from_str(&keypair.public_key().to_string()).unwrap(); @@ -360,7 +563,14 @@ frontend_session_code_ttl_seconds = 120 allowed_return_origins = [] [secrets] -creator_authority_key_env = "PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" +runtime_master_key_env = "PUBKY_LOCK_RUNTIME_MASTER_KEY" + +[deletion] +retry_max_attempts = 10 +retry_initial_backoff_seconds = 1 +retry_max_backoff_seconds = 300 +final_credential_issuance_window_seconds = 900 +final_read_window_seconds = 900 [logging] level = "info" diff --git a/locks-server/src/config/loading.rs b/locks-server/src/config/loading.rs index 145b241..18692f7 100644 --- a/locks-server/src/config/loading.rs +++ b/locks-server/src/config/loading.rs @@ -4,9 +4,9 @@ use super::defaults::{DEFAULT_CONFIG_FILE, DEFAULT_SECRET_FILE, DEFAULT_SERVICE_ use super::raw::RawConfig; use super::schema::{ ConfigError, ConfigPathResolution, ContentLocksConfig, CreatorAuthorityAcquisitionConfig, - DatabaseConfig, LockServerCredentialsConfig, LockServerRuntimeConfig, LoggingConfig, - PaykitConfig, PkdnsConfig, PubkyConfig, RateLimitsConfig, RuntimeConfig, RuntimeEnvironment, - SecretsConfig, WorkerConfig, + DatabaseConfig, DeletionConfig, DeletionWorkerConfig, LockServerCredentialsConfig, + LockServerRuntimeConfig, LoggingConfig, PaykitConfig, PkdnsConfig, PubkyConfig, + RateLimitsConfig, RuntimeConfig, RuntimeEnvironment, SecretsConfig, WorkerConfig, }; use super::secrets::{LockServerIdentityProvider, parse_lock_server_keypair_seed}; @@ -148,6 +148,8 @@ fn initialize_default_config( pkdns: PkdnsConfig::default(), rate_limits: RateLimitsConfig::default(), content_locks: ContentLocksConfig::default(), + deletion: DeletionConfig::default(), + deletion_worker: DeletionWorkerConfig::default(), paykit: Some(PaykitConfig { server_url: "http://127.0.0.1:3001/".to_owned(), minimum_confirmations: 0, @@ -190,7 +192,21 @@ frontend_session_code_ttl_seconds = {} # One-time callback code lifetime. Keep s allowed_return_origins = [] # Origins allowed to receive auth callback codes, e.g. ["https://pubky.app"]. Empty rejects all /connect return_to values; ["*"] is dev-only and unsafe for staging/prod. [secrets] -creator_authority_key_env = "{}" # Environment variable containing a 32-byte base64url key for encrypting creator authority at rest. Rotating requires data migration. +runtime_master_key_env = "{}" # Environment variable containing the 32-byte base64url runtime master key. Domain-separated keys encrypt creator authority and final credentials. Rotating requires data migration. + +[deletion] +retry_max_attempts = {} # Maximum attempts per deletion phase before stable retry_exhausted failure. +retry_initial_backoff_seconds = {} # Initial durable retry delay; must be positive and no greater than the maximum. +retry_max_backoff_seconds = {} # Maximum durable retry delay in seconds. +final_credential_issuance_window_seconds = {} # Bounded final-credential issuance window; must be 1..=3600. +final_read_window_seconds = {} # Bounded one-read-per-resource window; must be 1..=3600. + +[deletion_worker] +enabled = {} # true enables the in-process deletion worker; false leaves deletion jobs for another worker process. +poll_interval_ms = {} # Deletion queue polling interval in milliseconds. Must be > 0. +claim_timeout_seconds = {} # Seconds before another deletion worker may reclaim a stuck job. Must be > 0. +shutdown_timeout_seconds = {} # Maximum graceful deletion-worker shutdown wait in seconds. Must be > 0. +worker_id = "{}" # Stable, non-blank deletion-worker identity. Use a unique value per worker process. [logging] level = "{}" # Tracing level/filter, e.g. error, warn, info, debug, trace, or EnvFilter syntax. Higher verbosity may expose operational detail in logs. @@ -243,7 +259,17 @@ max_total_resource_bytes = {} # Maximum combined bytes across resources in one c config .creator_authority_acquisition .frontend_session_code_ttl_seconds, - config.secrets.creator_authority_key_env, + config.secrets.runtime_master_key_env, + config.deletion.retry_max_attempts, + config.deletion.retry_initial_backoff_seconds, + config.deletion.retry_max_backoff_seconds, + config.deletion.final_credential_issuance_window_seconds, + config.deletion.final_read_window_seconds, + config.deletion_worker.enabled, + config.deletion_worker.poll_interval_ms, + config.deletion_worker.claim_timeout_seconds, + config.deletion_worker.shutdown_timeout_seconds, + config.deletion_worker.worker_id, config.logging.level, config.pkdns.public_ip, config.pkdns.public_pubky_tls_port.unwrap_or(6287), diff --git a/locks-server/src/config/mod.rs b/locks-server/src/config/mod.rs index 0647f78..bc56164 100644 --- a/locks-server/src/config/mod.rs +++ b/locks-server/src/config/mod.rs @@ -10,11 +10,11 @@ mod validation; pub use loading::{load_existing_config_from_path, load_or_initialize_config, resolve_config_path}; pub use schema::{ ConfigError, ConfigPathResolution, ContentLocksConfig, CreatorAuthorityAcquisitionConfig, - CreatorAuthorityAcquisitionMethod, DatabaseConfig, LegacyConnectAcquisitionConfig, - LockServerCredentialsConfig, LockServerRuntimeConfig, LoggingConfig, - PAYKIT_CONNECT_TIMEOUT_SECONDS, PAYKIT_REQUEST_TIMEOUT_SECONDS, PaykitConfig, PkdnsConfig, - PubkyConfig, PubkyNetwork, RateLimitsConfig, RuntimeConfig, RuntimeEnvironment, SecretsConfig, - VerificationSubmissionRateLimitConfig, WorkerConfig, + CreatorAuthorityAcquisitionMethod, DatabaseConfig, DeletionConfig, DeletionWorkerConfig, + LegacyConnectAcquisitionConfig, LockServerCredentialsConfig, LockServerRuntimeConfig, + LoggingConfig, PAYKIT_CONNECT_TIMEOUT_SECONDS, PAYKIT_REQUEST_TIMEOUT_SECONDS, PaykitConfig, + PkdnsConfig, PubkyConfig, PubkyNetwork, RateLimitsConfig, RuntimeConfig, RuntimeEnvironment, + SecretsConfig, VerificationSubmissionRateLimitConfig, WorkerConfig, }; pub use secrets::{FilesystemLockServerIdentityProvider, LockServerIdentityProvider}; pub(crate) use secrets::{LockServerSigningKeyError, load_lock_server_signing_keypair}; diff --git a/locks-server/src/config/raw.rs b/locks-server/src/config/raw.rs index 922ea40..5669a29 100644 --- a/locks-server/src/config/raw.rs +++ b/locks-server/src/config/raw.rs @@ -7,14 +7,21 @@ use serde::Deserialize; use tracing_subscriber::EnvFilter; use url::Url; -use super::defaults::{DEFAULT_CREATOR_AUTHORITY_KEY_ENV, PUBLIC_KEY_PLACEHOLDER}; +use super::defaults::{ + DEFAULT_DELETION_RETRY_INITIAL_BACKOFF_SECONDS, DEFAULT_DELETION_RETRY_MAX_ATTEMPTS, + DEFAULT_DELETION_RETRY_MAX_BACKOFF_SECONDS, DEFAULT_DELETION_WORKER_CLAIM_TIMEOUT_SECONDS, + DEFAULT_DELETION_WORKER_ID, DEFAULT_DELETION_WORKER_POLL_INTERVAL_MS, + DEFAULT_DELETION_WORKER_SHUTDOWN_TIMEOUT_SECONDS, + DEFAULT_FINAL_CREDENTIAL_ISSUANCE_WINDOW_SECONDS, DEFAULT_FINAL_READ_WINDOW_SECONDS, + DEFAULT_RUNTIME_MASTER_KEY_ENV, PUBLIC_KEY_PLACEHOLDER, +}; use super::schema::{ ConfigError, ContentLocksConfig, CreatorAuthorityAcquisitionConfig, - CreatorAuthorityAcquisitionMethod, DatabaseConfig, LegacyConnectAcquisitionConfig, - LockServerCredentialsConfig, LockServerRuntimeConfig, LoggingConfig, - PAYKIT_REQUEST_TIMEOUT_SECONDS, PaykitConfig, PkdnsConfig, PubkyConfig, PubkyNetwork, - RateLimitsConfig, RuntimeConfig, RuntimeEnvironment, SecretsConfig, - VerificationSubmissionRateLimitConfig, WorkerConfig, + CreatorAuthorityAcquisitionMethod, DatabaseConfig, DeletionConfig, DeletionWorkerConfig, + LegacyConnectAcquisitionConfig, LockServerCredentialsConfig, LockServerRuntimeConfig, + LoggingConfig, MAX_DELETION_CREDENTIAL_WINDOW_SECONDS, PAYKIT_REQUEST_TIMEOUT_SECONDS, + PaykitConfig, PkdnsConfig, PubkyConfig, PubkyNetwork, RateLimitsConfig, RuntimeConfig, + RuntimeEnvironment, SecretsConfig, VerificationSubmissionRateLimitConfig, WorkerConfig, }; #[derive(Debug, Deserialize)] @@ -40,6 +47,10 @@ pub(super) struct RawConfig { #[serde(default)] content_locks: RawContentLocksConfig, #[serde(default)] + deletion: RawDeletionConfig, + #[serde(default)] + deletion_worker: RawDeletionWorkerConfig, + #[serde(default)] paykit: Option, } @@ -123,6 +134,9 @@ impl Default for RawPkdnsConfig { impl RawPkdnsConfig { fn into_pkdns_config(self) -> Result { + if self.key_republisher_interval_seconds == 0 { + return Err(ConfigError::InvalidPkarrRepublisherInterval); + } Ok(PkdnsConfig { public_ip: self.public_ip, public_pubky_tls_port: self.public_pubky_tls_port, @@ -248,20 +262,130 @@ fn validate_allowed_return_origin(value: String) -> Result #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct RawSecretsConfig { - #[serde(default = "default_creator_authority_key_env")] - creator_authority_key_env: String, + #[serde(default = "default_runtime_master_key_env")] + runtime_master_key_env: String, } impl Default for RawSecretsConfig { fn default() -> Self { Self { - creator_authority_key_env: default_creator_authority_key_env(), + runtime_master_key_env: default_runtime_master_key_env(), } } } -fn default_creator_authority_key_env() -> String { - DEFAULT_CREATOR_AUTHORITY_KEY_ENV.to_owned() +fn default_runtime_master_key_env() -> String { + DEFAULT_RUNTIME_MASTER_KEY_ENV.to_owned() +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RawDeletionConfig { + #[serde(default = "default_deletion_retry_max_attempts")] + retry_max_attempts: u32, + #[serde(default = "default_deletion_retry_initial_backoff_seconds")] + retry_initial_backoff_seconds: u64, + #[serde(default = "default_deletion_retry_max_backoff_seconds")] + retry_max_backoff_seconds: u64, + #[serde(default = "default_final_credential_issuance_window_seconds")] + final_credential_issuance_window_seconds: u64, + #[serde(default = "default_final_read_window_seconds")] + final_read_window_seconds: u64, +} + +impl Default for RawDeletionConfig { + fn default() -> Self { + Self { + retry_max_attempts: default_deletion_retry_max_attempts(), + retry_initial_backoff_seconds: default_deletion_retry_initial_backoff_seconds(), + retry_max_backoff_seconds: default_deletion_retry_max_backoff_seconds(), + final_credential_issuance_window_seconds: + default_final_credential_issuance_window_seconds(), + final_read_window_seconds: default_final_read_window_seconds(), + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RawDeletionWorkerConfig { + #[serde(default = "default_deletion_worker_enabled")] + enabled: bool, + #[serde(default = "default_deletion_worker_poll_interval_ms")] + poll_interval_ms: u64, + #[serde(default = "default_deletion_worker_claim_timeout_seconds")] + claim_timeout_seconds: u64, + #[serde(default = "default_deletion_worker_shutdown_timeout_seconds")] + shutdown_timeout_seconds: u64, + #[serde(default = "default_deletion_worker_id")] + worker_id: String, +} + +impl Default for RawDeletionWorkerConfig { + fn default() -> Self { + Self { + enabled: default_deletion_worker_enabled(), + poll_interval_ms: default_deletion_worker_poll_interval_ms(), + claim_timeout_seconds: default_deletion_worker_claim_timeout_seconds(), + shutdown_timeout_seconds: default_deletion_worker_shutdown_timeout_seconds(), + worker_id: default_deletion_worker_id(), + } + } +} + +impl RawDeletionWorkerConfig { + fn into_deletion_worker_config(self) -> Result { + if self.poll_interval_ms == 0 + || self.claim_timeout_seconds == 0 + || self.shutdown_timeout_seconds == 0 + || self.worker_id.trim().is_empty() + { + return Err(ConfigError::InvalidDeletionWorkerConfig); + } + Ok(DeletionWorkerConfig { + enabled: self.enabled, + poll_interval_ms: self.poll_interval_ms, + claim_timeout_seconds: self.claim_timeout_seconds, + shutdown_timeout_seconds: self.shutdown_timeout_seconds, + worker_id: self.worker_id, + }) + } +} + +fn default_deletion_worker_enabled() -> bool { + true +} + +fn default_deletion_worker_poll_interval_ms() -> u64 { + DEFAULT_DELETION_WORKER_POLL_INTERVAL_MS +} + +fn default_deletion_worker_claim_timeout_seconds() -> u64 { + DEFAULT_DELETION_WORKER_CLAIM_TIMEOUT_SECONDS +} + +fn default_deletion_worker_shutdown_timeout_seconds() -> u64 { + DEFAULT_DELETION_WORKER_SHUTDOWN_TIMEOUT_SECONDS +} + +fn default_deletion_worker_id() -> String { + DEFAULT_DELETION_WORKER_ID.to_owned() +} + +fn default_deletion_retry_max_attempts() -> u32 { + DEFAULT_DELETION_RETRY_MAX_ATTEMPTS +} +fn default_deletion_retry_initial_backoff_seconds() -> u64 { + DEFAULT_DELETION_RETRY_INITIAL_BACKOFF_SECONDS +} +fn default_deletion_retry_max_backoff_seconds() -> u64 { + DEFAULT_DELETION_RETRY_MAX_BACKOFF_SECONDS +} +fn default_final_credential_issuance_window_seconds() -> u64 { + DEFAULT_FINAL_CREDENTIAL_ISSUANCE_WINDOW_SECONDS +} +fn default_final_read_window_seconds() -> u64 { + DEFAULT_FINAL_READ_WINDOW_SECONDS } #[derive(Debug, Deserialize)] @@ -453,11 +577,40 @@ impl RawCreatorAuthorityAcquisitionConfig { impl RawSecretsConfig { fn into_secrets_config(self) -> Result { - if self.creator_authority_key_env.trim().is_empty() { - return Err(ConfigError::InvalidCreatorAuthorityKeyEnv); + if self.runtime_master_key_env.trim().is_empty() { + return Err(ConfigError::InvalidRuntimeMasterKeyEnv); } Ok(SecretsConfig { - creator_authority_key_env: self.creator_authority_key_env, + runtime_master_key_env: self.runtime_master_key_env, + }) + } +} + +impl RawDeletionConfig { + fn into_deletion_config(self) -> Result { + if self.retry_max_attempts == 0 + || self.retry_initial_backoff_seconds == 0 + || self.retry_max_backoff_seconds == 0 + { + return Err(ConfigError::InvalidDeletionRetry); + } + if self.retry_initial_backoff_seconds > self.retry_max_backoff_seconds { + return Err(ConfigError::InvalidDeletionRetryBackoffOrder); + } + if self.final_credential_issuance_window_seconds == 0 + || self.final_credential_issuance_window_seconds + > MAX_DELETION_CREDENTIAL_WINDOW_SECONDS + || self.final_read_window_seconds == 0 + || self.final_read_window_seconds > MAX_DELETION_CREDENTIAL_WINDOW_SECONDS + { + return Err(ConfigError::InvalidDeletionCredentialWindow); + } + Ok(DeletionConfig { + retry_max_attempts: self.retry_max_attempts, + retry_initial_backoff_seconds: self.retry_initial_backoff_seconds, + retry_max_backoff_seconds: self.retry_max_backoff_seconds, + final_credential_issuance_window_seconds: self.final_credential_issuance_window_seconds, + final_read_window_seconds: self.final_read_window_seconds, }) } } @@ -514,6 +667,8 @@ impl RawConfig { let pkdns = self.pkdns.into_pkdns_config()?; let rate_limits = self.rate_limits.into_rate_limits_config()?; let content_locks = self.content_locks.into_content_locks_config()?; + let deletion = self.deletion.into_deletion_config()?; + let deletion_worker = self.deletion_worker.into_deletion_worker_config()?; let paykit = self .paykit .map(RawPaykitConfig::into_paykit_config) @@ -557,6 +712,8 @@ impl RawConfig { pkdns, rate_limits, content_locks, + deletion, + deletion_worker, paykit, }) } diff --git a/locks-server/src/config/schema.rs b/locks-server/src/config/schema.rs index 3654c4c..6cd437f 100644 --- a/locks-server/src/config/schema.rs +++ b/locks-server/src/config/schema.rs @@ -5,7 +5,14 @@ use locks_core::ids::LockServerPubky; use serde::Deserialize; use thiserror::Error; -use super::defaults::DEFAULT_CREATOR_AUTHORITY_KEY_ENV; +use super::defaults::{ + DEFAULT_DELETION_RETRY_INITIAL_BACKOFF_SECONDS, DEFAULT_DELETION_RETRY_MAX_ATTEMPTS, + DEFAULT_DELETION_RETRY_MAX_BACKOFF_SECONDS, DEFAULT_DELETION_WORKER_CLAIM_TIMEOUT_SECONDS, + DEFAULT_DELETION_WORKER_ID, DEFAULT_DELETION_WORKER_POLL_INTERVAL_MS, + DEFAULT_DELETION_WORKER_SHUTDOWN_TIMEOUT_SECONDS, + DEFAULT_FINAL_CREDENTIAL_ISSUANCE_WINDOW_SECONDS, DEFAULT_FINAL_READ_WINDOW_SECONDS, + DEFAULT_RUNTIME_MASTER_KEY_ENV, +}; pub const PAYKIT_CONNECT_TIMEOUT_SECONDS: u64 = 5; pub const PAYKIT_REQUEST_TIMEOUT_SECONDS: u64 = 20; @@ -24,9 +31,56 @@ pub struct LockServerRuntimeConfig { pub pkdns: PkdnsConfig, pub rate_limits: RateLimitsConfig, pub content_locks: ContentLocksConfig, + pub deletion: DeletionConfig, + pub deletion_worker: DeletionWorkerConfig, pub paykit: Option, } +pub const MAX_DELETION_CREDENTIAL_WINDOW_SECONDS: u64 = 3600; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeletionConfig { + pub retry_max_attempts: u32, + pub retry_initial_backoff_seconds: u64, + pub retry_max_backoff_seconds: u64, + pub final_credential_issuance_window_seconds: u64, + pub final_read_window_seconds: u64, +} + +impl Default for DeletionConfig { + fn default() -> Self { + Self { + retry_max_attempts: DEFAULT_DELETION_RETRY_MAX_ATTEMPTS, + retry_initial_backoff_seconds: DEFAULT_DELETION_RETRY_INITIAL_BACKOFF_SECONDS, + retry_max_backoff_seconds: DEFAULT_DELETION_RETRY_MAX_BACKOFF_SECONDS, + final_credential_issuance_window_seconds: + DEFAULT_FINAL_CREDENTIAL_ISSUANCE_WINDOW_SECONDS, + final_read_window_seconds: DEFAULT_FINAL_READ_WINDOW_SECONDS, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeletionWorkerConfig { + pub enabled: bool, + pub poll_interval_ms: u64, + pub claim_timeout_seconds: u64, + pub shutdown_timeout_seconds: u64, + pub worker_id: String, +} + +impl Default for DeletionWorkerConfig { + fn default() -> Self { + Self { + enabled: true, + poll_interval_ms: DEFAULT_DELETION_WORKER_POLL_INTERVAL_MS, + claim_timeout_seconds: DEFAULT_DELETION_WORKER_CLAIM_TIMEOUT_SECONDS, + shutdown_timeout_seconds: DEFAULT_DELETION_WORKER_SHUTDOWN_TIMEOUT_SECONDS, + worker_id: DEFAULT_DELETION_WORKER_ID.to_owned(), + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct PaykitConfig { pub server_url: String, @@ -95,13 +149,13 @@ impl Default for PkdnsConfig { #[derive(Debug, Clone, PartialEq, Eq)] pub struct SecretsConfig { - pub creator_authority_key_env: String, + pub runtime_master_key_env: String, } impl Default for SecretsConfig { fn default() -> Self { Self { - creator_authority_key_env: DEFAULT_CREATOR_AUTHORITY_KEY_ENV.to_owned(), + runtime_master_key_env: DEFAULT_RUNTIME_MASTER_KEY_ENV.to_owned(), } } } @@ -310,8 +364,20 @@ pub enum ConfigError { InvalidContentLocksTotalResourceBytes, #[error("invalid logging.level filter: {0}")] InvalidLoggingLevel(String), - #[error("secrets.creator_authority_key_env must not be empty")] - InvalidCreatorAuthorityKeyEnv, + #[error("secrets.runtime_master_key_env must not be empty")] + InvalidRuntimeMasterKeyEnv, + #[error("deletion retry values must be greater than zero")] + InvalidDeletionRetry, + #[error( + "deletion.retry_initial_backoff_seconds must not exceed deletion.retry_max_backoff_seconds" + )] + InvalidDeletionRetryBackoffOrder, + #[error("deletion credential windows must be between 1 and 3600 seconds")] + InvalidDeletionCredentialWindow, + #[error( + "deletion_worker numeric settings must be greater than zero and deletion_worker.worker_id must not be blank" + )] + InvalidDeletionWorkerConfig, #[error( "creator_authority_acquisition.allowed_return_origins must contain http(s) origins without path, query, or fragment: {0}" )] @@ -320,6 +386,8 @@ pub enum ConfigError { "creator_authority_acquisition.allowed_return_origins must not be \"*\" when runtime.environment is production; list explicit origins" )] WildcardReturnOriginInProduction, + #[error("pkdns.key_republisher_interval_seconds must be greater than zero")] + InvalidPkarrRepublisherInterval, #[error("pkdns.pkarr_relays must contain valid http(s) URLs: {0}")] InvalidPkarrRelayUrl(String), #[error("paykit.server_url must be a valid http(s) URL: {0}")] diff --git a/locks-server/src/deletion_worker.rs b/locks-server/src/deletion_worker.rs new file mode 100644 index 0000000..3cd9d71 --- /dev/null +++ b/locks-server/src/deletion_worker.rs @@ -0,0 +1,1736 @@ +use std::time::Duration as StdDuration; + +use async_trait::async_trait; +use locks_service::application::{ + errors::ApplicationError, + models::{ClaimedContentLockDeletionJob, ContentLockDeletionFailureCode}, + ports::{Clock, ContentLockDeletionRepository}, + use_cases::{ + drain_lock_payments::DrainLockPaymentsUseCase, + execute_content_lock_deletion_phase::{ + ContentLockDeletionPhaseExecutor, ContentLockDeletionPhaseExecutorConfig, + ContentLockDeletionPhaseExecutorDependencies, ContentLockPaymentDrainExecutor, + DeletionDependencyEvidence, DeletionDependencySource, DeletionDependencyStatus, + DeletionPhaseExecution, DeletionPhaseExecutionOutcome, + }, + execute_forced_content_lock_deletion::{ + ExecuteForcedContentLockDeletionDependencies, ExecuteForcedContentLockDeletionUseCase, + ForcedContentLockDeletionOutcome, + }, + materialize_final_credentials::MaterializeFinalCredentialsUseCase, + no_paykit_deletion_drain::NoPaykitDeletionDrainUseCase, + }, +}; +use rand::Rng; +use time::{Duration, OffsetDateTime}; +use tokio::sync::watch; +use tracing::error; + +use crate::app_state::{AppState, WorkerKind, WorkerReadiness, WorkerReadinessEvidence}; + +/// Executes at most one bounded action for an already claimed deletion job. +#[async_trait] +pub trait ClaimedDeletionExecutor: Send + Sync { + async fn execute_claimed( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecution; +} + +#[derive(Clone)] +pub struct RuntimeClaimedDeletionExecutor { + state: AppState, +} + +impl RuntimeClaimedDeletionExecutor { + pub fn new(state: AppState) -> Self { + Self { state } + } + + async fn execute_graceful( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecution { + let materializer = MaterializeFinalCredentialsUseCase::new( + self.state.access_credentials().as_ref(), + self.state.credential_generator().as_ref(), + self.state.clock().as_ref(), + ); + let no_paykit = NoPaykitDeletionDrainUseCase::new( + self.state.content_lock_deletions().as_ref(), + self.state.clock().as_ref(), + ); + let real_paykit = match ( + self.state.payment_drains(), + self.state.payment_drain_client(), + self.state.config().paykit.as_ref(), + ) { + (Some(drains), Some(client), Some(config)) => Some(DrainLockPaymentsUseCase::new( + self.state.content_lock_deletions().as_ref(), + drains.as_ref(), + client.as_ref(), + self.state.entitlements().as_ref(), + self.state.clock().as_ref(), + self.state + .config() + .credentials + .lock_server_public_key + .clone(), + config.minimum_confirmations, + )), + _ => None, + }; + let payments: &dyn ContentLockPaymentDrainExecutor = + real_paykit.as_ref().map_or(&no_paykit, |drain| { + drain as &dyn ContentLockPaymentDrainExecutor + }); + let config = &self.state.config().deletion; + ContentLockDeletionPhaseExecutor::new( + ContentLockDeletionPhaseExecutorDependencies { + deletions: self.state.content_lock_deletions().as_ref(), + action_ownership: self.state.content_lock_deletion_action_ownership().as_ref(), + tombstones: self.state.content_lock_tombstones().as_ref(), + guarded_resources: self.state.guarded_resources().as_ref(), + access_credentials: self.state.access_credentials().as_ref(), + clock: self.state.clock().as_ref(), + payments, + final_credentials: &materializer, + }, + ContentLockDeletionPhaseExecutorConfig { + final_credential_issuance_window: Duration::seconds( + config.final_credential_issuance_window_seconds as i64, + ), + final_read_window: Duration::seconds(config.final_read_window_seconds as i64), + final_credential_batch_limit: 64, + }, + ) + .execute_with_evidence(claim, worker_id) + .await + } +} + +#[async_trait] +impl ClaimedDeletionExecutor for RuntimeClaimedDeletionExecutor { + async fn execute_claimed( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecution { + if claim.job.force_requested_at.is_some() { + let execution = ExecuteForcedContentLockDeletionUseCase::new( + ExecuteForcedContentLockDeletionDependencies { + action_ownership: self.state.content_lock_deletion_action_ownership().as_ref(), + tombstones: self.state.content_lock_tombstones().as_ref(), + guarded_resources: self.state.guarded_resources().as_ref(), + deletions: self.state.content_lock_deletions().as_ref(), + clock: self.state.clock().as_ref(), + }, + ) + .execute_with_evidence(claim, worker_id) + .await; + let phase_outcome = match execution.outcome { + ForcedContentLockDeletionOutcome::Completed => { + DeletionPhaseExecutionOutcome::Progressed + } + ForcedContentLockDeletionOutcome::Deferred => { + DeletionPhaseExecutionOutcome::Deferred + } + ForcedContentLockDeletionOutcome::ClaimLost => { + DeletionPhaseExecutionOutcome::ClaimLost + } + ForcedContentLockDeletionOutcome::TransientDependencyFailure => { + DeletionPhaseExecutionOutcome::TransientDependencyFailure + } + ForcedContentLockDeletionOutcome::FatalFailure => { + DeletionPhaseExecutionOutcome::FatalFailure + } + }; + return DeletionPhaseExecution::new(phase_outcome).with_evidence(execution.evidence); + } + self.execute_graceful(claim, worker_id).await + } +} + +/// Supplies a full-jitter delay in the inclusive range from zero through `cap`. +pub trait FullJitterSource: Send + Sync { + fn sample(&self, cap: StdDuration) -> StdDuration; +} + +/// Production full-jitter source. +#[derive(Debug, Default, Clone, Copy)] +pub struct RandomFullJitter; + +impl FullJitterSource for RandomFullJitter { + fn sample(&self, cap: StdDuration) -> StdDuration { + let cap_nanos = cap.as_nanos(); + if cap_nanos == 0 { + return StdDuration::ZERO; + } + let sampled = rand::thread_rng().gen_range(0..=cap_nanos); + StdDuration::new( + (sampled / 1_000_000_000).try_into().unwrap_or(u64::MAX), + (sampled % 1_000_000_000) as u32, + ) + } +} + +/// Runtime policy for the deletion worker core. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeletionWorkerConfig { + pub worker_id: String, + pub poll_interval: StdDuration, + pub claim_timeout: StdDuration, + pub retry_max_attempts: u32, + pub retry_initial_backoff: StdDuration, + pub retry_max_backoff: StdDuration, +} + +/// Secret-free result of one worker iteration. It intentionally carries no job IDs or paths. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeletionWorkerOutcome { + Idle, + Cancelled, + Progressed, + Deferred, + ClaimLost, + TerminalFailed, + RetryScheduled, + RetryExhausted, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct DeletionWorkerExecution { + outcome: DeletionWorkerOutcome, + evidence: DeletionDependencyEvidence, +} + +impl DeletionWorkerExecution { + fn new(outcome: DeletionWorkerOutcome, evidence: DeletionDependencyEvidence) -> Self { + Self { outcome, evidence } + } +} + +struct DeletionWorkerFailure { + error: ApplicationError, + evidence: DeletionDependencyEvidence, +} + +impl DeletionWorkerFailure { + fn repository( + error: ApplicationError, + source: DeletionDependencySource, + prior_evidence: DeletionDependencyEvidence, + ) -> Self { + Self { + error, + evidence: prior_evidence.merge(DeletionDependencyEvidence::unavailable(source)), + } + } + + fn fatal(error: ApplicationError) -> Self { + Self { + error, + evidence: DeletionDependencyEvidence::none(), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PollErrorDisposition { + RetryableRepository, + Fatal(&'static str), +} + +fn classify_poll_error(error: &ApplicationError) -> PollErrorDisposition { + match error { + ApplicationError::Storage { .. } => PollErrorDisposition::RetryableRepository, + ApplicationError::InvalidContentLockDeletionState { .. } => { + PollErrorDisposition::Fatal("invalid_deletion_state") + } + _ => PollErrorDisposition::Fatal("unexpected_application_error"), + } +} + +fn redacted_fatal_poll_error(class: &'static str) -> ApplicationError { + ApplicationError::InvalidContentLockDeletionState { + message: format!("deletion worker terminated after {class}"), + } +} + +#[derive(Debug, Default)] +struct DeletionReadinessRecovery { + degraded: [bool; DeletionDependencySource::ALL.len()], +} + +impl DeletionReadinessRecovery { + fn record_outcome( + &mut self, + outcome: DeletionWorkerOutcome, + evidence: DeletionDependencyEvidence, + readiness: &WorkerReadiness, + ) { + if outcome == DeletionWorkerOutcome::Cancelled { + return; + } + for source in DeletionDependencySource::ALL { + match evidence.status(source) { + Some(DeletionDependencyStatus::Healthy) => { + self.degraded[source_index(source)] = false + } + Some(DeletionDependencyStatus::Unavailable) => { + self.degraded[source_index(source)] = true + } + None => {} + } + } + readiness.record( + WorkerKind::Deletion, + if self.degraded.iter().any(|degraded| *degraded) { + WorkerReadinessEvidence::TransientDependencyFailure + } else { + WorkerReadinessEvidence::DependencySucceeded + }, + ); + } + + #[cfg(test)] + fn record_repository_failure( + &mut self, + source: DeletionDependencySource, + readiness: &WorkerReadiness, + ) { + self.record_outcome( + DeletionWorkerOutcome::RetryScheduled, + DeletionDependencyEvidence::unavailable(source), + readiness, + ); + } +} + +const fn source_index(source: DeletionDependencySource) -> usize { + match source { + DeletionDependencySource::PaymentProvider => 0, + DeletionDependencySource::PaymentDrainRepository => 1, + DeletionDependencySource::EntitlementRepository => 2, + DeletionDependencySource::PubkyWithdrawal => 3, + DeletionDependencySource::PubkyReadback => 4, + DeletionDependencySource::PubkyResource => 5, + DeletionDependencySource::PubkyForcePublic => 6, + DeletionDependencySource::RepositoryQueueClaim => 7, + DeletionDependencySource::RepositoryPhaseMutation => 8, + DeletionDependencySource::RepositoryDefer => 9, + DeletionDependencySource::RepositoryRetry => 10, + DeletionDependencySource::RepositoryTerminalMutation => 11, + DeletionDependencySource::RepositoryActionLock => 12, + DeletionDependencySource::RepositoryActionLockRelease => 13, + DeletionDependencySource::RepositoryForceReceipt => 14, + } +} + +/// Polling, retry, and cancellation core for content-lock deletion jobs. +pub struct DeletionWorker<'a> { + deletions: &'a dyn ContentLockDeletionRepository, + clock: &'a dyn Clock, + executor: &'a dyn ClaimedDeletionExecutor, + jitter: &'a dyn FullJitterSource, + config: DeletionWorkerConfig, +} + +impl<'a> DeletionWorker<'a> { + pub fn new( + deletions: &'a dyn ContentLockDeletionRepository, + clock: &'a dyn Clock, + executor: &'a dyn ClaimedDeletionExecutor, + jitter: &'a dyn FullJitterSource, + config: DeletionWorkerConfig, + ) -> Self { + Self { + deletions, + clock, + executor, + jitter, + config, + } + } + + /// Runs at most one claim. A sticky shutdown signal is checked before claiming and + /// again after claim acquisition, before any external deletion action can begin. + pub async fn run_once( + &self, + shutdown: &watch::Receiver, + ) -> Result { + self.run_once_with_evidence(shutdown) + .await + .map(|execution| execution.outcome) + .map_err(|failure| failure.error) + } + + async fn run_once_with_evidence( + &self, + shutdown: &watch::Receiver, + ) -> Result { + if *shutdown.borrow() { + return Ok(DeletionWorkerExecution::new( + DeletionWorkerOutcome::Cancelled, + DeletionDependencyEvidence::none(), + )); + } + + let claim_started_at = self.clock.now(); + let claim_expires_at = claim_started_at + to_time_duration(self.config.claim_timeout); + let repository_healthy = + DeletionDependencyEvidence::healthy(DeletionDependencySource::RepositoryQueueClaim); + let Some(claim) = self + .deletions + .claim_next( + &self.config.worker_id, + (claim_expires_at) - (claim_started_at), + ) + .await + .map_err(|error| { + DeletionWorkerFailure::repository( + error, + DeletionDependencySource::RepositoryQueueClaim, + DeletionDependencyEvidence::none(), + ) + })? + else { + return Ok(DeletionWorkerExecution::new( + DeletionWorkerOutcome::Idle, + repository_healthy, + )); + }; + + if *shutdown.borrow() { + let outcome = self + .release_cancelled_claim(&claim) + .await + .map_err(|error| { + DeletionWorkerFailure::repository( + error, + DeletionDependencySource::RepositoryDefer, + repository_healthy, + ) + })?; + let mutation_evidence = + mutation_success_evidence(outcome, DeletionDependencySource::RepositoryDefer); + return Ok(DeletionWorkerExecution::new( + outcome, + repository_healthy.merge(mutation_evidence), + )); + } + + let claim_for_write = claim.clone(); + let execution = self + .executor + .execute_claimed(claim, &self.config.worker_id) + .await; + let mut mutation_evidence = DeletionDependencyEvidence::none(); + let outcome = match execution.outcome { + DeletionPhaseExecutionOutcome::Progressed => DeletionWorkerOutcome::Progressed, + DeletionPhaseExecutionOutcome::TerminalFailed => DeletionWorkerOutcome::TerminalFailed, + DeletionPhaseExecutionOutcome::ClaimLost => DeletionWorkerOutcome::ClaimLost, + DeletionPhaseExecutionOutcome::Deferred => { + let outcome = self.defer_claim(&claim_for_write).await.map_err(|error| { + DeletionWorkerFailure::repository( + error, + DeletionDependencySource::RepositoryDefer, + repository_healthy.merge(execution.evidence), + ) + })?; + mutation_evidence = + mutation_success_evidence(outcome, DeletionDependencySource::RepositoryDefer); + outcome + } + DeletionPhaseExecutionOutcome::TransientDependencyFailure => { + let source = if claim_for_write.job.attempt_count >= self.config.retry_max_attempts + { + DeletionDependencySource::RepositoryTerminalMutation + } else { + DeletionDependencySource::RepositoryRetry + }; + let outcome = self + .retry_or_exhaust(claim_for_write) + .await + .map_err(|error| { + DeletionWorkerFailure::repository( + error, + source, + repository_healthy.merge(execution.evidence), + ) + })?; + mutation_evidence = mutation_success_evidence(outcome, source); + outcome + } + DeletionPhaseExecutionOutcome::FatalFailure => { + return Err(DeletionWorkerFailure::fatal(redacted_fatal_poll_error( + "fatal_execution_failure", + ))); + } + }; + Ok(DeletionWorkerExecution::new( + outcome, + execution + .evidence + .merge(repository_healthy) + .merge(mutation_evidence), + )) + } + + /// Runs until shutdown, sleeping only after idle polls. Shutdown prevents every future claim, + /// including when it arrives while claim acquisition is blocked. + pub async fn run_until_shutdown( + &self, + mut shutdown: watch::Receiver, + ) -> Result<(), ApplicationError> { + loop { + if *shutdown.borrow() { + return Ok(()); + } + + match self.run_once(&shutdown).await? { + DeletionWorkerOutcome::Cancelled => return Ok(()), + DeletionWorkerOutcome::Idle => { + tokio::select! { + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { + return Ok(()); + } + } + _ = tokio::time::sleep(self.config.poll_interval) => {} + } + } + DeletionWorkerOutcome::Progressed + | DeletionWorkerOutcome::Deferred + | DeletionWorkerOutcome::ClaimLost + | DeletionWorkerOutcome::TerminalFailed + | DeletionWorkerOutcome::RetryScheduled + | DeletionWorkerOutcome::RetryExhausted => {} + } + } + } + + pub async fn run_until_shutdown_with_readiness( + &self, + mut shutdown: watch::Receiver, + readiness: &WorkerReadiness, + ) -> Result<(), ApplicationError> { + let mut recovery = DeletionReadinessRecovery::default(); + loop { + if *shutdown.borrow() { + readiness.record(WorkerKind::Deletion, WorkerReadinessEvidence::Stopped); + return Ok(()); + } + + match self.run_once_with_evidence(&shutdown).await { + Ok(DeletionWorkerExecution { + outcome: DeletionWorkerOutcome::Cancelled, + .. + }) => { + readiness.record(WorkerKind::Deletion, WorkerReadinessEvidence::Stopped); + return Ok(()); + } + Ok( + execution @ DeletionWorkerExecution { + outcome: DeletionWorkerOutcome::Idle, + .. + }, + ) => { + recovery.record_outcome(execution.outcome, execution.evidence, readiness); + tokio::select! { + _ = shutdown.changed() => {} + _ = tokio::time::sleep(self.config.poll_interval) => {} + } + } + Ok(execution) => { + recovery.record_outcome(execution.outcome, execution.evidence, readiness) + } + Err(failure) => match classify_poll_error(&failure.error) { + PollErrorDisposition::RetryableRepository => { + recovery.record_outcome( + DeletionWorkerOutcome::RetryScheduled, + failure.evidence, + readiness, + ); + error!( + operation = "deletion_queue_poll_or_transition", + error_class = "repository_unavailable", + retrying = true, + "deletion worker repository operation failed" + ); + tokio::select! { + _ = shutdown.changed() => {} + _ = tokio::time::sleep(self.config.poll_interval) => {} + } + } + PollErrorDisposition::Fatal(error_class) => { + readiness.record( + WorkerKind::Deletion, + WorkerReadinessEvidence::UnexpectedExit, + ); + error!( + operation = "deletion_queue_poll_or_transition", + error_class, + retrying = false, + "deletion worker terminated after unexpected application error" + ); + return Err(redacted_fatal_poll_error(error_class)); + } + }, + } + } + } + + async fn release_cancelled_claim( + &self, + claim: &ClaimedContentLockDeletionJob, + ) -> Result { + let now = self.clock.now(); + self.defer_at(claim, now, now, DeletionWorkerOutcome::Cancelled) + .await + } + + async fn defer_claim( + &self, + claim: &ClaimedContentLockDeletionJob, + ) -> Result { + let now = self.clock.now(); + let due = now + to_time_duration(self.config.poll_interval); + self.defer_at(claim, now, due, DeletionWorkerOutcome::Deferred) + .await + } + + async fn defer_at( + &self, + claim: &ClaimedContentLockDeletionJob, + now: OffsetDateTime, + due: OffsetDateTime, + success: DeletionWorkerOutcome, + ) -> Result { + let updated = self + .deletions + .defer( + claim.job.job_id, + &self.config.worker_id, + claim.claim_token, + (due) - (now), + ) + .await?; + Ok(if updated.is_some() { + success + } else { + DeletionWorkerOutcome::ClaimLost + }) + } + + async fn retry_or_exhaust( + &self, + claim: ClaimedContentLockDeletionJob, + ) -> Result { + let now = self.clock.now(); + if claim.job.attempt_count >= self.config.retry_max_attempts { + let updated = self + .deletions + .finish( + claim.job.job_id, + &self.config.worker_id, + claim.claim_token, + Some(ContentLockDeletionFailureCode::RetryExhausted), + ) + .await?; + return Ok(if updated.is_some() { + DeletionWorkerOutcome::RetryExhausted + } else { + DeletionWorkerOutcome::ClaimLost + }); + } + + let cap = retry_cap( + self.config.retry_initial_backoff, + self.config.retry_max_backoff, + claim.job.attempt_count, + ); + // Keep repository scheduling bounded even if a custom source violates its contract. + let delay = self.jitter.sample(cap).min(cap); + let next_attempt_at = now + to_time_duration(delay); + let updated = self + .deletions + .schedule_retry( + claim.job.job_id, + &self.config.worker_id, + claim.claim_token, + (next_attempt_at) - (now), + ) + .await?; + Ok(if updated.is_some() { + DeletionWorkerOutcome::RetryScheduled + } else { + DeletionWorkerOutcome::ClaimLost + }) + } +} + +fn mutation_success_evidence( + outcome: DeletionWorkerOutcome, + source: DeletionDependencySource, +) -> DeletionDependencyEvidence { + if outcome == DeletionWorkerOutcome::ClaimLost { + DeletionDependencyEvidence::none() + } else { + DeletionDependencyEvidence::healthy(source) + } +} + +fn retry_cap(initial: StdDuration, maximum: StdDuration, attempt_count: u32) -> StdDuration { + let mut cap = initial.min(maximum); + for _ in 1..attempt_count { + cap = cap.checked_mul(2).unwrap_or(maximum).min(maximum); + if cap == maximum { + break; + } + } + cap +} + +fn to_time_duration(duration: StdDuration) -> Duration { + Duration::new( + i64::try_from(duration.as_secs()).unwrap_or(i64::MAX), + duration.subsec_nanos() as i32, + ) +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + str::FromStr, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::Duration as StdDuration, + }; + + use async_trait::async_trait; + use locks_core::{ + ids::{CreatorPubky, GuardedResourceHash, LockId}, + lock_policy::{ + AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, GuardedResource, LockLogic, + LockServerConfig, + }, + }; + use locks_service::application::{ + errors::ApplicationError, + models::{ + AdvanceContentLockDeletionPhaseResult, ClaimedContentLockDeletionJob, + ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionPhase, + ContentLockDeletionState, PrepareForceDeletionResult, + }, + ports::{Clock, ContentLockDeletionRepository}, + use_cases::execute_content_lock_deletion_phase::{ + DeletionDependencyEvidence, DeletionDependencySource, DeletionDependencyStatus, + DeletionPhaseExecution, DeletionPhaseExecutionOutcome, + }, + }; + use time::{OffsetDateTime, macros::datetime}; + use tokio::sync::{Notify, watch}; + use uuid::Uuid; + + use super::{ + ClaimedDeletionExecutor, DeletionReadinessRecovery, DeletionWorker, DeletionWorkerConfig, + DeletionWorkerOutcome, FullJitterSource, PollErrorDisposition, classify_poll_error, + retry_cap, + }; + use crate::app_state::{ReadinessStatus, WorkerKind, WorkerReadiness, WorkerReadinessState}; + + const NOW: OffsetDateTime = datetime!(2026-08-17 12:00:00 UTC); + const CREATOR: &str = "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy"; + + #[tokio::test] + async fn active_shutdown_performs_no_claim() { + let repository = FakeRepository::default(); + let executor = FakeExecutor::new(DeletionPhaseExecutionOutcome::Progressed); + let worker = worker(&repository, &executor, &FixedJitter(StdDuration::ZERO)); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + shutdown_tx.send(true).unwrap(); + + assert_eq!( + worker.run_once(&shutdown_rx).await.unwrap(), + DeletionWorkerOutcome::Cancelled + ); + assert_eq!(repository.claim_calls.load(Ordering::SeqCst), 0); + assert_eq!(executor.calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn shutdown_while_claim_is_blocked_releases_claim_without_external_work() { + let repository = Arc::new(FakeRepository::with_claim(claim(1))); + repository.block_claim.store(true, Ordering::SeqCst); + let executor = Arc::new(FakeExecutor::new(DeletionPhaseExecutionOutcome::Progressed)); + let jitter = Arc::new(FixedJitter(StdDuration::ZERO)); + let worker = Arc::new(OwnedWorker::new( + Arc::clone(&repository), + Arc::clone(&executor), + jitter, + )); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let run = { + let worker = Arc::clone(&worker); + tokio::spawn(async move { worker.run_once(&shutdown_rx).await }) + }; + repository.claim_entered.notified().await; + shutdown_tx.send(true).unwrap(); + repository.claim_release.notify_one(); + + assert_eq!( + run.await.unwrap().unwrap(), + DeletionWorkerOutcome::Cancelled + ); + assert_eq!(executor.calls.load(Ordering::SeqCst), 0); + let defers = repository.defers.lock().unwrap(); + assert_eq!(defers.len(), 1); + assert_eq!(defers[0].0, NOW); + assert_eq!(defers[0].1, NOW); + } + + #[tokio::test] + async fn shutdown_during_first_claim_prevents_a_second_claim() { + let repository = Arc::new(FakeRepository::with_claim(claim(1))); + repository.block_claim.store(true, Ordering::SeqCst); + let executor = Arc::new(FakeExecutor::new(DeletionPhaseExecutionOutcome::Progressed)); + let worker = Arc::new(OwnedWorker::new( + Arc::clone(&repository), + executor, + Arc::new(FixedJitter(StdDuration::ZERO)), + )); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let run = { + let worker = Arc::clone(&worker); + tokio::spawn(async move { worker.run_until_shutdown(shutdown_rx).await }) + }; + repository.claim_entered.notified().await; + shutdown_tx.send(true).unwrap(); + repository.claim_release.notify_one(); + + run.await.unwrap().unwrap(); + assert_eq!(repository.claim_calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn deferred_work_releases_claim_on_poll_schedule_without_retry_write() { + let repository = FakeRepository::with_claim(claim(3)); + let executor = FakeExecutor::new(DeletionPhaseExecutionOutcome::Deferred); + let worker = worker(&repository, &executor, &FixedJitter(StdDuration::ZERO)); + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + + assert_eq!( + worker.run_once(&shutdown_rx).await.unwrap(), + DeletionWorkerOutcome::Deferred + ); + assert!(repository.retries.lock().unwrap().is_empty()); + assert!(repository.finishes.lock().unwrap().is_empty()); + assert_eq!( + repository.defers.lock().unwrap().as_slice(), + &[(NOW, NOW + time::Duration::seconds(5))] + ); + } + + #[tokio::test] + async fn completed_terminal_and_lost_executor_outcomes_do_not_write_retry_state() { + for (execution, expected) in [ + ( + DeletionPhaseExecutionOutcome::Progressed, + DeletionWorkerOutcome::Progressed, + ), + ( + DeletionPhaseExecutionOutcome::TerminalFailed, + DeletionWorkerOutcome::TerminalFailed, + ), + ( + DeletionPhaseExecutionOutcome::ClaimLost, + DeletionWorkerOutcome::ClaimLost, + ), + ] { + let repository = FakeRepository::with_claim(claim(1)); + let executor = FakeExecutor::new(execution); + let worker = worker(&repository, &executor, &FixedJitter(StdDuration::ZERO)); + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + + assert_eq!(worker.run_once(&shutdown_rx).await.unwrap(), expected); + assert!(repository.defers.lock().unwrap().is_empty()); + assert!(repository.retries.lock().unwrap().is_empty()); + assert!(repository.finishes.lock().unwrap().is_empty()); + } + } + + #[tokio::test] + async fn idle_sleep_wakes_immediately_for_shutdown() { + let repository = Arc::new(FakeRepository::default()); + let executor = Arc::new(FakeExecutor::new(DeletionPhaseExecutionOutcome::Progressed)); + let worker = Arc::new(OwnedWorker::new( + Arc::clone(&repository), + executor, + Arc::new(FixedJitter(StdDuration::ZERO)), + )); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let run = { + let worker = Arc::clone(&worker); + tokio::spawn(async move { worker.run_until_shutdown(shutdown_rx).await }) + }; + while repository.claim_calls.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + + shutdown_tx.send(true).unwrap(); + tokio::time::timeout(StdDuration::from_millis(100), run) + .await + .expect("shutdown must interrupt the poll sleep") + .unwrap() + .unwrap(); + assert_eq!(repository.claim_calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn fatal_executor_failure_terminates_supervised_worker_without_retry_write() { + let repository = FakeRepository::with_claim(claim(1)); + let executor = FakeExecutor::new(DeletionPhaseExecutionOutcome::FatalFailure); + let worker = worker(&repository, &executor, &FixedJitter(StdDuration::ZERO)); + let readiness = WorkerReadiness::new(false, true); + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + + let error = worker + .run_until_shutdown_with_readiness(shutdown_rx, &readiness) + .await + .unwrap_err(); + + assert_eq!(readiness.status(), ReadinessStatus::NotReady); + assert!(error.to_string().contains("invalid_deletion_state")); + assert!(repository.retries.lock().unwrap().is_empty()); + assert!(repository.finishes.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn transient_executor_failure_schedules_retry_and_marks_dependency_degraded() { + let repository = FakeRepository::with_claim(claim(1)); + let executor = FakeExecutor::new(DeletionPhaseExecutionOutcome::TransientDependencyFailure); + let worker = worker(&repository, &executor, &FixedJitter(StdDuration::ZERO)); + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + + assert_eq!( + worker.run_once(&shutdown_rx).await.unwrap(), + DeletionWorkerOutcome::RetryScheduled + ); + assert_eq!(repository.retries.lock().unwrap().len(), 1); + } + + #[tokio::test] + async fn dependency_failure_exhausts_on_exact_max_attempt() { + let repository = FakeRepository::with_claim(claim(4)); + let executor = FakeExecutor::new(DeletionPhaseExecutionOutcome::TransientDependencyFailure); + let worker = worker(&repository, &executor, &FixedJitter(StdDuration::ZERO)); + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + + assert_eq!( + worker.run_once(&shutdown_rx).await.unwrap(), + DeletionWorkerOutcome::RetryExhausted + ); + assert!(repository.retries.lock().unwrap().is_empty()); + assert_eq!( + repository.finishes.lock().unwrap().as_slice(), + &[Some(ContentLockDeletionFailureCode::RetryExhausted)] + ); + } + + #[tokio::test] + async fn full_jitter_is_bounded_by_exponential_cap() { + assert_eq!( + retry_cap(StdDuration::from_secs(10), StdDuration::from_secs(25), 1), + StdDuration::from_secs(10) + ); + assert_eq!( + retry_cap(StdDuration::from_secs(10), StdDuration::from_secs(25), 2), + StdDuration::from_secs(20) + ); + assert_eq!( + retry_cap(StdDuration::from_secs(10), StdDuration::from_secs(25), 3), + StdDuration::from_secs(25) + ); + + let repository = FakeRepository::with_claim(claim(2)); + let executor = FakeExecutor::new(DeletionPhaseExecutionOutcome::TransientDependencyFailure); + let jitter = FixedJitter(StdDuration::from_secs(99)); + let worker = worker(&repository, &executor, &jitter); + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + worker.run_once(&shutdown_rx).await.unwrap(); + + assert_eq!( + repository.retries.lock().unwrap().as_slice(), + &[(NOW, NOW + time::Duration::seconds(20))] + ); + } + + #[tokio::test] + async fn retry_due_time_uses_jitter_not_poll_interval() { + let repository = FakeRepository::with_claim(claim(1)); + let executor = FakeExecutor::new(DeletionPhaseExecutionOutcome::TransientDependencyFailure); + let jitter = FixedJitter(StdDuration::from_secs(7)); + let worker = worker(&repository, &executor, &jitter); + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + + assert_eq!( + worker.run_once(&shutdown_rx).await.unwrap(), + DeletionWorkerOutcome::RetryScheduled + ); + assert_eq!( + repository.retries.lock().unwrap().as_slice(), + &[(NOW, NOW + time::Duration::seconds(7))] + ); + } + + #[tokio::test] + async fn stale_retry_defer_and_finish_writes_map_to_claim_lost() { + for (execution, attempts) in [ + (DeletionPhaseExecutionOutcome::Deferred, 1), + (DeletionPhaseExecutionOutcome::TransientDependencyFailure, 1), + (DeletionPhaseExecutionOutcome::TransientDependencyFailure, 4), + ] { + let repository = FakeRepository::with_claim(claim(attempts)); + repository.stale_writes.store(true, Ordering::SeqCst); + let executor = FakeExecutor::new(execution); + let worker = worker(&repository, &executor, &FixedJitter(StdDuration::ZERO)); + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + assert_eq!( + worker.run_once(&shutdown_rx).await.unwrap(), + DeletionWorkerOutcome::ClaimLost + ); + } + } + + #[tokio::test] + async fn successful_worker_mutations_emit_only_their_exact_healthy_slot() { + for (execution, attempts, expected_source) in [ + ( + DeletionPhaseExecutionOutcome::Deferred, + 1, + DeletionDependencySource::RepositoryDefer, + ), + ( + DeletionPhaseExecutionOutcome::TransientDependencyFailure, + 1, + DeletionDependencySource::RepositoryRetry, + ), + ( + DeletionPhaseExecutionOutcome::TransientDependencyFailure, + 4, + DeletionDependencySource::RepositoryTerminalMutation, + ), + ] { + let repository = FakeRepository::with_claim(claim(attempts)); + let executor = FakeExecutor::new(execution); + let worker = worker(&repository, &executor, &FixedJitter(StdDuration::ZERO)); + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + + let execution = match worker.run_once_with_evidence(&shutdown_rx).await { + Ok(execution) => execution, + Err(_) => panic!("worker mutation should succeed"), + }; + + assert_eq!( + execution.evidence.status(expected_source), + Some(DeletionDependencyStatus::Healthy) + ); + for skipped in [ + DeletionDependencySource::RepositoryPhaseMutation, + DeletionDependencySource::RepositoryDefer, + DeletionDependencySource::RepositoryRetry, + DeletionDependencySource::RepositoryTerminalMutation, + ] { + if skipped != expected_source { + assert_eq!(execution.evidence.status(skipped), None); + } + } + } + } + + #[tokio::test] + async fn stale_worker_mutations_are_evidence_free_for_the_skipped_slot() { + for (execution, attempts, skipped_source) in [ + ( + DeletionPhaseExecutionOutcome::Deferred, + 1, + DeletionDependencySource::RepositoryDefer, + ), + ( + DeletionPhaseExecutionOutcome::TransientDependencyFailure, + 1, + DeletionDependencySource::RepositoryRetry, + ), + ( + DeletionPhaseExecutionOutcome::TransientDependencyFailure, + 4, + DeletionDependencySource::RepositoryTerminalMutation, + ), + ] { + let repository = FakeRepository::with_claim(claim(attempts)); + repository.stale_writes.store(true, Ordering::SeqCst); + let executor = FakeExecutor::new(execution); + let worker = worker(&repository, &executor, &FixedJitter(StdDuration::ZERO)); + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + + let execution = match worker.run_once_with_evidence(&shutdown_rx).await { + Ok(execution) => execution, + Err(_) => panic!("worker mutation should succeed"), + }; + + assert_eq!(execution.outcome, DeletionWorkerOutcome::ClaimLost); + assert_eq!(execution.evidence.status(skipped_source), None); + } + } + + #[tokio::test] + async fn worker_mutation_failures_report_their_exact_repository_source() { + for (execution, attempts, write_failure, expected_source) in [ + ( + DeletionPhaseExecutionOutcome::Deferred, + 1, + RepositoryWriteFailure::Defer, + DeletionDependencySource::RepositoryDefer, + ), + ( + DeletionPhaseExecutionOutcome::TransientDependencyFailure, + 1, + RepositoryWriteFailure::Retry, + DeletionDependencySource::RepositoryRetry, + ), + ( + DeletionPhaseExecutionOutcome::TransientDependencyFailure, + 4, + RepositoryWriteFailure::Terminal, + DeletionDependencySource::RepositoryTerminalMutation, + ), + ] { + let repository = FakeRepository::with_claim(claim(attempts)); + *repository.write_failure.lock().unwrap() = Some(write_failure); + let executor = FakeExecutor::new(execution); + let worker = worker(&repository, &executor, &FixedJitter(StdDuration::ZERO)); + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + + let failure = match worker.run_once_with_evidence(&shutdown_rx).await { + Ok(_) => panic!("repository mutation should fail"), + Err(failure) => failure, + }; + + assert_eq!( + failure.evidence.status(expected_source), + Some(DeletionDependencyStatus::Unavailable) + ); + for unrelated in [ + DeletionDependencySource::RepositoryDefer, + DeletionDependencySource::RepositoryRetry, + DeletionDependencySource::RepositoryTerminalMutation, + ] { + if unrelated != expected_source { + assert_eq!(failure.evidence.status(unrelated), None); + } + } + } + } + + #[test] + fn healthy_business_deferral_establishes_readiness_without_degradation() { + let readiness = WorkerReadiness::new(false, true); + let mut recovery = DeletionReadinessRecovery::default(); + + recovery.record_outcome( + DeletionWorkerOutcome::Deferred, + DeletionDependencyEvidence::healthy(DeletionDependencySource::RepositoryQueueClaim), + &readiness, + ); + + assert_eq!(readiness.status(), ReadinessStatus::Ready); + assert_eq!( + readiness.worker_state(WorkerKind::Deletion), + WorkerReadinessState::Ready + ); + } + + #[test] + fn paykit_degradation_survives_unrelated_pubky_or_repository_progress() { + let readiness = WorkerReadiness::new(false, true); + let mut recovery = DeletionReadinessRecovery::default(); + + recovery.record_outcome( + DeletionWorkerOutcome::RetryScheduled, + DeletionDependencyEvidence::unavailable(DeletionDependencySource::PaymentProvider), + &readiness, + ); + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + + recovery.record_outcome( + DeletionWorkerOutcome::Progressed, + DeletionDependencyEvidence::healthy(DeletionDependencySource::PubkyReadback).merge( + DeletionDependencyEvidence::healthy(DeletionDependencySource::RepositoryQueueClaim), + ), + &readiness, + ); + + assert_eq!( + readiness.worker_state(WorkerKind::Deletion), + WorkerReadinessState::Degraded + ); + } + + #[test] + fn successful_active_paykit_drain_recovers_readiness_while_deferred() { + let readiness = WorkerReadiness::new(false, true); + let mut recovery = DeletionReadinessRecovery::default(); + + recovery.record_outcome( + DeletionWorkerOutcome::RetryScheduled, + DeletionDependencyEvidence::unavailable(DeletionDependencySource::PaymentProvider), + &readiness, + ); + recovery.record_outcome( + DeletionWorkerOutcome::Deferred, + DeletionDependencyEvidence::healthy(DeletionDependencySource::PaymentProvider).merge( + DeletionDependencyEvidence::healthy(DeletionDependencySource::RepositoryQueueClaim), + ), + &readiness, + ); + + assert_eq!(readiness.status(), ReadinessStatus::Ready); + } + + #[test] + fn dependency_sources_recover_independently() { + let readiness = WorkerReadiness::new(false, true); + let mut recovery = DeletionReadinessRecovery::default(); + + recovery.record_outcome( + DeletionWorkerOutcome::RetryScheduled, + DeletionDependencyEvidence::unavailable(DeletionDependencySource::PaymentProvider) + .merge(DeletionDependencyEvidence::unavailable( + DeletionDependencySource::PubkyReadback, + )), + &readiness, + ); + recovery.record_outcome( + DeletionWorkerOutcome::Progressed, + DeletionDependencyEvidence::healthy(DeletionDependencySource::PubkyReadback), + &readiness, + ); + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + + recovery.record_outcome( + DeletionWorkerOutcome::Deferred, + DeletionDependencyEvidence::healthy(DeletionDependencySource::PaymentProvider), + &readiness, + ); + assert_eq!(readiness.status(), ReadinessStatus::Ready); + } + + #[test] + fn repository_degradation_is_sticky_across_non_recovery_outcomes() { + for non_recovery in [ + DeletionWorkerOutcome::Deferred, + DeletionWorkerOutcome::ClaimLost, + DeletionWorkerOutcome::TerminalFailed, + ] { + let readiness = WorkerReadiness::new(false, true); + let mut recovery = DeletionReadinessRecovery::default(); + + recovery.record_repository_failure( + DeletionDependencySource::RepositoryQueueClaim, + &readiness, + ); + recovery.record_outcome(non_recovery, DeletionDependencyEvidence::none(), &readiness); + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + + recovery.record_outcome( + DeletionWorkerOutcome::Idle, + DeletionDependencyEvidence::healthy(DeletionDependencySource::RepositoryQueueClaim), + &readiness, + ); + assert_eq!(readiness.status(), ReadinessStatus::Ready); + } + } + + #[test] + fn idle_queue_poll_does_not_clear_repository_mutation_degradation() { + let readiness = WorkerReadiness::new(false, true); + let mut recovery = DeletionReadinessRecovery::default(); + + recovery.record_outcome( + DeletionWorkerOutcome::RetryScheduled, + DeletionDependencyEvidence::unavailable( + DeletionDependencySource::RepositoryPhaseMutation, + ), + &readiness, + ); + recovery.record_outcome( + DeletionWorkerOutcome::Idle, + DeletionDependencyEvidence::healthy(DeletionDependencySource::RepositoryQueueClaim), + &readiness, + ); + + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + } + + #[test] + fn forced_contention_does_not_clear_prior_pubky_degradation() { + let readiness = WorkerReadiness::new(false, true); + let mut recovery = DeletionReadinessRecovery::default(); + + recovery.record_outcome( + DeletionWorkerOutcome::RetryScheduled, + DeletionDependencyEvidence::unavailable(DeletionDependencySource::PubkyReadback), + &readiness, + ); + recovery.record_outcome( + DeletionWorkerOutcome::Deferred, + DeletionDependencyEvidence::none(), + &readiness, + ); + + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + } + + #[test] + fn retry_exhaustion_is_dependency_failure_evidence_even_without_prior_retry_tick() { + let readiness = WorkerReadiness::new(false, true); + let mut recovery = DeletionReadinessRecovery::default(); + + recovery.record_outcome( + DeletionWorkerOutcome::RetryExhausted, + DeletionDependencyEvidence::unavailable(DeletionDependencySource::PubkyReadback), + &readiness, + ); + + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + } + + #[test] + fn polling_error_classification_retries_only_storage_failures_without_details() { + let secret = "postgres://user:password@example.test/locks"; + assert_eq!( + classify_poll_error(&ApplicationError::Storage { + message: secret.to_owned(), + }), + PollErrorDisposition::RetryableRepository + ); + assert_eq!( + classify_poll_error(&ApplicationError::InvalidContentLockDeletionState { + message: secret.to_owned(), + }), + PollErrorDisposition::Fatal("invalid_deletion_state") + ); + assert_eq!( + classify_poll_error(&ApplicationError::MissingRecord { + record: "content_lock_deletion", + }), + PollErrorDisposition::Fatal("unexpected_application_error") + ); + + for class in ["invalid_deletion_state", "unexpected_application_error"] { + let redacted = super::redacted_fatal_poll_error(class).to_string(); + assert!(redacted.contains(class)); + assert!(!redacted.contains(secret)); + } + } + + #[tokio::test] + async fn unexpected_poll_error_terminates_worker_not_ready_with_redacted_error() { + let secret = "pubky-secret-path"; + let repository = + FakeRepository::with_claim_error(ApplicationError::InvalidContentLockDeletionState { + message: secret.to_owned(), + }); + let executor = FakeExecutor::new(DeletionPhaseExecutionOutcome::Progressed); + let worker = worker(&repository, &executor, &FixedJitter(StdDuration::ZERO)); + let readiness = WorkerReadiness::new(false, true); + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + + let error = worker + .run_until_shutdown_with_readiness(shutdown_rx, &readiness) + .await + .unwrap_err(); + + assert_eq!(readiness.status(), ReadinessStatus::NotReady); + assert!(error.to_string().contains("invalid_deletion_state")); + assert!(!error.to_string().contains(secret)); + } + + #[test] + fn public_outcome_debug_contains_no_job_identity() { + for outcome in [ + DeletionWorkerOutcome::Idle, + DeletionWorkerOutcome::Cancelled, + DeletionWorkerOutcome::Progressed, + DeletionWorkerOutcome::Deferred, + DeletionWorkerOutcome::ClaimLost, + DeletionWorkerOutcome::TerminalFailed, + DeletionWorkerOutcome::RetryScheduled, + DeletionWorkerOutcome::RetryExhausted, + ] { + let debug = format!("{outcome:?}"); + assert!(!debug.contains("pubky")); + assert!(!debug.contains('/')); + assert!(!debug.contains('-')); + } + } + + fn worker<'a>( + repository: &'a FakeRepository, + executor: &'a FakeExecutor, + jitter: &'a dyn FullJitterSource, + ) -> DeletionWorker<'a> { + DeletionWorker::new(repository, &FIXED_CLOCK, executor, jitter, config()) + } + + fn config() -> DeletionWorkerConfig { + DeletionWorkerConfig { + worker_id: "deletion-worker-test".to_owned(), + poll_interval: StdDuration::from_secs(5), + claim_timeout: StdDuration::from_secs(30), + retry_max_attempts: 4, + retry_initial_backoff: StdDuration::from_secs(10), + retry_max_backoff: StdDuration::from_secs(25), + } + } + + struct OwnedWorker { + repository: Arc, + executor: Arc, + jitter: Arc, + } + + impl OwnedWorker { + fn new( + repository: Arc, + executor: Arc, + jitter: Arc, + ) -> Self { + Self { + repository, + executor, + jitter, + } + } + + async fn run_once( + &self, + shutdown: &watch::Receiver, + ) -> Result { + DeletionWorker::new( + self.repository.as_ref(), + &FIXED_CLOCK, + self.executor.as_ref(), + self.jitter.as_ref(), + config(), + ) + .run_once(shutdown) + .await + } + + async fn run_until_shutdown( + &self, + shutdown: watch::Receiver, + ) -> Result<(), ApplicationError> { + DeletionWorker::new( + self.repository.as_ref(), + &FIXED_CLOCK, + self.executor.as_ref(), + self.jitter.as_ref(), + config(), + ) + .run_until_shutdown(shutdown) + .await + } + } + + struct FixedClock; + static FIXED_CLOCK: FixedClock = FixedClock; + impl Clock for FixedClock { + fn now(&self) -> OffsetDateTime { + NOW + } + } + + struct FixedJitter(StdDuration); + impl FullJitterSource for FixedJitter { + fn sample(&self, _cap: StdDuration) -> StdDuration { + self.0 + } + } + + struct FakeExecutor { + outcome: DeletionPhaseExecutionOutcome, + calls: AtomicUsize, + } + + impl FakeExecutor { + fn new(outcome: DeletionPhaseExecutionOutcome) -> Self { + Self { + outcome, + calls: AtomicUsize::new(0), + } + } + } + + #[async_trait] + impl ClaimedDeletionExecutor for FakeExecutor { + async fn execute_claimed( + &self, + _claim: ClaimedContentLockDeletionJob, + _worker_id: &str, + ) -> DeletionPhaseExecution { + self.calls.fetch_add(1, Ordering::SeqCst); + DeletionPhaseExecution::new(self.outcome) + } + } + + #[derive(Default)] + struct FakeRepository { + claim: Mutex>, + claim_error: Mutex>, + claim_calls: AtomicUsize, + block_claim: AtomicBool, + claim_entered: Notify, + claim_release: Notify, + stale_writes: AtomicBool, + defers: Mutex>, + retries: Mutex>, + finishes: Mutex>>, + write_failure: Mutex>, + } + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum RepositoryWriteFailure { + Defer, + Retry, + Terminal, + } + + impl FakeRepository { + fn with_claim(claim: ClaimedContentLockDeletionJob) -> Self { + Self { + claim: Mutex::new(Some(claim)), + ..Self::default() + } + } + + fn with_claim_error(error: ApplicationError) -> Self { + Self { + claim_error: Mutex::new(Some(error)), + ..Self::default() + } + } + + fn write_result(&self, job: ContentLockDeletionJob) -> Option { + (!self.stale_writes.load(Ordering::SeqCst)).then_some(job) + } + } + + #[async_trait] + impl ContentLockDeletionRepository for FakeRepository { + async fn begin_publication( + &self, + _: &CreatorPubky, + _: &LockId, + _: Uuid, + ) -> Result<(), ApplicationError> { + unreachable!() + } + async fn finish_publication( + &self, + _: &CreatorPubky, + _: &LockId, + _: Uuid, + ) -> Result { + unreachable!() + } + async fn abandon_publication( + &self, + _: &CreatorPubky, + _: &LockId, + _: Uuid, + ) -> Result { + unreachable!() + } + async fn publication_in_progress( + &self, + _: &CreatorPubky, + _: &LockId, + ) -> Result { + unreachable!() + } + async fn insert_job(&self, _: ContentLockDeletionJob) -> Result<(), ApplicationError> { + unreachable!() + } + async fn get_job( + &self, + _: &CreatorPubky, + _: &LockId, + ) -> Result, ApplicationError> { + unreachable!() + } + + async fn claim_next( + &self, + _: &str, + _: time::Duration, + ) -> Result, ApplicationError> { + self.claim_calls.fetch_add(1, Ordering::SeqCst); + if let Some(error) = self.claim_error.lock().unwrap().take() { + return Err(error); + } + if self.block_claim.load(Ordering::SeqCst) { + self.claim_entered.notify_one(); + self.claim_release.notified().await; + } + Ok(self.claim.lock().unwrap().take()) + } + + async fn schedule_retry( + &self, + _: Uuid, + _: &str, + _: Uuid, + retry_after: time::Duration, + ) -> Result, ApplicationError> { + if self.write_failure.lock().unwrap().take() == Some(RepositoryWriteFailure::Retry) { + return Err(ApplicationError::Storage { + message: "retry unavailable".to_owned(), + }); + } + self.retries.lock().unwrap().push((NOW, NOW + retry_after)); + Ok(self.write_result(job(1))) + } + + async fn defer( + &self, + _: Uuid, + _: &str, + _: Uuid, + defer_for: time::Duration, + ) -> Result, ApplicationError> { + if self.write_failure.lock().unwrap().take() == Some(RepositoryWriteFailure::Defer) { + return Err(ApplicationError::Storage { + message: "defer unavailable".to_owned(), + }); + } + self.defers.lock().unwrap().push((NOW, NOW + defer_for)); + Ok(self.write_result(job(1))) + } + + async fn advance_phase( + &self, + _: Uuid, + _: &str, + _: Uuid, + _: ContentLockDeletionPhase, + ) -> Result { + Ok(match self.write_result(job(1)) { + Some(job) => AdvanceContentLockDeletionPhaseResult::Advanced(Box::new(job)), + None => AdvanceContentLockDeletionPhaseResult::ClaimLost, + }) + } + + async fn finish( + &self, + _: Uuid, + _: &str, + _: Uuid, + code: Option, + ) -> Result, ApplicationError> { + if self.write_failure.lock().unwrap().take() == Some(RepositoryWriteFailure::Terminal) { + return Err(ApplicationError::Storage { + message: "terminal mutation unavailable".to_owned(), + }); + } + self.finishes.lock().unwrap().push(code); + Ok(self.write_result(job(1))) + } + + async fn resume_failed_job( + &self, + _: &CreatorPubky, + _: &LockId, + _: OffsetDateTime, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn prepare_force_deletion( + &self, + _: &CreatorPubky, + _: &LockId, + ) -> Result { + unreachable!() + } + async fn complete_force_deletion( + &self, + _: Uuid, + _: &str, + _: Uuid, + ) -> Result { + unreachable!() + } + async fn has_force_receipt( + &self, + _: &CreatorPubky, + _: &LockId, + ) -> Result { + unreachable!() + } + } + + fn claim(attempt_count: u32) -> ClaimedContentLockDeletionJob { + ClaimedContentLockDeletionJob { + job: job(attempt_count), + claim_token: Uuid::from_u128(2), + } + } + + fn job(attempt_count: u32) -> ContentLockDeletionJob { + let frozen_content_lock = content_lock(); + ContentLockDeletionJob { + job_id: Uuid::from_u128(1), + creator: frozen_content_lock.creator.clone(), + lock_id: frozen_content_lock.lock_id().unwrap(), + frozen_content_lock, + deletion_started_at: NOW, + state: ContentLockDeletionState::Running, + phase: ContentLockDeletionPhase::Withdraw, + attempt_count, + next_attempt_at: None, + force_requested_at: None, + failure_code: None, + } + } + + fn content_lock() -> ContentLock { + ContentLock { + version: CONTENT_LOCK_VERSION, + creator: CreatorPubky::from_str(CREATOR).unwrap(), + primary_resource: Some( + GuardedResource::new( + "/priv/locks.app/content/post.json".to_owned(), + GuardedResourceHash::from_bytes([7; 32]), + "application/json".to_owned(), + 42, + ) + .unwrap(), + ), + secondary_resources: BTreeMap::new(), + criteria: vec![], + lock_logic: LockLogic::All { criteria: vec![] }, + access_policy: AccessPolicy { + requested_credential_ttl_seconds: 900, + }, + lock_server: LockServerConfig { override_: None }, + created_at: NOW, + } + } +} diff --git a/locks-server/src/lib.rs b/locks-server/src/lib.rs index f1805f4..c5b23d7 100644 --- a/locks-server/src/lib.rs +++ b/locks-server/src/lib.rs @@ -1,6 +1,7 @@ pub mod api; pub mod app_state; pub mod config; +pub mod deletion_worker; pub mod paykit_http_client; pub mod pkdns; pub mod rate_limit; diff --git a/locks-server/src/main.rs b/locks-server/src/main.rs index 6fba9f7..c2406bf 100644 --- a/locks-server/src/main.rs +++ b/locks-server/src/main.rs @@ -1,14 +1,17 @@ -use std::env; -use std::error::Error; +use std::{env, error::Error, time::Duration}; use locks_server::api::routes::router; use locks_server::config::{FilesystemLockServerIdentityProvider, load_or_initialize_config}; +use locks_server::deletion_worker::{ + DeletionWorker, DeletionWorkerConfig, RandomFullJitter, RuntimeClaimedDeletionExecutor, +}; use locks_server::pkdns::LockServerKeyRepublisher; -use locks_server::runtime::{home_dir_from_env, parse_config_arg}; +use locks_server::runtime::{ + InitialStartupOutcome, RuntimeTasks, ShutdownFuture, bind_listener_then_run_initial, + home_dir_from_env, parse_config_arg, supervise, wait_for_shutdown, +}; use locks_server::storage::build_runtime_state; use locks_server::worker::VerificationWorker; -use tokio::net::TcpListener; -use tokio::sync::watch; use tower_http::trace::TraceLayer; use tracing::{error, info}; use tracing_subscriber::EnvFilter; @@ -21,41 +24,117 @@ async fn main() -> Result<(), Box> { let config = load_or_initialize_config(config_path, &home_dir, &identity_provider)?; init_tracing(&config.logging.level); let bind_addr = config.bind_addr; - let _key_republisher = LockServerKeyRepublisher::start_if_required(&config).await?; let state = build_runtime_state(config).await?; - let worker_enabled = state.config().worker.enabled; - let (shutdown_tx, shutdown_rx) = watch::channel(false); - let worker_handle = if worker_enabled { - let worker_state = state.clone(); - Some(tokio::spawn(async move { - let worker = VerificationWorker::from_state(&worker_state); - if let Err(error) = worker.run_until_shutdown(shutdown_rx).await { - error!(%error, "verification worker stopped with error"); + let shutdown_timeout = + Duration::from_secs(state.config().deletion_worker.shutdown_timeout_seconds); + // Validate publication state before network startup, but do not advertise the service until + // secret validation, database startup, migrations, and listener binding have all succeeded. + let key_republisher = LockServerKeyRepublisher::build_if_required(state.config())?; + let mut shutdown: ShutdownFuture = Box::pin(shutdown_signal()); + let startup = bind_listener_then_run_initial( + bind_addr, + shutdown_timeout, + &mut shutdown, + move || async move { + if let Some(republisher) = &key_republisher { + republisher + .publish_initial() + .await + .map_err(anyhow::Error::from)?; } - })) - } else { - None - }; - let app = router(state).layer(TraceLayer::new_for_http()); - let listener = TcpListener::bind(bind_addr).await?; - - info!(%bind_addr, "starting locks-server"); - axum::serve( - listener, - app.into_make_service_with_connect_info::(), + Ok(key_republisher) + }, ) - .with_graceful_shutdown(shutdown_signal()) .await?; - let _ = shutdown_tx.send(true); - if let Some(worker_handle) = worker_handle { - worker_handle.await?; + let InitialStartupOutcome::Ready { + listener, + initial: key_republisher, + } = startup + else { + return Ok(()); + }; + let app = router(state.clone()).layer(TraceLayer::new_for_http()); + + let mut tasks = RuntimeTasks::new(shutdown_timeout).with_http(Box::new(move |shutdown| { + Box::pin(async move { + axum::serve( + listener, + app.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(wait_for_shutdown(shutdown)) + .await + .map_err(anyhow::Error::from) + }) + })); + + if let Some(key_republisher) = key_republisher { + tasks = tasks.with_pkarr_republisher(Box::new(move |shutdown| { + Box::pin(async move { + key_republisher + .run_until_shutdown(shutdown) + .await + .map_err(anyhow::Error::from) + }) + })); } + + if state.config().worker.enabled { + let worker_state = state.clone(); + tasks = tasks.with_verification_worker(Box::new(move |context| { + Box::pin(async move { + VerificationWorker::from_state(&worker_state) + .run_until_shutdown_with_readiness( + context.shutdown(), + worker_state.worker_readiness(), + ) + .await + .map_err(anyhow::Error::from) + }) + })); + } + + if state.config().deletion_worker.enabled { + let worker_state = state.clone(); + tasks = tasks.with_deletion_worker(Box::new(move |context| { + Box::pin(async move { + let runtime = &worker_state.config().deletion_worker; + let retry = &worker_state.config().deletion; + let executor = RuntimeClaimedDeletionExecutor::new(worker_state.clone()); + let jitter = RandomFullJitter; + let config = DeletionWorkerConfig { + worker_id: runtime.worker_id.clone(), + poll_interval: Duration::from_millis(runtime.poll_interval_ms), + claim_timeout: Duration::from_secs(runtime.claim_timeout_seconds), + retry_max_attempts: retry.retry_max_attempts, + retry_initial_backoff: Duration::from_secs(retry.retry_initial_backoff_seconds), + retry_max_backoff: Duration::from_secs(retry.retry_max_backoff_seconds), + }; + DeletionWorker::new( + worker_state.content_lock_deletions().as_ref(), + worker_state.clock().as_ref(), + &executor, + &jitter, + config, + ) + .run_until_shutdown_with_readiness( + context.shutdown(), + worker_state.worker_readiness(), + ) + .await + .map_err(anyhow::Error::from) + }) + })); + } + + info!(%bind_addr, "starting locks-server"); + supervise(state.worker_readiness().clone(), shutdown, tasks).await?; Ok(()) } async fn shutdown_signal() { if let Err(error) = tokio::signal::ctrl_c().await { error!(%error, "failed to listen for shutdown signal"); + std::future::pending::<()>().await; } } diff --git a/locks-server/src/paykit_http_client.rs b/locks-server/src/paykit_http_client.rs index a5c6be8..0744804 100644 --- a/locks-server/src/paykit_http_client.rs +++ b/locks-server/src/paykit_http_client.rs @@ -4,6 +4,10 @@ use async_trait::async_trait; use base64::Engine; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use locks_core::ids::{BundleId, CreatorPubky}; +use locks_service::application::ports::{ + PaymentDrainCleanupToken, PaymentDrainClient, PaymentDrainClientError, PaymentDrainStatus, + PaymentDrainSummary, PaymentRequestState, PaymentRequestStatus, PaymentState, +}; use locks_service::infrastructure::verifiers::paykit_payment::{ PaykitPaymentStatus, PaykitPaymentStatusClient, PaykitPaymentStatusError, PaykitPaymentStatusKind, @@ -11,6 +15,8 @@ use locks_service::infrastructure::verifiers::paykit_payment::{ use pubky_common::crypto::Keypair; use reqwest::StatusCode; use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; +use time::format_description::well_known::Rfc3339; use url::Url; use crate::config::{ @@ -41,6 +47,8 @@ pub enum PaykitClientError { }, #[error("Paykit status response was invalid: {0}")] InvalidStatusResponse(reqwest::Error), + #[error("Paykit invoice response was invalid: {0}")] + InvalidInvoiceResponse(String), } #[derive(Debug, Clone)] @@ -54,9 +62,49 @@ pub struct PaykitHttpClient { pub struct PaykitInvoiceRequest { pub bundle_id: String, pub lock_resource: String, + pub payment_in: u64, pub reader: String, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PaykitInvoiceResponse { + pub invoice_created_at: OffsetDateTime, + pub payment_deadline: OffsetDateTime, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct PaykitInvoiceResponseBody { + invoice_created_at: String, + payment_deadline: String, +} + +#[derive(Debug, Serialize)] +struct PaymentDrainRequest<'a> { + lock_resource: &'a str, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct PaymentDrainResponseBody { + status: String, + accepted_count: u64, + terminal_count: u64, + cancellation_enqueued_count: u64, + cleanup_token: String, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct PaymentRequestStatusResponseBody { + request_state: String, + payment_state: String, + invoice_created_at: String, + payment_deadline: String, + confirmations: u32, + amount_matched: bool, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct PaykitStatusRequest { pub creator: String, @@ -109,17 +157,33 @@ impl PaykitHttpClient { pub async fn create_invoice( &self, request: &PaykitInvoiceRequest, - ) -> Result<(), PaykitClientError> { + ) -> Result { let response = self.signed_post("invoices", request).await?; - if response.status().is_success() { - Ok(()) - } else { - Err(PaykitClientError::NonSuccess { + if !response.status().is_success() { + return Err(PaykitClientError::NonSuccess { operation: "invoice creation", status: response.status(), - }) + }); } + + let body = response + .json::() + .await + .map_err(|error| PaykitClientError::InvalidInvoiceResponse(error.to_string()))?; + let invoice_created_at = OffsetDateTime::parse(&body.invoice_created_at, &Rfc3339) + .map_err(|error| PaykitClientError::InvalidInvoiceResponse(error.to_string()))?; + let payment_deadline = OffsetDateTime::parse(&body.payment_deadline, &Rfc3339) + .map_err(|error| PaykitClientError::InvalidInvoiceResponse(error.to_string()))?; + if payment_deadline < invoice_created_at { + return Err(PaykitClientError::InvalidInvoiceResponse( + "payment_deadline precedes invoice_created_at".to_owned(), + )); + } + Ok(PaykitInvoiceResponse { + invoice_created_at, + payment_deadline, + }) } pub async fn transaction_status( @@ -141,6 +205,97 @@ impl PaykitHttpClient { .map_err(PaykitClientError::InvalidStatusResponse) } + pub async fn start_payment_drain( + &self, + lock_resource: &str, + ) -> Result { + self.payment_drain("payment-request-drains", lock_resource) + .await? + .ok_or(PaymentDrainClientError::NotFound) + } + + pub async fn lookup_payment_drain( + &self, + lock_resource: &str, + ) -> Result, PaymentDrainClientError> { + self.payment_drain("payment-request-drain-lookups", lock_resource) + .await + } + + pub async fn payment_request_status( + &self, + creator: &str, + bundle_id: &str, + ) -> Result, PaymentDrainClientError> { + let response = self + .signed_post( + "payment-requests/status", + &PaykitStatusRequest { + creator: creator.to_owned(), + bundle_id: bundle_id.to_owned(), + }, + ) + .await + .map_err(|_| PaymentDrainClientError::Transport)?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + map_drain_error(response.status())?; + let body = response + .json::() + .await + .map_err(|_| PaymentDrainClientError::MalformedSuccess)?; + let invoice_created_at = OffsetDateTime::parse(&body.invoice_created_at, &Rfc3339) + .map_err(|_| PaymentDrainClientError::MalformedSuccess)?; + let payment_deadline = OffsetDateTime::parse(&body.payment_deadline, &Rfc3339) + .map_err(|_| PaymentDrainClientError::MalformedSuccess)?; + if payment_deadline < invoice_created_at { + return Err(PaymentDrainClientError::MalformedSuccess); + } + Ok(Some(PaymentRequestStatus { + request_state: PaymentRequestState::parse(&body.request_state) + .ok_or(PaymentDrainClientError::MalformedSuccess)?, + payment_state: PaymentState::parse(&body.payment_state) + .ok_or(PaymentDrainClientError::MalformedSuccess)?, + invoice_created_at, + payment_deadline, + confirmations: body.confirmations, + amount_matched: body.amount_matched, + })) + } + + async fn payment_drain( + &self, + path: &str, + lock_resource: &str, + ) -> Result, PaymentDrainClientError> { + let response = self + .signed_post(path, &PaymentDrainRequest { lock_resource }) + .await + .map_err(|_| PaymentDrainClientError::Transport)?; + if response.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + map_drain_error(response.status())?; + let body = response + .json::() + .await + .map_err(|_| PaymentDrainClientError::MalformedSuccess)?; + let status = PaymentDrainStatus::parse(&body.status) + .ok_or(PaymentDrainClientError::MalformedSuccess)?; + if (status == PaymentDrainStatus::Completed) != (body.accepted_count == 0) { + return Err(PaymentDrainClientError::MalformedSuccess); + } + Ok(Some(PaymentDrainSummary { + status, + accepted_count: body.accepted_count, + terminal_count: body.terminal_count, + cancellation_enqueued_count: body.cancellation_enqueued_count, + cleanup_token: PaymentDrainCleanupToken::parse(&body.cleanup_token) + .ok_or(PaymentDrainClientError::MalformedSuccess)?, + })) + } + fn endpoint(&self, path: &str) -> Url { let mut endpoint = self.server_url.clone(); endpoint.set_query(None); @@ -174,6 +329,15 @@ impl PaykitHttpClient { } } +fn map_drain_error(status: StatusCode) -> Result<(), PaymentDrainClientError> { + match status { + status if status.is_success() => Ok(()), + StatusCode::CONFLICT => Err(PaymentDrainClientError::Conflict), + status if status.is_server_error() => Err(PaymentDrainClientError::Server), + _ => Err(PaymentDrainClientError::Server), + } +} + fn bounded_http_client( connect_timeout: Duration, request_timeout: Duration, @@ -185,6 +349,32 @@ fn bounded_http_client( .map_err(PaykitClientError::Http) } +#[async_trait] +impl PaymentDrainClient for PaykitHttpClient { + async fn start_payment_drain( + &self, + lock_resource: &locks_core::ids::PubkyLockResource, + ) -> Result { + PaykitHttpClient::start_payment_drain(self, &lock_resource.to_string()).await + } + + async fn lookup_payment_drain( + &self, + lock_resource: &locks_core::ids::PubkyLockResource, + ) -> Result, PaymentDrainClientError> { + PaykitHttpClient::lookup_payment_drain(self, &lock_resource.to_string()).await + } + + async fn payment_request_status( + &self, + creator: &CreatorPubky, + bundle_id: &BundleId, + ) -> Result, PaymentDrainClientError> { + PaykitHttpClient::payment_request_status(self, &creator.to_string(), &bundle_id.to_string()) + .await + } +} + #[async_trait] impl PaykitPaymentStatusClient for PaykitHttpClient { async fn transaction_status( @@ -272,7 +462,7 @@ mod tests { assert_eq!( String::from_utf8(body).unwrap(), format!( - "{{\"bundle_id\":\"{BUNDLE_ID}\",\"lock_resource\":\"{LOCK_RESOURCE}\",\"reader\":\"{READER}\"}}" + "{{\"bundle_id\":\"{BUNDLE_ID}\",\"lock_resource\":\"{LOCK_RESOURCE}\",\"payment_in\":24,\"reader\":\"{READER}\"}}" ) ); } @@ -365,7 +555,16 @@ mod tests { let expected_body = canonical_body_bytes(&invoice_request()).unwrap(); let expected_signature = sign_body(&keypair, &expected_body); - client.create_invoice(&invoice_request()).await.unwrap(); + let response = client.create_invoice(&invoice_request()).await.unwrap(); + + assert_eq!( + response.invoice_created_at, + time::macros::datetime!(2026-08-12 10:00:00 UTC) + ); + assert_eq!( + response.payment_deadline, + time::macros::datetime!(2026-08-13 10:00:00 UTC) + ); let request = captured.single(); assert_eq!(request.path, "/invoices"); @@ -407,6 +606,172 @@ mod tests { assert_eq!(request.signature, Some(expected_signature)); } + #[tokio::test] + async fn payment_drain_calls_use_exact_signed_bodies_and_closed_responses() { + let captured = CapturedRequests::default(); + let server_url = spawn_test_server(captured.clone()).await; + let client = PaykitHttpClient::from_parts( + &server_url, + reqwest::Client::new(), + Keypair::from_secret(&[9_u8; 32]), + ) + .unwrap(); + + let started = client.start_payment_drain(LOCK_RESOURCE).await.unwrap(); + assert_eq!(started.status, PaymentDrainStatus::Active); + assert_eq!(started.accepted_count, 1); + assert_eq!( + started.cleanup_token.as_str(), + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + ); + let looked_up = client + .lookup_payment_drain(LOCK_RESOURCE) + .await + .unwrap() + .unwrap(); + assert_eq!(looked_up, started); + let status = client + .payment_request_status(CREATOR, BUNDLE_ID) + .await + .unwrap() + .unwrap(); + assert_eq!(status.request_state, PaymentRequestState::Accepted); + assert_eq!(status.payment_state, PaymentState::Detected); + assert!(status.amount_matched); + + let requests = captured.all(); + assert_eq!(requests.len(), 3); + assert_eq!(requests[0].path, "/payment-request-drains"); + assert_eq!(requests[1].path, "/payment-request-drain-lookups"); + assert_eq!(requests[2].path, "/payment-requests/status"); + assert_eq!( + String::from_utf8(requests[0].body.clone()).unwrap(), + format!("{{\"lock_resource\":\"{LOCK_RESOURCE}\"}}") + ); + assert_eq!(requests[0].body, requests[1].body); + assert_eq!( + String::from_utf8(requests[2].body.clone()).unwrap(), + format!("{{\"bundle_id\":\"{BUNDLE_ID}\",\"creator\":\"{CREATOR}\"}}") + ); + assert!(requests.iter().all(|request| request.signature.is_some())); + assert!(requests.iter().all(|request| { + !String::from_utf8_lossy(&request.body).contains("minimum_confirmations") + })); + } + + #[tokio::test] + async fn payment_drain_http_errors_preserve_not_found_conflict_and_server_failure() { + for (status, expected) in [ + ( + axum::http::StatusCode::NOT_FOUND, + PaymentDrainClientError::NotFound, + ), + ( + axum::http::StatusCode::CONFLICT, + PaymentDrainClientError::Conflict, + ), + ( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + PaymentDrainClientError::Server, + ), + ] { + let server_url = spawn_configured_drain_server(status, "{}").await; + let client = PaykitHttpClient::from_parts( + &server_url, + reqwest::Client::new(), + Keypair::from_secret(&[9_u8; 32]), + ) + .unwrap(); + assert_eq!( + client.start_payment_drain(LOCK_RESOURCE).await, + Err(expected) + ); + } + } + + #[tokio::test] + async fn payment_drain_rejects_malformed_unknown_or_inconsistent_success_bodies() { + for body in [ + "not-json", + r#"{"status":"active","accepted_count":0,"terminal_count":0,"cancellation_enqueued_count":0}"#, + r#"{"status":"complete","accepted_count":0,"terminal_count":0,"cancellation_enqueued_count":0,"cleanup_token":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}"#, + r#"{"status":"active","accepted_count":0,"terminal_count":0,"cancellation_enqueued_count":0,"cleanup_token":"short"}"#, + r#"{"status":"active","accepted_count":0,"terminal_count":0,"cancellation_enqueued_count":0,"cleanup_token":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","extra":true}"#, + r#"{"status":"completed","accepted_count":1,"terminal_count":0,"cancellation_enqueued_count":0,"cleanup_token":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}"#, + ] { + let server_url = spawn_configured_drain_server(axum::http::StatusCode::OK, body).await; + let client = PaykitHttpClient::from_parts( + &server_url, + reqwest::Client::new(), + Keypair::from_secret(&[9_u8; 32]), + ) + .unwrap(); + assert_eq!( + client.start_payment_drain(LOCK_RESOURCE).await, + Err(PaymentDrainClientError::MalformedSuccess) + ); + } + } + + #[tokio::test] + async fn payment_drain_accepts_completed_cancellation_only_aggregate() { + let server_url = spawn_configured_drain_server( + axum::http::StatusCode::OK, + r#"{"status":"completed","accepted_count":0,"terminal_count":0,"cancellation_enqueued_count":1,"cleanup_token":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}"#, + ) + .await; + let client = PaykitHttpClient::from_parts( + &server_url, + reqwest::Client::new(), + Keypair::from_secret(&[9_u8; 32]), + ) + .unwrap(); + + let summary = client.start_payment_drain(LOCK_RESOURCE).await.unwrap(); + assert_eq!(summary.status, PaymentDrainStatus::Completed); + assert_eq!(summary.accepted_count, 0); + assert_eq!(summary.terminal_count, 0); + assert_eq!(summary.cancellation_enqueued_count, 1); + } + + #[tokio::test] + async fn payment_drain_timeout_is_transport_failure() { + let server_url = spawn_hanging_test_server().await; + let client = PaykitHttpClient::from_parts( + &server_url, + bounded_http_client(Duration::from_millis(10), Duration::from_millis(25)).unwrap(), + Keypair::from_secret(&[9_u8; 32]), + ) + .unwrap(); + assert_eq!( + client.start_payment_drain(LOCK_RESOURCE).await, + Err(PaymentDrainClientError::Transport) + ); + } + + #[tokio::test] + async fn payment_request_status_rejects_unknown_or_reversed_success_body() { + for body in [ + r#"{"request_state":"unknown","payment_state":"detected","invoice_created_at":"2026-08-12T10:00:00Z","payment_deadline":"2026-08-13T10:00:00Z","confirmations":0,"amount_matched":true}"#, + r#"{"request_state":"accepted","payment_state":"unknown","invoice_created_at":"2026-08-12T10:00:00Z","payment_deadline":"2026-08-13T10:00:00Z","confirmations":0,"amount_matched":true}"#, + r#"{"request_state":"accepted","payment_state":"detected","invoice_created_at":"2026-08-13T10:00:00Z","payment_deadline":"2026-08-12T10:00:00Z","confirmations":0,"amount_matched":true}"#, + ] { + let server_url = + spawn_configured_payment_request_status_server(axum::http::StatusCode::OK, body) + .await; + let client = PaykitHttpClient::from_parts( + &server_url, + reqwest::Client::new(), + Keypair::from_secret(&[9_u8; 32]), + ) + .unwrap(); + assert_eq!( + client.payment_request_status(CREATOR, BUNDLE_ID).await, + Err(PaymentDrainClientError::MalformedSuccess) + ); + } + } + #[tokio::test] async fn create_invoice_times_out_when_paykit_does_not_respond() { let server_url = spawn_hanging_test_server().await; @@ -443,6 +808,30 @@ mod tests { assert!(matches!(error, PaykitClientError::Http(error) if error.is_timeout())); } + #[tokio::test] + async fn create_invoice_rejects_malformed_or_inconsistent_success_body() { + for body in [ + r#"{"invoice_created_at":"2026-08-12T10:00:00Z"}"#, + r#"{"invoice_created_at":"not-a-time","payment_deadline":"2026-08-13T10:00:00Z"}"#, + r#"{"invoice_created_at":"2026-08-13T10:00:00Z","payment_deadline":"2026-08-12T10:00:00Z"}"#, + r#"{"invoice_created_at":"2026-08-12T10:00:00Z","payment_deadline":"2026-08-13T10:00:00Z","extra":true}"#, + ] { + let server_url = + spawn_configured_invoice_server(axum::http::StatusCode::OK, body).await; + let client = PaykitHttpClient::from_parts( + &server_url, + reqwest::Client::new(), + Keypair::from_secret(&[9_u8; 32]), + ) + .unwrap(); + + assert!(matches!( + client.create_invoice(&invoice_request()).await, + Err(PaykitClientError::InvalidInvoiceResponse(_)) + )); + } + } + #[tokio::test] async fn status_not_found_and_invalid_success_body_use_retryable_client_error() { for (status, body) in [ @@ -474,6 +863,7 @@ mod tests { bundle_id: BUNDLE_ID.to_owned(), lock_resource: LOCK_RESOURCE.to_owned(), reader: READER.to_owned(), + payment_in: 24, } } @@ -490,6 +880,10 @@ mod tests { assert_eq!(requests.len(), 1); requests[0].clone() } + + fn all(&self) -> Vec { + self.0.lock().unwrap().clone() + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -503,6 +897,15 @@ mod tests { let app = Router::new() .route("/invoices", post(capture_invoice)) .route("/transactions/status", post(capture_status)) + .route("/payment-request-drains", post(capture_payment_drain)) + .route( + "/payment-request-drain-lookups", + post(capture_payment_drain_lookup), + ) + .route( + "/payment-requests/status", + post(capture_payment_request_status), + ) .with_state(captured); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -515,7 +918,23 @@ mod tests { async fn spawn_hanging_test_server() -> String { let app = Router::new() .route("/invoices", post(hang)) - .route("/transactions/status", post(hang)); + .route("/transactions/status", post(hang)) + .route("/payment-request-drains", post(hang)); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + format!("http://{addr}") + } + + async fn spawn_configured_invoice_server( + status: axum::http::StatusCode, + body: &'static str, + ) -> String { + let app = Router::new() + .route("/invoices", post(configured_status)) + .with_state(ConfiguredStatusResponse { status, body }); let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); tokio::spawn(async move { @@ -545,6 +964,36 @@ mod tests { format!("http://{addr}") } + async fn spawn_configured_drain_server( + status: axum::http::StatusCode, + body: &'static str, + ) -> String { + let app = Router::new() + .route("/payment-request-drains", post(configured_status)) + .with_state(ConfiguredStatusResponse { status, body }); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + format!("http://{addr}") + } + + async fn spawn_configured_payment_request_status_server( + status: axum::http::StatusCode, + body: &'static str, + ) -> String { + let app = Router::new() + .route("/payment-requests/status", post(configured_status)) + .with_state(ConfiguredStatusResponse { status, body }); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + format!("http://{addr}") + } + async fn configured_status( State(response): State, ) -> impl IntoResponse { @@ -568,7 +1017,10 @@ mod tests { .map(|value| value.to_str().unwrap().to_owned()), body: body.to_vec(), }); - (axum::http::StatusCode::CREATED, "accepted") + Json(json!({ + "invoice_created_at": "2026-08-12T10:00:00Z", + "payment_deadline": "2026-08-13T10:00:00Z", + })) } async fn capture_status( @@ -589,4 +1041,58 @@ mod tests { "amount_matched": true, })) } + + async fn capture_payment_drain( + State(captured): State, + headers: HeaderMap, + body: Bytes, + ) -> impl IntoResponse { + capture_request(&captured, &headers, body, "/payment-request-drains"); + drain_response() + } + + async fn capture_payment_drain_lookup( + State(captured): State, + headers: HeaderMap, + body: Bytes, + ) -> impl IntoResponse { + capture_request(&captured, &headers, body, "/payment-request-drain-lookups"); + drain_response() + } + + async fn capture_payment_request_status( + State(captured): State, + headers: HeaderMap, + body: Bytes, + ) -> impl IntoResponse { + capture_request(&captured, &headers, body, "/payment-requests/status"); + Json(json!({ + "request_state": "accepted", + "payment_state": "detected", + "invoice_created_at": "2026-08-12T10:00:00Z", + "payment_deadline": "2026-08-13T10:00:00Z", + "confirmations": 0, + "amount_matched": true, + })) + } + + fn capture_request(captured: &CapturedRequests, headers: &HeaderMap, body: Bytes, path: &str) { + captured.push(CapturedRequest { + path: path.to_owned(), + signature: headers + .get(SIGNATURE_HEADER) + .map(|value| value.to_str().unwrap().to_owned()), + body: body.to_vec(), + }); + } + + fn drain_response() -> Json { + Json(json!({ + "status": "active", + "accepted_count": 1, + "terminal_count": 0, + "cancellation_enqueued_count": 0, + "cleanup_token": "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + })) + } } diff --git a/locks-server/src/pkdns.rs b/locks-server/src/pkdns.rs index ae9f950..6877646 100644 --- a/locks-server/src/pkdns.rs +++ b/locks-server/src/pkdns.rs @@ -22,23 +22,30 @@ pub enum LockServerKeyRepublisherError { PacketBuild(String), #[error("failed to build PKARR client: {0}")] ClientBuild(String), + #[error("PKARR republisher interval must be greater than zero")] + InvalidInterval, #[error("failed to publish lock server PKARR packet: {0}")] Publish(String), } -/// Background task that publishes and periodically republishes the Lock Server key's PKARR record. +/// Owned task that publishes and periodically republishes the Lock Server key's PKARR record. #[derive(Debug)] pub struct LockServerKeyRepublisher { - join_handle: tokio::task::JoinHandle<()>, + client: pkarr::Client, + signed_packet: pkarr::SignedPacket, + interval: std::time::Duration, } impl LockServerKeyRepublisher { - pub async fn start_if_required( + pub fn build_if_required( config: &LockServerRuntimeConfig, ) -> Result, LockServerKeyRepublisherError> { if !requires_lock_server_pkarr(config) { return Ok(None); } + if config.pkdns.key_republisher_interval_seconds == 0 { + return Err(LockServerKeyRepublisherError::InvalidInterval); + } let keypair = load_lock_server_keypair(&config.credentials)?; let signed_packet = create_signed_packet(&config.pkdns, &keypair)?; @@ -51,30 +58,35 @@ impl LockServerKeyRepublisher { let client = builder .build() .map_err(|error| LockServerKeyRepublisherError::ClientBuild(error.to_string()))?; - publish_once(&client, &signed_packet).await?; - - let interval_seconds = config.pkdns.key_republisher_interval_seconds; - let join_handle = tokio::spawn(async move { - let mut interval = - tokio::time::interval(std::time::Duration::from_secs(interval_seconds)); - interval.tick().await; - loop { - interval.tick().await; - let _ = publish_once(&client, &signed_packet).await; - } - }); - - Ok(Some(Self { join_handle })) + Ok(Some(Self { + client, + signed_packet, + interval: std::time::Duration::from_secs(config.pkdns.key_republisher_interval_seconds), + })) } - pub fn stop(&self) { - self.join_handle.abort(); + pub async fn publish_initial(&self) -> Result<(), LockServerKeyRepublisherError> { + publish_once(&self.client, &self.signed_packet).await } -} -impl Drop for LockServerKeyRepublisher { - fn drop(&mut self) { - self.stop(); + pub async fn run_until_shutdown( + self, + mut shutdown: tokio::sync::watch::Receiver, + ) -> Result<(), LockServerKeyRepublisherError> { + let mut interval = tokio::time::interval(self.interval); + interval.tick().await; + loop { + tokio::select! { + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { + return Ok(()); + } + } + _ = interval.tick() => { + publish_once(&self.client, &self.signed_packet).await?; + } + } + } } } @@ -238,6 +250,20 @@ mod tests { )); } + #[tokio::test] + async fn zero_republisher_interval_is_rejected_before_secret_or_publication_work() { + let mut config = test_config(); + config.creator_authority_acquisition.enabled = true; + config.pkdns.key_republisher_interval_seconds = 0; + + let error = LockServerKeyRepublisher::build_if_required(&config).unwrap_err(); + + assert!(matches!( + error, + LockServerKeyRepublisherError::InvalidInterval + )); + } + #[test] fn pkarr_republisher_is_required_for_production_lock_server_identity() { let mut config = test_config(); @@ -292,6 +318,8 @@ mod tests { pkdns: PkdnsConfig::default(), rate_limits: RateLimitsConfig::default(), content_locks: ContentLocksConfig::default(), + deletion: crate::config::DeletionConfig::default(), + deletion_worker: crate::config::DeletionWorkerConfig::default(), paykit: None, } } diff --git a/locks-server/src/runtime.rs b/locks-server/src/runtime.rs index fbc11d3..312ff37 100644 --- a/locks-server/src/runtime.rs +++ b/locks-server/src/runtime.rs @@ -1,6 +1,425 @@ -use std::path::PathBuf; +use std::{ + collections::HashMap, future::Future, net::SocketAddr, path::PathBuf, pin::Pin, time::Duration, +}; -use crate::config::ConfigError; +use tokio::{ + net::TcpListener, + sync::watch, + task::{Id, JoinError, JoinSet}, +}; + +use crate::{ + app_state::{WorkerKind, WorkerReadiness, WorkerReadinessEvidence}, + config::ConfigError, +}; + +pub type RuntimeTaskFuture = Pin> + Send + 'static>>; +pub type ShutdownFuture = Pin + Send + 'static>>; +pub type HttpTaskFactory = + Box) -> RuntimeTaskFuture + Send + 'static>; +pub type LifecycleTaskFactory = + Box) -> RuntimeTaskFuture + Send + 'static>; +pub type WorkerTaskFactory = + Box RuntimeTaskFuture + Send + 'static>; + +#[derive(Debug)] +pub enum InitialStartupOutcome { + Ready { listener: TcpListener, initial: T }, + ShutdownRequested, +} + +#[derive(Debug, thiserror::Error)] +pub enum InitialStartupError { + #[error("failed to bind HTTP listener: {0}")] + Bind(#[source] std::io::Error), + #[error("initial startup task failed: {0}")] + Initial(#[source] anyhow::Error), + #[error("initial startup exceeded lifecycle timeout {0:?}")] + TimedOut(Duration), +} + +pub async fn bind_listener_then_run_initial( + bind_addr: SocketAddr, + lifecycle_timeout: Duration, + shutdown: &mut Pin>, + initial: F, +) -> Result, InitialStartupError> +where + S: Future + ?Sized, + F: FnOnce() -> Fut, + Fut: Future>, +{ + let listener = tokio::select! { + biased; + _ = shutdown.as_mut() => return Ok(InitialStartupOutcome::ShutdownRequested), + result = TcpListener::bind(bind_addr) => result.map_err(InitialStartupError::Bind)?, + }; + + let initial = tokio::time::timeout(lifecycle_timeout, initial()); + tokio::pin!(initial); + tokio::select! { + biased; + _ = shutdown.as_mut() => Ok(InitialStartupOutcome::ShutdownRequested), + result = &mut initial => match result { + Ok(Ok(initial)) => Ok(InitialStartupOutcome::Ready { listener, initial }), + Ok(Err(error)) => Err(InitialStartupError::Initial(error)), + Err(_) => Err(InitialStartupError::TimedOut(lifecycle_timeout)), + }, + } +} + +#[derive(Clone)] +pub struct WorkerTaskContext { + shutdown: watch::Receiver, + readiness: WorkerReadiness, + worker: WorkerKind, +} + +impl WorkerTaskContext { + pub fn shutdown(&self) -> watch::Receiver { + self.shutdown.clone() + } + + pub fn record(&self, evidence: WorkerReadinessEvidence) { + self.readiness.record(self.worker, evidence); + } +} + +#[derive(Default)] +pub struct RuntimeTasks { + shutdown_timeout: Duration, + http: Option, + pkarr_republisher: Option, + verification_worker: Option, + deletion_worker: Option, +} + +impl RuntimeTasks { + pub fn new(shutdown_timeout: Duration) -> Self { + Self { + shutdown_timeout, + ..Self::default() + } + } + + pub fn with_http(mut self, factory: HttpTaskFactory) -> Self { + self.http = Some(factory); + self + } + + pub fn with_pkarr_republisher(mut self, factory: LifecycleTaskFactory) -> Self { + self.pkarr_republisher = Some(factory); + self + } + + pub fn with_verification_worker(mut self, factory: WorkerTaskFactory) -> Self { + self.verification_worker = Some(factory); + self + } + + pub fn with_deletion_worker(mut self, factory: WorkerTaskFactory) -> Self { + self.deletion_worker = Some(factory); + self + } +} + +#[derive(Debug, thiserror::Error)] +pub enum RuntimeError { + #[error("{0} exited unexpectedly")] + UnexpectedExit(&'static str), + #[error("{task} failed: {error}")] + TaskFailed { + task: &'static str, + error: anyhow::Error, + }, + #[error("{0} panicked")] + TaskPanicked(&'static str), + #[error("runtime shutdown exceeded {0:?}")] + ShutdownTimedOut(Duration), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum OwnedTask { + Http, + PkarrRepublisher, + Verification, + Deletion, + Unknown, +} + +impl OwnedTask { + fn label(self) -> &'static str { + match self { + Self::Http => "HTTP server", + Self::PkarrRepublisher => "PKARR republisher", + Self::Verification => "verification worker", + Self::Deletion => "deletion worker", + Self::Unknown => "owned runtime task", + } + } + + fn worker(self) -> Option { + match self { + Self::Http | Self::PkarrRepublisher | Self::Unknown => None, + Self::Verification => Some(WorkerKind::Verification), + Self::Deletion => Some(WorkerKind::Deletion), + } + } +} + +pub async fn supervise( + readiness: WorkerReadiness, + mut shutdown: ShutdownFuture, + tasks: RuntimeTasks, +) -> Result<(), RuntimeError> { + let shutdown_timeout = tasks.shutdown_timeout; + let (worker_shutdown_tx, worker_shutdown_rx) = watch::channel(false); + let (service_shutdown_tx, service_shutdown_rx) = watch::channel(false); + let verification_enabled = tasks.verification_worker.is_some(); + let deletion_enabled = tasks.deletion_worker.is_some(); + let mut workers_remaining = usize::from(verification_enabled) + usize::from(deletion_enabled); + let mut roots = JoinSet::new(); + let mut root_tasks = HashMap::new(); + + if let Some(factory) = tasks.http { + spawn_root( + &mut roots, + &mut root_tasks, + OwnedTask::Http, + factory(service_shutdown_rx), + ); + } + if let Some(factory) = tasks.pkarr_republisher { + let shutdown = service_shutdown_tx.subscribe(); + spawn_root( + &mut roots, + &mut root_tasks, + OwnedTask::PkarrRepublisher, + factory(shutdown), + ); + } + if let Some(factory) = tasks.verification_worker { + let context = WorkerTaskContext { + shutdown: worker_shutdown_rx.clone(), + readiness: readiness.clone(), + worker: WorkerKind::Verification, + }; + spawn_root( + &mut roots, + &mut root_tasks, + OwnedTask::Verification, + factory(context), + ); + } + if let Some(factory) = tasks.deletion_worker { + let context = WorkerTaskContext { + shutdown: worker_shutdown_rx, + readiness: readiness.clone(), + worker: WorkerKind::Deletion, + }; + spawn_root( + &mut roots, + &mut root_tasks, + OwnedTask::Deletion, + factory(context), + ); + } + + let failure = tokio::select! { + biased; + _ = &mut shutdown => None, + completed = join_next_owned(&mut roots, &mut root_tasks) => { + if let Some((task, _)) = &completed + && task.worker().is_some() + { + workers_remaining = workers_remaining.saturating_sub(1); + } + Some(classify_premature(completed, &readiness)) + }, + }; + + begin_worker_shutdown( + &readiness, + verification_enabled, + deletion_enabled, + &worker_shutdown_tx, + ); + + let mut failure = failure; + let drain_result = tokio::time::timeout( + shutdown_timeout, + stop_workers_then_drain_services( + &mut roots, + &mut root_tasks, + workers_remaining, + &service_shutdown_tx, + &mut failure, + ), + ) + .await; + match drain_result { + Ok(()) => match failure { + Some(error) => Err(error), + None => Ok(()), + }, + Err(_) => { + roots.abort_all(); + while roots.join_next().await.is_some() {} + Err(failure.unwrap_or(RuntimeError::ShutdownTimedOut(shutdown_timeout))) + } + } +} + +fn spawn_root( + roots: &mut JoinSet<(OwnedTask, anyhow::Result<()>)>, + root_tasks: &mut HashMap, + task: OwnedTask, + future: RuntimeTaskFuture, +) { + let handle = roots.spawn(async move { (task, future.await) }); + root_tasks.insert(handle.id(), task); +} + +type OwnedCompletion = (OwnedTask, Result, JoinError>); + +async fn join_next_owned( + roots: &mut JoinSet<(OwnedTask, anyhow::Result<()>)>, + root_tasks: &mut HashMap, +) -> Option { + match roots.join_next_with_id().await { + Some(Ok((id, (task, result)))) => { + root_tasks.remove(&id); + Some((task, Ok(result))) + } + Some(Err(error)) => { + let task = root_tasks.remove(&error.id()).unwrap_or(OwnedTask::Unknown); + Some((task, Err(error))) + } + None => None, + } +} + +fn begin_worker_shutdown( + readiness: &WorkerReadiness, + verification_enabled: bool, + deletion_enabled: bool, + worker_shutdown_tx: &watch::Sender, +) { + if verification_enabled { + readiness.record(WorkerKind::Verification, WorkerReadinessEvidence::Stopping); + } + if deletion_enabled { + readiness.record(WorkerKind::Deletion, WorkerReadinessEvidence::Stopping); + } + worker_shutdown_tx.send_replace(true); +} + +async fn stop_workers_then_drain_services( + roots: &mut JoinSet<(OwnedTask, anyhow::Result<()>)>, + root_tasks: &mut HashMap, + mut workers_remaining: usize, + service_shutdown_tx: &watch::Sender, + failure: &mut Option, +) { + while workers_remaining > 0 { + match join_next_owned(roots, root_tasks).await { + Some((task, result)) if task.worker().is_some() => { + workers_remaining -= 1; + if let Some(error) = classify_shutdown_completion(task, result, false) { + failure.get_or_insert(error); + } + } + Some((task, result)) => { + if let Some(error) = classify_shutdown_completion(task, result, true) { + failure.get_or_insert(error); + } + } + None => { + failure.get_or_insert(RuntimeError::UnexpectedExit("runtime task set")); + break; + } + } + } + + // Worker root completion is explicit evidence that no future queue claim can begin. + // Only now may HTTP graceful draining and other service-root cancellation start. + service_shutdown_tx.send_replace(true); + drain(roots, root_tasks, failure).await; +} + +fn classify_premature( + completed: Option, + readiness: &WorkerReadiness, +) -> RuntimeError { + match completed { + Some((task, Ok(result))) => { + if let Some(worker) = task.worker() { + readiness.record(worker, WorkerReadinessEvidence::UnexpectedExit); + } + match result { + Ok(()) => RuntimeError::UnexpectedExit(task.label()), + Err(error) => RuntimeError::TaskFailed { + task: task.label(), + error, + }, + } + } + Some((task, Err(error))) => { + if let Some(worker) = task.worker() { + readiness.record(worker, WorkerReadinessEvidence::UnexpectedExit); + } + classify_join_error(task, error) + } + None => RuntimeError::UnexpectedExit("runtime task set"), + } +} + +fn classify_shutdown_completion( + task: OwnedTask, + result: Result, JoinError>, + success_is_unexpected: bool, +) -> Option { + match result { + Ok(Ok(())) if success_is_unexpected => Some(RuntimeError::UnexpectedExit(task.label())), + Ok(Ok(())) => None, + Ok(Err(error)) => Some(RuntimeError::TaskFailed { + task: task.label(), + error, + }), + Err(error) => Some(classify_join_error(task, error)), + } +} + +fn classify_join_error(task: OwnedTask, error: JoinError) -> RuntimeError { + if error.is_panic() { + RuntimeError::TaskPanicked(task.label()) + } else { + RuntimeError::UnexpectedExit(task.label()) + } +} + +async fn drain( + roots: &mut JoinSet<(OwnedTask, anyhow::Result<()>)>, + root_tasks: &mut HashMap, + failure: &mut Option, +) { + while let Some((task, result)) = join_next_owned(roots, root_tasks).await { + if let Some(error) = classify_shutdown_completion(task, result, false) { + failure.get_or_insert(error); + } + } +} + +pub async fn wait_for_shutdown(mut shutdown: watch::Receiver) { + if *shutdown.borrow() { + return; + } + while shutdown.changed().await.is_ok() { + if *shutdown.borrow() { + return; + } + } +} pub fn parse_config_arg(args: I) -> Result, ConfigError> where @@ -45,7 +464,348 @@ pub fn home_dir_from_env() -> Result { #[cfg(test)] mod tests { - use super::parse_config_arg; + use std::{ + sync::{Arc, Mutex}, + time::{Duration, Instant}, + }; + + use tokio::sync::{Notify, oneshot}; + + use crate::app_state::{ + WorkerKind, WorkerReadiness, WorkerReadinessEvidence, WorkerReadinessState, + }; + + use super::{RuntimeTasks, parse_config_arg, supervise, wait_for_shutdown}; + + #[tokio::test] + async fn premature_worker_success_fails_runtime_and_marks_worker_unexpected() { + let readiness = WorkerReadiness::new(true, false); + let observed = readiness.clone(); + let tasks = RuntimeTasks::new(Duration::from_millis(100)) + .with_verification_worker(Box::new(|_context| Box::pin(async { Ok(()) }))); + + let error = supervise(readiness, Box::pin(std::future::pending()), tasks) + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("verification worker exited unexpectedly") + ); + assert_eq!( + observed.worker_state(WorkerKind::Verification), + WorkerReadinessState::NotReady + ); + } + + #[tokio::test] + async fn worker_error_and_panic_fail_runtime() { + for tasks in [ + RuntimeTasks::new(Duration::from_millis(100)).with_verification_worker(Box::new( + |_context| Box::pin(async { anyhow::bail!("dependency broke") }), + )), + RuntimeTasks::new(Duration::from_millis(100)) + .with_verification_worker(Box::new(|_context| Box::pin(async { panic!("boom") }))), + ] { + let error = supervise( + WorkerReadiness::new(true, false), + Box::pin(std::future::pending()), + tasks, + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("failed") || error.to_string().contains("panicked")); + } + } + + #[tokio::test] + async fn worker_panic_starts_service_drain_promptly_and_preserves_panic_verdict() { + let http_started = Arc::new(Notify::new()); + let http_draining = Arc::new(Notify::new()); + let (panic_tx, panic_rx) = oneshot::channel(); + let tasks = RuntimeTasks::new(Duration::from_secs(5)) + .with_http(Box::new({ + let http_started = Arc::clone(&http_started); + let http_draining = Arc::clone(&http_draining); + move |shutdown| { + Box::pin(async move { + http_started.notify_one(); + wait_for_shutdown(shutdown).await; + http_draining.notify_one(); + Ok(()) + }) + } + })) + .with_verification_worker(Box::new(move |_context| { + Box::pin(async move { + let _ = panic_rx.await; + panic!("boom"); + }) + })); + let supervisor = tokio::spawn(supervise( + WorkerReadiness::new(true, false), + Box::pin(std::future::pending()), + tasks, + )); + + http_started.notified().await; + panic_tx.send(()).unwrap(); + let error = tokio::time::timeout(Duration::from_millis(200), supervisor) + .await + .expect("worker panic should not wait for the shutdown deadline") + .unwrap() + .unwrap_err(); + + tokio::time::timeout(Duration::from_millis(100), http_draining.notified()) + .await + .expect("worker panic should signal service draining"); + assert!(matches!( + error, + super::RuntimeError::TaskPanicked("verification worker") + )); + } + + #[tokio::test] + async fn pkarr_republisher_failure_fails_runtime() { + let tasks = RuntimeTasks::new(Duration::from_millis(100)).with_pkarr_republisher(Box::new( + |_shutdown| Box::pin(async { anyhow::bail!("publish failed") }), + )); + + let error = supervise( + WorkerReadiness::new(false, false), + Box::pin(std::future::pending()), + tasks, + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("PKARR republisher failed")); + } + + #[tokio::test] + async fn readiness_is_not_ready_before_workers_observe_sticky_cancellation() { + let readiness = WorkerReadiness::new(true, true); + let observed = readiness.clone(); + let observations = Arc::new(Mutex::new(Vec::new())); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let tasks = RuntimeTasks::new(Duration::from_millis(100)) + .with_verification_worker(observing_worker( + observed.clone(), + Arc::clone(&observations), + WorkerKind::Verification, + )) + .with_deletion_worker(observing_worker( + observed, + Arc::clone(&observations), + WorkerKind::Deletion, + )); + shutdown_tx.send(()).unwrap(); + + supervise( + readiness, + Box::pin(async { + let _ = shutdown_rx.await; + }), + tasks, + ) + .await + .unwrap(); + + assert_eq!( + *observations.lock().unwrap(), + vec![ + WorkerReadinessState::NotReady, + WorkerReadinessState::NotReady + ] + ); + } + + fn observing_worker( + readiness: WorkerReadiness, + observations: Arc>>, + kind: WorkerKind, + ) -> super::WorkerTaskFactory { + Box::new(move |context| { + Box::pin(async move { + wait_for_shutdown(context.shutdown()).await; + observations + .lock() + .unwrap() + .push(readiness.worker_state(kind)); + Ok(()) + }) + }) + } + + #[tokio::test] + async fn http_drain_waits_for_explicit_worker_claim_stop() { + let worker_stopping = Arc::new(Notify::new()); + let allow_worker_stop = Arc::new(Notify::new()); + let http_draining = Arc::new(Notify::new()); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let tasks = RuntimeTasks::new(Duration::from_secs(1)) + .with_http(Box::new({ + let http_draining = Arc::clone(&http_draining); + move |shutdown| { + Box::pin(async move { + wait_for_shutdown(shutdown).await; + http_draining.notify_one(); + Ok(()) + }) + } + })) + .with_verification_worker(Box::new({ + let worker_stopping = Arc::clone(&worker_stopping); + let allow_worker_stop = Arc::clone(&allow_worker_stop); + move |context| { + Box::pin(async move { + wait_for_shutdown(context.shutdown()).await; + worker_stopping.notify_one(); + allow_worker_stop.notified().await; + Ok(()) + }) + } + })); + let supervisor = tokio::spawn(supervise( + WorkerReadiness::new(true, false), + Box::pin(async { + let _ = shutdown_rx.await; + }), + tasks, + )); + + shutdown_tx.send(()).unwrap(); + worker_stopping.notified().await; + assert!( + tokio::time::timeout(Duration::from_millis(20), http_draining.notified()) + .await + .is_err(), + "HTTP draining started before the verification worker stopped claiming" + ); + allow_worker_stop.notify_one(); + supervisor.await.unwrap().unwrap(); + tokio::time::timeout(Duration::from_millis(100), http_draining.notified()) + .await + .unwrap(); + } + + #[tokio::test] + async fn graceful_shutdown_joins_all_roots_and_late_subscriber_exits() { + let joined = Arc::new(Mutex::new(Vec::new())); + let late_subscription_exited = Arc::new(Notify::new()); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let tasks = RuntimeTasks::new(Duration::from_secs(1)) + .with_http(joining_task(Arc::clone(&joined), "http")) + .with_verification_worker(Box::new({ + let joined = Arc::clone(&joined); + let late_subscription_exited = Arc::clone(&late_subscription_exited); + move |context| { + Box::pin(async move { + wait_for_shutdown(context.shutdown()).await; + wait_for_shutdown(context.shutdown()).await; + late_subscription_exited.notify_one(); + joined.lock().unwrap().push("verification"); + Ok(()) + }) + } + })) + .with_deletion_worker(Box::new({ + let joined = Arc::clone(&joined); + move |context| { + Box::pin(async move { + wait_for_shutdown(context.shutdown()).await; + joined.lock().unwrap().push("deletion"); + Ok(()) + }) + } + })); + shutdown_tx.send(()).unwrap(); + + supervise( + WorkerReadiness::new(true, true), + Box::pin(async { + let _ = shutdown_rx.await; + }), + tasks, + ) + .await + .unwrap(); + tokio::time::timeout( + Duration::from_millis(100), + late_subscription_exited.notified(), + ) + .await + .unwrap(); + let mut roots = joined.lock().unwrap().clone(); + roots.sort_unstable(); + assert_eq!(roots, vec!["deletion", "http", "verification"]); + } + + fn joining_task( + joined: Arc>>, + name: &'static str, + ) -> super::HttpTaskFactory { + Box::new(move |shutdown| { + Box::pin(async move { + wait_for_shutdown(shutdown).await; + joined.lock().unwrap().push(name); + Ok(()) + }) + }) + } + + #[tokio::test] + async fn blocked_root_is_aborted_and_drained_at_deadline() { + let dropped = Arc::new(Notify::new()); + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let tasks = + RuntimeTasks::new(Duration::from_millis(20)).with_verification_worker(Box::new({ + let dropped = Arc::clone(&dropped); + move |_context| { + Box::pin(async move { + let _guard = DropSignal(dropped); + std::future::pending::<()>().await; + Ok(()) + }) + } + })); + shutdown_tx.send(()).unwrap(); + let started = Instant::now(); + + let error = supervise( + WorkerReadiness::new(true, false), + Box::pin(async { + let _ = shutdown_rx.await; + }), + tasks, + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("shutdown exceeded")); + assert!(started.elapsed() < Duration::from_millis(250)); + tokio::time::timeout(Duration::from_millis(100), dropped.notified()) + .await + .unwrap(); + } + + struct DropSignal(Arc); + impl Drop for DropSignal { + fn drop(&mut self) { + self.0.notify_one(); + } + } + + #[test] + fn readiness_evidence_can_be_recorded_through_worker_context_contract() { + let readiness = WorkerReadiness::new(true, false); + readiness.record(WorkerKind::Verification, WorkerReadinessEvidence::Ready); + assert_eq!( + readiness.worker_state(WorkerKind::Verification), + WorkerReadinessState::Ready + ); + } #[test] fn parse_config_arg_accepts_absent_or_explicit_config() { diff --git a/locks-server/src/storage.rs b/locks-server/src/storage.rs index 94c963e..bdea208 100644 --- a/locks-server/src/storage.rs +++ b/locks-server/src/storage.rs @@ -1,6 +1,8 @@ +use locks_service::infrastructure::final_credentials::FinalCredentialCipher; use locks_service::infrastructure::postgres::{ CreatorAuthoritySecretCipher, PostgresError, run_migrations, }; +use locks_service::infrastructure::runtime_master_key::RuntimeMasterKey; use sqlx::postgres::PgPoolOptions; use crate::app_state::AppState; @@ -13,10 +15,10 @@ pub enum RuntimeStorageError { Connect(#[from] sqlx::Error), #[error("failed to run postgres runtime migrations: {0}")] Migrate(#[from] PostgresError), - #[error("creator authority encryption key env var is not set: {0}")] - MissingCreatorAuthorityEncryptionKeyEnv(String), - #[error("invalid creator authority encryption key in env var: {0}")] - InvalidCreatorAuthorityEncryptionKey(String), + #[error("runtime master key env var is not set: {0}")] + MissingRuntimeMasterKeyEnv(String), + #[error("invalid runtime master key in env var: {0}")] + InvalidRuntimeMasterKey(String), } /// Builds application state for the production-shaped runtime. @@ -30,7 +32,8 @@ pub enum RuntimeStorageError { pub async fn build_runtime_state( config: LockServerRuntimeConfig, ) -> Result { - let creator_authority_cipher = creator_authority_cipher_from_env(&config.secrets)?; + let (creator_authority_cipher, final_credential_cipher) = + runtime_ciphers_from_env(&config.secrets)?; let pool = connect_database(&config.database).await?; if config.database.run_migrations_on_startup { run_migrations(&pool).await?; @@ -40,22 +43,30 @@ pub async fn build_runtime_state( config, pool, creator_authority_cipher, + final_credential_cipher, )) } +#[cfg(test)] fn creator_authority_cipher_from_env( config: &SecretsConfig, ) -> Result { - let key = std::env::var(&config.creator_authority_key_env).map_err(|_| { - RuntimeStorageError::MissingCreatorAuthorityEncryptionKeyEnv( - config.creator_authority_key_env.clone(), - ) + runtime_ciphers_from_env(config).map(|(creator, _)| creator) +} + +fn runtime_ciphers_from_env( + config: &SecretsConfig, +) -> Result<(CreatorAuthoritySecretCipher, FinalCredentialCipher), RuntimeStorageError> { + let key = std::env::var(&config.runtime_master_key_env).map_err(|_| { + RuntimeStorageError::MissingRuntimeMasterKeyEnv(config.runtime_master_key_env.clone()) })?; - CreatorAuthoritySecretCipher::from_base64url_key(&key).map_err(|_| { - RuntimeStorageError::InvalidCreatorAuthorityEncryptionKey( - config.creator_authority_key_env.clone(), - ) - }) + let master_key = RuntimeMasterKey::from_base64url(&key).map_err(|_| { + RuntimeStorageError::InvalidRuntimeMasterKey(config.runtime_master_key_env.clone()) + })?; + Ok(( + CreatorAuthoritySecretCipher::new(master_key.creator_authority_key()), + FinalCredentialCipher::new(master_key.final_credential_key()), + )) } async fn connect_database(config: &DatabaseConfig) -> Result { @@ -81,7 +92,7 @@ mod tests { std::env::set_var(&env_name, URL_SAFE_NO_PAD.encode([7u8; 32])); } let config = SecretsConfig { - creator_authority_key_env: env_name.clone(), + runtime_master_key_env: env_name.clone(), }; let cipher = creator_authority_cipher_from_env(&config).unwrap(); @@ -99,14 +110,14 @@ mod tests { std::env::remove_var(&env_name); } let config = SecretsConfig { - creator_authority_key_env: env_name.clone(), + runtime_master_key_env: env_name.clone(), }; let error = creator_authority_cipher_from_env(&config).unwrap_err(); assert_eq!( error.to_string(), - format!("creator authority encryption key env var is not set: {env_name}") + format!("runtime master key env var is not set: {env_name}") ); } @@ -117,14 +128,14 @@ mod tests { std::env::set_var(&env_name, "not-a-valid-key"); } let config = SecretsConfig { - creator_authority_key_env: env_name.clone(), + runtime_master_key_env: env_name.clone(), }; let error = creator_authority_cipher_from_env(&config).unwrap_err(); assert!(matches!( error, - RuntimeStorageError::InvalidCreatorAuthorityEncryptionKey(ref name) if name == &env_name + RuntimeStorageError::InvalidRuntimeMasterKey(ref name) if name == &env_name )); let debug = format!("{error:?}"); assert!(!debug.contains("not-a-valid-key")); @@ -141,21 +152,21 @@ mod tests { } let mut config = TestServerApp::default_in_memory_config(); config.secrets = SecretsConfig { - creator_authority_key_env: env_name.clone(), + runtime_master_key_env: env_name.clone(), }; let error = build_runtime_state(config).await.unwrap_err(); assert!(matches!( error, - RuntimeStorageError::MissingCreatorAuthorityEncryptionKeyEnv(ref name) if name == &env_name + RuntimeStorageError::MissingRuntimeMasterKeyEnv(ref name) if name == &env_name )); assert!(!error.to_string().contains("postgres://")); } fn unique_env_name(suffix: &str) -> String { format!( - "LOCKS_TEST_CREATOR_AUTH_KEY_{}_{}", + "LOCKS_TEST_RUNTIME_MASTER_KEY_{}_{}", suffix, uuid::Uuid::new_v4().simple() ) diff --git a/locks-server/src/testing.rs b/locks-server/src/testing.rs index 758f9a2..a40165d 100644 --- a/locks-server/src/testing.rs +++ b/locks-server/src/testing.rs @@ -11,9 +11,11 @@ use locks_service::application::models::{ }; use locks_service::application::ports::LegacyCreatorConnectFlowClient; use locks_service::infrastructure::memory::{ + content_lock_tombstones::InMemoryContentLockTombstoneRepository, content_locks::InMemoryContentLockRepository, entitlements::InMemoryEntitlementRepository, guarded_resources::InMemoryGuardedResourceRepository, lock_service_pointers::InMemoryLockServicePointerRepository, + public_content_locks::InMemoryPublicContentLockStore, }; use locks_service::infrastructure::pubky::PubkyHomeserverStorageClient; use time::OffsetDateTime; @@ -40,10 +42,16 @@ impl TestServerApp { } pub fn new_in_memory(config: LockServerRuntimeConfig) -> Self { + let public_content_locks = InMemoryPublicContentLockStore::new(); Self { state: AppState::new_empty_in_memory_with_creator_repositories( config, - Arc::new(InMemoryContentLockRepository::new()), + Arc::new(InMemoryContentLockRepository::with_public_store( + public_content_locks.clone(), + )), + Arc::new(InMemoryContentLockTombstoneRepository::with_public_store( + public_content_locks, + )), Arc::new(InMemoryGuardedResourceRepository::new()), Arc::new(InMemoryLockServicePointerRepository::new()), Arc::new(InMemoryEntitlementRepository::new()), @@ -100,6 +108,8 @@ impl TestServerApp { pkdns: crate::config::PkdnsConfig::default(), rate_limits: RateLimitsConfig::default(), content_locks: ContentLocksConfig::default(), + deletion: crate::config::DeletionConfig::default(), + deletion_worker: crate::config::DeletionWorkerConfig::default(), paykit: None, } } diff --git a/locks-server/src/worker.rs b/locks-server/src/worker.rs index a85d8aa..47c7b4d 100644 --- a/locks-server/src/worker.rs +++ b/locks-server/src/worker.rs @@ -1,5 +1,12 @@ +use std::sync::atomic::{AtomicU8, Ordering}; + +use async_trait::async_trait; use locks_core::ids::{LockServerPubky, TaskId}; +use locks_core::verification::CriterionVerificationResult; use locks_service::application::errors::ApplicationError; +use locks_service::application::models::{ + ClaimedVerificationTask, CriterionVerificationRequest, VerificationTaskRecord, +}; use locks_service::application::ports::{ Clock, ContentLockRepository, CriterionVerifier, EntitlementRepository, VerificationTaskClaimer, VerificationTaskRepository, @@ -11,15 +18,333 @@ use locks_service::infrastructure::verifiers::registry::StaticCriterionVerifierR use tokio::sync::watch; use tracing::{debug, error, info}; -use crate::app_state::AppState; +use crate::app_state::{AppState, WorkerKind, WorkerReadiness, WorkerReadinessEvidence}; const PENDING_VERIFICATION_RETRY_DELAY_SECONDS: i64 = 30; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PaykitProviderEvidence { + None, + HealthyResponse, + Unavailable, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +enum OperationEvidence { + #[default] + None, + Succeeded, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +struct RepositoryOperationEvidence { + queue_poll: OperationEvidence, + mutation: OperationEvidence, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct VerificationPoll { + tick: WorkerTick, + paykit_provider: PaykitProviderEvidence, + repository: RepositoryOperationEvidence, +} + +impl VerificationPoll { + fn without_provider_evidence(tick: WorkerTick) -> Self { + Self { + tick, + paykit_provider: PaykitProviderEvidence::None, + repository: RepositoryOperationEvidence::default(), + } + } +} + +#[derive(Debug)] +struct VerificationPollError { + operation: RepositoryOperation, + error: ApplicationError, + repository: RepositoryOperationEvidence, +} + +#[derive(Default)] +struct RepositoryOperationObserver { + queue_poll_succeeded: AtomicU8, + mutation_succeeded: AtomicU8, +} + +impl RepositoryOperationObserver { + fn record_queue_poll_success(&self) { + self.queue_poll_succeeded.store(1, Ordering::Relaxed); + } + + fn record_mutation_success(&self) { + self.mutation_succeeded.store(1, Ordering::Relaxed); + } + + fn evidence(&self) -> RepositoryOperationEvidence { + RepositoryOperationEvidence { + queue_poll: if self.queue_poll_succeeded.load(Ordering::Relaxed) == 1 { + OperationEvidence::Succeeded + } else { + OperationEvidence::None + }, + mutation: if self.mutation_succeeded.load(Ordering::Relaxed) == 1 { + OperationEvidence::Succeeded + } else { + OperationEvidence::None + }, + } + } +} + +struct ObservedVerificationTaskClaimer<'a> { + inner: &'a dyn VerificationTaskClaimer, + observer: &'a RepositoryOperationObserver, +} + +#[async_trait] +impl VerificationTaskClaimer for ObservedVerificationTaskClaimer<'_> { + async fn begin_claimed_entitlement_publication( + &self, + task_id: &TaskId, + worker_id: &str, + claim_token: &uuid::Uuid, + ) -> Result { + let result = self + .inner + .begin_claimed_entitlement_publication(task_id, worker_id, claim_token) + .await; + if matches!(result, Ok(true)) { + self.observer.record_mutation_success(); + } + result + } + + async fn claim_next_verification_task( + &self, + worker_id: &str, + claim_ttl: time::Duration, + ) -> Result, ApplicationError> { + let result = self + .inner + .claim_next_verification_task(worker_id, claim_ttl) + .await; + if result.is_ok() { + self.observer.record_queue_poll_success(); + } + result + } + + async fn schedule_verification_task_retry( + &self, + task_id: &TaskId, + worker_id: &str, + claim_token: &uuid::Uuid, + retry_after: time::Duration, + ) -> Result, ApplicationError> { + let result = self + .inner + .schedule_verification_task_retry(task_id, worker_id, claim_token, retry_after) + .await; + if matches!(result, Ok(Some(_))) { + self.observer.record_mutation_success(); + } + result + } + + async fn persist_claimed_verification_task_transition( + &self, + task: VerificationTaskRecord, + worker_id: &str, + claim_token: &uuid::Uuid, + ) -> Result, ApplicationError> { + let result = self + .inner + .persist_claimed_verification_task_transition(task, worker_id, claim_token) + .await; + if matches!(result, Ok(Some(_))) { + self.observer.record_mutation_success(); + } + result + } +} + +struct ObservedPaykitVerifier<'a> { + inner: &'a dyn CriterionVerifier, + evidence: AtomicU8, +} + +impl<'a> ObservedPaykitVerifier<'a> { + const HEALTHY: u8 = 1; + const UNAVAILABLE: u8 = 2; + + fn new(inner: &'a dyn CriterionVerifier) -> Self { + Self { + inner, + evidence: AtomicU8::new(0), + } + } + + fn evidence(&self) -> PaykitProviderEvidence { + match self.evidence.load(Ordering::Relaxed) { + Self::HEALTHY => PaykitProviderEvidence::HealthyResponse, + Self::UNAVAILABLE => PaykitProviderEvidence::Unavailable, + _ => PaykitProviderEvidence::None, + } + } +} + +#[async_trait] +impl CriterionVerifier for ObservedPaykitVerifier<'_> { + async fn verify( + &self, + request: CriterionVerificationRequest, + ) -> Result { + let result = self.inner.verify(request).await; + match &result { + Err(ApplicationError::VerificationDependencyUnavailable) => { + self.evidence.store(Self::UNAVAILABLE, Ordering::Relaxed); + } + Ok(_) | Err(ApplicationError::VerificationPending) => { + self.evidence + .compare_exchange(0, Self::HEALTHY, Ordering::Relaxed, Ordering::Relaxed) + .ok(); + } + Err(_) => {} + } + result + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum VerificationPollErrorDisposition { + RetryableRepository, + Fatal(&'static str), +} + +fn classify_verification_poll_error(error: &ApplicationError) -> VerificationPollErrorDisposition { + match error { + ApplicationError::Storage { .. } => VerificationPollErrorDisposition::RetryableRepository, + ApplicationError::InvalidVerificationTaskState { .. } + | ApplicationError::InvalidVerificationTaskTransition { .. } + | ApplicationError::InvalidVerificationTaskFailureMessage => { + VerificationPollErrorDisposition::Fatal("invalid_verification_task_state") + } + _ => VerificationPollErrorDisposition::Fatal("unexpected_application_error"), + } +} + +fn redacted_fatal_verification_error(error_class: &'static str) -> ApplicationError { + ApplicationError::InvalidVerificationTaskState { + message: format!("verification worker terminated after {error_class}"), + } +} + +fn is_terminal_business_failure(error: &ApplicationError) -> bool { + matches!( + error, + ApplicationError::ContentLockUnavailable + | ApplicationError::EntitlementNotSatisfied + | ApplicationError::UnsupportedVerifierType { .. } + | ApplicationError::Verifier { .. } + | ApplicationError::ContentLockHashMismatch { .. } + | ApplicationError::ContentLockCanonicalization { .. } + | ApplicationError::EmptyContentLockCriteria + | ApplicationError::DuplicateContentLockCriterion { .. } + | ApplicationError::DuplicateVerificationResultCriterion { .. } + | ApplicationError::UnknownVerificationResultCriterion { .. } + ) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RepositoryOperation { + QueuePoll, + Mutation, +} + +#[derive(Debug, Default)] +struct VerificationReadinessRecovery { + paykit_provider_degraded: bool, + queue_poll_degraded: bool, + mutation_degraded: bool, +} + +impl VerificationReadinessRecovery { + fn record_poll(&mut self, poll: VerificationPoll, readiness: &WorkerReadiness) { + self.record_repository_evidence(poll.repository, readiness); + match poll.paykit_provider { + PaykitProviderEvidence::HealthyResponse => { + self.paykit_provider_degraded = false; + self.record_ready_if_healthy(readiness); + } + PaykitProviderEvidence::Unavailable => { + self.paykit_provider_degraded = true; + readiness.record( + WorkerKind::Verification, + WorkerReadinessEvidence::TransientDependencyFailure, + ); + } + PaykitProviderEvidence::None => {} + } + } + + fn record_repository_evidence( + &mut self, + evidence: RepositoryOperationEvidence, + readiness: &WorkerReadiness, + ) { + if evidence.queue_poll == OperationEvidence::Succeeded { + self.record_repository_success(RepositoryOperation::QueuePoll, readiness); + } + if evidence.mutation == OperationEvidence::Succeeded { + self.record_repository_success(RepositoryOperation::Mutation, readiness); + } + } + + fn record_repository_failure( + &mut self, + operation: RepositoryOperation, + readiness: &WorkerReadiness, + ) { + match operation { + RepositoryOperation::QueuePoll => self.queue_poll_degraded = true, + RepositoryOperation::Mutation => self.mutation_degraded = true, + } + readiness.record( + WorkerKind::Verification, + WorkerReadinessEvidence::TransientDependencyFailure, + ); + } + + fn record_repository_success( + &mut self, + operation: RepositoryOperation, + readiness: &WorkerReadiness, + ) { + match operation { + RepositoryOperation::QueuePoll => self.queue_poll_degraded = false, + RepositoryOperation::Mutation => self.mutation_degraded = false, + } + self.record_ready_if_healthy(readiness); + } + + fn record_ready_if_healthy(&self, readiness: &WorkerReadiness) { + if !self.paykit_provider_degraded && !self.queue_poll_degraded && !self.mutation_degraded { + readiness.record( + WorkerKind::Verification, + WorkerReadinessEvidence::DependencySucceeded, + ); + } + } +} + /// Result of one worker polling attempt. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum WorkerTick { + Cancelled, Idle, Completed(TaskId), + VerificationPendingRetryScheduled(TaskId), RetryScheduled(TaskId), Failed(TaskId), } @@ -96,24 +421,99 @@ impl<'a> VerificationWorker<'a> { } pub async fn run_once(&self) -> Result { + self.run_once_with_recovery_evidence() + .await + .map(|poll| poll.tick) + .map_err(|failure| failure.error) + } + + async fn run_once_with_recovery_evidence( + &self, + ) -> Result { + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + self.run_once_until_shutdown_with_recovery_evidence(&shutdown_rx) + .await + } + + pub async fn run_once_until_shutdown( + &self, + shutdown: &watch::Receiver, + ) -> Result { + self.run_once_until_shutdown_with_recovery_evidence(shutdown) + .await + .map(|poll| poll.tick) + .map_err(|failure| failure.error) + } + + async fn run_once_until_shutdown_with_recovery_evidence( + &self, + shutdown: &watch::Receiver, + ) -> Result { + let observer = RepositoryOperationObserver::default(); + let claimer = ObservedVerificationTaskClaimer { + inner: self.claimer, + observer: &observer, + }; + match self.run_once_until_shutdown_inner(shutdown, &claimer).await { + Ok(mut poll) => { + poll.repository = observer.evidence(); + Ok(poll) + } + Err((operation, error)) => Err(VerificationPollError { + operation, + error, + repository: observer.evidence(), + }), + } + } + + async fn run_once_until_shutdown_inner( + &self, + shutdown: &watch::Receiver, + claimer: &dyn VerificationTaskClaimer, + ) -> Result { + if *shutdown.borrow() { + return Ok(VerificationPoll::without_provider_evidence( + WorkerTick::Cancelled, + )); + } let now = self.clock.now(); let claim_expires_at = now + claim_timeout(self.claim_timeout_seconds); - let Some(claim) = self - .claimer - .claim_next_verification_task(&self.worker_id, now, claim_expires_at) - .await? + let Some(claim) = claimer + .claim_next_verification_task(&self.worker_id, (claim_expires_at) - (now)) + .await + .map_err(|error| (RepositoryOperation::QueuePoll, error))? else { - return Ok(WorkerTick::Idle); + return Ok(VerificationPoll::without_provider_evidence( + WorkerTick::Idle, + )); }; let task_id = claim.task.task_id; let claim_token = claim.claim_token; + if *shutdown.borrow() { + let _ = claimer + .schedule_verification_task_retry( + &task_id, + &self.worker_id, + &claim_token, + time::Duration::ZERO, + ) + .await + .map_err(|error| (RepositoryOperation::Mutation, error))?; + return Ok(VerificationPoll::without_provider_evidence( + WorkerTick::Cancelled, + )); + } debug!(%task_id, worker_id = %self.worker_id, "claimed verification task"); let mut verifiers = StaticCriterionVerifierRegistry::new(); if self.allow_dev_static_verifier { verifiers = verifiers.with_dev_static(self.dev_static_verifier); } - if let Some(paykit_payment_verifier) = self.paykit_payment_verifier { + let observed_paykit_verifier = self + .paykit_payment_verifier + .map(ObservedPaykitVerifier::new); + if let Some(paykit_payment_verifier) = observed_paykit_verifier.as_ref() { verifiers = verifiers.with_paykit_payment(paykit_payment_verifier); } let use_case = CompleteVerificationTaskUseCase::new( @@ -125,39 +525,80 @@ impl<'a> VerificationWorker<'a> { self.verified_by.clone(), ); - match use_case + let tick = match use_case .execute_claimed( CompleteVerificationTaskRequest { task_id }, claim, &self.worker_id, - self.claimer, + claimer, ) .await { Ok(completed) => { info!(%task_id, status = ?completed.status, "completed verification task"); - Ok(WorkerTick::Completed(task_id)) + WorkerTick::Completed(task_id) + } + Err(ApplicationError::VerificationDependencyUnavailable) => { + let retry_scheduled_at = self.clock.now(); + let next_attempt_at = retry_scheduled_at + retry_delay(); + let Some(_) = claimer + .schedule_verification_task_retry( + &task_id, + &self.worker_id, + &claim_token, + (next_attempt_at) - (retry_scheduled_at), + ) + .await + .map_err(|error| (RepositoryOperation::Mutation, error))? + else { + info!( + %task_id, + worker_id = %self.worker_id, + "verification task claim no longer owned; retry not scheduled" + ); + return Ok(VerificationPoll { + tick: WorkerTick::Idle, + paykit_provider: observed_paykit_verifier.as_ref().map_or( + PaykitProviderEvidence::None, + ObservedPaykitVerifier::evidence, + ), + repository: RepositoryOperationEvidence::default(), + }); + }; + debug!( + %task_id, + worker_id = %self.worker_id, + %next_attempt_at, + "scheduled verification task retry after dependency failure" + ); + WorkerTick::RetryScheduled(task_id) } Err(ApplicationError::VerificationPending) => { let retry_scheduled_at = self.clock.now(); let next_attempt_at = retry_scheduled_at + retry_delay(); - let Some(_) = self - .claimer + let Some(_) = claimer .schedule_verification_task_retry( &task_id, &self.worker_id, &claim_token, - retry_scheduled_at, - next_attempt_at, + (next_attempt_at) - (retry_scheduled_at), ) - .await? + .await + .map_err(|error| (RepositoryOperation::Mutation, error))? else { info!( %task_id, worker_id = %self.worker_id, "verification task claim no longer owned; retry not scheduled" ); - return Ok(WorkerTick::Idle); + return Ok(VerificationPoll { + tick: WorkerTick::Idle, + paykit_provider: observed_paykit_verifier.as_ref().map_or( + PaykitProviderEvidence::None, + ObservedPaykitVerifier::evidence, + ), + repository: RepositoryOperationEvidence::default(), + }); }; debug!( %task_id, @@ -165,7 +606,7 @@ impl<'a> VerificationWorker<'a> { %next_attempt_at, "scheduled verification task retry" ); - Ok(WorkerTick::RetryScheduled(task_id)) + WorkerTick::VerificationPendingRetryScheduled(task_id) } Err(ApplicationError::VerificationTaskClaimLost) => { info!( @@ -173,13 +614,27 @@ impl<'a> VerificationWorker<'a> { worker_id = %self.worker_id, "verification task claim no longer owned; terminal state not persisted" ); - Ok(WorkerTick::Idle) + WorkerTick::Idle } - Err(error) => { - error!(%task_id, error = %error, "verification task failed"); - Ok(WorkerTick::Failed(task_id)) + Err(error) if is_terminal_business_failure(&error) => { + error!( + %task_id, + error_class = "terminal_business_failure", + retrying = false, + "verification task failed" + ); + WorkerTick::Failed(task_id) } - } + Err(error) => return Err((RepositoryOperation::Mutation, error)), + }; + Ok(VerificationPoll { + tick, + paykit_provider: observed_paykit_verifier.as_ref().map_or( + PaykitProviderEvidence::None, + ObservedPaykitVerifier::evidence, + ), + repository: RepositoryOperationEvidence::default(), + }) } pub async fn run_until_shutdown( @@ -191,7 +646,8 @@ impl<'a> VerificationWorker<'a> { return Ok(()); } - match self.run_once().await? { + match self.run_once_until_shutdown(&shutdown).await? { + WorkerTick::Cancelled => return Ok(()), WorkerTick::Idle => { tokio::select! { _ = shutdown.changed() => { @@ -203,11 +659,91 @@ impl<'a> VerificationWorker<'a> { } } WorkerTick::Completed(_) + | WorkerTick::VerificationPendingRetryScheduled(_) | WorkerTick::RetryScheduled(_) | WorkerTick::Failed(_) => {} } } } + + pub async fn run_until_shutdown_with_readiness( + &self, + mut shutdown: watch::Receiver, + readiness: &WorkerReadiness, + ) -> Result<(), ApplicationError> { + let mut recovery = VerificationReadinessRecovery::default(); + loop { + if *shutdown.borrow() { + readiness.record(WorkerKind::Verification, WorkerReadinessEvidence::Stopped); + return Ok(()); + } + + match self + .run_once_until_shutdown_with_recovery_evidence(&shutdown) + .await + { + Ok(VerificationPoll { + tick: WorkerTick::Cancelled, + .. + }) => { + readiness.record(WorkerKind::Verification, WorkerReadinessEvidence::Stopped); + return Ok(()); + } + Ok( + poll @ VerificationPoll { + tick: WorkerTick::Idle, + .. + }, + ) => { + recovery.record_poll(poll, readiness); + tokio::select! { + _ = shutdown.changed() => {} + _ = tokio::time::sleep(self.poll_interval) => {} + } + } + Ok(poll) => recovery.record_poll(poll, readiness), + Err(failure) => { + recovery.record_repository_evidence(failure.repository, readiness); + match classify_verification_poll_error(&failure.error) { + VerificationPollErrorDisposition::RetryableRepository => { + recovery.record_repository_failure(failure.operation, readiness); + error!( + operation = match failure.operation { + RepositoryOperation::QueuePoll => "verification_queue_poll", + RepositoryOperation::Mutation => + "verification_repository_mutation", + }, + error_class = "repository_unavailable", + retrying = true, + "verification worker repository operation failed" + ); + tokio::select! { + _ = shutdown.changed() => {} + _ = tokio::time::sleep(self.poll_interval) => {} + } + } + VerificationPollErrorDisposition::Fatal(error_class) => { + readiness.record( + WorkerKind::Verification, + WorkerReadinessEvidence::UnexpectedExit, + ); + error!( + operation = match failure.operation { + RepositoryOperation::QueuePoll => "verification_queue_poll", + RepositoryOperation::Mutation => + "verification_repository_mutation", + }, + error_class, + retrying = false, + "verification worker terminated after unexpected application error" + ); + return Err(redacted_fatal_verification_error(error_class)); + } + } + } + } + } + } } fn claim_timeout(seconds: u64) -> time::Duration { @@ -237,10 +773,12 @@ mod tests { }; use locks_service::application::errors::ApplicationError; use locks_service::application::models::{ - CriterionVerificationRequest, VerificationTaskRecord, VerificationTaskStatus, + ClaimedVerificationTask, CriterionVerificationRequest, VerificationTaskRecord, + VerificationTaskStatus, }; use locks_service::application::ports::{ - ContentLockRepository, CriterionVerifier, EntitlementRepository, VerificationTaskRepository, + ContentLockRepository, CriterionVerifier, EntitlementRepository, VerificationTaskClaimer, + VerificationTaskRepository, }; use locks_service::infrastructure::memory::{ content_locks::InMemoryContentLockRepository, entitlements::InMemoryEntitlementRepository, @@ -248,16 +786,23 @@ mod tests { verification_tasks::InMemoryVerificationTaskRepository, }; use locks_service::infrastructure::verifiers::dev_static::DevStaticVerifier; + use locks_service::infrastructure::verifiers::paykit_payment::{ + PaykitPaymentStatus, PaykitPaymentStatusClient, PaykitPaymentStatusError, + PaykitPaymentStatusKind, PaykitPaymentVerifier, + }; use time::macros::datetime; - use tokio::sync::watch; + use tokio::sync::{Notify, watch}; - use crate::app_state::{AppState, SystemClock}; + use crate::app_state::{AppState, ReadinessStatus, SystemClock, WorkerReadiness}; use crate::config::{ ContentLocksConfig, DatabaseConfig, LockServerCredentialsConfig, LockServerRuntimeConfig, LoggingConfig, PubkyConfig, RateLimitsConfig, RuntimeConfig, RuntimeEnvironment, SecretsConfig, WorkerConfig, }; - use crate::worker::{VerificationWorker, WorkerTick, retry_delay}; + use crate::worker::{ + VerificationPollErrorDisposition, VerificationWorker, WorkerTick, + classify_verification_poll_error, redacted_fatal_verification_error, retry_delay, + }; const TASK_ID: &str = "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d10"; const BUNDLE_ID: &str = "000G40R40M30E209185GR38E1W"; @@ -318,11 +863,51 @@ mod tests { fixture.seed_task().await; let verifier = RetryOnceVerifier::default(); let worker = fixture.worker_with_verifier(&verifier); + let readiness = WorkerReadiness::new(true, false); + let mut recovery = super::VerificationReadinessRecovery::default(); + let poll = worker.run_once_with_recovery_evidence().await.unwrap(); assert_eq!( - worker.run_once().await.unwrap(), - WorkerTick::RetryScheduled(task_id()) + poll.tick, + WorkerTick::VerificationPendingRetryScheduled(task_id()) ); + assert_eq!( + poll.repository.queue_poll, + super::OperationEvidence::Succeeded + ); + assert_eq!( + poll.repository.mutation, + super::OperationEvidence::Succeeded + ); + recovery.record_poll(poll, &readiness); + assert_eq!(readiness.status(), ReadinessStatus::Ready); + let stored = fixture + .tasks + .get_verification_task(&task_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, VerificationTaskStatus::Pending); + assert_eq!(stored.failure_message, None); + assert_eq!(worker.run_once().await.unwrap(), WorkerTick::Idle); + } + + #[tokio::test] + async fn paykit_provider_outage_schedules_retry_and_degrades_readiness() { + let fixture = WorkerFixture::new(paykit_content_lock()).await; + fixture.seed_task().await; + let status_client = FakePaykitStatusClient::failing(); + let verifier = PaykitPaymentVerifier::new(&status_client, 1); + let worker = fixture.worker_with_paykit_verifier(&verifier); + let readiness = WorkerReadiness::new(true, false); + let mut recovery = super::VerificationReadinessRecovery::default(); + + let poll = worker.run_once_with_recovery_evidence().await.unwrap(); + + assert_eq!(poll.tick, WorkerTick::RetryScheduled(task_id())); + recovery.record_poll(poll, &readiness); + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + assert_eq!(status_client.calls.load(Ordering::SeqCst), 1); let stored = fixture .tasks .get_verification_task(&task_id()) @@ -334,6 +919,95 @@ mod tests { assert_eq!(worker.run_once().await.unwrap(), WorkerTick::Idle); } + #[tokio::test] + async fn healthy_paykit_pending_schedules_retry_without_degrading_readiness() { + let fixture = WorkerFixture::new(paykit_content_lock()).await; + fixture.seed_task().await; + let status_client = FakePaykitStatusClient::healthy_pending(); + let verifier = PaykitPaymentVerifier::new(&status_client, 1); + let worker = fixture.worker_with_paykit_verifier(&verifier); + let readiness = WorkerReadiness::new(true, false); + let mut recovery = super::VerificationReadinessRecovery::default(); + + let poll = worker.run_once_with_recovery_evidence().await.unwrap(); + + assert_eq!( + poll.tick, + WorkerTick::VerificationPendingRetryScheduled(task_id()) + ); + recovery.record_poll(poll, &readiness); + assert_eq!(readiness.status(), ReadinessStatus::Ready); + assert_eq!(status_client.calls.load(Ordering::SeqCst), 1); + let stored = fixture + .tasks + .get_verification_task(&task_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, VerificationTaskStatus::Pending); + assert_eq!(stored.failure_message, None); + assert_eq!(worker.run_once().await.unwrap(), WorkerTick::Idle); + } + + #[tokio::test] + async fn paykit_degradation_ignores_unrelated_work_until_healthy_paykit_response() { + let readiness = WorkerReadiness::new(true, false); + let mut recovery = super::VerificationReadinessRecovery::default(); + + let outage_fixture = WorkerFixture::new(paykit_content_lock()).await; + outage_fixture.seed_task().await; + let outage_client = FakePaykitStatusClient::failing(); + let outage_verifier = PaykitPaymentVerifier::new(&outage_client, 1); + let outage_worker = outage_fixture.worker_with_paykit_verifier(&outage_verifier); + recovery.record_poll( + outage_worker + .run_once_with_recovery_evidence() + .await + .unwrap(), + &readiness, + ); + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + + let completed_fixture = WorkerFixture::new(content_lock(true)).await; + completed_fixture.seed_task().await; + recovery.record_poll( + completed_fixture + .worker() + .run_once_with_recovery_evidence() + .await + .unwrap(), + &readiness, + ); + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + + let pending_fixture = WorkerFixture::new(content_lock(true)).await; + pending_fixture.seed_task().await; + let pending_verifier = RetryOnceVerifier::default(); + recovery.record_poll( + pending_fixture + .worker_with_verifier(&pending_verifier) + .run_once_with_recovery_evidence() + .await + .unwrap(), + &readiness, + ); + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + + let healthy_fixture = WorkerFixture::new(paykit_content_lock()).await; + healthy_fixture.seed_task().await; + let healthy_client = FakePaykitStatusClient::healthy_pending(); + let healthy_verifier = PaykitPaymentVerifier::new(&healthy_client, 1); + recovery.record_poll( + healthy_fixture + .worker_with_paykit_verifier(&healthy_verifier) + .run_once_with_recovery_evidence() + .await + .unwrap(), + &readiness, + ); + assert_eq!(readiness.status(), ReadinessStatus::Ready); + } + #[tokio::test] async fn worker_without_dev_static_registration_fails_dev_static_tasks() { let fixture = WorkerFixture::new(content_lock(true)).await; @@ -391,11 +1065,185 @@ mod tests { .unwrap(); } + #[tokio::test] + async fn shutdown_during_claim_releases_claim_without_verification_execution() { + let fixture = WorkerFixture::new(content_lock(true)).await; + fixture.seed_task().await; + let blocking = BlockingClaimer::new(&fixture.claimer); + let worker = fixture.worker_with_claimer(&blocking); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + + let (result, ()) = tokio::join!(worker.run_once_until_shutdown(&shutdown_rx), async { + blocking.claim_entered.notified().await; + shutdown_tx.send(true).unwrap(); + blocking.release_claim.notify_one(); + }); + + assert_eq!(result.unwrap(), WorkerTick::Cancelled); + assert_eq!(blocking.publication_calls.load(Ordering::SeqCst), 0); + assert_eq!(blocking.retry_calls.load(Ordering::SeqCst), 1); + } + #[test] fn pending_verification_retry_is_independent_of_queue_polling() { assert_eq!(retry_delay(), time::Duration::seconds(30)); } + #[tokio::test] + async fn unexpected_verification_error_terminates_not_ready_with_redacted_class() { + let fixture = WorkerFixture::new(content_lock(true)).await; + fixture.seed_task().await; + let verifier = UnexpectedVerifier("super-secret-verifier-detail".to_owned()); + let worker = fixture.worker_with_verifier(&verifier); + let readiness = WorkerReadiness::new(true, false); + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + + let error = worker + .run_until_shutdown_with_readiness(shutdown_rx, &readiness) + .await + .unwrap_err(); + + assert_eq!(readiness.status(), ReadinessStatus::NotReady); + assert!( + error + .to_string() + .contains("invalid_verification_task_state") + ); + assert!(!error.to_string().contains("super-secret-verifier-detail")); + } + + #[test] + fn ordinary_verification_pending_retry_does_not_degrade_readiness() { + let readiness = WorkerReadiness::new(true, false); + let mut recovery = super::VerificationReadinessRecovery::default(); + + recovery.record_repository_success(super::RepositoryOperation::QueuePoll, &readiness); + recovery.record_poll( + super::VerificationPoll::without_provider_evidence( + WorkerTick::VerificationPendingRetryScheduled(task_id()), + ), + &readiness, + ); + + assert_eq!(readiness.status(), ReadinessStatus::Ready); + } + + #[tokio::test] + async fn successful_idle_poll_reports_only_queue_poll_recovery() { + let fixture = WorkerFixture::empty(); + + let poll = fixture + .worker() + .run_once_with_recovery_evidence() + .await + .unwrap(); + + assert_eq!(poll.tick, WorkerTick::Idle); + assert_eq!( + poll.repository.queue_poll, + super::OperationEvidence::Succeeded + ); + assert_eq!(poll.repository.mutation, super::OperationEvidence::None); + } + + #[test] + fn repository_mutation_failure_ignores_outcomes_until_mutation_succeeds() { + let readiness = WorkerReadiness::new(true, false); + let mut recovery = super::VerificationReadinessRecovery::default(); + + recovery.record_repository_failure(super::RepositoryOperation::Mutation, &readiness); + for tick in [ + WorkerTick::Idle, + WorkerTick::Completed(task_id()), + WorkerTick::VerificationPendingRetryScheduled(task_id()), + WorkerTick::Failed(task_id()), + ] { + recovery.record_poll( + super::VerificationPoll::without_provider_evidence(tick), + &readiness, + ); + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + } + + recovery.record_repository_success(super::RepositoryOperation::Mutation, &readiness); + assert_eq!(readiness.status(), ReadinessStatus::Ready); + } + + #[test] + fn queue_poll_failure_recovers_only_from_successful_queue_poll() { + let readiness = WorkerReadiness::new(true, false); + let mut recovery = super::VerificationReadinessRecovery::default(); + + recovery.record_repository_failure(super::RepositoryOperation::QueuePoll, &readiness); + recovery.record_repository_success(super::RepositoryOperation::Mutation, &readiness); + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + + recovery.record_repository_success(super::RepositoryOperation::QueuePoll, &readiness); + assert_eq!(readiness.status(), ReadinessStatus::Ready); + } + + #[test] + fn provider_failure_stays_degraded_across_business_outcomes_until_provider_success() { + let readiness = WorkerReadiness::new(true, false); + let mut recovery = super::VerificationReadinessRecovery::default(); + + recovery.record_poll( + super::VerificationPoll { + tick: WorkerTick::RetryScheduled(task_id()), + paykit_provider: super::PaykitProviderEvidence::Unavailable, + repository: super::RepositoryOperationEvidence::default(), + }, + &readiness, + ); + recovery.record_poll( + super::VerificationPoll::without_provider_evidence(WorkerTick::Failed(task_id())), + &readiness, + ); + recovery.record_poll( + super::VerificationPoll::without_provider_evidence(WorkerTick::Idle), + &readiness, + ); + assert_eq!(readiness.status(), ReadinessStatus::Degraded); + + recovery.record_poll( + super::VerificationPoll { + tick: WorkerTick::VerificationPendingRetryScheduled(task_id()), + paykit_provider: super::PaykitProviderEvidence::HealthyResponse, + repository: super::RepositoryOperationEvidence::default(), + }, + &readiness, + ); + assert_eq!(readiness.status(), ReadinessStatus::Ready); + } + + #[test] + fn verification_errors_retry_only_storage_and_redact_fatal_details() { + let secret = "postgres://user:password@example.test/locks"; + assert_eq!( + classify_verification_poll_error(&ApplicationError::Storage { + message: secret.to_owned(), + }), + VerificationPollErrorDisposition::RetryableRepository + ); + assert_eq!( + classify_verification_poll_error(&ApplicationError::InvalidVerificationTaskState { + message: secret.to_owned(), + }), + VerificationPollErrorDisposition::Fatal("invalid_verification_task_state") + ); + assert_eq!( + classify_verification_poll_error(&ApplicationError::MissingRecord { + record: "verification_task", + }), + VerificationPollErrorDisposition::Fatal("unexpected_application_error") + ); + + let redacted = + redacted_fatal_verification_error("invalid_verification_task_state").to_string(); + assert!(redacted.contains("invalid_verification_task_state")); + assert!(!redacted.contains(secret)); + } + #[tokio::test] async fn worker_tick_debug_does_not_expose_submitted_proof_payload() { let fixture = WorkerFixture::new(content_lock_with_payload(json_secret_payload())).await; @@ -458,6 +1306,29 @@ mod tests { self.worker_with_verifier(&self.verifier) } + fn worker_with_claimer<'a>( + &'a self, + claimer: &'a dyn VerificationTaskClaimer, + ) -> VerificationWorker<'a> { + VerificationWorker::new( + self.tasks.as_ref(), + claimer, + &self.content_locks, + &self.entitlements, + &self.verifier, + None, + true, + &self.clock, + LockServerPubky::from_str( + "pubky7ir1ttte48bcp4zjychjyscicrwi1j34mtt91ptsafdbjmr8g9eo", + ) + .unwrap(), + "test-worker".to_owned(), + std::time::Duration::from_millis(10), + 60, + ) + } + fn worker_with_verifier<'a>( &'a self, verifier: &'a dyn CriterionVerifier, @@ -481,6 +1352,29 @@ mod tests { ) } + fn worker_with_paykit_verifier<'a>( + &'a self, + verifier: &'a dyn CriterionVerifier, + ) -> VerificationWorker<'a> { + VerificationWorker::new( + self.tasks.as_ref(), + &self.claimer, + &self.content_locks, + &self.entitlements, + &self.verifier, + Some(verifier), + false, + &self.clock, + LockServerPubky::from_str( + "pubky7ir1ttte48bcp4zjychjyscicrwi1j34mtt91ptsafdbjmr8g9eo", + ) + .unwrap(), + "test-worker".to_owned(), + std::time::Duration::from_millis(10), + 60, + ) + } + fn worker_without_dev_static_registration(&self) -> VerificationWorker<'_> { VerificationWorker::new( self.tasks.as_ref(), @@ -524,6 +1418,91 @@ mod tests { } } + struct BlockingClaimer<'a> { + inner: &'a dyn VerificationTaskClaimer, + claim_entered: Notify, + release_claim: Notify, + publication_calls: std::sync::atomic::AtomicUsize, + retry_calls: std::sync::atomic::AtomicUsize, + } + + impl<'a> BlockingClaimer<'a> { + fn new(inner: &'a dyn VerificationTaskClaimer) -> Self { + Self { + inner, + claim_entered: Notify::new(), + release_claim: Notify::new(), + publication_calls: std::sync::atomic::AtomicUsize::new(0), + retry_calls: std::sync::atomic::AtomicUsize::new(0), + } + } + } + + #[async_trait] + impl VerificationTaskClaimer for BlockingClaimer<'_> { + async fn begin_claimed_entitlement_publication( + &self, + task_id: &TaskId, + worker_id: &str, + claim_token: &uuid::Uuid, + ) -> Result { + self.publication_calls.fetch_add(1, Ordering::SeqCst); + self.inner + .begin_claimed_entitlement_publication(task_id, worker_id, claim_token) + .await + } + + async fn claim_next_verification_task( + &self, + worker_id: &str, + claim_ttl: time::Duration, + ) -> Result, ApplicationError> { + self.claim_entered.notify_one(); + self.release_claim.notified().await; + self.inner + .claim_next_verification_task(worker_id, claim_ttl) + .await + } + + async fn schedule_verification_task_retry( + &self, + task_id: &TaskId, + worker_id: &str, + claim_token: &uuid::Uuid, + retry_after: time::Duration, + ) -> Result, ApplicationError> { + self.retry_calls.fetch_add(1, Ordering::SeqCst); + self.inner + .schedule_verification_task_retry(task_id, worker_id, claim_token, retry_after) + .await + } + + async fn persist_claimed_verification_task_transition( + &self, + task: VerificationTaskRecord, + worker_id: &str, + claim_token: &uuid::Uuid, + ) -> Result, ApplicationError> { + self.inner + .persist_claimed_verification_task_transition(task, worker_id, claim_token) + .await + } + } + + struct UnexpectedVerifier(String); + + #[async_trait] + impl CriterionVerifier for UnexpectedVerifier { + async fn verify( + &self, + _request: CriterionVerificationRequest, + ) -> Result { + Err(ApplicationError::InvalidVerificationTaskState { + message: self.0.clone(), + }) + } + } + #[derive(Default)] struct RetryOnceVerifier { returned_pending: AtomicBool, @@ -542,6 +1521,43 @@ mod tests { } } + struct FakePaykitStatusClient { + response: Result, + calls: std::sync::atomic::AtomicUsize, + } + + impl FakePaykitStatusClient { + fn failing() -> Self { + Self { + response: Err(PaykitPaymentStatusError), + calls: std::sync::atomic::AtomicUsize::new(0), + } + } + + fn healthy_pending() -> Self { + Self { + response: Ok(PaykitPaymentStatus { + status: PaykitPaymentStatusKind::Detected, + confirmations: 0, + amount_matched: true, + }), + calls: std::sync::atomic::AtomicUsize::new(0), + } + } + } + + #[async_trait] + impl PaykitPaymentStatusClient for &FakePaykitStatusClient { + async fn transaction_status( + &self, + _creator: &CreatorPubky, + _bundle_id: &BundleId, + ) -> Result { + self.calls.fetch_add(1, Ordering::SeqCst); + self.response + } + } + fn task_for(content_lock: &ContentLock, payload: serde_json::Value) -> VerificationTaskRecord { VerificationTaskRecord { task_id: task_id(), @@ -569,7 +1585,7 @@ mod tests { reader_public_key: None, proofs: vec![Proof { criterion_id: "criterion-1".to_owned(), - verifier_type: VerifierType::DevStatic, + verifier_type: content_lock.criteria[0].verifier_type, payload, }], } @@ -579,6 +1595,17 @@ mod tests { content_lock_with_payload(serde_json::json!({ "satisfied": satisfied })) } + fn paykit_content_lock() -> ContentLock { + let mut content_lock = content_lock_with_payload(serde_json::json!({ + "recipient_pubky": creator().to_string(), + "amount": "50000", + "asset": "BTC", + "payment_in": 24 + })); + content_lock.criteria[0].verifier_type = VerifierType::PaykitPayment; + content_lock + } + fn content_lock_with_payload(params: serde_json::Value) -> ContentLock { ContentLock { version: CONTENT_LOCK_VERSION, @@ -644,6 +1671,8 @@ mod tests { pkdns: crate::config::PkdnsConfig::default(), rate_limits: RateLimitsConfig::default(), content_locks: ContentLocksConfig::default(), + deletion: crate::config::DeletionConfig::default(), + deletion_worker: crate::config::DeletionWorkerConfig::default(), paykit: None, } } diff --git a/locks-server/tests/startup_lifecycle.rs b/locks-server/tests/startup_lifecycle.rs new file mode 100644 index 0000000..250d6b0 --- /dev/null +++ b/locks-server/tests/startup_lifecycle.rs @@ -0,0 +1,84 @@ +use std::{ + net::SocketAddr, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::{Duration, Instant}, +}; + +use locks_server::runtime::{InitialStartupOutcome, bind_listener_then_run_initial}; +use tokio::sync::{Notify, oneshot}; + +#[tokio::test] +async fn listener_bind_failure_does_not_attempt_initial_publication() { + let occupied = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let bind_addr = occupied.local_addr().unwrap(); + let publication_calls = Arc::new(AtomicUsize::new(0)); + let observed_calls = Arc::clone(&publication_calls); + let mut shutdown = Box::pin(std::future::pending()); + + let error = bind_listener_then_run_initial( + bind_addr, + Duration::from_millis(100), + &mut shutdown, + move || async move { + observed_calls.fetch_add(1, Ordering::SeqCst); + Ok::<_, anyhow::Error>(()) + }, + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("bind")); + assert_eq!(publication_calls.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn blocked_initial_publication_stops_when_shutdown_is_requested() { + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let mut shutdown = Box::pin(async move { + let _ = shutdown_rx.await; + }); + let publication_started = Arc::new(Notify::new()); + let observed_start = Arc::clone(&publication_started); + let bind_addr: SocketAddr = "127.0.0.1:0".parse().unwrap(); + + let startup = bind_listener_then_run_initial( + bind_addr, + Duration::from_secs(1), + &mut shutdown, + move || async move { + observed_start.notify_one(); + std::future::pending::>().await + }, + ); + let request_shutdown = async { + publication_started.notified().await; + shutdown_tx.send(()).unwrap(); + }; + let (result, ()) = tokio::join!(startup, request_shutdown); + + assert!(matches!( + result.unwrap(), + InitialStartupOutcome::ShutdownRequested + )); +} + +#[tokio::test] +async fn blocked_initial_publication_is_bounded_by_lifecycle_timeout() { + let mut shutdown = Box::pin(std::future::pending()); + let started = Instant::now(); + + let error = bind_listener_then_run_initial( + "127.0.0.1:0".parse().unwrap(), + Duration::from_millis(20), + &mut shutdown, + || async { std::future::pending::>().await }, + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("initial startup exceeded")); + assert!(started.elapsed() < Duration::from_millis(250)); +} diff --git a/locks-service/migrations/0010_content_lock_ownership.sql b/locks-service/migrations/0010_content_lock_ownership.sql new file mode 100644 index 0000000..518ace4 --- /dev/null +++ b/locks-service/migrations/0010_content_lock_ownership.sql @@ -0,0 +1,8 @@ +CREATE TABLE content_lock_ownership ( + creator TEXT NOT NULL, + guarded_path TEXT NOT NULL, + lock_id TEXT NOT NULL, + status TEXT NOT NULL, + CONSTRAINT content_lock_ownership_creator_path_unique UNIQUE (creator, guarded_path), + CONSTRAINT content_lock_ownership_status_valid CHECK (status IN ('reserved', 'published')) +); diff --git a/locks-service/migrations/0011_content_lock_deletions.sql b/locks-service/migrations/0011_content_lock_deletions.sql new file mode 100644 index 0000000..418315e --- /dev/null +++ b/locks-service/migrations/0011_content_lock_deletions.sql @@ -0,0 +1,69 @@ +CREATE TABLE content_lock_deletion_jobs ( + job_id UUID PRIMARY KEY, + creator TEXT NOT NULL, + lock_id TEXT NOT NULL, + frozen_content_lock JSONB NOT NULL, + deletion_started_at TIMESTAMPTZ NOT NULL, + state TEXT NOT NULL DEFAULT 'queued', + phase TEXT NOT NULL DEFAULT 'withdraw', + attempt_count BIGINT NOT NULL DEFAULT 0, + next_attempt_at TIMESTAMPTZ, + force_requested_at TIMESTAMPTZ, + failure_code TEXT, + claimed_by TEXT, + claim_token UUID, + claim_expires_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT content_lock_deletion_jobs_creator_lock_unique UNIQUE (creator, lock_id), + CONSTRAINT content_lock_deletion_jobs_state_valid CHECK ( + state IN ('queued', 'running', 'completed', 'failed') + ), + CONSTRAINT content_lock_deletion_jobs_phase_valid CHECK ( + phase IN ( + 'withdraw', + 'start_payment_drain', + 'drain_payments', + 'drain_existing_credentials', + 'issue_final_credentials', + 'drain_final_reads', + 'delete_content', + 'delete_tombstone', + 'purge_operational_state' + ) + ), + CONSTRAINT content_lock_deletion_jobs_attempt_count_valid CHECK (attempt_count >= 0), + CONSTRAINT content_lock_deletion_jobs_claim_valid CHECK ( + (state = 'running' + AND claimed_by IS NOT NULL + AND claim_token IS NOT NULL + AND claim_expires_at IS NOT NULL + AND next_attempt_at IS NULL) + OR + (state <> 'running' + AND claimed_by IS NULL + AND claim_token IS NULL + AND claim_expires_at IS NULL) + ), + CONSTRAINT content_lock_deletion_jobs_failure_valid CHECK ( + (state = 'failed' AND failure_code IN ( + 'tombstone_missing', + 'tombstone_replaced', + 'retry_exhausted', + 'state_corrupt' + )) + OR + (state <> 'failed' AND failure_code IS NULL) + ) +); + +CREATE INDEX content_lock_deletion_jobs_due_idx + ON content_lock_deletion_jobs (deletion_started_at) + WHERE state IN ('queued', 'running'); + +CREATE TABLE content_lock_force_deletion_receipts ( + creator TEXT NOT NULL, + lock_id TEXT NOT NULL, + forced_at TIMESTAMPTZ NOT NULL, + CONSTRAINT content_lock_force_deletion_receipts_pkey PRIMARY KEY (creator, lock_id) +); diff --git a/locks-service/migrations/0012_content_lock_deletion_task_snapshot.sql b/locks-service/migrations/0012_content_lock_deletion_task_snapshot.sql new file mode 100644 index 0000000..15ac984 --- /dev/null +++ b/locks-service/migrations/0012_content_lock_deletion_task_snapshot.sql @@ -0,0 +1,18 @@ +CREATE TABLE content_lock_deletion_task_snapshot ( + deletion_job_id UUID NOT NULL REFERENCES content_lock_deletion_jobs(job_id) ON DELETE CASCADE, + verification_task_id UUID NOT NULL REFERENCES verification_tasks(task_id), + CONSTRAINT content_lock_deletion_task_snapshot_pkey + PRIMARY KEY (deletion_job_id, verification_task_id), + CONSTRAINT content_lock_deletion_task_snapshot_task_unique + UNIQUE (verification_task_id) +); + +CREATE TABLE paykit_task_admissions ( + verification_task_id UUID PRIMARY KEY REFERENCES verification_tasks(task_id) ON DELETE CASCADE, + ready BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + ready_at TIMESTAMPTZ, + CONSTRAINT paykit_task_admissions_ready_time_valid CHECK ( + (ready AND ready_at IS NOT NULL) OR (NOT ready AND ready_at IS NULL) + ) +); diff --git a/locks-service/migrations/0013_content_lock_publication_intents.sql b/locks-service/migrations/0013_content_lock_publication_intents.sql new file mode 100644 index 0000000..7d9712b --- /dev/null +++ b/locks-service/migrations/0013_content_lock_publication_intents.sql @@ -0,0 +1,8 @@ +CREATE TABLE content_lock_publication_intents ( + creator TEXT NOT NULL, + lock_id TEXT NOT NULL, + publication_token UUID NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT content_lock_publication_intents_pkey PRIMARY KEY (creator, lock_id), + CONSTRAINT content_lock_publication_intents_token_unique UNIQUE (publication_token) +); \ No newline at end of file diff --git a/locks-service/migrations/0014_paykit_invoice_windows.sql b/locks-service/migrations/0014_paykit_invoice_windows.sql new file mode 100644 index 0000000..87b1b9b --- /dev/null +++ b/locks-service/migrations/0014_paykit_invoice_windows.sql @@ -0,0 +1,28 @@ +ALTER TABLE paykit_task_admissions +ADD COLUMN payment_in_hours BIGINT, +ADD COLUMN invoice_created_at TIMESTAMPTZ, +ADD COLUMN payment_deadline TIMESTAMPTZ; + +-- Rows created before this migration have no authoritative payment window to +-- backfill. Preserve that all-NULL legacy state so the application can fail +-- closed instead of fabricating invoice facts. Every post-migration admission +-- writes a positive payment_in_hours and becomes ready only with a complete +-- immutable timestamp window. +ALTER TABLE paykit_task_admissions +ADD CONSTRAINT paykit_task_admissions_invoice_window_valid + CHECK ( + (payment_in_hours IS NULL + AND invoice_created_at IS NULL + AND payment_deadline IS NULL) + OR + (payment_in_hours > 0 + AND NOT ready + AND invoice_created_at IS NULL + AND payment_deadline IS NULL) + OR + (payment_in_hours > 0 + AND ready + AND invoice_created_at IS NOT NULL + AND payment_deadline IS NOT NULL + AND invoice_created_at <= payment_deadline) + ); diff --git a/locks-service/migrations/0015_content_lock_payment_drains.sql b/locks-service/migrations/0015_content_lock_payment_drains.sql new file mode 100644 index 0000000..b9913a2 --- /dev/null +++ b/locks-service/migrations/0015_content_lock_payment_drains.sql @@ -0,0 +1,92 @@ +ALTER TABLE content_lock_deletion_task_snapshot + ADD COLUMN creator TEXT, + ADD COLUMN bundle_id TEXT, + ADD COLUMN pubky_lock_resource TEXT, + ADD COLUMN criterion_id TEXT, + ADD COLUMN status_at_cutoff TEXT, + ADD COLUMN paykit_admission_required BOOLEAN, + ADD COLUMN payment_in_hours BIGINT, + ADD COLUMN invoice_created_at TIMESTAMPTZ, + ADD COLUMN payment_deadline TIMESTAMPTZ, + ADD COLUMN resolved_status TEXT, + ADD COLUMN resolved_at TIMESTAMPTZ; + +ALTER TABLE verification_tasks + ADD COLUMN entitlement_publication_claim_token UUID, + ADD COLUMN deletion_job_id UUID REFERENCES content_lock_deletion_jobs(job_id); + +COMMENT ON COLUMN verification_tasks.entitlement_publication_claim_token IS + 'Claim token that crossed the external entitlement-publication boundary.'; +COMMENT ON COLUMN verification_tasks.deletion_job_id IS + 'Task-row ownership fence set by graceful deletion admission.'; + +ALTER TABLE content_lock_deletion_task_snapshot + ADD CONSTRAINT content_lock_deletion_task_snapshot_cutoff_status_valid + CHECK (status_at_cutoff IN ('pending', 'in_progress', 'completed', 'failed', 'expired')), + ADD CONSTRAINT content_lock_deletion_task_snapshot_identity_all_or_none + CHECK ( + (creator IS NULL AND bundle_id IS NULL AND pubky_lock_resource IS NULL + AND criterion_id IS NULL AND status_at_cutoff IS NULL) + OR + (creator IS NOT NULL AND bundle_id IS NOT NULL + AND pubky_lock_resource IS NOT NULL AND status_at_cutoff IS NOT NULL) + ), + ADD CONSTRAINT content_lock_deletion_task_snapshot_admission_shape_valid + CHECK ( + (paykit_admission_required IS NULL + AND payment_in_hours IS NULL + AND invoice_created_at IS NULL + AND payment_deadline IS NULL) + OR + (paykit_admission_required = FALSE + AND payment_in_hours IS NULL + AND invoice_created_at IS NULL + AND payment_deadline IS NULL) + OR + (paykit_admission_required = TRUE + AND payment_in_hours IS NULL + AND invoice_created_at IS NULL + AND payment_deadline IS NULL) + OR + (paykit_admission_required = TRUE + AND payment_in_hours > 0 + AND invoice_created_at IS NOT NULL + AND payment_deadline IS NOT NULL + AND invoice_created_at <= payment_deadline) + ), + ADD CONSTRAINT content_lock_deletion_task_snapshot_resolution_valid + CHECK ( + (resolved_status IS NULL AND resolved_at IS NULL) + OR + (resolved_status IN ('completed', 'failed', 'expired') AND resolved_at IS NOT NULL) + ), + ADD CONSTRAINT content_lock_deletion_task_snapshot_bundle_unique + UNIQUE (deletion_job_id, creator, bundle_id); + +CREATE INDEX content_lock_deletion_task_snapshot_resolution_idx + ON content_lock_deletion_task_snapshot (deletion_job_id, resolved_status); + +CREATE TABLE content_lock_payment_drains ( + deletion_job_id UUID PRIMARY KEY + REFERENCES content_lock_deletion_jobs(job_id) ON DELETE CASCADE, + status TEXT NOT NULL, + accepted_count BIGINT NOT NULL, + terminal_count BIGINT NOT NULL, + cancellation_enqueued_count BIGINT NOT NULL, + cleanup_token TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + CONSTRAINT content_lock_payment_drains_status_valid + CHECK (status IN ('active', 'completed')), + CONSTRAINT content_lock_payment_drains_counts_valid + CHECK ( + accepted_count >= 0 + AND terminal_count >= 0 + AND cancellation_enqueued_count >= 0 + ), + CONSTRAINT content_lock_payment_drains_cleanup_token_shape + CHECK ( + length(cleanup_token) = 43 + AND cleanup_token ~ '^[A-Za-z0-9_-]{43}$' + ) +); diff --git a/locks-service/migrations/0016_content_lock_access_drains.sql b/locks-service/migrations/0016_content_lock_access_drains.sql new file mode 100644 index 0000000..18c4a40 --- /dev/null +++ b/locks-service/migrations/0016_content_lock_access_drains.sql @@ -0,0 +1,112 @@ +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM content_lock_deletion_jobs + WHERE state IN ('queued', 'running', 'failed') + ) THEN + RAISE EXCEPTION USING + MESSAGE = 'migration 0016 cannot classify pre-existing resumable deletion jobs; drain or explicitly reset pre-0016 deletion jobs before retrying', + HINT = 'see docs/RUNTIME.md for the required drain/reset procedure'; + END IF; +END +$$; + +ALTER TABLE content_lock_deletion_jobs + ADD COLUMN final_issuance_started_at TIMESTAMPTZ, + ADD COLUMN final_credential_issuance_deadline TIMESTAMPTZ, + ADD COLUMN final_read_deadline TIMESTAMPTZ, + ADD CONSTRAINT content_lock_deletion_jobs_final_window_shape CHECK ( + (final_issuance_started_at IS NULL + AND final_credential_issuance_deadline IS NULL + AND final_read_deadline IS NULL) + OR + (final_issuance_started_at IS NOT NULL + AND final_credential_issuance_deadline IS NOT NULL + AND final_read_deadline IS NOT NULL + AND final_issuance_started_at < final_credential_issuance_deadline + AND final_credential_issuance_deadline < final_read_deadline) + ); + +ALTER TABLE content_lock_deletion_task_snapshot + ADD COLUMN had_active_credential_at_cutoff BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN final_credential_eligible_at TIMESTAMPTZ, + ADD COLUMN final_credential_issued_at TIMESTAMPTZ, + ADD CONSTRAINT content_lock_deletion_task_snapshot_final_eligibility_valid CHECK ( + final_credential_eligible_at IS NULL + OR ( + had_active_credential_at_cutoff = FALSE + AND paykit_admission_required = TRUE + AND resolved_status = 'completed' + ) + ), + ADD CONSTRAINT content_lock_deletion_task_snapshot_final_issuance_valid CHECK ( + final_credential_issued_at IS NULL + OR final_credential_eligible_at IS NOT NULL + ); + +ALTER TABLE access_credentials + ADD COLUMN deletion_job_id UUID + REFERENCES content_lock_deletion_jobs(job_id) ON DELETE CASCADE; + +CREATE INDEX access_credentials_deletion_active_idx + ON access_credentials (deletion_job_id, expires_at) + WHERE deletion_job_id IS NOT NULL; + +CREATE TABLE content_lock_access_drain_credentials ( + credential_id UUID PRIMARY KEY, + deletion_job_id UUID NOT NULL + REFERENCES content_lock_deletion_jobs(job_id) ON DELETE CASCADE, + lookup_key BYTEA NOT NULL UNIQUE + REFERENCES access_credentials(lookup_key) ON DELETE CASCADE, + creator TEXT NOT NULL, + bundle_id TEXT NOT NULL, + credential_kind TEXT NOT NULL, + encrypted_bearer TEXT, + issued_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + CONSTRAINT content_lock_access_drain_credentials_kind_valid CHECK ( + credential_kind IN ('ordinary', 'final') + ), + CONSTRAINT content_lock_access_drain_credentials_envelope_valid CHECK ( + (credential_kind = 'ordinary' AND encrypted_bearer IS NULL) + OR + (credential_kind = 'final' + AND encrypted_bearer IS NOT NULL + AND encrypted_bearer LIKE 'v1.xchacha20poly1305:%') + ), + CONSTRAINT content_lock_access_drain_credentials_expiry_valid CHECK ( + issued_at < expires_at + ) +); + +CREATE UNIQUE INDEX content_lock_access_drain_one_final_per_bundle_idx + ON content_lock_access_drain_credentials (deletion_job_id, creator, bundle_id) + WHERE credential_kind = 'final'; + +CREATE INDEX content_lock_access_drain_credentials_job_expiry_idx + ON content_lock_access_drain_credentials (deletion_job_id, expires_at); + +CREATE TABLE content_lock_access_drain_reads ( + credential_id UUID NOT NULL + REFERENCES content_lock_access_drain_credentials(credential_id) ON DELETE CASCADE, + guarded_path TEXT NOT NULL, + claim_token UUID, + claim_expires_at TIMESTAMPTZ, + consumed_at TIMESTAMPTZ, + CONSTRAINT content_lock_access_drain_reads_pkey + PRIMARY KEY (credential_id, guarded_path), + CONSTRAINT content_lock_access_drain_reads_claim_shape CHECK ( + (claim_token IS NULL AND claim_expires_at IS NULL) + OR + (claim_token IS NOT NULL AND claim_expires_at IS NOT NULL) + ), + CONSTRAINT content_lock_access_drain_reads_consumed_shape CHECK ( + consumed_at IS NULL + OR (claim_token IS NULL AND claim_expires_at IS NULL) + ) +); + +CREATE INDEX content_lock_access_drain_reads_claim_idx + ON content_lock_access_drain_reads (claim_expires_at) + WHERE consumed_at IS NULL AND claim_token IS NOT NULL; diff --git a/locks-service/migrations/0017_content_lock_deletion_resource_replaced.sql b/locks-service/migrations/0017_content_lock_deletion_resource_replaced.sql new file mode 100644 index 0000000..a2b3b70 --- /dev/null +++ b/locks-service/migrations/0017_content_lock_deletion_resource_replaced.sql @@ -0,0 +1,15 @@ +ALTER TABLE content_lock_deletion_jobs + DROP CONSTRAINT content_lock_deletion_jobs_failure_valid; + +ALTER TABLE content_lock_deletion_jobs + ADD CONSTRAINT content_lock_deletion_jobs_failure_valid CHECK ( + (state = 'failed' AND failure_code IN ( + 'tombstone_missing', + 'tombstone_replaced', + 'resource_replaced', + 'retry_exhausted', + 'state_corrupt' + )) + OR + (state <> 'failed' AND failure_code IS NULL) + ); diff --git a/locks-service/src/application/errors.rs b/locks-service/src/application/errors.rs index 748fcb2..6417919 100644 --- a/locks-service/src/application/errors.rs +++ b/locks-service/src/application/errors.rs @@ -12,6 +12,21 @@ pub enum ApplicationError { /// Stable record kind for diagnostics. record: &'static str, }, + /// A guarded path already has an in-flight or published ownership record. + #[error("content lock path conflict")] + ContentLockPathConflict { + /// Full creator-scoped guarded path for structured internal handling. + guarded_path: String, + }, + /// A graceful deletion cutoff already blocks new proof Bundle IDs for the lock. + #[error("content lock deletion in progress")] + ContentLockDeletionInProgress, + /// Persisted content-lock deletion state violates its internal invariants. + #[error("invalid content lock deletion state: {message}")] + InvalidContentLockDeletionState { + /// Secret-free invariant failure detail. + message: String, + }, /// An update-only operation targeted a missing record. #[error("missing {record} record")] MissingRecord { @@ -33,6 +48,9 @@ pub enum ApplicationError { /// Criterion verifier is not terminal yet and should be retried later. #[error("verification pending")] VerificationPending, + /// A verification provider is transiently unavailable and should be retried later. + #[error("verification dependency unavailable")] + VerificationDependencyUnavailable, /// Submitted payment proof does not match its canonical content lock criterion. #[error("invalid paykit payment submission")] InvalidPaykitPaymentSubmission, @@ -76,6 +94,12 @@ pub enum ApplicationError { /// Human-readable credential generation failure detail. message: String, }, + /// Final deletion credential envelope could not be encrypted or decrypted. + #[error("final credential secret error: {message}")] + FinalCredentialSecret { + /// Stable secret-free failure detail. + message: String, + }, /// Creator-granted homeserver authority is missing, expired, revoked, or unusable. #[error("creator authority unavailable")] CreatorAuthorityUnavailable, diff --git a/locks-service/src/application/models/access.rs b/locks-service/src/application/models/access.rs index 454c9bb..9e75db9 100644 --- a/locks-service/src/application/models/access.rs +++ b/locks-service/src/application/models/access.rs @@ -1,7 +1,11 @@ use std::fmt; -use locks_core::ids::{BundleId, CreatorPubky}; +use locks_core::{ + ids::{BundleId, CreatorPubky}, + lock_policy::GuardedResource, +}; use time::OffsetDateTime; +use uuid::Uuid; use crate::application::errors::ApplicationError; @@ -36,6 +40,82 @@ impl fmt::Debug for AccessCredential { } } +/// Versioned encrypted bearer envelope persisted for exact final-credential replay. +#[derive(Clone, PartialEq, Eq)] +pub struct EncryptedFinalCredential(String); + +impl EncryptedFinalCredential { + pub fn new(value: impl Into) -> Self { + Self(value.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for EncryptedFinalCredential { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("EncryptedFinalCredential") + .field(&"") + .finish() + } +} + +/// Immutable identity bound into final-credential AEAD associated data. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FinalCredentialContext { + pub deletion_job_id: Uuid, + pub creator: CreatorPubky, + pub bundle_id: BundleId, +} + +/// Secret-free identity of an eligible deletion snapshot awaiting final credential issuance. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FinalCredentialMaterialization { + pub creator: CreatorPubky, + pub bundle_id: BundleId, +} + +/// Immutable final-access windows established by the storage serialization winner. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FinalAccessWindows { + pub issuance_started_at: OffsetDateTime, + pub credential_issuance_deadline: OffsetDateTime, + pub read_deadline: OffsetDateTime, +} + +/// Closed result of claim-fenced final-access window initialization. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InitializeFinalAccessWindowsResult { + Initialized(FinalAccessWindows), + ClaimLost, +} + +/// A deletion credential returned only after its encrypted bearer is durable. +#[derive(Clone, PartialEq, Eq)] +pub struct IssuedDeletionCredential { + pub credential: AccessCredential, + pub expires_at: OffsetDateTime, +} + +impl fmt::Debug for IssuedDeletionCredential { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("IssuedDeletionCredential") + .field("credential", &"") + .field("expires_at", &self.expires_at) + .finish() + } +} + +/// Frozen-manifest authorization prepared before guarded-resource I/O. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeletionReadAuthorization { + pub claim_token: Option, + pub creator: CreatorPubky, + pub resource: GuardedResource, +} + /// Non-bearer lookup key derived from an access credential. /// /// Stores use this BLAKE3 digest instead of raw bearer credential strings. diff --git a/locks-service/src/application/models/content_lock_deletion.rs b/locks-service/src/application/models/content_lock_deletion.rs new file mode 100644 index 0000000..0cd2e46 --- /dev/null +++ b/locks-service/src/application/models/content_lock_deletion.rs @@ -0,0 +1,212 @@ +use std::str::FromStr; + +use locks_core::{ + ids::{CreatorPubky, LockId}, + lock_policy::ContentLock, +}; +use time::OffsetDateTime; +use uuid::Uuid; + +use crate::application::errors::ApplicationError; + +/// Internal deletion workflow state. Public API status conversion is intentionally separate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContentLockDeletionState { + Queued, + Running, + Completed, + Failed, +} + +/// Closed creator-visible failure vocabulary. Raw dependency errors never cross this boundary. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContentLockDeletionFailureCode { + TombstoneMissing, + TombstoneReplaced, + ResourceReplaced, + RetryExhausted, + StateCorrupt, +} + +impl ContentLockDeletionFailureCode { + /// Returns the exact stable public/database value. + pub fn as_str(self) -> &'static str { + match self { + Self::TombstoneMissing => "tombstone_missing", + Self::TombstoneReplaced => "tombstone_replaced", + Self::ResourceReplaced => "resource_replaced", + Self::RetryExhausted => "retry_exhausted", + Self::StateCorrupt => "state_corrupt", + } + } +} + +impl FromStr for ContentLockDeletionFailureCode { + type Err = ApplicationError; + + fn from_str(value: &str) -> Result { + match value { + "tombstone_missing" => Ok(Self::TombstoneMissing), + "tombstone_replaced" => Ok(Self::TombstoneReplaced), + "resource_replaced" => Ok(Self::ResourceReplaced), + "retry_exhausted" => Ok(Self::RetryExhausted), + "state_corrupt" => Ok(Self::StateCorrupt), + _ => Err(ApplicationError::InvalidContentLockDeletionState { + message: "unknown content lock deletion failure code".to_owned(), + }), + } + } +} + +/// Internal orchestration phase. These values are not public API. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContentLockDeletionPhase { + Withdraw, + StartPaymentDrain, + DrainPayments, + DrainExistingCredentials, + IssueFinalCredentials, + DrainFinalReads, + DeleteContent, + DeleteTombstone, + PurgeOperationalState, +} + +impl ContentLockDeletionPhase { + /// Returns true only for the immediate forward workflow transition. + pub fn permits(self, next: Self) -> bool { + matches!( + (self, next), + (Self::Withdraw, Self::StartPaymentDrain) + | (Self::StartPaymentDrain, Self::DrainPayments) + | (Self::DrainPayments, Self::DrainExistingCredentials) + | (Self::DrainExistingCredentials, Self::IssueFinalCredentials) + | (Self::IssueFinalCredentials, Self::DrainFinalReads) + | (Self::DrainFinalReads, Self::DeleteContent) + | (Self::DeleteContent, Self::DeleteTombstone) + | (Self::DeleteTombstone, Self::PurgeOperationalState) + ) + } +} + +/// Durable graceful content-lock deletion job and immutable frozen manifest. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContentLockDeletionJob { + pub job_id: Uuid, + pub creator: CreatorPubky, + pub lock_id: LockId, + pub frozen_content_lock: ContentLock, + pub deletion_started_at: OffsetDateTime, + pub state: ContentLockDeletionState, + pub phase: ContentLockDeletionPhase, + pub attempt_count: u32, + pub next_attempt_at: Option, + pub force_requested_at: Option, + pub failure_code: Option, +} + +/// Claimed job plus the fresh lease-incarnation token required for fenced writes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClaimedContentLockDeletionJob { + pub job: ContentLockDeletionJob, + pub claim_token: Uuid, +} + +/// Transactionally authoritative result of a requested deletion phase advance. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AdvanceContentLockDeletionPhaseResult { + /// The exact live claim advanced and the returned job reflects the committed phase. + Advanced(Box), + /// The claim was no longer live when the repository serialized the transition. + ClaimLost, + /// The claim remains live, but healthy access obligations must drain before advancing. + ObligationsPending, + /// The claim remains live, but a closed failure requires immediate fail-closed terminalization. + TerminalFailure(ContentLockDeletionFailureCode), +} + +impl AdvanceContentLockDeletionPhaseResult { + pub fn advanced(self) -> Option { + match self { + Self::Advanced(job) => Some(*job), + Self::ClaimLost | Self::ObligationsPending | Self::TerminalFailure(_) => None, + } + } +} + +/// Durable decision made while serializing force deletion against graceful lifecycle state. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PrepareForceDeletionResult { + /// A pre-existing publication intent must reconcile before force can begin. + PublicationInProgress, + /// An active graceful job was durably marked for asynchronous force processing. + Active(ContentLockDeletionJob), + /// A permanent force receipt was established. A terminal frozen job is returned when present. + Synchronous(Option), +} + +impl ContentLockDeletionJob { + /// Creates a queued deletion job from a canonical frozen content lock. + pub fn new( + job_id: Uuid, + frozen_content_lock: ContentLock, + deletion_started_at: OffsetDateTime, + ) -> Result { + let lock_id = frozen_content_lock.lock_id().map_err(|error| { + ApplicationError::ContentLockCanonicalization { + message: error.to_string(), + } + })?; + Ok(Self { + job_id, + creator: frozen_content_lock.creator.clone(), + lock_id, + frozen_content_lock, + deletion_started_at, + state: ContentLockDeletionState::Queued, + phase: ContentLockDeletionPhase::Withdraw, + attempt_count: 0, + next_attempt_at: None, + force_requested_at: None, + failure_code: None, + }) + } + + /// Recomputes the frozen lock identity and verifies the durable key fields. + pub fn validate_frozen_identity(&self) -> Result<(), ApplicationError> { + let actual = self.frozen_content_lock.lock_id().map_err(|error| { + ApplicationError::ContentLockCanonicalization { + message: error.to_string(), + } + })?; + if actual != self.lock_id || self.frozen_content_lock.creator != self.creator { + return Err(ApplicationError::InvalidContentLockDeletionState { + message: "frozen content lock identity does not match deletion job".to_owned(), + }); + } + Ok(()) + } + + /// Validates lifecycle fields against whether persistence has a complete active lease. + pub fn validate_state(&self, has_active_lease: bool) -> Result<(), ApplicationError> { + let valid = match self.state { + ContentLockDeletionState::Queued => !has_active_lease && self.failure_code.is_none(), + ContentLockDeletionState::Running => { + has_active_lease && self.next_attempt_at.is_none() && self.failure_code.is_none() + } + ContentLockDeletionState::Completed => { + !has_active_lease && self.next_attempt_at.is_none() && self.failure_code.is_none() + } + ContentLockDeletionState::Failed => { + !has_active_lease && self.next_attempt_at.is_none() && self.failure_code.is_some() + } + }; + if valid { + Ok(()) + } else { + Err(ApplicationError::InvalidContentLockDeletionState { + message: "deletion lifecycle fields are inconsistent".to_owned(), + }) + } + } +} diff --git a/locks-service/src/application/models/content_lock_ownership.rs b/locks-service/src/application/models/content_lock_ownership.rs new file mode 100644 index 0000000..3692729 --- /dev/null +++ b/locks-service/src/application/models/content_lock_ownership.rs @@ -0,0 +1,46 @@ +use locks_core::ids::{CreatorPubky, LockId}; + +use crate::application::errors::ApplicationError; + +/// Durable lifecycle status for exclusive guarded-path ownership. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContentLockOwnershipStatus { + /// The path is reserved for an intended lock before public publication. + Reserved, + /// The intended lock was published successfully. + Published, +} + +impl ContentLockOwnershipStatus { + /// Returns the stable Postgres representation. + pub fn as_str(self) -> &'static str { + match self { + Self::Reserved => "reserved", + Self::Published => "published", + } + } + + /// Parses a status loaded from persistence. + pub fn from_storage(value: &str) -> Result { + match value { + "reserved" => Ok(Self::Reserved), + "published" => Ok(Self::Published), + _ => Err(ApplicationError::Storage { + message: format!("invalid content lock ownership status: {value}"), + }), + } + } +} + +/// Exclusive ownership of one creator-scoped guarded path by an intended lock. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContentLockOwnership { + /// Creator who owns the guarded path. + pub creator: CreatorPubky, + /// Full canonical guarded-resource path. + pub guarded_path: String, + /// Canonical Lock ID intended to own the path. + pub lock_id: LockId, + /// Reservation/publication lifecycle status. + pub status: ContentLockOwnershipStatus, +} diff --git a/locks-service/src/application/models/mod.rs b/locks-service/src/application/models/mod.rs index 68d2597..907bc24 100644 --- a/locks-service/src/application/models/mod.rs +++ b/locks-service/src/application/models/mod.rs @@ -1,10 +1,14 @@ mod access; +mod content_lock_deletion; +mod content_lock_ownership; mod creator_authority; mod frontend_session; mod guarded_resource; mod verification; pub use access::*; +pub use content_lock_deletion::*; +pub use content_lock_ownership::*; pub use creator_authority::*; pub use frontend_session::*; pub use guarded_resource::*; diff --git a/locks-service/src/application/ports/access.rs b/locks-service/src/application/ports/access.rs index 07539f1..1699555 100644 --- a/locks-service/src/application/ports/access.rs +++ b/locks-service/src/application/ports/access.rs @@ -1,10 +1,25 @@ use async_trait::async_trait; +use locks_core::ids::{BundleId, CreatorPubky, LockId}; +use time::{Duration, OffsetDateTime}; +use uuid::Uuid; use crate::application::errors::ApplicationError; use crate::application::models::{ - AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, + AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, DeletionReadAuthorization, + FinalCredentialMaterialization, InitializeFinalAccessWindowsResult, IssuedDeletionCredential, }; +/// Exact worker ownership and candidate material for one final-credential winner operation. +pub struct FinalCredentialWorkerIssueRequest<'a> { + pub deletion_job_id: Uuid, + pub worker_id: &'a str, + pub claim_token: Uuid, + pub creator: &'a CreatorPubky, + pub bundle_id: &'a BundleId, + pub now: OffsetDateTime, + pub candidate: AccessCredential, +} + /// Store for issued opaque access credentials. #[async_trait] pub trait AccessCredentialStore: Send + Sync { @@ -13,6 +28,7 @@ pub trait AccessCredentialStore: Send + Sync { /// Returns `DuplicateRecord` if the credential lookup key already exists. async fn insert_access_credential( &self, + lock_id: &LockId, lookup_key: AccessCredentialLookupKey, record: AccessCredentialRecord, ) -> Result<(), ApplicationError>; @@ -32,6 +48,95 @@ pub trait AccessCredentialStore: Send + Sync { &self, lookup_key: &AccessCredentialLookupKey, ) -> Result<(), ApplicationError>; + + async fn initialize_final_access_windows( + &self, + _deletion_job_id: Uuid, + _worker_id: &str, + _claim_token: Uuid, + _issuance_window: Duration, + _read_window: Duration, + ) -> Result { + Ok(InitializeFinalAccessWindowsResult::ClaimLost) + } + + /// Enumerates eligible snapshots awaiting final credential materialization under an exact + /// live deletion-worker claim. Implementations return deterministic bundle ordering. + async fn final_credentials_to_materialize( + &self, + _deletion_job_id: Uuid, + _worker_id: &str, + _claim_token: Uuid, + _limit: usize, + ) -> Result, ApplicationError> { + Ok(Vec::new()) + } + + async fn issue_or_replay_final_credential( + &self, + _creator: &CreatorPubky, + _bundle_id: &BundleId, + _now: OffsetDateTime, + _candidate: AccessCredential, + ) -> Result, ApplicationError> { + Ok(None) + } + + /// Issues or replays one final credential only while the exact deletion-worker claim remains + /// live. Implementations revalidate ownership and fresh time in the winner transaction. + async fn issue_or_replay_final_credential_for_worker( + &self, + _request: FinalCredentialWorkerIssueRequest<'_>, + ) -> Result, ApplicationError> { + Ok(None) + } + + /// Reports whether this deletion Bundle may issue or replay its final credential now. + async fn final_credential_available( + &self, + _creator: &CreatorPubky, + _bundle_id: &BundleId, + _now: OffsetDateTime, + ) -> Result { + Ok(false) + } + + async fn prepare_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + _claim_duration: Duration, + ) -> Result, ApplicationError> { + Ok(None) + } + + /// Reports whether a credential was enrolled in deletion, regardless of + /// whether deletion access is currently usable. + async fn deletion_credential_enrolled( + &self, + _lookup_key: &AccessCredentialLookupKey, + ) -> Result { + Ok(false) + } + + async fn release_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + _claim_token: Uuid, + _now: OffsetDateTime, + ) -> Result { + Ok(false) + } + + async fn consume_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + _claim_token: Uuid, + ) -> Result { + Ok(false) + } } /// Generator for opaque access credentials. diff --git a/locks-service/src/application/ports/content_lock_deletion.rs b/locks-service/src/application/ports/content_lock_deletion.rs new file mode 100644 index 0000000..3b9fce5 --- /dev/null +++ b/locks-service/src/application/ports/content_lock_deletion.rs @@ -0,0 +1,139 @@ +use async_trait::async_trait; +use locks_core::ids::{CreatorPubky, LockId}; +use time::OffsetDateTime; +use uuid::Uuid; + +use crate::application::{ + errors::ApplicationError, + models::{ + AdvanceContentLockDeletionPhaseResult, ClaimedContentLockDeletionJob, + ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionPhase, + PrepareForceDeletionResult, + }, +}; + +/// Durable repository and fenced worker lease boundary for content-lock deletion jobs. +#[async_trait] +pub trait ContentLockDeletionRepository: Send + Sync { + /// Reserves canonical publication under the same per-lock fence used by force deletion. + async fn begin_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result<(), ApplicationError>; + + /// Finalizes the exact publication reservation after external publication and ownership commit. + async fn finish_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result; + + /// Removes the exact unfinalized reservation after a safely compensated publication failure. + async fn abandon_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result; + + /// Checks publication admission under the canonical per-lock fence. + async fn publication_in_progress( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result; + + async fn insert_job(&self, job: ContentLockDeletionJob) -> Result<(), ApplicationError>; + + async fn get_job( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result, ApplicationError>; + + async fn claim_next( + &self, + worker_id: &str, + claim_ttl: time::Duration, + ) -> Result, ApplicationError>; + + async fn schedule_retry( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + retry_after: time::Duration, + ) -> Result, ApplicationError>; + + /// Releases a healthy-poll claim and schedules its next observation without + /// charging the claim-acquisition increment to the transient failure budget. + async fn defer( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + defer_for: time::Duration, + ) -> Result, ApplicationError>; + + async fn advance_phase( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + next_phase: ContentLockDeletionPhase, + ) -> Result; + + /// Expires every unresolved frozen non-Paykit task under the exact live deletion claim. + async fn expire_unresolved_non_paykit_tasks( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + ) -> Result { + let _ = (job_id, worker_id, claim_token); + Err(ApplicationError::InvalidContentLockDeletionState { + message: "non-Paykit deletion drain is not supported by this repository".to_owned(), + }) + } + + /// Persists terminal completion or a stable secret-free failure under the exact lease. + async fn finish( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + failure_code: Option, + ) -> Result, ApplicationError>; + + /// Requeues a failed job with its frozen manifest unless a permanent force receipt exists. + async fn resume_failed_job( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + resumed_at: OffsetDateTime, + ) -> Result, ApplicationError>; + + /// Atomically escalates an active job or establishes the permanent synchronous-force receipt. + async fn prepare_force_deletion( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result; + + /// Finalizes force deletion only for the exact live worker claim that observed the effects. + async fn complete_force_deletion( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + ) -> Result; + + async fn has_force_receipt( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result; +} diff --git a/locks-service/src/application/ports/content_lock_deletion_action_ownership.rs b/locks-service/src/application/ports/content_lock_deletion_action_ownership.rs new file mode 100644 index 0000000..3cb3c10 --- /dev/null +++ b/locks-service/src/application/ports/content_lock_deletion_action_ownership.rs @@ -0,0 +1,40 @@ +use async_trait::async_trait; +use uuid::Uuid; + +use crate::application::{errors::ApplicationError, models::ContentLockDeletionPhase}; + +/// Exact expected live claim and lifecycle state for one external action lane. +#[derive(Debug, Clone, Copy)] +pub struct ContentLockDeletionActionClaim<'a> { + pub job_id: Uuid, + pub worker_id: &'a str, + pub claim_token: Uuid, + pub expected_phase: ContentLockDeletionPhase, + pub force: bool, +} + +/// Owns one job's external side-effect lane until explicitly released or dropped. +#[async_trait] +pub trait ContentLockDeletionActionGuard: Send { + /// Releases ownership. Implementations must not return a still-locked + /// connection or equivalent resource to shared storage. + async fn release(self: Box) -> Result<(), ApplicationError>; +} + +/// Closed result of post-lock live-claim validation. +pub enum ContentLockDeletionActionAcquireResult { + Acquired(Box), + Busy, + ClaimLost, +} + +/// Nonblocking per-job ownership boundary for deletion external actions. +#[async_trait] +pub trait ContentLockDeletionActionOwnership: Send + Sync { + /// Acquires the per-job lane, then validates the exact claim against + /// storage-authoritative time and expected lifecycle state. + async fn try_acquire( + &self, + claim: ContentLockDeletionActionClaim<'_>, + ) -> Result; +} diff --git a/locks-service/src/application/ports/content_lock_ownership.rs b/locks-service/src/application/ports/content_lock_ownership.rs new file mode 100644 index 0000000..323a7ee --- /dev/null +++ b/locks-service/src/application/ports/content_lock_ownership.rs @@ -0,0 +1,48 @@ +use async_trait::async_trait; +use locks_core::ids::{CreatorPubky, LockId}; + +use crate::application::errors::ApplicationError; +use crate::application::models::ContentLockOwnership; + +/// Repository for exclusive creator-scoped guarded-path ownership. +#[async_trait] +pub trait ContentLockOwnershipRepository: Send + Sync { + /// Atomically reserves every path for the intended lock. + /// + /// Exact retry for the same published Lock ID is idempotent. An existing + /// reservation, or a path owned by a different Lock ID, returns + /// `ContentLockPathConflict` and reserves none of the previously unowned + /// paths in the request. Blocking in-flight reservations prevents one + /// publisher from compensating another publisher's ownership. + async fn reserve_paths( + &self, + creator: &CreatorPubky, + guarded_paths: &[String], + lock_id: &LockId, + ) -> Result<(), ApplicationError>; + + /// Marks the intended lock's complete path set as successfully published. + async fn mark_paths_published( + &self, + creator: &CreatorPubky, + guarded_paths: &[String], + lock_id: &LockId, + ) -> Result<(), ApplicationError>; + + /// Best-effort publication-failure compensation for matching reserved rows. + /// + /// Published ownership is deliberately retained. + async fn compensate_reserved_paths( + &self, + creator: &CreatorPubky, + guarded_paths: &[String], + lock_id: &LockId, + ) -> Result<(), ApplicationError>; + + /// Reads current ownership for a creator-scoped guarded path. + async fn get_path_ownership( + &self, + creator: &CreatorPubky, + guarded_path: &str, + ) -> Result, ApplicationError>; +} diff --git a/locks-service/src/application/ports/content_lock_tombstone.rs b/locks-service/src/application/ports/content_lock_tombstone.rs new file mode 100644 index 0000000..c61feee --- /dev/null +++ b/locks-service/src/application/ports/content_lock_tombstone.rs @@ -0,0 +1,74 @@ +use async_trait::async_trait; +use locks_core::content_lock_deletion::ContentLockDeletionTombstone; +use locks_core::ids::{ContentLockPath, CreatorPubky}; +use locks_core::lock_policy::ContentLock; + +use crate::application::errors::ApplicationError; + +/// Byte-for-byte state of a canonical public deletion tombstone. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TombstoneReadback { + /// The public bytes exactly match the canonical expected tombstone bytes. + Exact, + /// The canonical public lock path is absent. + Missing, + /// The path contains bytes other than the canonical expected tombstone. + Replaced, +} + +/// Exact public tombstone publication and readback boundary. +#[async_trait] +pub trait ContentLockTombstoneRepository: Send + Sync { + /// Reconciles withdrawal from the current canonical public bytes. + /// + /// An exact tombstone is replay success without a write. The implementation compares current + /// bytes with the frozen original before publishing and fails closed on bytes already observed + /// as missing or replaced. Pubky 0.9.3 has no conditional write, so this is deliberately not a + /// CAS guarantee: an out-of-band replacement racing between the comparison and unconditional + /// tombstone PUT can be overwritten under the product's documented TOCTOU exception. + async fn withdraw_content_lock( + &self, + creator: CreatorPubky, + content_lock_path: ContentLockPath, + frozen_original: &ContentLock, + tombstone: &ContentLockDeletionTombstone, + ) -> Result; + + /// Classifies raw bytes at the canonical public lock path without parsing them as a lock. + async fn read_tombstone( + &self, + creator: &CreatorPubky, + content_lock_path: &ContentLockPath, + expected: &ContentLockDeletionTombstone, + ) -> Result; + + /// Force-only operation that unconditionally deletes whatever bytes occupy the canonical + /// public lock path and succeeds only after a raw read verifies that the path is absent. + /// + /// This deliberately does not compare or parse the current bytes: an original Content Lock, + /// the expected tombstone, or any replacement is deleted. + async fn force_delete_content_lock_and_verify_absent( + &self, + creator: &CreatorPubky, + content_lock_path: &ContentLockPath, + ) -> Result<(), ApplicationError>; +} + +pub(crate) fn canonical_tombstone_bytes( + tombstone: &ContentLockDeletionTombstone, +) -> Result, ApplicationError> { + serde_json::to_vec(tombstone).map_err(|error| ApplicationError::Storage { + message: format!("failed to serialize content lock deletion tombstone: {error}"), + }) +} + +pub(crate) fn classify_tombstone_bytes( + actual: Option<&[u8]>, + expected: &[u8], +) -> TombstoneReadback { + match actual { + None => TombstoneReadback::Missing, + Some(actual) if actual == expected => TombstoneReadback::Exact, + Some(_) => TombstoneReadback::Replaced, + } +} diff --git a/locks-service/src/application/ports/entitlement.rs b/locks-service/src/application/ports/entitlement.rs index 9158df8..e635e7a 100644 --- a/locks-service/src/application/ports/entitlement.rs +++ b/locks-service/src/application/ports/entitlement.rs @@ -35,3 +35,22 @@ pub trait EntitlementRepository: Send + Sync { bundle_id: &BundleId, ) -> Result<(), ApplicationError>; } + +pub(crate) fn same_entitlement_decision( + existing: &VerifiedProofBundle, + candidate: &VerifiedProofBundle, +) -> bool { + if existing.verification_result.criteria.len() != candidate.verification_result.criteria.len() { + return false; + } + let mut normalized_candidate = candidate.clone(); + for (candidate_result, existing_result) in normalized_candidate + .verification_result + .criteria + .iter_mut() + .zip(&existing.verification_result.criteria) + { + candidate_result.verified_at = existing_result.verified_at; + } + existing == &normalized_candidate +} diff --git a/locks-service/src/application/ports/guarded_resources.rs b/locks-service/src/application/ports/guarded_resources.rs index b5241a8..e304770 100644 --- a/locks-service/src/application/ports/guarded_resources.rs +++ b/locks-service/src/application/ports/guarded_resources.rs @@ -4,6 +4,14 @@ use locks_core::ids::{CreatorPubky, GuardedResourceHash}; use crate::application::errors::ApplicationError; use crate::application::models::GuardedResourceRecord; +/// Closed, non-destructive classification of a frozen guarded-resource generation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GuardedResourceReadback { + Exact, + Missing, + Replaced, +} + /// Repository for guarded resource bytes used by the first retrieval/access slice. #[async_trait] pub trait GuardedResourceRepository: Send + Sync { @@ -42,4 +50,20 @@ pub trait GuardedResourceRepository: Send + Sync { creator: &CreatorPubky, path: &str, ) -> Result; + + /// Reads and classifies the current generation without deleting any bytes. + async fn read_guarded_resource_generation( + &self, + creator: &CreatorPubky, + path: &str, + expected_hash: &GuardedResourceHash, + ) -> Result { + Ok( + match self.get_current_guarded_resource(creator, path).await? { + None => GuardedResourceReadback::Missing, + Some(record) if record.hash == *expected_hash => GuardedResourceReadback::Exact, + Some(_) => GuardedResourceReadback::Replaced, + }, + ) + } } diff --git a/locks-service/src/application/ports/lock_policy.rs b/locks-service/src/application/ports/lock_policy.rs index 03a2438..f8f1f5a 100644 --- a/locks-service/src/application/ports/lock_policy.rs +++ b/locks-service/src/application/ports/lock_policy.rs @@ -25,6 +25,14 @@ pub trait ContentLockRepository: Send + Sync { creator: &CreatorPubky, content_lock_path: &ContentLockPath, ) -> Result, ApplicationError>; + + /// Deletes the public content lock at the canonical creator-owned path. + /// Returns true when a record existed and false when already absent. + async fn delete_content_lock( + &self, + creator: &CreatorPubky, + content_lock_path: &ContentLockPath, + ) -> Result; } /// Repository for creator-owned Lock Service Pointer config objects. diff --git a/locks-service/src/application/ports/mod.rs b/locks-service/src/application/ports/mod.rs index afa0e29..d5ce366 100644 --- a/locks-service/src/application/ports/mod.rs +++ b/locks-service/src/application/ports/mod.rs @@ -1,17 +1,69 @@ pub mod semantics {} mod access; +mod content_lock_deletion; +pub mod content_lock_deletion_action_ownership; +mod content_lock_ownership; +pub mod content_lock_tombstone; mod creator_authority; mod entitlement; mod guarded_resources; mod lock_policy; +mod payment_drain; +mod payment_drain_repository; mod runtime; mod verification; pub use access::*; +pub use content_lock_deletion::*; +pub use content_lock_deletion_action_ownership::*; +pub use content_lock_ownership::*; +pub use content_lock_tombstone::*; pub use creator_authority::*; pub use entitlement::*; pub use guarded_resources::*; pub use lock_policy::*; +pub use payment_drain::*; +pub use payment_drain_repository::*; pub use runtime::*; pub use verification::*; + +#[cfg(test)] +mod payment_drain_contract_tests { + use super::{ + PaymentDrainCleanupToken, PaymentDrainClient, PaymentDrainClientError, PaymentDrainStatus, + PaymentRequestState, PaymentState, + }; + + fn assert_object_safe(_: &dyn PaymentDrainClient) {} + + #[test] + fn payment_drain_port_is_object_safe_and_closed_values_parse_strictly() { + let _ = assert_object_safe; + assert_eq!( + PaymentDrainStatus::parse("active"), + Some(PaymentDrainStatus::Active) + ); + assert_eq!(PaymentDrainStatus::parse("complete"), None); + assert_eq!( + PaymentRequestState::parse("proposal_expired"), + Some(PaymentRequestState::ProposalExpired) + ); + assert_eq!(PaymentRequestState::parse("unknown"), None); + assert_eq!(PaymentState::parse("expired"), Some(PaymentState::Expired)); + assert_eq!(PaymentState::parse("late"), None); + let token = + PaymentDrainCleanupToken::parse("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA").unwrap(); + assert_eq!( + token.as_str(), + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + ); + assert!(PaymentDrainCleanupToken::parse("short").is_none()); + assert_eq!(format!("{token:?}"), "PaymentDrainCleanupToken()"); + let _ = PaymentDrainClientError::NotFound; + let _ = PaymentDrainClientError::Conflict; + let _ = PaymentDrainClientError::MalformedSuccess; + let _ = PaymentDrainClientError::Transport; + let _ = PaymentDrainClientError::Server; + } +} diff --git a/locks-service/src/application/ports/payment_drain.rs b/locks-service/src/application/ports/payment_drain.rs new file mode 100644 index 0000000..4b5e55c --- /dev/null +++ b/locks-service/src/application/ports/payment_drain.rs @@ -0,0 +1,143 @@ +use std::fmt; + +use async_trait::async_trait; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use locks_core::ids::{BundleId, CreatorPubky, PubkyLockResource}; +use time::OffsetDateTime; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PaymentDrainStatus { + Active, + Completed, +} + +impl PaymentDrainStatus { + pub fn parse(value: &str) -> Option { + match value { + "active" => Some(Self::Active), + "completed" => Some(Self::Completed), + _ => None, + } + } +} + +#[derive(Clone, PartialEq, Eq)] +pub struct PaymentDrainCleanupToken(String); + +impl PaymentDrainCleanupToken { + pub fn parse(value: &str) -> Option { + if value.len() != 43 || value.contains('=') { + return None; + } + let decoded: [u8; 32] = URL_SAFE_NO_PAD.decode(value).ok()?.try_into().ok()?; + (URL_SAFE_NO_PAD.encode(decoded) == value).then(|| Self(value.to_owned())) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for PaymentDrainCleanupToken { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("PaymentDrainCleanupToken()") + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PaymentDrainSummary { + pub status: PaymentDrainStatus, + pub accepted_count: u64, + pub terminal_count: u64, + pub cancellation_enqueued_count: u64, + pub cleanup_token: PaymentDrainCleanupToken, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PaymentRequestState { + Proposed, + ProposalExpired, + Accepted, + Rejected, + Canceled, + ProofSubmitted, + ActiveRecurring, + RecoveryRequired, + InvalidConflict, +} + +impl PaymentRequestState { + pub fn parse(value: &str) -> Option { + match value { + "proposed" => Some(Self::Proposed), + "proposal_expired" => Some(Self::ProposalExpired), + "accepted" => Some(Self::Accepted), + "rejected" => Some(Self::Rejected), + "canceled" => Some(Self::Canceled), + "proof_submitted" => Some(Self::ProofSubmitted), + "active_recurring" => Some(Self::ActiveRecurring), + "recovery_required" => Some(Self::RecoveryRequired), + "invalid_conflict" => Some(Self::InvalidConflict), + _ => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PaymentState { + Undetected, + Detected, + Confirmed, + Expired, +} + +impl PaymentState { + pub fn parse(value: &str) -> Option { + match value { + "undetected" => Some(Self::Undetected), + "detected" => Some(Self::Detected), + "confirmed" => Some(Self::Confirmed), + "expired" => Some(Self::Expired), + _ => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PaymentRequestStatus { + pub request_state: PaymentRequestState, + pub payment_state: PaymentState, + pub invoice_created_at: OffsetDateTime, + pub payment_deadline: OffsetDateTime, + pub confirmations: u32, + pub amount_matched: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PaymentDrainClientError { + NotFound, + Conflict, + MalformedSuccess, + Transport, + Server, +} + +#[async_trait] +pub trait PaymentDrainClient: Send + Sync { + async fn start_payment_drain( + &self, + lock_resource: &PubkyLockResource, + ) -> Result; + + async fn lookup_payment_drain( + &self, + lock_resource: &PubkyLockResource, + ) -> Result, PaymentDrainClientError>; + + async fn payment_request_status( + &self, + creator: &CreatorPubky, + bundle_id: &BundleId, + ) -> Result, PaymentDrainClientError>; +} diff --git a/locks-service/src/application/ports/payment_drain_repository.rs b/locks-service/src/application/ports/payment_drain_repository.rs new file mode 100644 index 0000000..89de2b4 --- /dev/null +++ b/locks-service/src/application/ports/payment_drain_repository.rs @@ -0,0 +1,78 @@ +use async_trait::async_trait; +use locks_core::ids::{BundleId, CreatorPubky, PubkyLockResource, TaskId}; +use time::OffsetDateTime; +use uuid::Uuid; + +use crate::application::errors::ApplicationError; +use crate::application::models::VerificationTaskStatus; + +use super::PaymentDrainSummary; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PaymentDrainObligation { + pub task_id: TaskId, + pub creator: CreatorPubky, + pub bundle_id: BundleId, + pub lock_resource: PubkyLockResource, + pub criterion_id: String, + pub invoice_created_at: OffsetDateTime, + pub payment_deadline: OffsetDateTime, + pub status: VerificationTaskStatus, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PaymentDrainTerminalTransition { + pub status: VerificationTaskStatus, + pub entitlement_publication_token: Option, +} + +#[async_trait] +pub trait PaymentDrainRepository: Send + Sync { + async fn store_payment_drain( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + summary: &PaymentDrainSummary, + ) -> Result; + + async fn get_payment_drain( + &self, + deletion_job_id: Uuid, + ) -> Result, ApplicationError>; + + async fn reconcile_payment_drain( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + summary: &PaymentDrainSummary, + ) -> Result; + + async fn list_obligations( + &self, + deletion_job_id: Uuid, + ) -> Result, ApplicationError>; + + async fn begin_entitlement_publication( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + task_id: &TaskId, + ) -> Result, ApplicationError>; + + async fn persist_terminal_obligation( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + task_id: &TaskId, + transition: PaymentDrainTerminalTransition, + ) -> Result; + + async fn all_obligations_terminal( + &self, + deletion_job_id: Uuid, + ) -> Result; +} diff --git a/locks-service/src/application/ports/verification.rs b/locks-service/src/application/ports/verification.rs index ad706b9..078bfd7 100644 --- a/locks-service/src/application/ports/verification.rs +++ b/locks-service/src/application/ports/verification.rs @@ -15,6 +15,8 @@ pub trait VerificationTaskRepository: Send + Sync { /// /// Returns `DuplicateRecord` if a task with the same Task ID or public /// verification attempt handle (`creator`, `bundle_id`) already exists. + /// PostgreSQL additionally returns `ContentLockDeletionInProgress` when the + /// authoritative deletion cutoff already exists for a new Bundle ID. async fn insert_verification_task( &self, task: VerificationTaskRecord, @@ -28,6 +30,28 @@ pub trait VerificationTaskRepository: Send + Sync { task: VerificationTaskRecord, ) -> Result<(), ApplicationError>; + /// Atomically updates a prevalidated set of verification tasks. + /// + /// Repositories that support multi-record updates must either apply every update or none. + /// The default supports the trivially atomic single-record case only. + async fn update_verification_tasks_atomically( + &self, + mut tasks: Vec, + ) -> Result<(), ApplicationError> { + if tasks.len() == 1 { + return self + .update_verification_task(tasks.pop().expect("length checked")) + .await; + } + if tasks.is_empty() { + return Ok(()); + } + Err(ApplicationError::Storage { + message: "atomic verification task batch update is not implemented by this repository" + .to_owned(), + }) + } + /// Loads a verification task by task ID. /// /// Returns `Ok(None)` when no task exists. @@ -60,16 +84,24 @@ pub trait VerificationTaskRepository: Send + Sync { /// Worker-facing port for claiming verification task leases. #[async_trait] pub trait VerificationTaskClaimer: Send + Sync { + /// Fences entitlement publication before external storage I/O. + /// + /// Returns false when the exact lease is lost or deletion already owns the task. + async fn begin_claimed_entitlement_publication( + &self, + task_id: &TaskId, + worker_id: &str, + claim_token: &uuid::Uuid, + ) -> Result; + /// Claims one pending or expired in-progress verification task for a worker. /// /// Returns `Ok(None)` when no task is claimable. Every successful claim includes a fresh - /// opaque token. The `now` parameter defines expiration comparison time, and - /// `claim_expires_at` is the new lease expiry assigned to the claimed task. + /// opaque token. Implementations sample authoritative time after fencing the selected task. async fn claim_next_verification_task( &self, worker_id: &str, - now: time::OffsetDateTime, - claim_expires_at: time::OffsetDateTime, + claim_ttl: time::Duration, ) -> Result, ApplicationError>; /// Returns an actively owned in-progress task to pending with a durable retry due time. @@ -82,8 +114,7 @@ pub trait VerificationTaskClaimer: Send + Sync { task_id: &TaskId, worker_id: &str, claim_token: &uuid::Uuid, - now: time::OffsetDateTime, - next_attempt_at: time::OffsetDateTime, + retry_after: time::Duration, ) -> Result, ApplicationError>; /// Persists a terminal task transition only for the exact active lease incarnation. @@ -94,7 +125,6 @@ pub trait VerificationTaskClaimer: Send + Sync { task: VerificationTaskRecord, worker_id: &str, claim_token: &uuid::Uuid, - now: time::OffsetDateTime, ) -> Result, ApplicationError>; } diff --git a/locks-service/src/application/use_cases/complete_verification_task.rs b/locks-service/src/application/use_cases/complete_verification_task.rs index 9e00837..ee318f0 100644 --- a/locks-service/src/application/use_cases/complete_verification_task.rs +++ b/locks-service/src/application/use_cases/complete_verification_task.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use time::OffsetDateTime; -use locks_core::ids::{LockServerPubky, TaskId}; +use locks_core::ids::{BundleId, CreatorPubky, LockServerPubky, TaskId}; use locks_core::lock_policy::ContentLock; use locks_core::verification::{ EntitlementLifetime, VERIFIED_PROOF_BUNDLE_VERSION, VerificationResult, VerifiedProofBundle, @@ -15,7 +15,7 @@ use crate::application::models::{ }; use crate::application::ports::{ Clock, ContentLockRepository, CriterionVerifierRegistry, EntitlementRepository, - VerificationTaskClaimer, VerificationTaskRepository, + VerificationTaskClaimer, VerificationTaskRepository, same_entitlement_decision, }; use crate::application::use_cases::entitlement_check::verify_content_lock_identity; @@ -51,7 +51,54 @@ struct ClaimFencedTaskRepository<'a> { claim: ClaimedVerificationTask, claimer: &'a dyn VerificationTaskClaimer, worker_id: &'a str, - clock: &'a dyn Clock, +} + +struct ClaimFencedEntitlementRepository<'a> { + inner: &'a dyn EntitlementRepository, + claim: &'a ClaimedVerificationTask, + claimer: &'a dyn VerificationTaskClaimer, + worker_id: &'a str, +} + +#[async_trait] +impl EntitlementRepository for ClaimFencedEntitlementRepository<'_> { + async fn insert_verified_proof_bundle( + &self, + entitlement: VerifiedProofBundle, + ) -> Result<(), ApplicationError> { + if !self + .claimer + .begin_claimed_entitlement_publication( + &self.claim.task.task_id, + self.worker_id, + &self.claim.claim_token, + ) + .await? + { + return Err(ApplicationError::VerificationTaskClaimLost); + } + self.inner.insert_verified_proof_bundle(entitlement).await + } + + async fn get_verified_proof_bundle( + &self, + creator: &CreatorPubky, + bundle_id: &BundleId, + ) -> Result, ApplicationError> { + self.inner + .get_verified_proof_bundle(creator, bundle_id) + .await + } + + async fn delete_verified_proof_bundle( + &self, + creator: &CreatorPubky, + bundle_id: &BundleId, + ) -> Result<(), ApplicationError> { + self.inner + .delete_verified_proof_bundle(creator, bundle_id) + .await + } } #[async_trait] @@ -72,7 +119,6 @@ impl VerificationTaskRepository for ClaimFencedTaskRepository<'_> { task, self.worker_id, &self.claim.claim_token, - self.clock.now(), ) .await? .ok_or(ApplicationError::VerificationTaskClaimLost)?; @@ -155,7 +201,11 @@ impl<'a> CompleteVerificationTaskUseCase<'a> { let verification_result = match self.verify_criteria(&task, &content_lock).await { Ok(verification_result) => verification_result, Err(error) => { - if matches!(error, ApplicationError::VerificationPending) { + if matches!( + error, + ApplicationError::VerificationPending + | ApplicationError::VerificationDependencyUnavailable + ) { return Err(error); } self.persist_failed_task(task, viewer_safe_failure_message(&error).to_owned()) @@ -213,19 +263,8 @@ impl<'a> CompleteVerificationTaskUseCase<'a> { .await { Ok(Some(existing)) if same_entitlement_decision(&existing, &entitlement) => {} - Ok(_) => { - self.persist_failed_task(task, viewer_safe_failure_message(&error).to_owned()) - .await?; - return Err(error); - } - Err(lookup_error) => { - self.persist_failed_task( - task, - viewer_safe_failure_message(&lookup_error).to_owned(), - ) - .await?; - return Err(lookup_error); - } + Ok(Some(_)) => return Err(error), + Ok(None) | Err(_) => return Err(ApplicationError::VerificationPending), } } @@ -257,15 +296,20 @@ impl<'a> CompleteVerificationTaskUseCase<'a> { }); } let fenced_tasks = ClaimFencedTaskRepository { - claim, + claim: claim.clone(), + claimer, + worker_id, + }; + let fenced_entitlements = ClaimFencedEntitlementRepository { + inner: self.entitlements, + claim: &claim, claimer, worker_id, - clock: self.clock, }; CompleteVerificationTaskUseCase::new( &fenced_tasks, self.content_locks, - self.entitlements, + &fenced_entitlements, self.verifiers, self.clock, self.verified_by.clone(), @@ -356,9 +400,13 @@ fn viewer_safe_failure_message(error: &ApplicationError) -> &'static str { } ApplicationError::Storage { .. } | ApplicationError::DuplicateRecord { .. } + | ApplicationError::ContentLockPathConflict { .. } + | ApplicationError::ContentLockDeletionInProgress + | ApplicationError::InvalidContentLockDeletionState { .. } | ApplicationError::MissingRecord { .. } | ApplicationError::InvalidVerificationTaskTransition { .. } | ApplicationError::VerificationPending + | ApplicationError::VerificationDependencyUnavailable | ApplicationError::InvalidVerificationTaskState { .. } | ApplicationError::VerificationTaskClaimLost | ApplicationError::InvalidVerificationTaskFailureMessage @@ -366,6 +414,7 @@ fn viewer_safe_failure_message(error: &ApplicationError) -> &'static str { | ApplicationError::RateLimited | ApplicationError::UnsupportedCredentialTtl { .. } | ApplicationError::CredentialGeneration { .. } + | ApplicationError::FinalCredentialSecret { .. } | ApplicationError::CreatorAuthorityUnavailable | ApplicationError::CreatorAuthoritySecret { .. } | ApplicationError::InvalidCreatorAuthorityAuthKind { .. } @@ -386,26 +435,6 @@ fn viewer_safe_failure_message(error: &ApplicationError) -> &'static str { } } -fn same_entitlement_decision( - existing: &VerifiedProofBundle, - candidate: &VerifiedProofBundle, -) -> bool { - if existing.verification_result.criteria.len() != candidate.verification_result.criteria.len() { - return false; - } - - let mut normalized_candidate = candidate.clone(); - for (candidate_result, existing_result) in normalized_candidate - .verification_result - .criteria - .iter_mut() - .zip(&existing.verification_result.criteria) - { - candidate_result.verified_at = existing_result.verified_at; - } - existing == &normalized_candidate -} - #[cfg(test)] mod tests { use std::str::FromStr; @@ -473,6 +502,7 @@ mod tests { datetime!(2026-05-29 12:01:00 UTC), datetime!(2026-05-29 12:02:00 UTC), datetime!(2026-05-29 12:03:00 UTC), + datetime!(2026-05-29 12:04:00 UTC), ]); let use_case = CompleteVerificationTaskUseCase::new( &tasks, @@ -580,6 +610,82 @@ mod tests { ); } + #[tokio::test] + async fn ambiguous_entitlement_write_without_read_back_stays_pending() { + let content_lock = content_lock_fixture(true); + let tasks = FakeTasks::new(Some(pending_task_for(&content_lock))); + let content_locks = FakeContentLocks::new(Some(content_lock)); + let entitlements = FakeEntitlements::with_ambiguous_write_and_absent_read_back(); + let verifier = FakeVerifier { + satisfied: true, + error: None, + }; + let registry = FakeRegistry { + verifier: Some(&verifier), + }; + let clock = SequenceClock::new(vec![ + datetime!(2026-05-29 12:01:00 UTC), + datetime!(2026-05-29 12:02:00 UTC), + ]); + + let result = CompleteVerificationTaskUseCase::new( + &tasks, + &content_locks, + &entitlements, + ®istry, + &clock, + lock_server(), + ) + .execute(CompleteVerificationTaskRequest { task_id: task_id() }) + .await; + + assert_eq!(result, Err(ApplicationError::VerificationPending)); + assert!( + !tasks + .updates() + .iter() + .any(|task| task.status == VerificationTaskStatus::Failed) + ); + } + + #[tokio::test] + async fn ambiguous_entitlement_write_with_failed_read_back_stays_pending() { + let content_lock = content_lock_fixture(true); + let tasks = FakeTasks::new(Some(pending_task_for(&content_lock))); + let content_locks = FakeContentLocks::new(Some(content_lock)); + let entitlements = FakeEntitlements::with_ambiguous_write_and_failed_read_back(); + let verifier = FakeVerifier { + satisfied: true, + error: None, + }; + let registry = FakeRegistry { + verifier: Some(&verifier), + }; + let clock = SequenceClock::new(vec![ + datetime!(2026-05-29 12:01:00 UTC), + datetime!(2026-05-29 12:02:00 UTC), + ]); + + let result = CompleteVerificationTaskUseCase::new( + &tasks, + &content_locks, + &entitlements, + ®istry, + &clock, + lock_server(), + ) + .execute(CompleteVerificationTaskRequest { task_id: task_id() }) + .await; + + assert_eq!(result, Err(ApplicationError::VerificationPending)); + assert!( + !tasks + .updates() + .iter() + .any(|task| task.status == VerificationTaskStatus::Failed) + ); + } + #[tokio::test] async fn current_claim_owner_completes_after_equivalent_entitlement_was_published() { let content_lock = content_lock_fixture(true); @@ -614,8 +720,7 @@ mod tests { let claim = claimer .claim_next_verification_task( "worker-a", - datetime!(2026-05-29 12:04:00 UTC), - datetime!(2026-05-29 13:00:00 UTC), + (datetime!(2026-05-29 13:00:00 UTC)) - (datetime!(2026-05-29 12:04:00 UTC)), ) .await .unwrap() @@ -625,6 +730,7 @@ mod tests { datetime!(2026-05-29 12:05:00 UTC), datetime!(2026-05-29 12:06:00 UTC), datetime!(2026-05-29 12:07:00 UTC), + datetime!(2026-05-29 12:08:00 UTC), ]); let completed = CompleteVerificationTaskUseCase::new( &tasks, @@ -704,10 +810,10 @@ mod tests { record: "verified_proof_bundle" }) ); - assert_eq!( - retry_tasks.updates().last().unwrap().status, - VerificationTaskStatus::Failed - ); + let updates = retry_tasks.updates(); + let stored = updates.last().unwrap(); + assert_eq!(stored.status, VerificationTaskStatus::InProgress); + assert_eq!(stored.failure_message, None); } #[tokio::test] @@ -1124,11 +1230,19 @@ mod tests { #[async_trait] impl VerificationTaskClaimer for LostClaimClaimer { + async fn begin_claimed_entitlement_publication( + &self, + _task_id: &TaskId, + _worker_id: &str, + _claim_token: &uuid::Uuid, + ) -> Result { + Ok(true) + } + async fn claim_next_verification_task( &self, _worker_id: &str, - _now: OffsetDateTime, - _claim_expires_at: OffsetDateTime, + _claim_ttl: time::Duration, ) -> Result, ApplicationError> { unreachable!("completion must not claim tasks") } @@ -1138,8 +1252,7 @@ mod tests { _task_id: &TaskId, _worker_id: &str, _claim_token: &uuid::Uuid, - _now: OffsetDateTime, - _next_attempt_at: OffsetDateTime, + _retry_after: time::Duration, ) -> Result, ApplicationError> { unreachable!("completion must not schedule retries") } @@ -1149,7 +1262,6 @@ mod tests { _task: VerificationTaskRecord, _worker_id: &str, _claim_token: &uuid::Uuid, - _now: OffsetDateTime, ) -> Result, ApplicationError> { Ok(None) } @@ -1233,12 +1345,22 @@ mod tests { ) -> Result, ApplicationError> { Ok(self.content_lock.clone()) } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _content_lock_path: &ContentLockPath, + ) -> Result { + unreachable!("completion must not delete content locks") + } } #[derive(Default)] struct FakeEntitlements { stored: Mutex>, fail_after_store: bool, + hide_stored: bool, + fail_read: bool, } impl FakeEntitlements { @@ -1246,6 +1368,26 @@ mod tests { Self { stored: Mutex::new(Vec::new()), fail_after_store: true, + hide_stored: false, + fail_read: false, + } + } + + fn with_ambiguous_write_and_absent_read_back() -> Self { + Self { + stored: Mutex::new(Vec::new()), + fail_after_store: true, + hide_stored: true, + fail_read: false, + } + } + + fn with_ambiguous_write_and_failed_read_back() -> Self { + Self { + stored: Mutex::new(Vec::new()), + fail_after_store: true, + hide_stored: false, + fail_read: true, } } @@ -1284,6 +1426,14 @@ mod tests { creator: &CreatorPubky, bundle_id: &BundleId, ) -> Result, ApplicationError> { + if self.fail_read { + return Err(ApplicationError::Storage { + message: "entitlement read unavailable".to_owned(), + }); + } + if self.hide_stored { + return Ok(None); + } Ok(self .stored .lock() diff --git a/locks-service/src/application/use_cases/create_content_lock.rs b/locks-service/src/application/use_cases/create_content_lock.rs index 5eb0496..417d5a0 100644 --- a/locks-service/src/application/use_cases/create_content_lock.rs +++ b/locks-service/src/application/use_cases/create_content_lock.rs @@ -5,9 +5,13 @@ use locks_core::lock_policy::{ AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, Criterion, GuardedResource, LockLogic, LockServerConfig, SecondaryGuardedResource, }; +use uuid::Uuid; use crate::application::errors::ApplicationError; -use crate::application::ports::{Clock, ContentLockRepository, GuardedResourceRepository}; +use crate::application::ports::{ + Clock, ContentLockDeletionRepository, ContentLockOwnershipRepository, ContentLockRepository, + GuardedResourceRepository, +}; /// Request to create a local content lock for an already-registered guarded resource. #[derive(Debug, Clone, PartialEq, Eq)] @@ -42,6 +46,8 @@ pub struct CreatedContentLock { /// Creates local content locks after verifying current guarded resource metadata. pub struct CreateContentLockUseCase<'a> { content_locks: &'a dyn ContentLockRepository, + content_lock_deletions: &'a dyn ContentLockDeletionRepository, + content_lock_ownership: &'a dyn ContentLockOwnershipRepository, guarded_resources: &'a dyn GuardedResourceRepository, clock: &'a dyn Clock, } @@ -50,11 +56,15 @@ impl<'a> CreateContentLockUseCase<'a> { /// Creates a content-lock use case from its application ports. pub fn new( content_locks: &'a dyn ContentLockRepository, + content_lock_deletions: &'a dyn ContentLockDeletionRepository, + content_lock_ownership: &'a dyn ContentLockOwnershipRepository, guarded_resources: &'a dyn GuardedResourceRepository, clock: &'a dyn Clock, ) -> Self { Self { content_locks, + content_lock_deletions, + content_lock_ownership, guarded_resources, clock, } @@ -115,14 +125,85 @@ impl<'a> CreateContentLockUseCase<'a> { message: error.to_string(), } })?; + let guarded_paths = resource_descriptors(&content_lock) + .into_iter() + .map(|resource| resource.path) + .collect::>(); + + self.content_lock_ownership + .reserve_paths(&request.creator, &guarded_paths, &lock_id) + .await?; + + let publication_token = Uuid::new_v4(); + if let Err(error) = self + .content_lock_deletions + .begin_publication(&request.creator, &lock_id, publication_token) + .await + { + let _ = self + .content_lock_ownership + .compensate_reserved_paths(&request.creator, &guarded_paths, &lock_id) + .await; + return Err(error); + } - self.content_locks + if let Err(error) = self + .content_locks .upsert_content_lock( - request.creator, + request.creator.clone(), content_lock_path.clone(), content_lock.clone(), ) + .await + { + match self + .content_locks + .get_content_lock(&request.creator, &content_lock_path) + .await + { + Ok(Some(published)) if published == content_lock => { + if self + .content_lock_ownership + .mark_paths_published(&request.creator, &guarded_paths, &lock_id) + .await + .is_ok() + { + let _ = self + .content_lock_deletions + .finish_publication(&request.creator, &lock_id, publication_token) + .await; + } + } + Ok(None) => { + if self + .content_lock_ownership + .compensate_reserved_paths(&request.creator, &guarded_paths, &lock_id) + .await + .is_ok() + { + let _ = self + .content_lock_deletions + .abandon_publication(&request.creator, &lock_id, publication_token) + .await; + } + } + Ok(Some(_)) | Err(_) => {} + } + return Err(error); + } + + self.content_lock_ownership + .mark_paths_published(&request.creator, &guarded_paths, &lock_id) .await?; + if !self + .content_lock_deletions + .finish_publication(&request.creator, &lock_id, publication_token) + .await? + { + return Err(ApplicationError::Storage { + message: "content lock publication intent was lost".to_owned(), + }); + } Ok(CreatedContentLock { lock_id, @@ -169,6 +250,7 @@ fn resource_descriptors(content_lock: &ContentLock) -> Vec { mod tests { use std::str::FromStr; + use async_trait::async_trait; use serde_json::json; use time::OffsetDateTime; use time::macros::datetime; @@ -177,8 +259,15 @@ mod tests { use locks_core::lock_policy::VerifierType; use super::*; - use crate::application::models::GuardedResourceRecord; - use crate::application::ports::{Clock, ContentLockRepository, GuardedResourceRepository}; + use crate::application::models::{ + ContentLockOwnershipStatus, GuardedResourceRecord, PrepareForceDeletionResult, + }; + use crate::application::ports::{ + Clock, ContentLockDeletionRepository, ContentLockOwnershipRepository, + ContentLockRepository, GuardedResourceRepository, + }; + use crate::infrastructure::memory::content_lock_deletions::InMemoryContentLockDeletionRepository; + use crate::infrastructure::memory::content_lock_ownership::InMemoryContentLockOwnershipRepository; use crate::infrastructure::memory::content_locks::InMemoryContentLockRepository; use crate::infrastructure::memory::guarded_resources::InMemoryGuardedResourceRepository; @@ -219,6 +308,50 @@ mod tests { .unwrap(), Some(result.content_lock) ); + let ownership = fixture + .content_lock_ownership + .get_path_ownership(&creator(), "/priv/locks.app/content/hello.txt") + .await + .unwrap() + .unwrap(); + assert_eq!(ownership.lock_id, result.lock_id); + assert_eq!(ownership.status.as_str(), "published"); + } + + #[tokio::test] + async fn permanent_force_receipt_blocks_canonical_lock_republication() { + let fixture = Fixture::seeded().await; + let request = content_lock_request(registered_guarded_resource()); + let content_lock = ContentLock { + version: CONTENT_LOCK_VERSION, + creator: request.creator.clone(), + primary_resource: request.primary_resource.clone(), + secondary_resources: request.secondary_resources.clone(), + criteria: request.criteria.clone(), + lock_logic: request.lock_logic.clone(), + access_policy: request.access_policy.clone(), + lock_server: request.lock_server.clone(), + created_at: fixture.clock.now(), + }; + let lock_id = content_lock.lock_id().unwrap(); + fixture + .content_lock_deletions + .prepare_force_deletion(&request.creator, &lock_id) + .await + .unwrap(); + + let result = fixture.use_case().execute(request).await; + + assert_eq!(result, Err(ApplicationError::ContentLockDeletionInProgress)); + assert_eq!(fixture.content_locks_len().await, 0); + assert_eq!( + fixture + .content_lock_ownership + .get_path_ownership(&creator(), "/priv/locks.app/content/hello.txt") + .await + .unwrap(), + None + ); } #[tokio::test] @@ -304,7 +437,7 @@ mod tests { } #[tokio::test] - async fn changed_criteria_create_different_lock_id_and_path() { + async fn changed_criteria_rejects_path_owned_by_different_lock() { let fixture = Fixture::seeded().await; let use_case = fixture.use_case(); let first_request = content_lock_request(registered_guarded_resource()); @@ -312,12 +445,21 @@ mod tests { second_request.criteria[0].params = json!({ "satisfied": false }); let first = use_case.execute(first_request).await.unwrap(); - let second = use_case.execute(second_request).await.unwrap(); + let second = use_case.execute(second_request).await; - assert_ne!(second.lock_id, first.lock_id); - assert_ne!(second.content_lock_path, first.content_lock_path); - assert_ne!(second.content_lock, first.content_lock); - assert_eq!(fixture.content_locks_len().await, 2); + assert!(matches!( + second, + Err(ApplicationError::ContentLockPathConflict { ref guarded_path }) + if guarded_path == "/priv/locks.app/content/hello.txt" + )); + assert_eq!(fixture.content_locks_len().await, 1); + let ownership = fixture + .content_lock_ownership + .get_path_ownership(&creator(), "/priv/locks.app/content/hello.txt") + .await + .unwrap() + .unwrap(); + assert_eq!(ownership.lock_id, first.lock_id); } #[tokio::test] @@ -330,6 +472,7 @@ mod tests { "recipient_pubky": creator().to_string(), "amount": "0", "asset": "BTC", + "payment_in": 24, }); let result = use_case.execute(request).await; @@ -353,7 +496,8 @@ mod tests { params: json!({ "recipient_pubky": creator().to_string(), "amount": "50000", - "asset": "BTC" + "asset": "BTC", + "payment_in": 24 }), }); request.lock_logic = LockLogic::All { @@ -369,8 +513,335 @@ mod tests { assert_eq!(fixture.content_locks_len().await, 0); } + #[tokio::test] + async fn publication_failure_compensates_reserved_path_ownership() { + let fixture = Fixture::seeded().await; + let use_case = CreateContentLockUseCase::new( + &FailingContentLockRepository, + &fixture.content_lock_deletions, + &fixture.content_lock_ownership, + &fixture.guarded_resources, + &fixture.clock, + ); + + let result = use_case + .execute(content_lock_request(registered_guarded_resource())) + .await; + + assert!(matches!( + result, + Err(ApplicationError::Storage { ref message }) if message == "publication failed" + )); + assert_eq!( + fixture + .content_lock_ownership + .get_path_ownership(&creator(), "/priv/locks.app/content/hello.txt") + .await + .unwrap(), + None + ); + } + + #[tokio::test] + async fn ambiguous_publication_error_reconciles_committed_lock_without_releasing_ownership() { + let fixture = Fixture::seeded().await; + let content_locks = AmbiguousContentLockRepository::default(); + let use_case = CreateContentLockUseCase::new( + &content_locks, + &fixture.content_lock_deletions, + &fixture.content_lock_ownership, + &fixture.guarded_resources, + &fixture.clock, + ); + let request = content_lock_request(registered_guarded_resource()); + let expected = ContentLock { + version: CONTENT_LOCK_VERSION, + creator: request.creator.clone(), + primary_resource: request.primary_resource.clone(), + secondary_resources: request.secondary_resources.clone(), + criteria: request.criteria.clone(), + lock_logic: request.lock_logic.clone(), + access_policy: request.access_policy.clone(), + lock_server: request.lock_server.clone(), + created_at: fixture.clock.now(), + }; + let lock_id = expected.lock_id().unwrap(); + let path = expected.content_lock_path().unwrap(); + + let result = use_case.execute(request).await; + + assert!(matches!( + result, + Err(ApplicationError::Storage { ref message }) if message == "publication response lost" + )); + assert_eq!( + content_locks + .get_content_lock(&creator(), &path) + .await + .unwrap(), + Some(expected) + ); + assert_eq!( + fixture + .content_lock_ownership + .get_path_ownership(&creator(), "/priv/locks.app/content/hello.txt") + .await + .unwrap() + .unwrap() + .status, + ContentLockOwnershipStatus::Published + ); + assert!( + !fixture + .content_lock_deletions + .publication_in_progress(&creator(), &lock_id) + .await + .unwrap() + ); + } + + #[tokio::test] + async fn unreconciled_publication_error_retains_reserved_ownership_and_deletion_fence() { + let fixture = Fixture::seeded().await; + let content_locks = UnreconciledContentLockRepository; + let use_case = CreateContentLockUseCase::new( + &content_locks, + &fixture.content_lock_deletions, + &fixture.content_lock_ownership, + &fixture.guarded_resources, + &fixture.clock, + ); + let request = content_lock_request(registered_guarded_resource()); + let lock_id = ContentLock { + version: CONTENT_LOCK_VERSION, + creator: request.creator.clone(), + primary_resource: request.primary_resource.clone(), + secondary_resources: request.secondary_resources.clone(), + criteria: request.criteria.clone(), + lock_logic: request.lock_logic.clone(), + access_policy: request.access_policy.clone(), + lock_server: request.lock_server.clone(), + created_at: fixture.clock.now(), + } + .lock_id() + .unwrap(); + + let result = use_case.execute(request).await; + + assert!(matches!( + result, + Err(ApplicationError::Storage { ref message }) if message == "publication response lost" + )); + assert_eq!( + fixture + .content_lock_ownership + .get_path_ownership(&creator(), "/priv/locks.app/content/hello.txt") + .await + .unwrap() + .unwrap() + .status, + ContentLockOwnershipStatus::Reserved + ); + assert!( + fixture + .content_lock_deletions + .publication_in_progress(&creator(), &lock_id) + .await + .unwrap() + ); + assert_eq!( + fixture + .content_lock_deletions + .prepare_force_deletion(&creator(), &lock_id) + .await + .unwrap(), + PrepareForceDeletionResult::PublicationInProgress + ); + } + + #[tokio::test] + async fn publication_intent_fences_force_during_external_upsert() { + let fixture = Fixture::seeded().await; + let probe = PublicationRaceProbe { + deletions: &fixture.content_lock_deletions, + }; + let use_case = CreateContentLockUseCase::new( + &probe, + &fixture.content_lock_deletions, + &fixture.content_lock_ownership, + &fixture.guarded_resources, + &fixture.clock, + ); + + let created = use_case + .execute(content_lock_request(registered_guarded_resource())) + .await + .unwrap(); + + assert_eq!( + fixture + .content_lock_deletions + .prepare_force_deletion(&creator(), &created.lock_id) + .await + .unwrap(), + PrepareForceDeletionResult::Synchronous(None) + ); + } + + struct PublicationRaceProbe<'a> { + deletions: &'a InMemoryContentLockDeletionRepository, + } + + #[async_trait] + impl ContentLockRepository for PublicationRaceProbe<'_> { + async fn upsert_content_lock( + &self, + creator: CreatorPubky, + _path: ContentLockPath, + content_lock: ContentLock, + ) -> Result<(), ApplicationError> { + let lock_id = content_lock.lock_id().unwrap(); + assert_eq!( + self.deletions + .prepare_force_deletion(&creator, &lock_id) + .await?, + PrepareForceDeletionResult::PublicationInProgress + ); + assert!(!self.deletions.has_force_receipt(&creator, &lock_id).await?); + Ok(()) + } + + async fn get_content_lock( + &self, + _creator: &CreatorPubky, + _path: &ContentLockPath, + ) -> Result, ApplicationError> { + Ok(None) + } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _path: &ContentLockPath, + ) -> Result { + unreachable!("creation must not delete content locks") + } + } + + #[derive(Default)] + struct AmbiguousContentLockRepository { + published: tokio::sync::RwLock>, + } + + #[async_trait] + impl ContentLockRepository for AmbiguousContentLockRepository { + async fn upsert_content_lock( + &self, + creator: CreatorPubky, + path: ContentLockPath, + content_lock: ContentLock, + ) -> Result<(), ApplicationError> { + *self.published.write().await = Some((creator, path, content_lock)); + Err(ApplicationError::Storage { + message: "publication response lost".to_owned(), + }) + } + + async fn get_content_lock( + &self, + creator: &CreatorPubky, + path: &ContentLockPath, + ) -> Result, ApplicationError> { + Ok(self + .published + .read() + .await + .as_ref() + .filter(|(stored_creator, stored_path, _)| { + stored_creator == creator && stored_path == path + }) + .map(|(_, _, content_lock)| content_lock.clone())) + } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _path: &ContentLockPath, + ) -> Result { + unreachable!("creation must not delete content locks") + } + } + + struct UnreconciledContentLockRepository; + + #[async_trait] + impl ContentLockRepository for UnreconciledContentLockRepository { + async fn upsert_content_lock( + &self, + _creator: CreatorPubky, + _path: ContentLockPath, + _content_lock: ContentLock, + ) -> Result<(), ApplicationError> { + Err(ApplicationError::Storage { + message: "publication response lost".to_owned(), + }) + } + + async fn get_content_lock( + &self, + _creator: &CreatorPubky, + _path: &ContentLockPath, + ) -> Result, ApplicationError> { + Err(ApplicationError::Storage { + message: "publication reconciliation failed".to_owned(), + }) + } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _path: &ContentLockPath, + ) -> Result { + unreachable!("creation must not delete content locks") + } + } + + struct FailingContentLockRepository; + + #[async_trait] + impl ContentLockRepository for FailingContentLockRepository { + async fn upsert_content_lock( + &self, + _creator: CreatorPubky, + _path: ContentLockPath, + _content_lock: ContentLock, + ) -> Result<(), ApplicationError> { + Err(ApplicationError::Storage { + message: "publication failed".to_owned(), + }) + } + + async fn get_content_lock( + &self, + _creator: &CreatorPubky, + _path: &ContentLockPath, + ) -> Result, ApplicationError> { + Ok(None) + } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _path: &ContentLockPath, + ) -> Result { + unreachable!("creation must not delete content locks") + } + } + struct Fixture { content_locks: InMemoryContentLockRepository, + content_lock_deletions: InMemoryContentLockDeletionRepository, + content_lock_ownership: InMemoryContentLockOwnershipRepository, guarded_resources: InMemoryGuardedResourceRepository, clock: FixedClock, } @@ -379,6 +850,8 @@ mod tests { fn empty() -> Self { Self { content_locks: InMemoryContentLockRepository::new(), + content_lock_deletions: InMemoryContentLockDeletionRepository::new(), + content_lock_ownership: InMemoryContentLockOwnershipRepository::new(), guarded_resources: InMemoryGuardedResourceRepository::new(), clock: FixedClock(datetime!(2026-06-03 12:00:00 UTC)), } @@ -403,7 +876,13 @@ mod tests { } fn use_case(&self) -> CreateContentLockUseCase<'_> { - CreateContentLockUseCase::new(&self.content_locks, &self.guarded_resources, &self.clock) + CreateContentLockUseCase::new( + &self.content_locks, + &self.content_lock_deletions, + &self.content_lock_ownership, + &self.guarded_resources, + &self.clock, + ) } async fn content_locks_len(&self) -> usize { diff --git a/locks-service/src/application/use_cases/credential_flow_tests.rs b/locks-service/src/application/use_cases/credential_flow_tests.rs index eedda7b..ecb0c07 100644 --- a/locks-service/src/application/use_cases/credential_flow_tests.rs +++ b/locks-service/src/application/use_cases/credential_flow_tests.rs @@ -1,3 +1,4 @@ +use std::collections::VecDeque; use std::str::FromStr; use std::sync::Mutex; @@ -7,7 +8,7 @@ use time::OffsetDateTime; use time::macros::datetime; use locks_core::ids::{ - BundleId, ContentLockPath, CreatorPubky, GuardedResourceHash, LockServerPubky, + BundleId, ContentLockPath, CreatorPubky, GuardedResourceHash, LockId, LockServerPubky, PubkyLockResource, TaskId, }; use locks_core::lock_policy::{ @@ -22,7 +23,7 @@ use locks_core::verification::{ use crate::application::errors::ApplicationError; use crate::application::models::{ AccessCredential, AccessCredentialLookupKey, AccessCredentialPolicy, AccessCredentialRecord, - GuardedResourceRecord, VerificationTaskRecord, + GuardedResourceRecord, IssuedDeletionCredential, VerificationTaskRecord, }; use crate::application::ports::{ AccessCredentialGenerator, AccessCredentialStore, Clock, ContentLockRepository, @@ -37,6 +38,36 @@ use crate::application::use_cases::validate_access_credential::{ const BUNDLE_ID: &str = "000G40R40M30E209185GR38E1W"; +#[tokio::test] +async fn final_credential_winner_uses_fresh_time_after_generation() { + let before_generation = datetime!(2026-05-29 12:00:00 UTC); + let after_generation = datetime!(2026-05-29 12:00:01 UTC); + let entitlements = FakeEntitlements::new(None); + let content_locks = FakeContentLocks::new(None); + let store = FakeAccessCredentialStore::with_final_credential(); + let generator = FakeGenerator::new(AccessCredential::new("final-bearer")); + let clock = SequenceClock::new([before_generation, after_generation]); + let use_case = IssueAccessCredentialUseCase::new( + &entitlements, + &content_locks, + &store, + &generator, + &clock, + AccessCredentialPolicy::new(3600), + ); + + let issued = use_case + .execute(IssueAccessCredentialRequest { + creator: creator(), + bundle_id: bundle_id(), + }) + .await + .unwrap(); + + assert_eq!(issued.credential.as_str(), "final-bearer"); + assert_eq!(store.final_issue_time(), Some(after_generation)); +} + #[tokio::test] async fn issue_access_credential_rechecks_entitlement_and_stores_lookup_key() { let content_lock = content_lock_fixture(900); @@ -502,6 +533,14 @@ impl ContentLockRepository for FakeContentLocks { ) -> Result, ApplicationError> { Ok(self.content_lock.lock().unwrap().clone()) } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _content_lock_path: &ContentLockPath, + ) -> Result { + unreachable!("credential flow must not delete content locks") + } } struct FakeEntitlements { @@ -548,6 +587,8 @@ impl EntitlementRepository for FakeEntitlements { struct FakeAccessCredentialStore { record: Mutex>, deleted: Mutex>, + final_available: bool, + final_issue_time: Mutex>, } impl FakeAccessCredentialStore { @@ -555,9 +596,22 @@ impl FakeAccessCredentialStore { Self { record: Mutex::new(Some((lookup_key, record))), deleted: Mutex::new(None), + final_available: false, + final_issue_time: Mutex::new(None), + } + } + + fn with_final_credential() -> Self { + Self { + final_available: true, + ..Self::default() } } + fn final_issue_time(&self) -> Option { + *self.final_issue_time.lock().unwrap() + } + fn stored_record(&self) -> (AccessCredentialLookupKey, AccessCredentialRecord) { self.record.lock().unwrap().clone().unwrap() } @@ -575,6 +629,7 @@ impl FakeAccessCredentialStore { impl AccessCredentialStore for FakeAccessCredentialStore { async fn insert_access_credential( &self, + _lock_id: &LockId, lookup_key: AccessCredentialLookupKey, record: AccessCredentialRecord, ) -> Result<(), ApplicationError> { @@ -603,6 +658,29 @@ impl AccessCredentialStore for FakeAccessCredentialStore { *self.record.lock().unwrap() = None; Ok(()) } + + async fn final_credential_available( + &self, + _creator: &CreatorPubky, + _bundle_id: &BundleId, + _now: OffsetDateTime, + ) -> Result { + Ok(self.final_available) + } + + async fn issue_or_replay_final_credential( + &self, + _creator: &CreatorPubky, + _bundle_id: &BundleId, + now: OffsetDateTime, + candidate: AccessCredential, + ) -> Result, ApplicationError> { + *self.final_issue_time.lock().unwrap() = Some(now); + Ok(Some(IssuedDeletionCredential { + credential: candidate, + expires_at: now + time::Duration::minutes(1), + })) + } } struct FakeGenerator { @@ -647,6 +725,28 @@ impl Clock for FakeClock { } } +struct SequenceClock { + values: Mutex>, +} + +impl SequenceClock { + fn new(values: impl IntoIterator) -> Self { + Self { + values: Mutex::new(values.into_iter().collect()), + } + } +} + +impl Clock for SequenceClock { + fn now(&self) -> OffsetDateTime { + self.values + .lock() + .unwrap() + .pop_front() + .expect("sequence clock exhausted") + } +} + #[allow(dead_code)] struct UnusedPortImplementations; diff --git a/locks-service/src/application/use_cases/drain_lock_payments.rs b/locks-service/src/application/use_cases/drain_lock_payments.rs new file mode 100644 index 0000000..aa9a57c --- /dev/null +++ b/locks-service/src/application/use_cases/drain_lock_payments.rs @@ -0,0 +1,814 @@ +use locks_core::ids::{ContentLockPath, LockServerPubky, PubkyLockResource}; +use locks_core::lock_policy::VerifierType; +use locks_core::verification::{ + CriterionVerificationResult, EntitlementLifetime, VERIFIED_PROOF_BUNDLE_VERSION, + VerificationResult, VerifiedProofBundle, +}; + +use crate::application::errors::ApplicationError; +use crate::application::models::{ + AdvanceContentLockDeletionPhaseResult, ClaimedContentLockDeletionJob, ContentLockDeletionPhase, + VerificationTaskStatus, +}; +use crate::application::ports::{ + Clock, ContentLockDeletionRepository, EntitlementRepository, PaymentDrainClient, + PaymentDrainClientError, PaymentDrainRepository, PaymentDrainStatus, PaymentDrainSummary, + PaymentDrainTerminalTransition, PaymentRequestState, PaymentRequestStatus, PaymentState, + same_entitlement_decision, +}; + +use super::execute_content_lock_deletion_phase::{ + DeletionDependencyEvidence, DeletionDependencySource, DeletionExecutionErrorClass, + DeletionPhaseExecution, DeletionPhaseExecutionOutcome, classify_deletion_execution_error, +}; + +pub struct DrainLockPaymentsUseCase<'a> { + deletions: &'a dyn ContentLockDeletionRepository, + drains: &'a dyn PaymentDrainRepository, + paykit: &'a dyn PaymentDrainClient, + entitlements: &'a dyn EntitlementRepository, + clock: &'a dyn Clock, + verified_by: LockServerPubky, + minimum_confirmations: u32, +} + +impl<'a> DrainLockPaymentsUseCase<'a> { + pub fn new( + deletions: &'a dyn ContentLockDeletionRepository, + drains: &'a dyn PaymentDrainRepository, + paykit: &'a dyn PaymentDrainClient, + entitlements: &'a dyn EntitlementRepository, + clock: &'a dyn Clock, + verified_by: LockServerPubky, + minimum_confirmations: u32, + ) -> Self { + Self { + deletions, + drains, + paykit, + entitlements, + clock, + verified_by, + minimum_confirmations, + } + } + + pub async fn execute_claimed( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> Result { + self.execute_claimed_observed(claim, worker_id, &mut DeletionDependencyEvidence::none()) + .await + } + + pub async fn execute_claimed_with_evidence( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecution { + let mut evidence = DeletionDependencyEvidence::none(); + match self + .execute_claimed_observed(claim, worker_id, &mut evidence) + .await + { + Ok(true) => DeletionPhaseExecution::new(DeletionPhaseExecutionOutcome::Progressed) + .with_evidence(evidence), + Ok(false) => DeletionPhaseExecution::new(DeletionPhaseExecutionOutcome::Deferred) + .with_evidence(evidence), + Err(error) => { + let outcome = match classify_deletion_execution_error(&error) { + DeletionExecutionErrorClass::TransientDependency => { + DeletionPhaseExecutionOutcome::TransientDependencyFailure + } + DeletionExecutionErrorClass::Fatal => { + DeletionPhaseExecutionOutcome::FatalFailure + } + }; + DeletionPhaseExecution::new(outcome).with_evidence(evidence) + } + } + } + + async fn execute_claimed_observed( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + evidence: &mut DeletionDependencyEvidence, + ) -> Result { + match claim.job.phase { + ContentLockDeletionPhase::StartPaymentDrain => { + self.start(claim, worker_id, evidence).await + } + ContentLockDeletionPhase::DrainPayments => self.drain(claim, worker_id, evidence).await, + _ => Err(ApplicationError::InvalidContentLockDeletionState { + message: "payment drain use case requires a payment drain phase".to_owned(), + }), + } + } + + async fn start( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + evidence: &mut DeletionDependencyEvidence, + ) -> Result { + let lock_resource = lock_resource(&claim); + let summary = match self.paykit.start_payment_drain(&lock_resource).await { + Ok(summary) => { + observe_healthy(evidence, DeletionDependencySource::PaymentProvider); + summary + } + Err(PaymentDrainClientError::Conflict) => { + observe_healthy(evidence, DeletionDependencySource::PaymentProvider); + match self.paykit.lookup_payment_drain(&lock_resource).await { + Ok(Some(summary)) => { + observe_healthy(evidence, DeletionDependencySource::PaymentProvider); + summary + } + Ok(None) => { + observe_healthy(evidence, DeletionDependencySource::PaymentProvider); + return Err(ApplicationError::Verifier { + message: "Paykit payment drain conflict could not be reconciled" + .to_owned(), + }); + } + Err(error) => { + observe_unavailable(evidence, DeletionDependencySource::PaymentProvider); + return Err(map_client_error(error)); + } + } + } + Err(error) => { + observe_unavailable(evidence, DeletionDependencySource::PaymentProvider); + return Err(map_client_error(error)); + } + }; + let _now = self.clock.now(); + match self + .drains + .store_payment_drain(claim.job.job_id, worker_id, claim.claim_token, &summary) + .await + { + Ok(true) => observe_healthy(evidence, DeletionDependencySource::PaymentDrainRepository), + Ok(false) => return Ok(false), + Err(error) => { + observe_unavailable(evidence, DeletionDependencySource::PaymentDrainRepository); + return Err(error); + } + } + match self + .deletions + .advance_phase( + claim.job.job_id, + worker_id, + claim.claim_token, + ContentLockDeletionPhase::DrainPayments, + ) + .await + { + Ok(AdvanceContentLockDeletionPhaseResult::Advanced(_)) => { + observe_healthy(evidence, DeletionDependencySource::RepositoryPhaseMutation); + Ok(true) + } + Ok(AdvanceContentLockDeletionPhaseResult::ClaimLost) => Ok(false), + Ok(_) => { + observe_healthy(evidence, DeletionDependencySource::RepositoryPhaseMutation); + Ok(false) + } + Err(error) => { + observe_unavailable(evidence, DeletionDependencySource::RepositoryPhaseMutation); + Err(error) + } + } + } + + async fn drain( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + evidence: &mut DeletionDependencyEvidence, + ) -> Result { + let lock_resource = lock_resource(&claim); + let summary = match self.paykit.lookup_payment_drain(&lock_resource).await { + Ok(Some(summary)) => { + observe_healthy(evidence, DeletionDependencySource::PaymentProvider); + summary + } + Ok(None) => { + observe_healthy(evidence, DeletionDependencySource::PaymentProvider); + return Err(ApplicationError::Verifier { + message: "Paykit payment drain not found".to_owned(), + }); + } + Err(error) => { + observe_unavailable(evidence, DeletionDependencySource::PaymentProvider); + return Err(map_client_error(error)); + } + }; + let persisted = match self.drains.get_payment_drain(claim.job.job_id).await { + Ok(Some(persisted)) => { + observe_healthy(evidence, DeletionDependencySource::PaymentDrainRepository); + persisted + } + Ok(None) => { + observe_healthy(evidence, DeletionDependencySource::PaymentDrainRepository); + return Err(ApplicationError::InvalidContentLockDeletionState { + message: "payment drain cleanup token is missing".to_owned(), + }); + } + Err(error) => { + observe_unavailable(evidence, DeletionDependencySource::PaymentDrainRepository); + return Err(error); + } + }; + validate_aggregate_progress(&persisted, &summary)?; + let _now = self.clock.now(); + match self + .drains + .reconcile_payment_drain(claim.job.job_id, worker_id, claim.claim_token, &summary) + .await + { + Ok(true) => observe_healthy(evidence, DeletionDependencySource::PaymentDrainRepository), + Ok(false) => return Ok(false), + Err(error) => { + observe_unavailable(evidence, DeletionDependencySource::PaymentDrainRepository); + return Err(error); + } + } + + let obligations = match self.drains.list_obligations(claim.job.job_id).await { + Ok(obligations) => { + observe_healthy(evidence, DeletionDependencySource::PaymentDrainRepository); + obligations + } + Err(error) => { + observe_unavailable(evidence, DeletionDependencySource::PaymentDrainRepository); + return Err(error); + } + }; + for obligation in obligations { + if matches!( + obligation.status, + VerificationTaskStatus::Completed + | VerificationTaskStatus::Expired + | VerificationTaskStatus::Failed + ) { + continue; + } + let status = match self + .paykit + .payment_request_status(&obligation.creator, &obligation.bundle_id) + .await + { + Ok(Some(status)) => { + observe_healthy(evidence, DeletionDependencySource::PaymentProvider); + status + } + Ok(None) => { + observe_healthy(evidence, DeletionDependencySource::PaymentProvider); + return Err(ApplicationError::Verifier { + message: "Paykit payment request not found".to_owned(), + }); + } + Err(error) => { + observe_unavailable(evidence, DeletionDependencySource::PaymentProvider); + return Err(map_client_error(error)); + } + }; + if status.invoice_created_at != obligation.invoice_created_at + || status.payment_deadline != obligation.payment_deadline + { + return Err(ApplicationError::InvalidVerificationTaskState { + message: "Paykit payment window changed during deletion".to_owned(), + }); + } + let Some(next_status) = classify_payment_task(status, self.minimum_confirmations)? + else { + continue; + }; + let now = self.clock.now(); + let mut entitlement_publication_token = None; + if next_status == VerificationTaskStatus::Completed { + let publication_token = match self + .drains + .begin_entitlement_publication( + claim.job.job_id, + worker_id, + claim.claim_token, + &obligation.task_id, + ) + .await + { + Ok(Some(token)) => { + observe_healthy(evidence, DeletionDependencySource::PaymentDrainRepository); + token + } + Ok(None) => return Ok(false), + Err(error) => { + observe_unavailable( + evidence, + DeletionDependencySource::PaymentDrainRepository, + ); + return Err(error); + } + }; + let entitlement = VerifiedProofBundle { + version: VERIFIED_PROOF_BUNDLE_VERSION, + bundle_id: obligation.bundle_id.clone(), + pubky_lock_resource: obligation.lock_resource.clone(), + verification_result: VerificationResult { + criteria: vec![CriterionVerificationResult { + criterion_id: obligation.criterion_id.clone(), + satisfied: true, + verified_at: now, + verified_by: self.verified_by.clone(), + verifier_type: VerifierType::PaykitPayment, + }], + }, + entitlement_lifetime: EntitlementLifetime::Unbounded, + }; + persist_entitlement(self.entitlements, entitlement, evidence).await?; + entitlement_publication_token = Some(publication_token); + } + match self + .drains + .persist_terminal_obligation( + claim.job.job_id, + worker_id, + claim.claim_token, + &obligation.task_id, + PaymentDrainTerminalTransition { + status: next_status, + entitlement_publication_token, + }, + ) + .await + { + Ok(true) => { + observe_healthy(evidence, DeletionDependencySource::PaymentDrainRepository) + } + Ok(false) => return Ok(false), + Err(error) => { + observe_unavailable(evidence, DeletionDependencySource::PaymentDrainRepository); + return Err(error); + } + } + } + + if summary.status != PaymentDrainStatus::Completed { + return Ok(false); + } + match self.drains.all_obligations_terminal(claim.job.job_id).await { + Ok(true) => observe_healthy(evidence, DeletionDependencySource::PaymentDrainRepository), + Ok(false) => { + observe_healthy(evidence, DeletionDependencySource::PaymentDrainRepository); + return Ok(false); + } + Err(error) => { + observe_unavailable(evidence, DeletionDependencySource::PaymentDrainRepository); + return Err(error); + } + } + match self + .deletions + .advance_phase( + claim.job.job_id, + worker_id, + claim.claim_token, + ContentLockDeletionPhase::DrainExistingCredentials, + ) + .await + { + Ok(AdvanceContentLockDeletionPhaseResult::Advanced(_)) => { + observe_healthy(evidence, DeletionDependencySource::RepositoryPhaseMutation); + Ok(true) + } + Ok(AdvanceContentLockDeletionPhaseResult::ClaimLost) => Ok(false), + Ok(_) => { + observe_healthy(evidence, DeletionDependencySource::RepositoryPhaseMutation); + Ok(false) + } + Err(error) => { + observe_unavailable(evidence, DeletionDependencySource::RepositoryPhaseMutation); + Err(error) + } + } + } +} + +fn lock_resource(claim: &ClaimedContentLockDeletionJob) -> PubkyLockResource { + PubkyLockResource::new( + claim.job.creator.clone(), + ContentLockPath::from_lock_id(claim.job.lock_id.clone()), + ) +} + +async fn persist_entitlement( + entitlements: &dyn EntitlementRepository, + entitlement: VerifiedProofBundle, + evidence: &mut DeletionDependencyEvidence, +) -> Result<(), ApplicationError> { + if let Err(error) = entitlements + .insert_verified_proof_bundle(entitlement.clone()) + .await + { + let expected_duplicate = matches!( + &error, + ApplicationError::DuplicateRecord { record } + if *record == "verified_proof_bundle" + ); + if !expected_duplicate { + observe_unavailable(evidence, DeletionDependencySource::EntitlementRepository); + } + let existing = match entitlements + .get_verified_proof_bundle( + entitlement.pubky_lock_resource.creator(), + &entitlement.bundle_id, + ) + .await + { + Ok(existing) => { + observe_healthy(evidence, DeletionDependencySource::EntitlementRepository); + existing + } + Err(read_error) => { + observe_unavailable(evidence, DeletionDependencySource::EntitlementRepository); + return Err(read_error); + } + }; + if !existing + .as_ref() + .is_some_and(|existing| same_entitlement_decision(existing, &entitlement)) + { + return Err(error); + } + } else { + observe_healthy(evidence, DeletionDependencySource::EntitlementRepository); + } + Ok(()) +} + +fn observe_healthy(evidence: &mut DeletionDependencyEvidence, source: DeletionDependencySource) { + *evidence = evidence.merge(DeletionDependencyEvidence::healthy(source)); +} + +fn observe_unavailable( + evidence: &mut DeletionDependencyEvidence, + source: DeletionDependencySource, +) { + *evidence = evidence.merge(DeletionDependencyEvidence::unavailable(source)); +} + +fn map_client_error(error: PaymentDrainClientError) -> ApplicationError { + ApplicationError::Verifier { + message: match error { + PaymentDrainClientError::NotFound => "Paykit payment drain not found", + PaymentDrainClientError::Conflict => "Paykit payment drain conflict", + PaymentDrainClientError::MalformedSuccess => { + "Paykit payment drain returned malformed success" + } + PaymentDrainClientError::Transport => "Paykit payment drain transport failure", + PaymentDrainClientError::Server => "Paykit payment drain server failure", + } + .to_owned(), + } +} + +fn validate_aggregate_progress( + persisted: &PaymentDrainSummary, + current: &PaymentDrainSummary, +) -> Result<(), ApplicationError> { + let accepted_delta = persisted.accepted_count.checked_sub(current.accepted_count); + let terminal_delta = current.terminal_count.checked_sub(persisted.terminal_count); + if persisted.cleanup_token != current.cleanup_token + || persisted.cancellation_enqueued_count != current.cancellation_enqueued_count + || accepted_delta.is_none() + || terminal_delta.is_none() + || accepted_delta != terminal_delta + || (current.status == PaymentDrainStatus::Completed && current.accepted_count != 0) + || (current.status == PaymentDrainStatus::Active && current.accepted_count == 0) + || (persisted.status == PaymentDrainStatus::Completed + && current.status != PaymentDrainStatus::Completed) + { + return Err(ApplicationError::InvalidContentLockDeletionState { + message: "Paykit payment drain aggregate changed incompatibly for deletion job" + .to_owned(), + }); + } + Ok(()) +} + +pub(crate) fn classify_payment_task( + status: PaymentRequestStatus, + minimum_confirmations: u32, +) -> Result, ApplicationError> { + if matches!( + status.request_state, + PaymentRequestState::RecoveryRequired + | PaymentRequestState::InvalidConflict + | PaymentRequestState::ProofSubmitted + | PaymentRequestState::ActiveRecurring + ) { + return Err(ApplicationError::InvalidVerificationTaskState { + message: "Paykit payment request entered an unsupported failure state".to_owned(), + }); + } + if matches!( + status.request_state, + PaymentRequestState::Rejected + | PaymentRequestState::Canceled + | PaymentRequestState::ProposalExpired + ) || (status.request_state == PaymentRequestState::Accepted + && status.payment_state == PaymentState::Expired) + { + return Ok(Some(VerificationTaskStatus::Expired)); + } + + if status.request_state != PaymentRequestState::Accepted || !status.amount_matched { + return Ok(None); + } + + if minimum_confirmations == 0 + && matches!( + status.payment_state, + PaymentState::Detected | PaymentState::Confirmed + ) + { + return Ok(Some(VerificationTaskStatus::Completed)); + } + + Ok((status.payment_state == PaymentState::Confirmed + && status.confirmations >= minimum_confirmations) + .then_some(VerificationTaskStatus::Completed)) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use async_trait::async_trait; + use base64::Engine; + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use time::macros::datetime; + + use locks_core::ids::{BundleId, ContentLockPath, CreatorPubky, PubkyLockResource}; + use locks_core::verification::{ + EntitlementLifetime, VERIFIED_PROOF_BUNDLE_VERSION, VerificationResult, VerifiedProofBundle, + }; + + use crate::application::errors::ApplicationError; + use crate::application::models::VerificationTaskStatus; + use crate::application::ports::{ + EntitlementRepository, PaymentDrainCleanupToken, PaymentDrainStatus, PaymentDrainSummary, + PaymentRequestState, PaymentRequestStatus, PaymentState, + }; + + use crate::application::use_cases::execute_content_lock_deletion_phase::DeletionDependencyStatus; + + use super::{ + DeletionDependencyEvidence, DeletionDependencySource, classify_payment_task, + persist_entitlement, validate_aggregate_progress, + }; + + #[tokio::test] + async fn exact_duplicate_entitlement_replay_finishes_with_healthy_evidence() { + let entitlement = entitlement(); + let repository = DuplicateEntitlementRepository(entitlement.clone()); + let mut evidence = DeletionDependencyEvidence::none(); + + persist_entitlement(&repository, entitlement, &mut evidence) + .await + .unwrap(); + + assert_eq!( + evidence.status(DeletionDependencySource::EntitlementRepository), + Some(DeletionDependencyStatus::Healthy) + ); + } + + struct DuplicateEntitlementRepository(VerifiedProofBundle); + + #[async_trait] + impl EntitlementRepository for DuplicateEntitlementRepository { + async fn insert_verified_proof_bundle( + &self, + _: VerifiedProofBundle, + ) -> Result<(), ApplicationError> { + Err(ApplicationError::DuplicateRecord { + record: "verified_proof_bundle", + }) + } + + async fn get_verified_proof_bundle( + &self, + _: &CreatorPubky, + _: &BundleId, + ) -> Result, ApplicationError> { + Ok(Some(self.0.clone())) + } + + async fn delete_verified_proof_bundle( + &self, + _: &CreatorPubky, + _: &BundleId, + ) -> Result<(), ApplicationError> { + unreachable!() + } + } + + fn entitlement() -> VerifiedProofBundle { + let creator = + CreatorPubky::from_str("pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy") + .unwrap(); + let path = ContentLockPath::from_str( + "/pub/locks.app/000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG.json", + ) + .unwrap(); + VerifiedProofBundle { + version: VERIFIED_PROOF_BUNDLE_VERSION, + bundle_id: BundleId::from_str("000G40R40M30E209185GR38E1W").unwrap(), + pubky_lock_resource: PubkyLockResource::new(creator, path), + verification_result: VerificationResult { criteria: vec![] }, + entitlement_lifetime: EntitlementLifetime::Unbounded, + } + } + + #[test] + fn aggregate_progress_accepts_only_equal_monotonic_transfer() { + let initial = summary(PaymentDrainStatus::Active, 2, 3, 1, 1); + assert!( + validate_aggregate_progress(&initial, &summary(PaymentDrainStatus::Active, 1, 4, 1, 1)) + .is_ok() + ); + assert!( + validate_aggregate_progress( + &initial, + &summary(PaymentDrainStatus::Completed, 0, 5, 1, 1) + ) + .is_ok() + ); + + for invalid in [ + summary(PaymentDrainStatus::Active, 3, 2, 1, 1), + summary(PaymentDrainStatus::Active, 1, 3, 1, 1), + summary(PaymentDrainStatus::Active, 1, 5, 1, 1), + summary(PaymentDrainStatus::Active, 1, 4, 2, 1), + summary(PaymentDrainStatus::Active, 1, 4, 1, 2), + summary(PaymentDrainStatus::Completed, 1, 4, 1, 1), + ] { + assert!(validate_aggregate_progress(&initial, &invalid).is_err()); + } + let completed = summary(PaymentDrainStatus::Completed, 0, 5, 1, 1); + assert!( + validate_aggregate_progress( + &completed, + &summary(PaymentDrainStatus::Active, 0, 5, 1, 1) + ) + .is_err() + ); + } + + fn summary( + status: PaymentDrainStatus, + accepted_count: u64, + terminal_count: u64, + cancellation_enqueued_count: u64, + token_byte: u8, + ) -> PaymentDrainSummary { + PaymentDrainSummary { + status, + accepted_count, + terminal_count, + cancellation_enqueued_count, + cleanup_token: PaymentDrainCleanupToken::parse( + &URL_SAFE_NO_PAD.encode([token_byte; 32]), + ) + .unwrap(), + } + } + + #[test] + fn terminal_unpaid_states_expire_without_failure() { + for request_state in [ + PaymentRequestState::Rejected, + PaymentRequestState::Canceled, + PaymentRequestState::ProposalExpired, + ] { + assert_eq!( + classify_payment_task(status(request_state, PaymentState::Undetected, 0, false), 6), + Ok(Some(VerificationTaskStatus::Expired)) + ); + } + assert_eq!( + classify_payment_task( + status( + PaymentRequestState::Accepted, + PaymentState::Expired, + 0, + false + ), + 6, + ), + Ok(Some(VerificationTaskStatus::Expired)) + ); + } + + #[test] + fn confirmations_are_applied_only_by_locks() { + assert_eq!( + classify_payment_task( + status( + PaymentRequestState::Accepted, + PaymentState::Detected, + 0, + true + ), + 0, + ), + Ok(Some(VerificationTaskStatus::Completed)) + ); + assert_eq!( + classify_payment_task( + status( + PaymentRequestState::Accepted, + PaymentState::Detected, + 0, + true + ), + 1, + ), + Ok(None) + ); + assert_eq!( + classify_payment_task( + status( + PaymentRequestState::Accepted, + PaymentState::Confirmed, + 5, + true + ), + 6, + ), + Ok(None) + ); + assert_eq!( + classify_payment_task( + status( + PaymentRequestState::Accepted, + PaymentState::Confirmed, + 6, + true + ), + 6, + ), + Ok(Some(VerificationTaskStatus::Completed)) + ); + } + + #[test] + fn timely_matched_payment_stays_pending_after_deadline_until_confirmed() { + let mut value = status( + PaymentRequestState::Accepted, + PaymentState::Detected, + 0, + true, + ); + value.payment_deadline = datetime!(2026-08-11 10:00:00 UTC); + assert_eq!(classify_payment_task(value, 6), Ok(None)); + } + + #[test] + fn classification_failure_states_fail_closed() { + for request_state in [ + PaymentRequestState::RecoveryRequired, + PaymentRequestState::InvalidConflict, + PaymentRequestState::ProofSubmitted, + PaymentRequestState::ActiveRecurring, + ] { + assert!( + classify_payment_task( + status(request_state, PaymentState::Undetected, 0, false), + 6, + ) + .is_err() + ); + } + } + + fn status( + request_state: PaymentRequestState, + payment_state: PaymentState, + confirmations: u32, + amount_matched: bool, + ) -> PaymentRequestStatus { + PaymentRequestStatus { + request_state, + payment_state, + invoice_created_at: datetime!(2026-08-12 10:00:00 UTC), + payment_deadline: datetime!(2026-08-13 10:00:00 UTC), + confirmations, + amount_matched, + } + } +} diff --git a/locks-service/src/application/use_cases/entitlement_check.rs b/locks-service/src/application/use_cases/entitlement_check.rs index 1cb3eed..6657cb6 100644 --- a/locks-service/src/application/use_cases/entitlement_check.rs +++ b/locks-service/src/application/use_cases/entitlement_check.rs @@ -1,4 +1,4 @@ -use locks_core::ids::{BundleId, ContentLockPath, CreatorPubky}; +use locks_core::ids::{BundleId, ContentLockPath, CreatorPubky, LockId}; use locks_core::lock_policy::ContentLock; use locks_core::verification::VerifiedProofBundle; @@ -10,6 +10,8 @@ use crate::application::ports::{ContentLockRepository, EntitlementRepository}; pub(super) struct ValidEntitlement { /// Current hash-verified content lock referenced by the entitlement. pub content_lock: ContentLock, + /// Canonical Lock ID verified against the entitlement path. + pub lock_id: LockId, } /// Loads and validates current entitlement state for credential issuance/validation. @@ -25,7 +27,7 @@ pub(super) async fn load_valid_entitlement( .ok_or(ApplicationError::EntitlementNotFound)?; let content_lock = load_current_content_lock(content_locks, &verified_proof_bundle).await?; - verify_content_lock_identity( + let lock_id = verify_content_lock_identity( &content_lock, verified_proof_bundle .pubky_lock_resource @@ -36,7 +38,10 @@ pub(super) async fn load_valid_entitlement( return Err(ApplicationError::EntitlementNotSatisfied); } - Ok(ValidEntitlement { content_lock }) + Ok(ValidEntitlement { + content_lock, + lock_id, + }) } async fn load_current_content_lock( @@ -57,7 +62,7 @@ async fn load_current_content_lock( pub(super) fn verify_content_lock_identity( content_lock: &ContentLock, content_lock_path: &ContentLockPath, -) -> Result<(), ApplicationError> { +) -> Result { let actual = content_lock .lock_id() @@ -67,7 +72,7 @@ pub(super) fn verify_content_lock_identity( let expected = content_lock_path.lock_id().clone(); if actual == expected { - Ok(()) + Ok(actual) } else { Err(ApplicationError::ContentLockHashMismatch { expected, actual }) } diff --git a/locks-service/src/application/use_cases/execute_content_lock_deletion_phase.rs b/locks-service/src/application/use_cases/execute_content_lock_deletion_phase.rs new file mode 100644 index 0000000..37f58d3 --- /dev/null +++ b/locks-service/src/application/use_cases/execute_content_lock_deletion_phase.rs @@ -0,0 +1,1025 @@ +use std::collections::BTreeMap; + +use async_trait::async_trait; +use locks_core::{ + content_lock_deletion::ContentLockDeletionTombstone, + ids::{ContentLockPath, GuardedResourceHash}, +}; +use time::Duration; + +use crate::application::{ + errors::ApplicationError, + models::{ + AdvanceContentLockDeletionPhaseResult, ClaimedContentLockDeletionJob, + ContentLockDeletionFailureCode, ContentLockDeletionPhase, + InitializeFinalAccessWindowsResult, + }, + ports::{ + AccessCredentialStore, Clock, ContentLockDeletionActionAcquireResult, + ContentLockDeletionActionClaim, ContentLockDeletionActionOwnership, + ContentLockDeletionRepository, ContentLockTombstoneRepository, GuardedResourceReadback, + GuardedResourceRepository, TombstoneReadback, + }, +}; + +use super::{ + drain_lock_payments::DrainLockPaymentsUseCase, + materialize_final_credentials::{ + MaterializeFinalCredentialsOutcome, MaterializeFinalCredentialsRequest, + MaterializeFinalCredentialsUseCase, + }, +}; + +/// Closed result of one bounded deletion-phase execution. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeletionPhaseExecutionOutcome { + Progressed, + Deferred, + ClaimLost, + TerminalFailed, + TransientDependencyFailure, + FatalFailure, +} + +/// Internal, identifier-free dependency classes observed by deletion execution. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeletionDependencySource { + PaymentProvider, + PaymentDrainRepository, + EntitlementRepository, + PubkyWithdrawal, + PubkyReadback, + PubkyResource, + PubkyForcePublic, + RepositoryQueueClaim, + RepositoryPhaseMutation, + RepositoryDefer, + RepositoryRetry, + RepositoryTerminalMutation, + RepositoryActionLock, + RepositoryActionLockRelease, + RepositoryForceReceipt, +} + +/// Health observed from one dependency during a bounded execution. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeletionDependencyStatus { + Healthy, + Unavailable, +} + +/// Closed dependency evidence carried from the real phase invocation to worker readiness. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct DeletionDependencyEvidence { + statuses: [Option; DeletionDependencySource::ALL.len()], +} + +impl DeletionDependencyEvidence { + pub const fn none() -> Self { + Self { + statuses: [None; DeletionDependencySource::ALL.len()], + } + } + + pub fn healthy(source: DeletionDependencySource) -> Self { + Self::observed(source, DeletionDependencyStatus::Healthy) + } + + pub fn unavailable(source: DeletionDependencySource) -> Self { + Self::observed(source, DeletionDependencyStatus::Unavailable) + } + + pub fn status(self, source: DeletionDependencySource) -> Option { + self.statuses[source.index()] + } + + pub fn merge(mut self, other: Self) -> Self { + for source in DeletionDependencySource::ALL { + let index = source.index(); + self.statuses[index] = match (self.statuses[index], other.statuses[index]) { + (Some(DeletionDependencyStatus::Unavailable), _) + | (_, Some(DeletionDependencyStatus::Unavailable)) => { + Some(DeletionDependencyStatus::Unavailable) + } + (Some(status), None) | (None, Some(status)) => Some(status), + ( + Some(DeletionDependencyStatus::Healthy), + Some(DeletionDependencyStatus::Healthy), + ) => Some(DeletionDependencyStatus::Healthy), + (None, None) => None, + }; + } + self + } + + fn observed(source: DeletionDependencySource, status: DeletionDependencyStatus) -> Self { + let mut evidence = Self::none(); + evidence.statuses[source.index()] = Some(status); + evidence + } +} + +impl DeletionDependencySource { + pub const ALL: [Self; 15] = [ + Self::PaymentProvider, + Self::PaymentDrainRepository, + Self::EntitlementRepository, + Self::PubkyWithdrawal, + Self::PubkyReadback, + Self::PubkyResource, + Self::PubkyForcePublic, + Self::RepositoryQueueClaim, + Self::RepositoryPhaseMutation, + Self::RepositoryDefer, + Self::RepositoryRetry, + Self::RepositoryTerminalMutation, + Self::RepositoryActionLock, + Self::RepositoryActionLockRelease, + Self::RepositoryForceReceipt, + ]; + + pub const fn index(self) -> usize { + match self { + Self::PaymentProvider => 0, + Self::PaymentDrainRepository => 1, + Self::EntitlementRepository => 2, + Self::PubkyWithdrawal => 3, + Self::PubkyReadback => 4, + Self::PubkyResource => 5, + Self::PubkyForcePublic => 6, + Self::RepositoryQueueClaim => 7, + Self::RepositoryPhaseMutation => 8, + Self::RepositoryDefer => 9, + Self::RepositoryRetry => 10, + Self::RepositoryTerminalMutation => 11, + Self::RepositoryActionLock => 12, + Self::RepositoryActionLockRelease => 13, + Self::RepositoryForceReceipt => 14, + } + } +} + +/// One closed execution outcome plus source-aware, secret-free dependency evidence. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DeletionPhaseExecution { + pub outcome: DeletionPhaseExecutionOutcome, + pub evidence: DeletionDependencyEvidence, +} + +impl DeletionPhaseExecution { + pub fn new(outcome: DeletionPhaseExecutionOutcome) -> Self { + Self { + outcome, + evidence: DeletionDependencyEvidence::none(), + } + } + + pub fn with_evidence(mut self, evidence: DeletionDependencyEvidence) -> Self { + self.evidence = self.evidence.merge(evidence); + self + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum DeletionExecutionErrorClass { + TransientDependency, + Fatal, +} + +pub(super) fn classify_deletion_execution_error( + error: &ApplicationError, +) -> DeletionExecutionErrorClass { + match error { + ApplicationError::Storage { .. } | ApplicationError::Verifier { .. } => { + DeletionExecutionErrorClass::TransientDependency + } + _ => DeletionExecutionErrorClass::Fatal, + } +} + +fn error_outcome(error: &ApplicationError) -> DeletionPhaseExecutionOutcome { + match classify_deletion_execution_error(error) { + DeletionExecutionErrorClass::TransientDependency => { + DeletionPhaseExecutionOutcome::TransientDependencyFailure + } + DeletionExecutionErrorClass::Fatal => DeletionPhaseExecutionOutcome::FatalFailure, + } +} + +fn error_execution( + error: &ApplicationError, + source: DeletionDependencySource, +) -> DeletionPhaseExecution { + let outcome = error_outcome(error); + DeletionPhaseExecution::new(outcome) + .with_evidence(DeletionDependencyEvidence::unavailable(source)) +} + +/// Bounded final-access settings used by the phase executor. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ContentLockDeletionPhaseExecutorConfig { + pub final_credential_issuance_window: Duration, + pub final_read_window: Duration, + pub final_credential_batch_limit: usize, +} + +/// Object-safe payment-drain collaborator for phase-executor tests and runtime composition. +#[async_trait] +pub trait ContentLockPaymentDrainExecutor: Send + Sync { + /// Returns only evidence recorded by the concrete remote/repository operations it invoked. + async fn execute_claimed( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecution; +} + +#[async_trait] +impl ContentLockPaymentDrainExecutor for DrainLockPaymentsUseCase<'_> { + async fn execute_claimed( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecution { + DrainLockPaymentsUseCase::execute_claimed_with_evidence(self, claim, worker_id).await + } +} + +/// Object-safe bounded final-credential materialization collaborator. +#[async_trait] +pub trait FinalCredentialMaterializer: Send + Sync { + async fn materialize( + &self, + request: MaterializeFinalCredentialsRequest<'_>, + ) -> Result; +} + +#[async_trait] +impl FinalCredentialMaterializer for MaterializeFinalCredentialsUseCase<'_> { + async fn materialize( + &self, + request: MaterializeFinalCredentialsRequest<'_>, + ) -> Result { + self.execute(request).await + } +} + +/// Dependencies for one graceful-deletion phase executor. +pub struct ContentLockDeletionPhaseExecutorDependencies<'a> { + pub deletions: &'a dyn ContentLockDeletionRepository, + pub action_ownership: &'a dyn ContentLockDeletionActionOwnership, + pub tombstones: &'a dyn ContentLockTombstoneRepository, + pub guarded_resources: &'a dyn GuardedResourceRepository, + pub access_credentials: &'a dyn AccessCredentialStore, + pub clock: &'a dyn Clock, + pub payments: &'a dyn ContentLockPaymentDrainExecutor, + pub final_credentials: &'a dyn FinalCredentialMaterializer, +} + +/// Executes at most one bounded graceful-deletion phase while holding its action guard. +pub struct ContentLockDeletionPhaseExecutor<'a> { + dependencies: ContentLockDeletionPhaseExecutorDependencies<'a>, + config: ContentLockDeletionPhaseExecutorConfig, +} + +impl<'a> ContentLockDeletionPhaseExecutor<'a> { + pub fn new( + dependencies: ContentLockDeletionPhaseExecutorDependencies<'a>, + config: ContentLockDeletionPhaseExecutorConfig, + ) -> Self { + Self { + dependencies, + config, + } + } + + pub async fn execute( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecutionOutcome { + self.execute_with_evidence(claim, worker_id).await.outcome + } + + pub async fn execute_with_evidence( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecution { + let guard = match self + .dependencies + .action_ownership + .try_acquire(ContentLockDeletionActionClaim { + job_id: claim.job.job_id, + worker_id, + claim_token: claim.claim_token, + expected_phase: claim.job.phase, + force: false, + }) + .await + { + Ok(ContentLockDeletionActionAcquireResult::Acquired(guard)) => guard, + Ok(ContentLockDeletionActionAcquireResult::Busy) => { + return DeletionPhaseExecution::new(DeletionPhaseExecutionOutcome::Deferred); + } + Ok(ContentLockDeletionActionAcquireResult::ClaimLost) => { + return DeletionPhaseExecution::new(DeletionPhaseExecutionOutcome::ClaimLost); + } + Err(error) => { + return error_execution(&error, DeletionDependencySource::RepositoryActionLock); + } + }; + + let execution = self + .execute_guarded_with_evidence(claim, worker_id) + .await + .with_evidence(DeletionDependencyEvidence::healthy( + DeletionDependencySource::RepositoryActionLock, + )); + if let Err(error) = guard.release().await { + return error_execution( + &error, + DeletionDependencySource::RepositoryActionLockRelease, + ) + .with_evidence(execution.evidence); + } + execution.with_evidence(DeletionDependencyEvidence::healthy( + DeletionDependencySource::RepositoryActionLockRelease, + )) + } + + async fn execute_guarded_with_evidence( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecution { + let phase = claim.job.phase; + if matches!( + phase, + ContentLockDeletionPhase::StartPaymentDrain | ContentLockDeletionPhase::DrainPayments + ) { + return self + .dependencies + .payments + .execute_claimed(claim, worker_id) + .await; + } + if phase == ContentLockDeletionPhase::Withdraw { + return self.withdraw_with_evidence(&claim, worker_id).await; + } + if phase == ContentLockDeletionPhase::DeleteContent { + return self + .verify_frozen_content_with_evidence(&claim, worker_id) + .await; + } + if phase == ContentLockDeletionPhase::DeleteTombstone { + return self + .verify_tombstone_for_purge_with_evidence(&claim, worker_id) + .await; + } + + match phase { + ContentLockDeletionPhase::PurgeOperationalState => { + DeletionPhaseExecution::new(DeletionPhaseExecutionOutcome::Deferred) + } + _ => self.repository_execution(self.execute_guarded(claim, worker_id).await), + } + } + + async fn execute_guarded( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecutionOutcome { + match claim.job.phase { + ContentLockDeletionPhase::Withdraw => self.withdraw(&claim, worker_id).await, + ContentLockDeletionPhase::StartPaymentDrain + | ContentLockDeletionPhase::DrainPayments => { + self.dependencies + .payments + .execute_claimed(claim, worker_id) + .await + .outcome + } + ContentLockDeletionPhase::DrainExistingCredentials => { + self.advance_or_defer( + &claim, + worker_id, + ContentLockDeletionPhase::IssueFinalCredentials, + ) + .await + } + ContentLockDeletionPhase::IssueFinalCredentials => { + self.issue_final_credentials(&claim, worker_id).await + } + ContentLockDeletionPhase::DrainFinalReads => { + self.advance_or_defer(&claim, worker_id, ContentLockDeletionPhase::DeleteContent) + .await + } + ContentLockDeletionPhase::DeleteContent => { + self.verify_frozen_content(&claim, worker_id).await + } + ContentLockDeletionPhase::DeleteTombstone => { + self.verify_tombstone_for_purge(&claim, worker_id).await + } + ContentLockDeletionPhase::PurgeOperationalState => { + DeletionPhaseExecutionOutcome::Deferred + } + } + } + + async fn withdraw_with_evidence( + &self, + claim: &ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecution { + let tombstone = tombstone(claim); + let content_lock_path = ContentLockPath::from_lock_id(claim.job.lock_id.clone()); + let readback = self + .dependencies + .tombstones + .withdraw_content_lock( + claim.job.creator.clone(), + content_lock_path, + &claim.job.frozen_content_lock, + &tombstone, + ) + .await; + let pubky_healthy = + DeletionDependencyEvidence::healthy(DeletionDependencySource::PubkyWithdrawal); + match readback { + Ok(TombstoneReadback::Exact) => self + .repository_execution( + self.advance( + claim, + worker_id, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await, + ) + .with_evidence(pubky_healthy), + Ok(TombstoneReadback::Missing) => self + .repository_execution( + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneMissing, + ) + .await, + ) + .with_evidence(pubky_healthy), + Ok(TombstoneReadback::Replaced) => self + .repository_execution( + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneReplaced, + ) + .await, + ) + .with_evidence(pubky_healthy), + Err(error) => error_execution(&error, DeletionDependencySource::PubkyWithdrawal), + } + } + + fn repository_execution( + &self, + outcome: DeletionPhaseExecutionOutcome, + ) -> DeletionPhaseExecution { + let evidence = match outcome { + DeletionPhaseExecutionOutcome::TransientDependencyFailure => { + DeletionDependencyEvidence::unavailable( + DeletionDependencySource::RepositoryPhaseMutation, + ) + } + DeletionPhaseExecutionOutcome::Progressed + | DeletionPhaseExecutionOutcome::Deferred + | DeletionPhaseExecutionOutcome::TerminalFailed => DeletionDependencyEvidence::healthy( + DeletionDependencySource::RepositoryPhaseMutation, + ), + DeletionPhaseExecutionOutcome::ClaimLost + | DeletionPhaseExecutionOutcome::FatalFailure => DeletionDependencyEvidence::none(), + }; + DeletionPhaseExecution::new(outcome).with_evidence(evidence) + } + + async fn withdraw( + &self, + claim: &ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecutionOutcome { + let tombstone = tombstone(claim); + let content_lock_path = ContentLockPath::from_lock_id(claim.job.lock_id.clone()); + let readback = self + .dependencies + .tombstones + .withdraw_content_lock( + claim.job.creator.clone(), + content_lock_path, + &claim.job.frozen_content_lock, + &tombstone, + ) + .await; + match readback { + Ok(TombstoneReadback::Exact) => { + self.advance( + claim, + worker_id, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + } + Ok(TombstoneReadback::Missing) => { + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneMissing, + ) + .await + } + Ok(TombstoneReadback::Replaced) => { + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneReplaced, + ) + .await + } + Err(error) => error_outcome(&error), + } + } + + async fn issue_final_credentials( + &self, + claim: &ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecutionOutcome { + let windows = match self + .dependencies + .access_credentials + .initialize_final_access_windows( + claim.job.job_id, + worker_id, + claim.claim_token, + self.config.final_credential_issuance_window, + self.config.final_read_window, + ) + .await + { + Ok(InitializeFinalAccessWindowsResult::Initialized(windows)) => windows, + Ok(InitializeFinalAccessWindowsResult::ClaimLost) => { + return DeletionPhaseExecutionOutcome::ClaimLost; + } + Err(error) => return error_outcome(&error), + }; + + let materialized = self + .dependencies + .final_credentials + .materialize(MaterializeFinalCredentialsRequest { + deletion_job_id: claim.job.job_id, + worker_id, + claim_token: claim.claim_token, + now: windows.issuance_started_at, + batch_limit: self.config.final_credential_batch_limit, + }) + .await; + match materialized { + Ok(outcome) + if self.config.final_credential_batch_limit > 0 + && outcome.materialized_count >= self.config.final_credential_batch_limit => + { + DeletionPhaseExecutionOutcome::Deferred + } + Ok(_) => { + self.advance_or_defer(claim, worker_id, ContentLockDeletionPhase::DrainFinalReads) + .await + } + Err(error) => error_outcome(&error), + } + } + + async fn verify_frozen_content_with_evidence( + &self, + claim: &ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecution { + let tombstone = tombstone(claim); + let content_lock_path = ContentLockPath::from_lock_id(claim.job.lock_id.clone()); + let mut resources = BTreeMap::::new(); + for (path, resource) in &claim.job.frozen_content_lock.secondary_resources { + resources.insert(path.clone(), resource.hash); + } + if let Some(primary) = &claim.job.frozen_content_lock.primary_resource { + resources.insert(primary.path.clone(), primary.hash); + } + + let pubky_healthy = + DeletionDependencyEvidence::healthy(DeletionDependencySource::PubkyReadback); + let mut resource_was_read = false; + for (path, expected_hash) in resources { + match self + .dependencies + .tombstones + .read_tombstone(&claim.job.creator, &content_lock_path, &tombstone) + .await + { + Ok(TombstoneReadback::Exact) => {} + Ok(TombstoneReadback::Missing) => { + return self + .repository_execution( + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneMissing, + ) + .await, + ) + .with_evidence(pubky_healthy); + } + Ok(TombstoneReadback::Replaced) => { + return self + .repository_execution( + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneReplaced, + ) + .await, + ) + .with_evidence(pubky_healthy); + } + Err(error) => { + return error_execution(&error, DeletionDependencySource::PubkyReadback); + } + } + match self + .dependencies + .guarded_resources + .read_guarded_resource_generation(&claim.job.creator, &path, &expected_hash) + .await + { + Ok(GuardedResourceReadback::Exact) => resource_was_read = true, + Ok(GuardedResourceReadback::Missing) => { + return self + .repository_execution( + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::StateCorrupt, + ) + .await, + ) + .with_evidence(pubky_healthy) + .with_evidence(DeletionDependencyEvidence::healthy( + DeletionDependencySource::PubkyResource, + )); + } + Ok(GuardedResourceReadback::Replaced) => { + return self + .repository_execution( + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::ResourceReplaced, + ) + .await, + ) + .with_evidence(pubky_healthy) + .with_evidence(DeletionDependencyEvidence::healthy( + DeletionDependencySource::PubkyResource, + )); + } + Err(error) => { + return error_execution(&error, DeletionDependencySource::PubkyResource) + .with_evidence(pubky_healthy); + } + } + } + + let resource_evidence = if resource_was_read { + DeletionDependencyEvidence::healthy(DeletionDependencySource::PubkyResource) + } else { + DeletionDependencyEvidence::none() + }; + + match self + .dependencies + .tombstones + .read_tombstone(&claim.job.creator, &content_lock_path, &tombstone) + .await + { + Ok(TombstoneReadback::Exact) => self + .repository_execution( + self.advance(claim, worker_id, ContentLockDeletionPhase::DeleteTombstone) + .await, + ) + .with_evidence(pubky_healthy) + .with_evidence(resource_evidence), + Ok(TombstoneReadback::Missing) => self + .repository_execution( + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneMissing, + ) + .await, + ) + .with_evidence(pubky_healthy) + .with_evidence(resource_evidence), + Ok(TombstoneReadback::Replaced) => self + .repository_execution( + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneReplaced, + ) + .await, + ) + .with_evidence(pubky_healthy) + .with_evidence(resource_evidence), + Err(error) => error_execution(&error, DeletionDependencySource::PubkyReadback) + .with_evidence(resource_evidence), + } + } + + async fn verify_tombstone_for_purge_with_evidence( + &self, + claim: &ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecution { + let tombstone = tombstone(claim); + let content_lock_path = ContentLockPath::from_lock_id(claim.job.lock_id.clone()); + let readback = self + .dependencies + .tombstones + .read_tombstone(&claim.job.creator, &content_lock_path, &tombstone) + .await; + let pubky_healthy = + DeletionDependencyEvidence::healthy(DeletionDependencySource::PubkyReadback); + match readback { + Ok(TombstoneReadback::Exact) => self + .repository_execution( + self.advance( + claim, + worker_id, + ContentLockDeletionPhase::PurgeOperationalState, + ) + .await, + ) + .with_evidence(pubky_healthy), + Ok(TombstoneReadback::Missing) => self + .repository_execution( + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneMissing, + ) + .await, + ) + .with_evidence(pubky_healthy), + Ok(TombstoneReadback::Replaced) => self + .repository_execution( + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneReplaced, + ) + .await, + ) + .with_evidence(pubky_healthy), + Err(error) => error_execution(&error, DeletionDependencySource::PubkyReadback), + } + } + + async fn verify_frozen_content( + &self, + claim: &ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecutionOutcome { + let tombstone = tombstone(claim); + let content_lock_path = ContentLockPath::from_lock_id(claim.job.lock_id.clone()); + let mut resources = BTreeMap::::new(); + for (path, resource) in &claim.job.frozen_content_lock.secondary_resources { + resources.insert(path.clone(), resource.hash); + } + if let Some(primary) = &claim.job.frozen_content_lock.primary_resource { + resources.insert(primary.path.clone(), primary.hash); + } + + for (path, expected_hash) in resources { + match self + .dependencies + .tombstones + .read_tombstone(&claim.job.creator, &content_lock_path, &tombstone) + .await + { + Ok(TombstoneReadback::Exact) => {} + Ok(TombstoneReadback::Missing) => { + return self + .finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneMissing, + ) + .await; + } + Ok(TombstoneReadback::Replaced) => { + return self + .finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneReplaced, + ) + .await; + } + Err(error) => return error_outcome(&error), + } + match self + .dependencies + .guarded_resources + .read_guarded_resource_generation(&claim.job.creator, &path, &expected_hash) + .await + { + Ok(GuardedResourceReadback::Exact) => {} + Ok(GuardedResourceReadback::Missing) => { + return self + .finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::StateCorrupt, + ) + .await; + } + Ok(GuardedResourceReadback::Replaced) => { + return self + .finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::ResourceReplaced, + ) + .await; + } + Err(error) => return error_outcome(&error), + } + } + + match self + .dependencies + .tombstones + .read_tombstone(&claim.job.creator, &content_lock_path, &tombstone) + .await + { + Ok(TombstoneReadback::Exact) => {} + Ok(TombstoneReadback::Missing) => { + return self + .finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneMissing, + ) + .await; + } + Ok(TombstoneReadback::Replaced) => { + return self + .finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneReplaced, + ) + .await; + } + Err(error) => return error_outcome(&error), + } + + self.advance(claim, worker_id, ContentLockDeletionPhase::DeleteTombstone) + .await + } + + async fn verify_tombstone_for_purge( + &self, + claim: &ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecutionOutcome { + let tombstone = tombstone(claim); + let content_lock_path = ContentLockPath::from_lock_id(claim.job.lock_id.clone()); + match self + .dependencies + .tombstones + .read_tombstone(&claim.job.creator, &content_lock_path, &tombstone) + .await + { + Ok(TombstoneReadback::Exact) => { + self.advance( + claim, + worker_id, + ContentLockDeletionPhase::PurgeOperationalState, + ) + .await + } + Ok(TombstoneReadback::Missing) => { + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneMissing, + ) + .await + } + Ok(TombstoneReadback::Replaced) => { + self.finish_terminal( + claim, + worker_id, + ContentLockDeletionFailureCode::TombstoneReplaced, + ) + .await + } + Err(error) => error_outcome(&error), + } + } + + async fn advance( + &self, + claim: &ClaimedContentLockDeletionJob, + worker_id: &str, + next_phase: ContentLockDeletionPhase, + ) -> DeletionPhaseExecutionOutcome { + match self + .dependencies + .deletions + .advance_phase(claim.job.job_id, worker_id, claim.claim_token, next_phase) + .await + { + Ok(AdvanceContentLockDeletionPhaseResult::Advanced(_)) => { + DeletionPhaseExecutionOutcome::Progressed + } + Ok(AdvanceContentLockDeletionPhaseResult::ClaimLost) => { + DeletionPhaseExecutionOutcome::ClaimLost + } + Ok(AdvanceContentLockDeletionPhaseResult::ObligationsPending) => { + DeletionPhaseExecutionOutcome::Deferred + } + Ok(AdvanceContentLockDeletionPhaseResult::TerminalFailure(failure_code)) => { + self.finish_terminal(claim, worker_id, failure_code).await + } + Err(error) => error_outcome(&error), + } + } + + async fn advance_or_defer( + &self, + claim: &ClaimedContentLockDeletionJob, + worker_id: &str, + next_phase: ContentLockDeletionPhase, + ) -> DeletionPhaseExecutionOutcome { + match self + .dependencies + .deletions + .advance_phase(claim.job.job_id, worker_id, claim.claim_token, next_phase) + .await + { + Ok(AdvanceContentLockDeletionPhaseResult::Advanced(_)) => { + DeletionPhaseExecutionOutcome::Progressed + } + Ok(AdvanceContentLockDeletionPhaseResult::ClaimLost) => { + DeletionPhaseExecutionOutcome::ClaimLost + } + Ok(AdvanceContentLockDeletionPhaseResult::ObligationsPending) => { + DeletionPhaseExecutionOutcome::Deferred + } + Ok(AdvanceContentLockDeletionPhaseResult::TerminalFailure(failure_code)) => { + self.finish_terminal(claim, worker_id, failure_code).await + } + Err(error) => error_outcome(&error), + } + } + + async fn finish_terminal( + &self, + claim: &ClaimedContentLockDeletionJob, + worker_id: &str, + failure_code: ContentLockDeletionFailureCode, + ) -> DeletionPhaseExecutionOutcome { + match self + .dependencies + .deletions + .finish( + claim.job.job_id, + worker_id, + claim.claim_token, + Some(failure_code), + ) + .await + { + Ok(Some(_)) => DeletionPhaseExecutionOutcome::TerminalFailed, + Ok(None) => DeletionPhaseExecutionOutcome::ClaimLost, + Err(error) => error_outcome(&error), + } + } +} + +fn tombstone(claim: &ClaimedContentLockDeletionJob) -> ContentLockDeletionTombstone { + ContentLockDeletionTombstone::new(claim.job.lock_id.clone(), claim.job.deletion_started_at) +} + +#[cfg(test)] +mod tests; diff --git a/locks-service/src/application/use_cases/execute_content_lock_deletion_phase/tests.rs b/locks-service/src/application/use_cases/execute_content_lock_deletion_phase/tests.rs new file mode 100644 index 0000000..27cacd6 --- /dev/null +++ b/locks-service/src/application/use_cases/execute_content_lock_deletion_phase/tests.rs @@ -0,0 +1,1104 @@ +use std::{ + collections::{BTreeMap, VecDeque}, + str::FromStr, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + }, +}; + +use async_trait::async_trait; +use locks_core::{ + content_lock_deletion::ContentLockDeletionTombstone, + ids::{ContentLockPath, CreatorPubky, GuardedResourceHash, LockId}, + lock_policy::{ + AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, GuardedResource, LockLogic, + LockServerConfig, SecondaryGuardedResource, + }, +}; +use time::{Duration, OffsetDateTime, macros::datetime}; +use uuid::Uuid; + +use super::*; +use crate::application::{ + models::{ + AccessCredentialLookupKey, AccessCredentialRecord, ContentLockDeletionJob, + FinalAccessWindows, GuardedResourceRecord, PrepareForceDeletionResult, + }, + ports::{ + ContentLockDeletionActionAcquireResult, ContentLockDeletionActionClaim, + ContentLockDeletionActionGuard, GuardedResourceReadback, + }, +}; + +const NOW: OffsetDateTime = datetime!(2026-08-17 12:00:00 UTC); + +#[tokio::test] +async fn withdraw_exact_progresses_and_releases_guard() { + let h = Harness::new(); + let outcome = h + .executor() + .execute(claim(ContentLockDeletionPhase::Withdraw), "worker") + .await; + assert_eq!(outcome, DeletionPhaseExecutionOutcome::Progressed); + assert_eq!( + h.deletions.advances(), + vec![ContentLockDeletionPhase::StartPaymentDrain] + ); + assert!(h.actions.released.load(Ordering::SeqCst)); +} + +#[tokio::test] +async fn withdraw_reclaim_preserves_replacement_after_phase_advance_failure() { + let h = Harness::new(); + *h.deletions.advance_error.lock().unwrap() = Some(ApplicationError::Storage { + message: "phase persistence unavailable after publication".to_owned(), + }); + + let first = h + .executor() + .execute(claim(ContentLockDeletionPhase::Withdraw), "worker-a") + .await; + assert_eq!( + first, + DeletionPhaseExecutionOutcome::TransientDependencyFailure + ); + assert_eq!(h.tombstones.withdraw_count(), 1); + assert!(h.deletions.advances().is_empty()); + + h.tombstones.replace_public_bytes(); + + let mut reclaimed = claim(ContentLockDeletionPhase::Withdraw); + reclaimed.claim_token = Uuid::from_u128(3); + let second = h.executor().execute(reclaimed, "worker-b").await; + + assert_eq!(second, DeletionPhaseExecutionOutcome::TerminalFailed); + assert_eq!(h.tombstones.withdraw_count(), 1); + assert!(h.tombstones.replacement_is_present()); + assert_eq!( + h.deletions.finishes(), + vec![ContentLockDeletionFailureCode::TombstoneReplaced] + ); + assert_eq!( + h.deletions.advances(), + Vec::::new() + ); +} + +#[tokio::test] +async fn missing_tombstone_finishes_exact_claim_with_stable_code() { + let h = Harness::new(); + h.tombstones.set_reads([TombstoneReadback::Missing]); + let outcome = h + .executor() + .execute(claim(ContentLockDeletionPhase::DeleteContent), "worker") + .await; + assert_eq!(outcome, DeletionPhaseExecutionOutcome::TerminalFailed); + assert_eq!( + h.deletions.finishes(), + vec![ContentLockDeletionFailureCode::TombstoneMissing] + ); + assert!(h.resources.observed().is_empty()); +} + +#[tokio::test] +async fn replaced_tombstone_is_terminal() { + let h = Harness::new(); + h.tombstones.set_reads([TombstoneReadback::Replaced]); + let outcome = h + .executor() + .execute(claim(ContentLockDeletionPhase::DeleteTombstone), "worker") + .await; + assert_eq!(outcome, DeletionPhaseExecutionOutcome::TerminalFailed); + assert_eq!( + h.deletions.finishes(), + vec![ContentLockDeletionFailureCode::TombstoneReplaced] + ); + assert_eq!(h.tombstones.read_count(), 1); +} + +#[tokio::test] +async fn exact_checks_verify_every_sorted_deduplicated_content_generation() { + let h = Harness::new(); + h.tombstones.set_reads([TombstoneReadback::Exact; 4]); + let outcome = h + .executor() + .execute(claim(ContentLockDeletionPhase::DeleteContent), "worker") + .await; + assert_eq!(outcome, DeletionPhaseExecutionOutcome::Progressed); + assert_eq!( + h.resources.observed(), + vec![ + "/priv/locks.app/content/a".to_owned(), + "/priv/locks.app/content/m".to_owned(), + "/priv/locks.app/content/z".to_owned(), + ] + ); + assert_eq!(h.tombstones.read_count(), 4); + assert_eq!( + h.deletions.advances(), + vec![ContentLockDeletionPhase::DeleteTombstone] + ); +} + +#[tokio::test] +async fn preexisting_guarded_resource_replacement_is_terminal_and_not_deleted() { + let h = Harness::new(); + h.resources + .set_outcomes([GuardedResourceReadback::Replaced]); + h.tombstones.set_reads([TombstoneReadback::Exact]); + + let outcome = h + .executor() + .execute(claim(ContentLockDeletionPhase::DeleteContent), "worker") + .await; + + assert_eq!(outcome, DeletionPhaseExecutionOutcome::TerminalFailed); + assert_eq!( + h.deletions.finishes(), + vec![ContentLockDeletionFailureCode::ResourceReplaced] + ); + assert!(h.deletions.advances().is_empty()); + assert_eq!(h.resources.observed().len(), 1); +} + +#[tokio::test] +async fn final_tombstone_loss_after_content_deletion_fails_before_phase_advance() { + let h = Harness::new(); + h.tombstones.set_reads([ + TombstoneReadback::Exact, + TombstoneReadback::Exact, + TombstoneReadback::Exact, + TombstoneReadback::Missing, + ]); + let outcome = h + .executor() + .execute(claim(ContentLockDeletionPhase::DeleteContent), "worker") + .await; + + assert_eq!(outcome, DeletionPhaseExecutionOutcome::TerminalFailed); + assert!(h.deletions.advances().is_empty()); + assert_eq!( + h.deletions.finishes(), + vec![ContentLockDeletionFailureCode::TombstoneMissing] + ); +} + +#[tokio::test] +async fn exact_tombstone_remains_published_at_purge_handoff() { + let h = Harness::new(); + h.tombstones.set_reads([TombstoneReadback::Exact]); + let outcome = h + .executor() + .execute(claim(ContentLockDeletionPhase::DeleteTombstone), "worker") + .await; + assert_eq!(outcome, DeletionPhaseExecutionOutcome::Progressed); + assert_eq!( + h.deletions.advances(), + vec![ContentLockDeletionPhase::PurgeOperationalState] + ); +} + +#[tokio::test] +async fn busy_action_guard_defers_without_side_effects() { + let h = Harness::new(); + h.actions.busy.store(true, Ordering::SeqCst); + let outcome = h + .executor() + .execute(claim(ContentLockDeletionPhase::Withdraw), "worker") + .await; + assert_eq!(outcome, DeletionPhaseExecutionOutcome::Deferred); + assert!(h.deletions.advances().is_empty()); + assert_eq!(h.tombstones.withdraw_count(), 0); +} + +#[tokio::test] +async fn action_ownership_unexpected_error_is_fatal() { + let h = Harness::new(); + *h.actions.error.lock().unwrap() = Some(ApplicationError::MissingRecord { + record: "deletion_action_guard", + }); + + let outcome = h + .executor() + .execute(claim(ContentLockDeletionPhase::Withdraw), "worker") + .await; + + assert_eq!(outcome, DeletionPhaseExecutionOutcome::FatalFailure); + assert_eq!(h.tombstones.withdraw_count(), 0); +} + +#[tokio::test] +async fn tombstone_storage_error_is_transient_but_unexpected_error_is_fatal() { + for (error, expected) in [ + ( + ApplicationError::Storage { + message: "temporary homeserver outage".to_owned(), + }, + DeletionPhaseExecutionOutcome::TransientDependencyFailure, + ), + ( + ApplicationError::MissingRecord { + record: "content_lock_tombstone", + }, + DeletionPhaseExecutionOutcome::FatalFailure, + ), + ] { + let h = Harness::new(); + *h.tombstones.withdraw_error.lock().unwrap() = Some(error); + + let outcome = h + .executor() + .execute(claim(ContentLockDeletionPhase::Withdraw), "worker") + .await; + + assert_eq!(outcome, expected); + } +} + +#[tokio::test] +async fn final_credential_crypto_error_is_fatal() { + let h = Harness::new(); + *h.finals.error.lock().unwrap() = Some(ApplicationError::FinalCredentialSecret { + message: "secret detail".to_owned(), + }); + + let outcome = h + .executor() + .execute( + claim(ContentLockDeletionPhase::IssueFinalCredentials), + "worker", + ) + .await; + + assert_eq!(outcome, DeletionPhaseExecutionOutcome::FatalFailure); +} + +#[tokio::test] +async fn invalid_state_transition_is_fatal_instead_of_business_deferral() { + let h = Harness::new(); + *h.deletions.advance_error.lock().unwrap() = + Some(ApplicationError::InvalidContentLockDeletionState { + message: "invariant detail".to_owned(), + }); + + let outcome = h + .executor() + .execute( + claim(ContentLockDeletionPhase::DrainExistingCredentials), + "worker", + ) + .await; + + assert_eq!(outcome, DeletionPhaseExecutionOutcome::FatalFailure); +} + +#[tokio::test] +async fn repository_obligations_pending_is_healthy_deferral() { + let h = Harness::new(); + h.deletions + .obligations_pending + .store(true, Ordering::SeqCst); + + let outcome = h + .executor() + .execute( + claim(ContentLockDeletionPhase::DrainExistingCredentials), + "worker", + ) + .await; + + assert_eq!(outcome, DeletionPhaseExecutionOutcome::Deferred); +} + +#[tokio::test] +async fn missed_final_credential_issuance_deadline_terminalizes_without_advancing() { + let h = Harness::new(); + h.deletions + .issuance_deadline_missed + .store(true, Ordering::SeqCst); + + let outcome = h + .executor() + .execute( + claim(ContentLockDeletionPhase::IssueFinalCredentials), + "worker", + ) + .await; + + assert_eq!(outcome, DeletionPhaseExecutionOutcome::TerminalFailed); + assert_eq!( + h.deletions.finishes(), + vec![ContentLockDeletionFailureCode::StateCorrupt] + ); + assert!(h.deletions.advances().is_empty()); + assert!(h.resources.observed().is_empty()); +} + +#[tokio::test] +async fn transient_provider_error_remains_retryable_across_payment_boundary() { + let h = Harness::new(); + *h.payments.error.lock().unwrap() = Some(ApplicationError::Verifier { + message: "provider detail".to_owned(), + }); + + let outcome = h + .executor() + .execute(claim(ContentLockDeletionPhase::DrainPayments), "worker") + .await; + + assert_eq!( + outcome, + DeletionPhaseExecutionOutcome::TransientDependencyFailure + ); +} + +#[tokio::test] +async fn paykit_failure_carries_only_paykit_unavailable_evidence() { + let h = Harness::new(); + *h.payments.error.lock().unwrap() = Some(ApplicationError::Verifier { + message: "provider detail".to_owned(), + }); + + let execution = h + .executor() + .execute_with_evidence(claim(ContentLockDeletionPhase::DrainPayments), "worker") + .await; + + assert_eq!( + execution.outcome, + DeletionPhaseExecutionOutcome::TransientDependencyFailure + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::PaymentProvider), + Some(DeletionDependencyStatus::Unavailable) + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::PubkyReadback), + None + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::RepositoryPhaseMutation), + None + ); +} + +#[tokio::test] +async fn successful_active_paykit_drain_is_healthy_evidence_while_deferred() { + let h = Harness::new(); + + let execution = h + .executor() + .execute_with_evidence(claim(ContentLockDeletionPhase::DrainPayments), "worker") + .await; + + assert_eq!(execution.outcome, DeletionPhaseExecutionOutcome::Deferred); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::PaymentProvider), + Some(DeletionDependencyStatus::Healthy) + ); +} + +#[tokio::test] +async fn pubky_success_followed_by_repository_failure_preserves_source_independence() { + let h = Harness::new(); + *h.deletions.advance_error.lock().unwrap() = Some(ApplicationError::Storage { + message: "local persistence unavailable".to_owned(), + }); + + let execution = h + .executor() + .execute_with_evidence(claim(ContentLockDeletionPhase::Withdraw), "worker") + .await; + + assert_eq!( + execution.outcome, + DeletionPhaseExecutionOutcome::TransientDependencyFailure + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::PubkyReadback), + None + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::PubkyWithdrawal), + Some(DeletionDependencyStatus::Healthy) + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::RepositoryPhaseMutation), + Some(DeletionDependencyStatus::Unavailable) + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::PaymentProvider), + None + ); +} + +#[tokio::test] +async fn delete_content_pubky_success_followed_by_advance_failure_preserves_stage_evidence() { + let h = Harness::new(); + h.tombstones.set_reads([TombstoneReadback::Exact; 4]); + *h.deletions.advance_error.lock().unwrap() = Some(ApplicationError::Storage { + message: "phase persistence unavailable".to_owned(), + }); + + let execution = h + .executor() + .execute_with_evidence(claim(ContentLockDeletionPhase::DeleteContent), "worker") + .await; + + assert_eq!( + execution.outcome, + DeletionPhaseExecutionOutcome::TransientDependencyFailure + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::PubkyReadback), + Some(DeletionDependencyStatus::Healthy) + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::PubkyResource), + Some(DeletionDependencyStatus::Healthy) + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::RepositoryPhaseMutation), + Some(DeletionDependencyStatus::Unavailable) + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::RepositoryQueueClaim), + None + ); +} + +#[tokio::test] +async fn busy_action_guard_defers_without_dependency_health_evidence() { + let h = Harness::new(); + h.actions.busy.store(true, Ordering::SeqCst); + + let execution = h + .executor() + .execute_with_evidence(claim(ContentLockDeletionPhase::Withdraw), "worker") + .await; + + assert_eq!(execution.outcome, DeletionPhaseExecutionOutcome::Deferred); + for source in [ + DeletionDependencySource::PaymentProvider, + DeletionDependencySource::PubkyReadback, + DeletionDependencySource::RepositoryQueueClaim, + DeletionDependencySource::RepositoryPhaseMutation, + ] { + assert_eq!(execution.evidence.status(source), None); + } +} + +#[tokio::test] +async fn repository_none_is_claim_lost_and_guard_is_released() { + let h = Harness::new(); + h.deletions.claim_live.store(false, Ordering::SeqCst); + let outcome = h + .executor() + .execute(claim(ContentLockDeletionPhase::Withdraw), "worker") + .await; + assert_eq!(outcome, DeletionPhaseExecutionOutcome::ClaimLost); + assert!(h.actions.released.load(Ordering::SeqCst)); +} + +#[tokio::test] +async fn purge_is_deferred_task_ten_handoff() { + let h = Harness::new(); + let outcome = h + .executor() + .execute( + claim(ContentLockDeletionPhase::PurgeOperationalState), + "worker", + ) + .await; + assert_eq!(outcome, DeletionPhaseExecutionOutcome::Deferred); + assert!(h.deletions.advances().is_empty()); + assert!(h.actions.released.load(Ordering::SeqCst)); +} + +#[tokio::test] +async fn operation_evidence_does_not_infer_phase_health_from_purge_or_claim_loss() { + let h = Harness::new(); + let purge = h + .executor() + .execute_with_evidence( + claim(ContentLockDeletionPhase::PurgeOperationalState), + "worker", + ) + .await; + assert_eq!(purge.outcome, DeletionPhaseExecutionOutcome::Deferred); + assert_eq!( + purge + .evidence + .status(DeletionDependencySource::RepositoryPhaseMutation), + None + ); + + h.deletions.claim_live.store(false, Ordering::SeqCst); + let lost = h + .executor() + .execute_with_evidence(claim(ContentLockDeletionPhase::Withdraw), "worker") + .await; + assert_eq!(lost.outcome, DeletionPhaseExecutionOutcome::ClaimLost); + assert_eq!( + lost.evidence + .status(DeletionDependencySource::RepositoryPhaseMutation), + None + ); + assert_eq!( + lost.evidence + .status(DeletionDependencySource::PubkyWithdrawal), + Some(DeletionDependencyStatus::Healthy) + ); +} + +struct Harness { + deletions: FakeDeletions, + actions: FakeActions, + tombstones: FakeTombstones, + resources: FakeResources, + access: FakeAccess, + clock: FixedClock, + payments: FakePayments, + finals: FakeFinals, +} + +impl Harness { + fn new() -> Self { + Self { + deletions: FakeDeletions::new(), + actions: FakeActions::new(), + tombstones: FakeTombstones::new(), + resources: FakeResources::default(), + access: FakeAccess, + clock: FixedClock, + payments: FakePayments::default(), + finals: FakeFinals::default(), + } + } + + fn executor(&self) -> ContentLockDeletionPhaseExecutor<'_> { + ContentLockDeletionPhaseExecutor::new( + ContentLockDeletionPhaseExecutorDependencies { + deletions: &self.deletions, + action_ownership: &self.actions, + tombstones: &self.tombstones, + guarded_resources: &self.resources, + access_credentials: &self.access, + clock: &self.clock, + payments: &self.payments, + final_credentials: &self.finals, + }, + ContentLockDeletionPhaseExecutorConfig { + final_credential_issuance_window: Duration::minutes(15), + final_read_window: Duration::minutes(15), + final_credential_batch_limit: 10, + }, + ) + } +} + +#[derive(Default)] +struct FakeDeletions { + claim_live: AtomicBool, + obligations_pending: AtomicBool, + issuance_deadline_missed: AtomicBool, + advances: Mutex>, + finishes: Mutex>, + advance_error: Mutex>, +} +impl FakeDeletions { + fn new() -> Self { + Self { + claim_live: AtomicBool::new(true), + ..Self::default() + } + } + fn advances(&self) -> Vec { + self.advances.lock().unwrap().clone() + } + fn finishes(&self) -> Vec { + self.finishes.lock().unwrap().clone() + } + fn result(&self) -> Option { + self.claim_live + .load(Ordering::SeqCst) + .then(|| claim(ContentLockDeletionPhase::Withdraw).job) + } +} + +#[async_trait] +impl ContentLockDeletionRepository for FakeDeletions { + async fn begin_publication( + &self, + _: &CreatorPubky, + _: &LockId, + _: Uuid, + ) -> Result<(), ApplicationError> { + unreachable!() + } + async fn finish_publication( + &self, + _: &CreatorPubky, + _: &LockId, + _: Uuid, + ) -> Result { + unreachable!() + } + async fn abandon_publication( + &self, + _: &CreatorPubky, + _: &LockId, + _: Uuid, + ) -> Result { + unreachable!() + } + async fn publication_in_progress( + &self, + _: &CreatorPubky, + _: &LockId, + ) -> Result { + unreachable!() + } + async fn insert_job(&self, _: ContentLockDeletionJob) -> Result<(), ApplicationError> { + unreachable!() + } + async fn get_job( + &self, + _: &CreatorPubky, + _: &LockId, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn claim_next( + &self, + _: &str, + _: time::Duration, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn schedule_retry( + &self, + _: Uuid, + _: &str, + _: Uuid, + _: time::Duration, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn defer( + &self, + _: Uuid, + _: &str, + _: Uuid, + _: time::Duration, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn advance_phase( + &self, + _: Uuid, + _: &str, + _: Uuid, + next: ContentLockDeletionPhase, + ) -> Result { + if let Some(error) = self.advance_error.lock().unwrap().take() { + return Err(error); + } + if self.obligations_pending.load(Ordering::SeqCst) { + return Ok(AdvanceContentLockDeletionPhaseResult::ObligationsPending); + } + if self.issuance_deadline_missed.load(Ordering::SeqCst) { + return Ok(AdvanceContentLockDeletionPhaseResult::TerminalFailure( + ContentLockDeletionFailureCode::StateCorrupt, + )); + } + self.advances.lock().unwrap().push(next); + Ok(match self.result() { + Some(job) => AdvanceContentLockDeletionPhaseResult::Advanced(Box::new(job)), + None => AdvanceContentLockDeletionPhaseResult::ClaimLost, + }) + } + async fn finish( + &self, + _: Uuid, + _: &str, + _: Uuid, + code: Option, + ) -> Result, ApplicationError> { + self.finishes.lock().unwrap().push(code.unwrap()); + Ok(self.result()) + } + async fn resume_failed_job( + &self, + _: &CreatorPubky, + _: &LockId, + _: OffsetDateTime, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn prepare_force_deletion( + &self, + _: &CreatorPubky, + _: &LockId, + ) -> Result { + unreachable!() + } + async fn complete_force_deletion( + &self, + _: Uuid, + _: &str, + _: Uuid, + ) -> Result { + unreachable!() + } + async fn has_force_receipt( + &self, + _: &CreatorPubky, + _: &LockId, + ) -> Result { + unreachable!() + } +} + +struct FakeGuard(Arc); +#[async_trait] +impl ContentLockDeletionActionGuard for FakeGuard { + async fn release(self: Box) -> Result<(), ApplicationError> { + self.0.store(true, Ordering::SeqCst); + Ok(()) + } +} +struct FakeActions { + busy: AtomicBool, + released: Arc, + error: Mutex>, +} +impl FakeActions { + fn new() -> Self { + Self { + busy: AtomicBool::new(false), + released: Arc::new(AtomicBool::new(false)), + error: Mutex::new(None), + } + } +} +#[async_trait] +impl ContentLockDeletionActionOwnership for FakeActions { + async fn try_acquire( + &self, + _: ContentLockDeletionActionClaim<'_>, + ) -> Result { + if let Some(error) = self.error.lock().unwrap().take() { + return Err(error); + } + if self.busy.load(Ordering::SeqCst) { + Ok(ContentLockDeletionActionAcquireResult::Busy) + } else { + Ok(ContentLockDeletionActionAcquireResult::Acquired(Box::new( + FakeGuard(self.released.clone()), + ))) + } + } +} + +struct FakeTombstones { + public_state: Mutex, + withdraw_error: Mutex>, + reads: Mutex>, + withdraw_count: Mutex, + read_count: Mutex, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum FakePublicLockState { + Original, + Tombstone, + Replacement, +} + +impl FakeTombstones { + fn new() -> Self { + Self { + public_state: Mutex::new(FakePublicLockState::Original), + withdraw_error: Mutex::new(None), + reads: Mutex::new(VecDeque::new()), + withdraw_count: Mutex::new(0), + read_count: Mutex::new(0), + } + } + + fn set_reads(&self, values: [TombstoneReadback; N]) { + *self.reads.lock().unwrap() = values.into(); + } + fn withdraw_count(&self) -> usize { + *self.withdraw_count.lock().unwrap() + } + fn replace_public_bytes(&self) { + *self.public_state.lock().unwrap() = FakePublicLockState::Replacement; + } + fn replacement_is_present(&self) -> bool { + *self.public_state.lock().unwrap() == FakePublicLockState::Replacement + } + fn read_count(&self) -> usize { + *self.read_count.lock().unwrap() + } +} +#[async_trait] +impl ContentLockTombstoneRepository for FakeTombstones { + async fn withdraw_content_lock( + &self, + _: CreatorPubky, + _: ContentLockPath, + _: &ContentLock, + _: &ContentLockDeletionTombstone, + ) -> Result { + if let Some(error) = self.withdraw_error.lock().unwrap().take() { + return Err(error); + } + let mut state = self.public_state.lock().unwrap(); + match *state { + FakePublicLockState::Original => { + *self.withdraw_count.lock().unwrap() += 1; + *state = FakePublicLockState::Tombstone; + Ok(TombstoneReadback::Exact) + } + FakePublicLockState::Tombstone => Ok(TombstoneReadback::Exact), + FakePublicLockState::Replacement => Ok(TombstoneReadback::Replaced), + } + } + async fn read_tombstone( + &self, + _: &CreatorPubky, + _: &ContentLockPath, + _: &ContentLockDeletionTombstone, + ) -> Result { + *self.read_count.lock().unwrap() += 1; + Ok(self.reads.lock().unwrap().pop_front().unwrap()) + } + + async fn force_delete_content_lock_and_verify_absent( + &self, + _: &CreatorPubky, + _: &ContentLockPath, + ) -> Result<(), ApplicationError> { + unreachable!() + } +} + +#[derive(Default)] +struct FakeResources { + observed: Mutex>, + outcomes: Mutex>, +} +impl FakeResources { + fn observed(&self) -> Vec { + self.observed.lock().unwrap().clone() + } + + fn set_outcomes(&self, outcomes: impl IntoIterator) { + *self.outcomes.lock().unwrap() = outcomes.into_iter().collect(); + } +} +#[async_trait] +impl GuardedResourceRepository for FakeResources { + async fn upsert_guarded_resource( + &self, + _: GuardedResourceRecord, + ) -> Result<(), ApplicationError> { + unreachable!() + } + async fn get_guarded_resource( + &self, + _: &CreatorPubky, + _: &str, + _: &GuardedResourceHash, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn get_current_guarded_resource( + &self, + _: &CreatorPubky, + _: &str, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn delete_guarded_resource( + &self, + _: &CreatorPubky, + path: &str, + ) -> Result { + self.observed.lock().unwrap().push(path.to_owned()); + Ok(false) + } + async fn read_guarded_resource_generation( + &self, + _: &CreatorPubky, + path: &str, + _: &GuardedResourceHash, + ) -> Result { + self.observed.lock().unwrap().push(path.to_owned()); + Ok(self + .outcomes + .lock() + .unwrap() + .pop_front() + .unwrap_or(GuardedResourceReadback::Exact)) + } +} + +struct FakeAccess; +#[async_trait] +impl AccessCredentialStore for FakeAccess { + async fn insert_access_credential( + &self, + _: &LockId, + _: AccessCredentialLookupKey, + _: AccessCredentialRecord, + ) -> Result<(), ApplicationError> { + unreachable!() + } + async fn get_access_credential( + &self, + _: &AccessCredentialLookupKey, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn delete_access_credential( + &self, + _: &AccessCredentialLookupKey, + ) -> Result<(), ApplicationError> { + unreachable!() + } + async fn initialize_final_access_windows( + &self, + _: Uuid, + _: &str, + _: Uuid, + _: Duration, + _: Duration, + ) -> Result { + Ok(InitializeFinalAccessWindowsResult::Initialized( + FinalAccessWindows { + issuance_started_at: NOW, + credential_issuance_deadline: NOW + Duration::minutes(15), + read_deadline: NOW + Duration::minutes(30), + }, + )) + } +} +struct FixedClock; +impl Clock for FixedClock { + fn now(&self) -> OffsetDateTime { + NOW + } +} +#[derive(Default)] +struct FakePayments { + error: Mutex>, +} +#[async_trait] +impl ContentLockPaymentDrainExecutor for FakePayments { + async fn execute_claimed( + &self, + _: ClaimedContentLockDeletionJob, + _: &str, + ) -> DeletionPhaseExecution { + if let Some(error) = self.error.lock().unwrap().take() { + return error_execution(&error, DeletionDependencySource::PaymentProvider); + } + DeletionPhaseExecution::new(DeletionPhaseExecutionOutcome::Deferred).with_evidence( + DeletionDependencyEvidence::healthy(DeletionDependencySource::PaymentProvider), + ) + } +} +#[derive(Default)] +struct FakeFinals { + error: Mutex>, +} +#[async_trait] +impl FinalCredentialMaterializer for FakeFinals { + async fn materialize( + &self, + _: MaterializeFinalCredentialsRequest<'_>, + ) -> Result { + if let Some(error) = self.error.lock().unwrap().take() { + return Err(error); + } + Ok(MaterializeFinalCredentialsOutcome { + materialized_count: 0, + }) + } +} + +fn claim(phase: ContentLockDeletionPhase) -> ClaimedContentLockDeletionJob { + let mut secondary_resources = BTreeMap::new(); + for path in [ + "/priv/locks.app/content/z", + "/priv/locks.app/content/a", + "/priv/locks.app/content/m", + ] { + secondary_resources.insert( + path.to_owned(), + SecondaryGuardedResource { + hash: GuardedResourceHash::from_bytes([7; 32]), + content_type: "text/plain".to_owned(), + size: 1, + }, + ); + } + let lock = ContentLock { + version: CONTENT_LOCK_VERSION, + creator: CreatorPubky::from_str( + "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy", + ) + .unwrap(), + primary_resource: Some( + GuardedResource::new( + "/priv/locks.app/content/m", + GuardedResourceHash::from_bytes([8; 32]), + "text/plain", + 1, + ) + .unwrap(), + ), + secondary_resources, + criteria: vec![], + lock_logic: LockLogic::All { criteria: vec![] }, + access_policy: AccessPolicy { + requested_credential_ttl_seconds: 900, + }, + lock_server: LockServerConfig { override_: None }, + created_at: NOW, + }; + let mut job = ContentLockDeletionJob::new(Uuid::from_u128(1), lock, NOW).unwrap(); + job.phase = phase; + ClaimedContentLockDeletionJob { + job, + claim_token: Uuid::from_u128(2), + } +} diff --git a/locks-service/src/application/use_cases/execute_forced_content_lock_deletion.rs b/locks-service/src/application/use_cases/execute_forced_content_lock_deletion.rs new file mode 100644 index 0000000..b40a019 --- /dev/null +++ b/locks-service/src/application/use_cases/execute_forced_content_lock_deletion.rs @@ -0,0 +1,220 @@ +use std::collections::BTreeSet; + +use locks_core::ids::ContentLockPath; + +use crate::application::{ + errors::ApplicationError, + models::ClaimedContentLockDeletionJob, + ports::{ + Clock, ContentLockDeletionActionAcquireResult, ContentLockDeletionActionClaim, + ContentLockDeletionActionOwnership, ContentLockDeletionRepository, + ContentLockTombstoneRepository, GuardedResourceRepository, + }, +}; + +use super::execute_content_lock_deletion_phase::{ + DeletionDependencyEvidence, DeletionDependencySource, DeletionExecutionErrorClass, + classify_deletion_execution_error, +}; + +/// Closed, secret-free result of one active-force external execution attempt. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ForcedContentLockDeletionOutcome { + Completed, + Deferred, + ClaimLost, + TransientDependencyFailure, + FatalFailure, +} + +fn error_outcome(error: &ApplicationError) -> ForcedContentLockDeletionOutcome { + match classify_deletion_execution_error(error) { + DeletionExecutionErrorClass::TransientDependency => { + ForcedContentLockDeletionOutcome::TransientDependencyFailure + } + DeletionExecutionErrorClass::Fatal => ForcedContentLockDeletionOutcome::FatalFailure, + } +} + +/// One force outcome plus exact, identifier-free dependency observations. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ForcedContentLockDeletionExecution { + pub outcome: ForcedContentLockDeletionOutcome, + pub evidence: DeletionDependencyEvidence, +} + +impl ForcedContentLockDeletionExecution { + fn new(outcome: ForcedContentLockDeletionOutcome) -> Self { + Self { + outcome, + evidence: DeletionDependencyEvidence::none(), + } + } + + fn with_evidence(mut self, evidence: DeletionDependencyEvidence) -> Self { + self.evidence = self.evidence.merge(evidence); + self + } +} + +fn error_execution( + error: &ApplicationError, + source: DeletionDependencySource, +) -> ForcedContentLockDeletionExecution { + ForcedContentLockDeletionExecution::new(error_outcome(error)) + .with_evidence(DeletionDependencyEvidence::unavailable(source)) +} + +/// External dependencies required by active-force execution. +pub struct ExecuteForcedContentLockDeletionDependencies<'a> { + pub action_ownership: &'a dyn ContentLockDeletionActionOwnership, + pub tombstones: &'a dyn ContentLockTombstoneRepository, + pub guarded_resources: &'a dyn GuardedResourceRepository, + pub deletions: &'a dyn ContentLockDeletionRepository, + pub clock: &'a dyn Clock, +} + +/// Executes the force path for an already-claimed active deletion job. +pub struct ExecuteForcedContentLockDeletionUseCase<'a> { + dependencies: ExecuteForcedContentLockDeletionDependencies<'a>, +} + +impl<'a> ExecuteForcedContentLockDeletionUseCase<'a> { + pub fn new(dependencies: ExecuteForcedContentLockDeletionDependencies<'a>) -> Self { + Self { dependencies } + } + + pub async fn execute( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> ForcedContentLockDeletionOutcome { + self.execute_with_evidence(claim, worker_id).await.outcome + } + + pub async fn execute_with_evidence( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> ForcedContentLockDeletionExecution { + if claim.job.force_requested_at.is_none() { + return ForcedContentLockDeletionExecution::new( + ForcedContentLockDeletionOutcome::Deferred, + ); + } + + let guard = match self + .dependencies + .action_ownership + .try_acquire(ContentLockDeletionActionClaim { + job_id: claim.job.job_id, + worker_id, + claim_token: claim.claim_token, + expected_phase: claim.job.phase, + force: true, + }) + .await + { + Ok(ContentLockDeletionActionAcquireResult::Acquired(guard)) => guard, + Ok(ContentLockDeletionActionAcquireResult::Busy) => { + return ForcedContentLockDeletionExecution::new( + ForcedContentLockDeletionOutcome::Deferred, + ); + } + Ok(ContentLockDeletionActionAcquireResult::ClaimLost) => { + return ForcedContentLockDeletionExecution::new( + ForcedContentLockDeletionOutcome::ClaimLost, + ); + } + Err(error) => { + return error_execution(&error, DeletionDependencySource::RepositoryActionLock); + } + }; + + let execution = self.execute_guarded(&claim, worker_id).await.with_evidence( + DeletionDependencyEvidence::healthy(DeletionDependencySource::RepositoryActionLock), + ); + if let Err(error) = guard.release().await { + return error_execution( + &error, + DeletionDependencySource::RepositoryActionLockRelease, + ) + .with_evidence(execution.evidence); + } + execution.with_evidence(DeletionDependencyEvidence::healthy( + DeletionDependencySource::RepositoryActionLockRelease, + )) + } + + async fn execute_guarded( + &self, + claim: &ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> ForcedContentLockDeletionExecution { + let public_path = ContentLockPath::from_lock_id(claim.job.lock_id.clone()); + if let Err(error) = self + .dependencies + .tombstones + .force_delete_content_lock_and_verify_absent(&claim.job.creator, &public_path) + .await + { + return error_execution(&error, DeletionDependencySource::PubkyForcePublic); + } + + let mut evidence = + DeletionDependencyEvidence::healthy(DeletionDependencySource::PubkyForcePublic); + for path in frozen_resource_paths(claim) { + evidence = evidence.merge( + match self + .dependencies + .guarded_resources + .delete_guarded_resource(&claim.job.creator, &path) + .await + { + Ok(_) => { + DeletionDependencyEvidence::healthy(DeletionDependencySource::PubkyResource) + } + Err(_) => DeletionDependencyEvidence::unavailable( + DeletionDependencySource::PubkyResource, + ), + }, + ); + } + + let execution = match self + .dependencies + .deletions + .complete_force_deletion(claim.job.job_id, worker_id, claim.claim_token) + .await + { + Ok(true) => { + ForcedContentLockDeletionExecution::new(ForcedContentLockDeletionOutcome::Completed) + .with_evidence(DeletionDependencyEvidence::healthy( + DeletionDependencySource::RepositoryForceReceipt, + )) + } + Ok(false) => { + ForcedContentLockDeletionExecution::new(ForcedContentLockDeletionOutcome::ClaimLost) + } + Err(error) => error_execution(&error, DeletionDependencySource::RepositoryForceReceipt), + }; + execution.with_evidence(evidence) + } +} + +fn frozen_resource_paths(claim: &ClaimedContentLockDeletionJob) -> BTreeSet { + let mut paths = claim + .job + .frozen_content_lock + .secondary_resources + .keys() + .cloned() + .collect::>(); + if let Some(primary) = &claim.job.frozen_content_lock.primary_resource { + paths.insert(primary.path.clone()); + } + paths +} + +#[cfg(test)] +mod tests; diff --git a/locks-service/src/application/use_cases/execute_forced_content_lock_deletion/tests.rs b/locks-service/src/application/use_cases/execute_forced_content_lock_deletion/tests.rs new file mode 100644 index 0000000..1a13d0a --- /dev/null +++ b/locks-service/src/application/use_cases/execute_forced_content_lock_deletion/tests.rs @@ -0,0 +1,660 @@ +use std::{ + collections::{BTreeMap, HashSet}, + str::FromStr, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + }, +}; + +use async_trait::async_trait; +use locks_core::{ + content_lock_deletion::ContentLockDeletionTombstone, + ids::{ContentLockPath, CreatorPubky, GuardedResourceHash, LockId}, + lock_policy::{ + AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, GuardedResource, LockLogic, + LockServerConfig, SecondaryGuardedResource, + }, +}; +use time::{OffsetDateTime, macros::datetime}; +use uuid::Uuid; + +use super::*; +use crate::application::use_cases::execute_content_lock_deletion_phase::{ + DeletionDependencyEvidence, DeletionDependencySource, DeletionDependencyStatus, +}; +use crate::application::{ + errors::ApplicationError, + models::{ + AdvanceContentLockDeletionPhaseResult, ClaimedContentLockDeletionJob, + ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionPhase, + GuardedResourceRecord, PrepareForceDeletionResult, + }, + ports::{ + ContentLockDeletionActionGuard, ContentLockDeletionActionOwnership, + ContentLockDeletionRepository, ContentLockTombstoneRepository, GuardedResourceRepository, + TombstoneReadback, + }, +}; + +const NOW: OffsetDateTime = datetime!(2026-08-17 12:00:00 UTC); + +#[test] +fn outcomes_have_closed_path_free_debug_output() { + assert_eq!( + format!("{:?}", ForcedContentLockDeletionOutcome::Completed), + "Completed" + ); + assert_eq!( + format!("{:?}", ForcedContentLockDeletionOutcome::Deferred), + "Deferred" + ); + assert_eq!( + format!("{:?}", ForcedContentLockDeletionOutcome::ClaimLost), + "ClaimLost" + ); + assert_eq!( + format!( + "{:?}", + ForcedContentLockDeletionOutcome::TransientDependencyFailure + ), + "TransientDependencyFailure" + ); + assert_eq!( + format!("{:?}", ForcedContentLockDeletionOutcome::FatalFailure), + "FatalFailure" + ); +} + +#[tokio::test] +async fn deletes_public_path_first_then_attempts_sorted_deduplicated_resources_and_completes() { + let h = Harness::new(); + + let outcome = h.use_case().execute(force_claim(), "worker").await; + + assert_eq!(outcome, ForcedContentLockDeletionOutcome::Completed); + assert_eq!( + h.operations(), + vec![ + "public".to_owned(), + "resource:/priv/locks.app/content/a".to_owned(), + "resource:/priv/locks.app/content/m".to_owned(), + "resource:/priv/locks.app/content/z".to_owned(), + "complete".to_owned(), + "release".to_owned(), + ] + ); +} + +#[tokio::test] +async fn resource_errors_are_best_effort_and_still_complete_the_force_receipt() { + let h = Harness::new(); + h.resources + .fail_paths + .lock() + .unwrap() + .insert("/priv/locks.app/content/m".to_owned()); + + let outcome = h.use_case().execute(force_claim(), "worker").await; + + assert_eq!(outcome, ForcedContentLockDeletionOutcome::Completed); + assert_eq!(h.deletions.complete_calls.load(Ordering::SeqCst), 1); + assert!( + h.operations() + .iter() + .any(|op| op == "resource:/priv/locks.app/content/z") + ); +} + +#[tokio::test] +async fn stale_claim_cannot_complete_after_all_external_attempts() { + let h = Harness::new(); + h.deletions.claim_live.store(false, Ordering::SeqCst); + + let outcome = h.use_case().execute(force_claim(), "worker").await; + + assert_eq!(outcome, ForcedContentLockDeletionOutcome::ClaimLost); + assert_eq!(h.deletions.complete_calls.load(Ordering::SeqCst), 1); + assert_eq!(h.operations().last().map(String::as_str), Some("release")); +} + +#[tokio::test] +async fn reclaimed_force_claim_replays_deletions_after_receipt_persistence_failure() { + let h = Harness::new(); + h.deletions.fail_complete_once.store(true, Ordering::SeqCst); + + let first = h.use_case().execute(force_claim(), "worker-a").await; + assert_eq!( + first, + ForcedContentLockDeletionOutcome::TransientDependencyFailure + ); + assert_eq!(h.deletions.complete_calls.load(Ordering::SeqCst), 1); + + let mut reclaimed = force_claim(); + reclaimed.claim_token = Uuid::from_u128(3); + let second = h.use_case().execute(reclaimed, "worker-b").await; + + assert_eq!(second, ForcedContentLockDeletionOutcome::Completed); + assert_eq!(h.deletions.complete_calls.load(Ordering::SeqCst), 2); + assert_eq!( + h.operations() + .iter() + .filter(|operation| operation.as_str() == "public") + .count(), + 2 + ); +} + +#[tokio::test] +async fn busy_guard_defers_without_effects() { + let h = Harness::new(); + h.actions.busy.store(true, Ordering::SeqCst); + + let outcome = h.use_case().execute(force_claim(), "worker").await; + + assert_eq!(outcome, ForcedContentLockDeletionOutcome::Deferred); + assert!(h.operations().is_empty()); + assert_eq!(h.deletions.complete_calls.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn post_lock_stale_force_claim_is_lost_without_external_action() { + let h = Harness::new(); + h.actions.claim_live.store(false, Ordering::SeqCst); + let outcome = h.use_case().execute(force_claim(), "worker").await; + assert_eq!(outcome, ForcedContentLockDeletionOutcome::ClaimLost); + assert!(h.operations().is_empty()); + assert_eq!(h.deletions.complete_calls.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn public_absence_failure_stops_before_resources_and_completion() { + let h = Harness::new(); + h.tombstones.fail.store(true, Ordering::SeqCst); + + let outcome = h.use_case().execute(force_claim(), "worker").await; + + assert_eq!( + outcome, + ForcedContentLockDeletionOutcome::TransientDependencyFailure + ); + assert_eq!( + h.operations(), + vec!["public".to_owned(), "release".to_owned()] + ); + assert_eq!(h.deletions.complete_calls.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn busy_guard_has_no_healthy_dependency_evidence() { + let h = Harness::new(); + h.actions.busy.store(true, Ordering::SeqCst); + + let execution = h + .use_case() + .execute_with_evidence(force_claim(), "worker") + .await; + + assert_eq!( + execution.outcome, + ForcedContentLockDeletionOutcome::Deferred + ); + assert_eq!(execution.evidence, DeletionDependencyEvidence::none()); +} + +#[tokio::test] +async fn force_receipt_failure_reports_pubky_healthy_and_repository_mutation_unavailable() { + let h = Harness::new(); + h.deletions.fail_complete_once.store(true, Ordering::SeqCst); + + let execution = h + .use_case() + .execute_with_evidence(force_claim(), "worker") + .await; + + assert_eq!( + execution.outcome, + ForcedContentLockDeletionOutcome::TransientDependencyFailure + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::PubkyForcePublic), + Some(DeletionDependencyStatus::Healthy) + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::RepositoryForceReceipt), + Some(DeletionDependencyStatus::Unavailable) + ); +} + +#[tokio::test] +async fn force_pubky_failure_does_not_degrade_repository_mutation() { + let h = Harness::new(); + h.tombstones.fail.store(true, Ordering::SeqCst); + + let execution = h + .use_case() + .execute_with_evidence(force_claim(), "worker") + .await; + + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::PubkyForcePublic), + Some(DeletionDependencyStatus::Unavailable) + ); + assert_ne!( + execution + .evidence + .status(DeletionDependencySource::RepositoryForceReceipt), + Some(DeletionDependencyStatus::Unavailable) + ); +} + +#[tokio::test] +async fn stale_force_receipt_and_guard_release_do_not_infer_force_receipt_health() { + let h = Harness::new(); + h.deletions.claim_live.store(false, Ordering::SeqCst); + + let execution = h + .use_case() + .execute_with_evidence(force_claim(), "worker") + .await; + + assert_eq!( + execution.outcome, + ForcedContentLockDeletionOutcome::ClaimLost + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::RepositoryForceReceipt), + None + ); + assert_eq!( + execution + .evidence + .status(DeletionDependencySource::RepositoryActionLockRelease), + Some(DeletionDependencyStatus::Healthy) + ); +} + +struct Harness { + operations: Arc>>, + deletions: FakeDeletions, + actions: FakeActions, + tombstones: FakeTombstones, + resources: FakeResources, + clock: FixedClock, +} + +impl Harness { + fn new() -> Self { + let operations = Arc::new(Mutex::new(Vec::new())); + Self { + deletions: FakeDeletions::new(Arc::clone(&operations)), + actions: FakeActions::new(Arc::clone(&operations)), + tombstones: FakeTombstones::new(Arc::clone(&operations)), + resources: FakeResources::new(Arc::clone(&operations)), + clock: FixedClock, + operations, + } + } + + fn use_case(&self) -> ExecuteForcedContentLockDeletionUseCase<'_> { + ExecuteForcedContentLockDeletionUseCase::new(ExecuteForcedContentLockDeletionDependencies { + action_ownership: &self.actions, + tombstones: &self.tombstones, + guarded_resources: &self.resources, + deletions: &self.deletions, + clock: &self.clock, + }) + } + + fn operations(&self) -> Vec { + self.operations.lock().unwrap().clone() + } +} + +struct FixedClock; +impl crate::application::ports::Clock for FixedClock { + fn now(&self) -> OffsetDateTime { + NOW + } +} + +struct FakeGuard(Arc>>); +#[async_trait] +impl ContentLockDeletionActionGuard for FakeGuard { + async fn release(self: Box) -> Result<(), ApplicationError> { + self.0.lock().unwrap().push("release".to_owned()); + Ok(()) + } +} + +struct FakeActions { + busy: AtomicBool, + claim_live: AtomicBool, + operations: Arc>>, +} +impl FakeActions { + fn new(operations: Arc>>) -> Self { + Self { + busy: AtomicBool::new(false), + claim_live: AtomicBool::new(true), + operations, + } + } +} +#[async_trait] +impl ContentLockDeletionActionOwnership for FakeActions { + async fn try_acquire( + &self, + _: ContentLockDeletionActionClaim<'_>, + ) -> Result { + if self.busy.load(Ordering::SeqCst) { + Ok(ContentLockDeletionActionAcquireResult::Busy) + } else if !self.claim_live.load(Ordering::SeqCst) { + Ok(ContentLockDeletionActionAcquireResult::ClaimLost) + } else { + Ok(ContentLockDeletionActionAcquireResult::Acquired(Box::new( + FakeGuard(Arc::clone(&self.operations)), + ))) + } + } +} + +struct FakeTombstones { + operations: Arc>>, + fail: AtomicBool, +} +impl FakeTombstones { + fn new(operations: Arc>>) -> Self { + Self { + operations, + fail: AtomicBool::new(false), + } + } +} +#[async_trait] +impl ContentLockTombstoneRepository for FakeTombstones { + async fn withdraw_content_lock( + &self, + _: CreatorPubky, + _: ContentLockPath, + _: &ContentLock, + _: &ContentLockDeletionTombstone, + ) -> Result { + unreachable!() + } + async fn read_tombstone( + &self, + _: &CreatorPubky, + _: &ContentLockPath, + _: &ContentLockDeletionTombstone, + ) -> Result { + unreachable!() + } + + async fn force_delete_content_lock_and_verify_absent( + &self, + _: &CreatorPubky, + _: &ContentLockPath, + ) -> Result<(), ApplicationError> { + self.operations.lock().unwrap().push("public".to_owned()); + if self.fail.load(Ordering::SeqCst) { + Err(ApplicationError::Storage { + message: "public dependency".to_owned(), + }) + } else { + Ok(()) + } + } +} + +struct FakeResources { + operations: Arc>>, + fail_paths: Mutex>, +} +impl FakeResources { + fn new(operations: Arc>>) -> Self { + Self { + operations, + fail_paths: Mutex::new(HashSet::new()), + } + } +} +#[async_trait] +impl GuardedResourceRepository for FakeResources { + async fn upsert_guarded_resource( + &self, + _: GuardedResourceRecord, + ) -> Result<(), ApplicationError> { + unreachable!() + } + async fn get_guarded_resource( + &self, + _: &CreatorPubky, + _: &str, + _: &GuardedResourceHash, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn get_current_guarded_resource( + &self, + _: &CreatorPubky, + _: &str, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn delete_guarded_resource( + &self, + _: &CreatorPubky, + path: &str, + ) -> Result { + self.operations + .lock() + .unwrap() + .push(format!("resource:{path}")); + if self.fail_paths.lock().unwrap().contains(path) { + Err(ApplicationError::Storage { + message: "resource dependency".to_owned(), + }) + } else { + Ok(false) + } + } +} + +struct FakeDeletions { + operations: Arc>>, + claim_live: AtomicBool, + complete_calls: AtomicUsize, + fail_complete_once: AtomicBool, +} +use std::sync::atomic::AtomicUsize; +impl FakeDeletions { + fn new(operations: Arc>>) -> Self { + Self { + operations, + claim_live: AtomicBool::new(true), + complete_calls: AtomicUsize::new(0), + fail_complete_once: AtomicBool::new(false), + } + } +} +#[async_trait] +impl ContentLockDeletionRepository for FakeDeletions { + async fn begin_publication( + &self, + _: &CreatorPubky, + _: &LockId, + _: Uuid, + ) -> Result<(), ApplicationError> { + unreachable!() + } + async fn finish_publication( + &self, + _: &CreatorPubky, + _: &LockId, + _: Uuid, + ) -> Result { + unreachable!() + } + async fn abandon_publication( + &self, + _: &CreatorPubky, + _: &LockId, + _: Uuid, + ) -> Result { + unreachable!() + } + async fn publication_in_progress( + &self, + _: &CreatorPubky, + _: &LockId, + ) -> Result { + unreachable!() + } + async fn insert_job(&self, _: ContentLockDeletionJob) -> Result<(), ApplicationError> { + unreachable!() + } + async fn get_job( + &self, + _: &CreatorPubky, + _: &LockId, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn claim_next( + &self, + _: &str, + _: time::Duration, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn schedule_retry( + &self, + _: Uuid, + _: &str, + _: Uuid, + _: time::Duration, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn defer( + &self, + _: Uuid, + _: &str, + _: Uuid, + _: time::Duration, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn advance_phase( + &self, + _: Uuid, + _: &str, + _: Uuid, + _: ContentLockDeletionPhase, + ) -> Result { + unreachable!() + } + async fn finish( + &self, + _: Uuid, + _: &str, + _: Uuid, + _: Option, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn resume_failed_job( + &self, + _: &CreatorPubky, + _: &LockId, + _: OffsetDateTime, + ) -> Result, ApplicationError> { + unreachable!() + } + async fn prepare_force_deletion( + &self, + _: &CreatorPubky, + _: &LockId, + ) -> Result { + unreachable!() + } + async fn complete_force_deletion( + &self, + _: Uuid, + _: &str, + _: Uuid, + ) -> Result { + self.complete_calls.fetch_add(1, Ordering::SeqCst); + self.operations.lock().unwrap().push("complete".to_owned()); + if self.fail_complete_once.swap(false, Ordering::SeqCst) { + return Err(ApplicationError::Storage { + message: "force receipt persistence unavailable after deletion".to_owned(), + }); + } + Ok(self.claim_live.load(Ordering::SeqCst)) + } + async fn has_force_receipt( + &self, + _: &CreatorPubky, + _: &LockId, + ) -> Result { + unreachable!() + } +} + +fn force_claim() -> ClaimedContentLockDeletionJob { + let mut secondary_resources = BTreeMap::new(); + for path in [ + "/priv/locks.app/content/z", + "/priv/locks.app/content/a", + "/priv/locks.app/content/m", + ] { + secondary_resources.insert( + path.to_owned(), + SecondaryGuardedResource { + hash: GuardedResourceHash::from_bytes([7; 32]), + content_type: "text/plain".to_owned(), + size: 1, + }, + ); + } + let lock = ContentLock { + version: CONTENT_LOCK_VERSION, + creator: CreatorPubky::from_str( + "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy", + ) + .unwrap(), + primary_resource: Some( + GuardedResource::new( + "/priv/locks.app/content/m", + GuardedResourceHash::from_bytes([8; 32]), + "text/plain", + 1, + ) + .unwrap(), + ), + secondary_resources, + criteria: vec![], + lock_logic: LockLogic::All { criteria: vec![] }, + access_policy: AccessPolicy { + requested_credential_ttl_seconds: 900, + }, + lock_server: LockServerConfig { override_: None }, + created_at: NOW, + }; + let mut job = ContentLockDeletionJob::new(Uuid::from_u128(1), lock, NOW).unwrap(); + job.force_requested_at = Some(NOW); + ClaimedContentLockDeletionJob { + job, + claim_token: Uuid::from_u128(2), + } +} diff --git a/locks-service/src/application/use_cases/issue_access_credential.rs b/locks-service/src/application/use_cases/issue_access_credential.rs index 1d4e0b7..32d5feb 100644 --- a/locks-service/src/application/use_cases/issue_access_credential.rs +++ b/locks-service/src/application/use_cases/issue_access_credential.rs @@ -65,6 +65,33 @@ impl<'a> IssueAccessCredentialUseCase<'a> { &self, request: IssueAccessCredentialRequest, ) -> Result { + let now = self.clock.now(); + if self + .credential_store + .final_credential_available(&request.creator, &request.bundle_id, now) + .await? + { + let candidate = self + .credential_generator + .generate_access_credential() + .await?; + let final_issue_now = self.clock.now(); + if let Some(final_credential) = self + .credential_store + .issue_or_replay_final_credential( + &request.creator, + &request.bundle_id, + final_issue_now, + candidate, + ) + .await? + { + return Ok(IssuedAccessCredential { + credential: final_credential.credential, + expires_at: final_credential.expires_at, + }); + } + } let valid_entitlement = load_valid_entitlement( self.entitlements, self.content_locks, @@ -79,7 +106,7 @@ impl<'a> IssueAccessCredentialUseCase<'a> { .access_policy .requested_credential_ttl_seconds, )?; - let expires_at = self.clock.now() + Duration::seconds(requested_ttl_seconds as i64); + let expires_at = now + Duration::seconds(requested_ttl_seconds as i64); let credential = self .credential_generator .generate_access_credential() @@ -88,6 +115,7 @@ impl<'a> IssueAccessCredentialUseCase<'a> { self.credential_store .insert_access_credential( + &valid_entitlement.lock_id, lookup_key, AccessCredentialRecord { creator: request.creator, diff --git a/locks-service/src/application/use_cases/materialize_final_credentials.rs b/locks-service/src/application/use_cases/materialize_final_credentials.rs new file mode 100644 index 0000000..e5ba8c2 --- /dev/null +++ b/locks-service/src/application/use_cases/materialize_final_credentials.rs @@ -0,0 +1,326 @@ +use time::OffsetDateTime; +use uuid::Uuid; + +use crate::application::{ + errors::ApplicationError, + ports::{ + AccessCredentialGenerator, AccessCredentialStore, Clock, FinalCredentialWorkerIssueRequest, + }, +}; + +/// Exact worker claim and bounded batch used to materialize final deletion credentials. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MaterializeFinalCredentialsRequest<'a> { + pub deletion_job_id: Uuid, + pub worker_id: &'a str, + pub claim_token: Uuid, + pub now: OffsetDateTime, + pub batch_limit: usize, +} + +/// Secret-free result of one bounded materialization pass. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MaterializeFinalCredentialsOutcome { + pub materialized_count: usize, +} + +/// Materializes final credentials selected under a live deletion-worker claim. +pub struct MaterializeFinalCredentialsUseCase<'a> { + store: &'a dyn AccessCredentialStore, + generator: &'a dyn AccessCredentialGenerator, + clock: &'a dyn Clock, +} + +impl<'a> MaterializeFinalCredentialsUseCase<'a> { + pub fn new( + store: &'a dyn AccessCredentialStore, + generator: &'a dyn AccessCredentialGenerator, + clock: &'a dyn Clock, + ) -> Self { + Self { + store, + generator, + clock, + } + } + + pub async fn execute( + &self, + request: MaterializeFinalCredentialsRequest<'_>, + ) -> Result { + let pending = self + .store + .final_credentials_to_materialize( + request.deletion_job_id, + request.worker_id, + request.claim_token, + request.batch_limit, + ) + .await?; + let mut materialized_count = 0; + for item in pending { + let candidate = self.generator.generate_access_credential().await?; + let fresh_now = self.clock.now(); + if self + .store + .issue_or_replay_final_credential_for_worker(FinalCredentialWorkerIssueRequest { + deletion_job_id: request.deletion_job_id, + worker_id: request.worker_id, + claim_token: request.claim_token, + creator: &item.creator, + bundle_id: &item.bundle_id, + now: fresh_now, + candidate, + }) + .await? + .is_some() + { + materialized_count += 1; + } + } + Ok(MaterializeFinalCredentialsOutcome { materialized_count }) + } +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, str::FromStr, sync::Mutex}; + + use async_trait::async_trait; + use locks_core::ids::{BundleId, CreatorPubky, LockId}; + use time::macros::datetime; + use uuid::Uuid; + + use super::{MaterializeFinalCredentialsRequest, MaterializeFinalCredentialsUseCase}; + use crate::application::{ + errors::ApplicationError, + models::{ + AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, + FinalCredentialMaterialization, IssuedDeletionCredential, + }, + ports::{ + AccessCredentialGenerator, AccessCredentialStore, Clock, + FinalCredentialWorkerIssueRequest, + }, + }; + + const NOW: time::OffsetDateTime = datetime!(2026-08-17 12:00:00 UTC); + + #[tokio::test] + async fn materializes_every_eligible_snapshot_in_deterministic_store_order() { + let creator = creator(); + let first_bundle = BundleId::from_str("000G40R40M30E209185GR38E1V").unwrap(); + let second_bundle = BundleId::from_str("000G40R40M30E209185GR38E1W").unwrap(); + let store = FakeStore::with_pending(vec![ + FinalCredentialMaterialization { + creator: creator.clone(), + bundle_id: first_bundle.clone(), + }, + FinalCredentialMaterialization { + creator: creator.clone(), + bundle_id: second_bundle.clone(), + }, + ]); + let generator = SequenceGenerator::new(["first-secret", "second-secret"]); + let request = request(10); + + let clock = FixedClock(NOW + time::Duration::seconds(1)); + let outcome = MaterializeFinalCredentialsUseCase::new(&store, &generator, &clock) + .execute(request) + .await + .unwrap(); + + assert_eq!(outcome.materialized_count, 2); + assert_eq!( + store.calls(), + vec![ + (first_bundle, "first-secret".to_owned()), + (second_bundle, "second-secret".to_owned()), + ] + ); + assert_eq!(generator.generated_count(), 2); + assert!(!format!("{outcome:?}").contains("secret")); + } + + #[tokio::test] + async fn retry_after_materialization_does_not_generate_or_persist_a_second_bearer() { + let creator = creator(); + let bundle_id = BundleId::from_str("000G40R40M30E209185GR38E1W").unwrap(); + let store = FakeStore::with_pending(vec![FinalCredentialMaterialization { + creator, + bundle_id: bundle_id.clone(), + }]); + let generator = SequenceGenerator::new(["persisted-secret", "must-not-be-generated"]); + + let clock = FixedClock(NOW + time::Duration::seconds(1)); + let first = MaterializeFinalCredentialsUseCase::new(&store, &generator, &clock) + .execute(request(1)) + .await + .unwrap(); + let retry = MaterializeFinalCredentialsUseCase::new(&store, &generator, &clock) + .execute(request(1)) + .await + .unwrap(); + + assert_eq!(first.materialized_count, 1); + assert_eq!(retry.materialized_count, 0); + assert_eq!(generator.generated_count(), 1); + assert_eq!( + store.calls(), + vec![(bundle_id, "persisted-secret".to_owned())] + ); + } + + fn request(batch_limit: usize) -> MaterializeFinalCredentialsRequest<'static> { + MaterializeFinalCredentialsRequest { + deletion_job_id: Uuid::from_u128(7), + worker_id: "worker-final", + claim_token: Uuid::from_u128(8), + now: NOW, + batch_limit, + } + } + + fn creator() -> CreatorPubky { + CreatorPubky::from_str("pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy").unwrap() + } + + #[derive(Debug, Clone, Copy)] + struct FixedClock(time::OffsetDateTime); + + impl Clock for FixedClock { + fn now(&self) -> time::OffsetDateTime { + self.0 + } + } + + #[derive(Debug)] + struct SequenceGenerator { + values: Mutex>, + generated: Mutex, + } + + impl SequenceGenerator { + fn new(values: [&str; N]) -> Self { + Self { + values: Mutex::new( + values + .into_iter() + .rev() + .map(AccessCredential::new) + .collect(), + ), + generated: Mutex::new(0), + } + } + + fn generated_count(&self) -> usize { + *self.generated.lock().unwrap() + } + } + + #[async_trait] + impl AccessCredentialGenerator for SequenceGenerator { + async fn generate_access_credential(&self) -> Result { + *self.generated.lock().unwrap() += 1; + self.values.lock().unwrap().pop().ok_or_else(|| { + ApplicationError::CredentialGeneration { + message: "test generator exhausted".to_owned(), + } + }) + } + } + + #[derive(Debug)] + struct FakeStore { + pending: Mutex>, + winners: Mutex>, + calls: Mutex>, + } + + impl FakeStore { + fn with_pending(pending: Vec) -> Self { + Self { + pending: Mutex::new(pending), + winners: Mutex::new(HashMap::new()), + calls: Mutex::new(Vec::new()), + } + } + + fn calls(&self) -> Vec<(BundleId, String)> { + self.calls.lock().unwrap().clone() + } + } + + #[async_trait] + impl AccessCredentialStore for FakeStore { + async fn insert_access_credential( + &self, + _lock_id: &LockId, + _lookup_key: AccessCredentialLookupKey, + _record: AccessCredentialRecord, + ) -> Result<(), ApplicationError> { + unreachable!() + } + + async fn get_access_credential( + &self, + _lookup_key: &AccessCredentialLookupKey, + ) -> Result, ApplicationError> { + unreachable!() + } + + async fn delete_access_credential( + &self, + _lookup_key: &AccessCredentialLookupKey, + ) -> Result<(), ApplicationError> { + unreachable!() + } + + async fn final_credentials_to_materialize( + &self, + _deletion_job_id: Uuid, + _worker_id: &str, + _claim_token: Uuid, + limit: usize, + ) -> Result, ApplicationError> { + let mut pending = self.pending.lock().unwrap(); + let take = pending.len().min(limit); + Ok(pending.drain(..take).collect()) + } + + async fn issue_or_replay_final_credential_for_worker( + &self, + request: FinalCredentialWorkerIssueRequest<'_>, + ) -> Result, ApplicationError> { + let FinalCredentialWorkerIssueRequest { + deletion_job_id, + worker_id, + claim_token, + bundle_id, + now, + candidate, + .. + } = request; + assert_eq!(deletion_job_id, Uuid::from_u128(7)); + assert_eq!(worker_id, "worker-final"); + assert_eq!(claim_token, Uuid::from_u128(8)); + assert_eq!(now, NOW + time::Duration::seconds(1)); + self.calls + .lock() + .unwrap() + .push((bundle_id.clone(), candidate.as_str().to_owned())); + let winner = self + .winners + .lock() + .unwrap() + .entry(bundle_id.clone()) + .or_insert(candidate) + .clone(); + Ok(Some(IssuedDeletionCredential { + credential: winner, + expires_at: NOW + time::Duration::minutes(30), + })) + } + } +} diff --git a/locks-service/src/application/use_cases/mod.rs b/locks-service/src/application/use_cases/mod.rs index 1218395..30ab185 100644 --- a/locks-service/src/application/use_cases/mod.rs +++ b/locks-service/src/application/use_cases/mod.rs @@ -4,11 +4,16 @@ pub mod create_content_lock; #[cfg(test)] mod credential_flow_tests; pub mod delete_guarded_resource; +pub mod drain_lock_payments; mod entitlement_check; pub mod exchange_frontend_session_code; +pub mod execute_content_lock_deletion_phase; +pub mod execute_forced_content_lock_deletion; pub mod get_creator_authority_status; pub mod get_verification_task; pub mod issue_access_credential; +pub mod materialize_final_credentials; +pub mod no_paykit_deletion_drain; pub mod proxy_read_guarded_resource; pub mod register_guarded_resource; pub mod require_creator_authority_for_pubky_io; diff --git a/locks-service/src/application/use_cases/no_paykit_deletion_drain.rs b/locks-service/src/application/use_cases/no_paykit_deletion_drain.rs new file mode 100644 index 0000000..3bfa020 --- /dev/null +++ b/locks-service/src/application/use_cases/no_paykit_deletion_drain.rs @@ -0,0 +1,171 @@ +use async_trait::async_trait; + +use crate::application::{ + errors::ApplicationError, + models::{ + AdvanceContentLockDeletionPhaseResult, ClaimedContentLockDeletionJob, + ContentLockDeletionPhase, + }, + ports::{Clock, ContentLockDeletionRepository}, +}; + +use super::execute_content_lock_deletion_phase::{ + ContentLockPaymentDrainExecutor, DeletionDependencyEvidence, DeletionDependencySource, + DeletionExecutionErrorClass, DeletionPhaseExecution, DeletionPhaseExecutionOutcome, + classify_deletion_execution_error, +}; + +#[async_trait] +pub trait NoPaykitDeletionDrainExecutor: Send + Sync { + async fn execute_claimed( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> Result; +} + +pub struct NoPaykitDeletionDrainUseCase<'a> { + deletions: &'a dyn ContentLockDeletionRepository, + clock: &'a dyn Clock, +} + +impl<'a> NoPaykitDeletionDrainUseCase<'a> { + pub fn new(deletions: &'a dyn ContentLockDeletionRepository, clock: &'a dyn Clock) -> Self { + Self { deletions, clock } + } + + pub async fn execute_claimed( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> Result { + let next_phase = match claim.job.phase { + ContentLockDeletionPhase::StartPaymentDrain => ContentLockDeletionPhase::DrainPayments, + ContentLockDeletionPhase::DrainPayments => { + ContentLockDeletionPhase::DrainExistingCredentials + } + _ => { + return Err(ApplicationError::InvalidContentLockDeletionState { + message: "non-Paykit drain requires a payment drain phase".to_owned(), + }); + } + }; + let _now = self.clock.now(); + if !self + .deletions + .expire_unresolved_non_paykit_tasks(claim.job.job_id, worker_id, claim.claim_token) + .await? + { + return Ok(false); + } + Ok(matches!( + self.deletions + .advance_phase(claim.job.job_id, worker_id, claim.claim_token, next_phase) + .await?, + AdvanceContentLockDeletionPhaseResult::Advanced(_) + )) + } +} + +#[async_trait] +impl NoPaykitDeletionDrainExecutor for NoPaykitDeletionDrainUseCase<'_> { + async fn execute_claimed( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> Result { + NoPaykitDeletionDrainUseCase::execute_claimed(self, claim, worker_id).await + } +} + +#[async_trait] +impl ContentLockPaymentDrainExecutor for NoPaykitDeletionDrainUseCase<'_> { + async fn execute_claimed( + &self, + claim: ClaimedContentLockDeletionJob, + worker_id: &str, + ) -> DeletionPhaseExecution { + let next_phase = match claim.job.phase { + ContentLockDeletionPhase::StartPaymentDrain => ContentLockDeletionPhase::DrainPayments, + ContentLockDeletionPhase::DrainPayments => { + ContentLockDeletionPhase::DrainExistingCredentials + } + _ => return DeletionPhaseExecution::new(DeletionPhaseExecutionOutcome::FatalFailure), + }; + let _now = self.clock.now(); + let mut evidence = DeletionDependencyEvidence::none(); + match self + .deletions + .expire_unresolved_non_paykit_tasks(claim.job.job_id, worker_id, claim.claim_token) + .await + { + Ok(false) => { + return DeletionPhaseExecution::new(DeletionPhaseExecutionOutcome::ClaimLost); + } + Ok(true) => { + evidence = evidence.merge(DeletionDependencyEvidence::healthy( + DeletionDependencySource::PaymentDrainRepository, + )); + } + Err(error) => { + return observed_error( + &error, + DeletionDependencySource::PaymentDrainRepository, + evidence, + ); + } + } + + match self + .deletions + .advance_phase(claim.job.job_id, worker_id, claim.claim_token, next_phase) + .await + { + Ok(AdvanceContentLockDeletionPhaseResult::Advanced(_)) => { + observed_phase(DeletionPhaseExecutionOutcome::Progressed, evidence) + } + Ok(AdvanceContentLockDeletionPhaseResult::ClaimLost) => { + DeletionPhaseExecution::new(DeletionPhaseExecutionOutcome::ClaimLost) + .with_evidence(evidence) + } + Ok(AdvanceContentLockDeletionPhaseResult::ObligationsPending) => { + observed_phase(DeletionPhaseExecutionOutcome::Deferred, evidence) + } + Ok(AdvanceContentLockDeletionPhaseResult::TerminalFailure(_)) => { + observed_phase(DeletionPhaseExecutionOutcome::TerminalFailed, evidence) + } + Err(error) => observed_error( + &error, + DeletionDependencySource::RepositoryPhaseMutation, + evidence, + ), + } + } +} + +fn observed_phase( + outcome: DeletionPhaseExecutionOutcome, + evidence: DeletionDependencyEvidence, +) -> DeletionPhaseExecution { + DeletionPhaseExecution::new(outcome) + .with_evidence(evidence) + .with_evidence(DeletionDependencyEvidence::healthy( + DeletionDependencySource::RepositoryPhaseMutation, + )) +} + +fn observed_error( + error: &ApplicationError, + source: DeletionDependencySource, + evidence: DeletionDependencyEvidence, +) -> DeletionPhaseExecution { + let outcome = match classify_deletion_execution_error(error) { + DeletionExecutionErrorClass::TransientDependency => { + DeletionPhaseExecutionOutcome::TransientDependencyFailure + } + DeletionExecutionErrorClass::Fatal => DeletionPhaseExecutionOutcome::FatalFailure, + }; + DeletionPhaseExecution::new(outcome) + .with_evidence(evidence) + .with_evidence(DeletionDependencyEvidence::unavailable(source)) +} diff --git a/locks-service/src/application/use_cases/proxy_read_guarded_resource.rs b/locks-service/src/application/use_cases/proxy_read_guarded_resource.rs index f9b7388..e6546a0 100644 --- a/locks-service/src/application/use_cases/proxy_read_guarded_resource.rs +++ b/locks-service/src/application/use_cases/proxy_read_guarded_resource.rs @@ -1,5 +1,5 @@ use crate::application::errors::ApplicationError; -use crate::application::models::AccessCredential; +use crate::application::models::{AccessCredential, AccessCredentialLookupKey}; use crate::application::ports::{ AccessCredentialStore, Clock, ContentLockRepository, EntitlementRepository, GuardedResourceRepository, @@ -30,6 +30,14 @@ pub struct ProxiedGuardedResource { pub hash: GuardedResourceHash, /// Guarded resource bytes. pub bytes: Vec, + deletion_claim: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct DeletionReadClaim { + lookup_key: AccessCredentialLookupKey, + path: String, + claim_token: uuid::Uuid, } /// Validates an access credential and returns the currently guarded resource bytes. @@ -64,6 +72,63 @@ impl<'a> ProxyReadGuardedResourceUseCase<'a> { &self, request: ProxyReadGuardedResourceRequest, ) -> Result { + let lookup_key = AccessCredentialLookupKey::derive(&request.credential); + if let Some(authorization) = self + .credential_store + .prepare_deletion_read(&lookup_key, &request.path, time::Duration::seconds(30)) + .await? + { + let claim_token = authorization.claim_token; + let guarded_resource = authorization.resource; + let prepared_response = async { + let guarded_record = self + .guarded_resources + .get_current_guarded_resource(&authorization.creator, &guarded_resource.path) + .await? + .ok_or(ApplicationError::GuardedResourceUnavailable)?; + if guarded_record.hash != guarded_resource.hash + || guarded_record.content_type != guarded_resource.content_type + || guarded_record.size != guarded_resource.size + { + return Err(ApplicationError::GuardedResourceUnavailable); + } + Ok(ProxiedGuardedResource { + path: guarded_resource.path, + content_type: guarded_record.content_type, + hash: guarded_resource.hash, + bytes: guarded_record.bytes, + deletion_claim: claim_token.map(|claim_token| DeletionReadClaim { + lookup_key: lookup_key.clone(), + path: request.path.clone(), + claim_token, + }), + }) + } + .await; + match prepared_response { + Ok(response) => return Ok(response), + Err(error) => { + if let Some(claim_token) = claim_token { + self.credential_store + .release_deletion_read( + &lookup_key, + &request.path, + claim_token, + self.clock.now(), + ) + .await?; + } + return Err(error); + } + } + } + if self + .credential_store + .deletion_credential_enrolled(&lookup_key) + .await? + { + return Err(ApplicationError::GuardedResourceUnavailable); + } let validation = ValidateAccessCredentialUseCase::new( self.credential_store, self.entitlements, @@ -105,18 +170,60 @@ impl<'a> ProxyReadGuardedResourceUseCase<'a> { content_type: guarded_record.content_type, hash: guarded_resource.hash, bytes: guarded_record.bytes, + deletion_claim: None, }) } + + /// Permanently consumes the exact final-read claim after a complete HTTP 200 + /// response has been constructed. A lost claim prevents response return. + pub async fn consume_prepared_deletion_read( + &self, + response: &ProxiedGuardedResource, + ) -> Result<(), ApplicationError> { + let Some(claim) = &response.deletion_claim else { + return Ok(()); + }; + if !self + .credential_store + .consume_deletion_read(&claim.lookup_key, &claim.path, claim.claim_token) + .await? + { + return Err(ApplicationError::InvalidAccessCredential); + } + Ok(()) + } + + /// Releases the exact final-read claim when HTTP response construction fails. + pub async fn release_prepared_deletion_read( + &self, + response: &ProxiedGuardedResource, + ) -> Result<(), ApplicationError> { + let Some(claim) = &response.deletion_claim else { + return Ok(()); + }; + self.credential_store + .release_deletion_read( + &claim.lookup_key, + &claim.path, + claim.claim_token, + self.clock.now(), + ) + .await?; + Ok(()) + } } #[cfg(test)] mod tests { use std::collections::BTreeMap; use std::str::FromStr; + use std::sync::atomic::{AtomicUsize, Ordering}; + use async_trait::async_trait; use serde_json::json; use time::OffsetDateTime; use time::macros::datetime; + use uuid::Uuid; use locks_core::ids::{ BundleId, CreatorPubky, GuardedResourceHash, LockServerPubky, PubkyLockResource, @@ -133,7 +240,8 @@ mod tests { use super::{ProxyReadGuardedResourceRequest, ProxyReadGuardedResourceUseCase}; use crate::application::errors::ApplicationError; use crate::application::models::{ - AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, GuardedResourceRecord, + AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, + DeletionReadAuthorization, GuardedResourceRecord, }; use crate::application::ports::{ AccessCredentialStore, Clock, ContentLockRepository, EntitlementRepository, @@ -257,6 +365,205 @@ mod tests { assert_eq!(result, Err(ApplicationError::GuardedResourceUnavailable)); } + #[tokio::test] + async fn deletion_read_stays_claimed_until_response_boundary_consumes_it() { + let fixture = Fixture::seed().await; + let credentials = DeletionReadStore::new(); + let empty_public_locks = InMemoryContentLockRepository::new(); + let use_case = ProxyReadGuardedResourceUseCase::new( + &credentials, + &fixture.entitlements, + &empty_public_locks, + &fixture.guarded_resources, + &fixture.clock, + ); + + let response = use_case + .execute(ProxyReadGuardedResourceRequest { + credential: fixture.credential, + path: "/priv/locks.app/content/resource.txt".to_owned(), + }) + .await + .unwrap(); + + assert_eq!(response.bytes, b"guarded bytes".to_vec()); + assert_eq!(credentials.consumes.load(Ordering::SeqCst), 0); + use_case + .consume_prepared_deletion_read(&response) + .await + .unwrap(); + assert_eq!(credentials.consumes.load(Ordering::SeqCst), 1); + assert_eq!(credentials.releases.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn deletion_read_can_be_released_when_response_construction_fails() { + let fixture = Fixture::seed().await; + let credentials = DeletionReadStore::new(); + let use_case = ProxyReadGuardedResourceUseCase::new( + &credentials, + &fixture.entitlements, + &fixture.content_locks, + &fixture.guarded_resources, + &fixture.clock, + ); + let response = use_case + .execute(ProxyReadGuardedResourceRequest { + credential: fixture.credential, + path: "/priv/locks.app/content/resource.txt".to_owned(), + }) + .await + .unwrap(); + + use_case + .release_prepared_deletion_read(&response) + .await + .unwrap(); + + assert_eq!(credentials.releases.load(Ordering::SeqCst), 1); + assert_eq!(credentials.consumes.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn deletion_read_releases_claim_when_upstream_resource_is_unavailable() { + let fixture = Fixture::seed_without_guarded_resource().await; + let credentials = DeletionReadStore::new(); + let use_case = ProxyReadGuardedResourceUseCase::new( + &credentials, + &fixture.entitlements, + &fixture.content_locks, + &fixture.guarded_resources, + &fixture.clock, + ); + + let result = use_case + .execute(ProxyReadGuardedResourceRequest { + credential: fixture.credential, + path: "/priv/locks.app/content/resource.txt".to_owned(), + }) + .await; + + assert_eq!(result, Err(ApplicationError::GuardedResourceUnavailable)); + assert_eq!(credentials.releases.load(Ordering::SeqCst), 1); + assert_eq!(credentials.consumes.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn deletion_credential_denied_path_does_not_fall_through_to_public_lock() { + let fixture = Fixture::seed().await; + let credentials = DeletionReadStore::new(); + let empty_public_locks = InMemoryContentLockRepository::new(); + let use_case = ProxyReadGuardedResourceUseCase::new( + &credentials, + &fixture.entitlements, + &empty_public_locks, + &fixture.guarded_resources, + &fixture.clock, + ); + + let result = use_case + .execute(ProxyReadGuardedResourceRequest { + credential: fixture.credential, + path: "/priv/locks.app/content/not-frozen.txt".to_owned(), + }) + .await; + + assert_eq!(result, Err(ApplicationError::GuardedResourceUnavailable)); + assert_eq!(credentials.releases.load(Ordering::SeqCst), 0); + assert_eq!(credentials.consumes.load(Ordering::SeqCst), 0); + } + + struct DeletionReadStore { + ordinary: InMemoryAccessCredentialStore, + claim_token: Uuid, + releases: AtomicUsize, + consumes: AtomicUsize, + } + + impl DeletionReadStore { + fn new() -> Self { + Self { + ordinary: InMemoryAccessCredentialStore::new(), + claim_token: Uuid::new_v4(), + releases: AtomicUsize::new(0), + consumes: AtomicUsize::new(0), + } + } + } + + #[async_trait] + impl AccessCredentialStore for DeletionReadStore { + async fn insert_access_credential( + &self, + lock_id: &locks_core::ids::LockId, + lookup_key: AccessCredentialLookupKey, + record: AccessCredentialRecord, + ) -> Result<(), ApplicationError> { + self.ordinary + .insert_access_credential(lock_id, lookup_key, record) + .await + } + + async fn get_access_credential( + &self, + lookup_key: &AccessCredentialLookupKey, + ) -> Result, ApplicationError> { + self.ordinary.get_access_credential(lookup_key).await + } + + async fn delete_access_credential( + &self, + lookup_key: &AccessCredentialLookupKey, + ) -> Result<(), ApplicationError> { + self.ordinary.delete_access_credential(lookup_key).await + } + + async fn prepare_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + path: &str, + _claim_duration: time::Duration, + ) -> Result, ApplicationError> { + Ok((path == "/priv/locks.app/content/resource.txt").then(|| { + DeletionReadAuthorization { + claim_token: Some(self.claim_token), + creator: creator(), + resource: content_lock_fixture().primary_resource.unwrap(), + } + })) + } + + async fn deletion_credential_enrolled( + &self, + _lookup_key: &AccessCredentialLookupKey, + ) -> Result { + Ok(true) + } + + async fn release_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + claim_token: Uuid, + _now: OffsetDateTime, + ) -> Result { + assert_eq!(claim_token, self.claim_token); + self.releases.fetch_add(1, Ordering::SeqCst); + Ok(true) + } + + async fn consume_deletion_read( + &self, + _lookup_key: &AccessCredentialLookupKey, + _path: &str, + claim_token: Uuid, + ) -> Result { + assert_eq!(claim_token, self.claim_token); + self.consumes.fetch_add(1, Ordering::SeqCst); + Ok(true) + } + } + struct Fixture { credentials: InMemoryAccessCredentialStore, entitlements: InMemoryEntitlementRepository, @@ -313,6 +620,7 @@ mod tests { .unwrap(); credentials .insert_access_credential( + &content_lock.lock_id().unwrap(), AccessCredentialLookupKey::derive(&credential), AccessCredentialRecord { creator: creator(), diff --git a/locks-service/src/application/use_cases/submit_proof_bundle.rs b/locks-service/src/application/use_cases/submit_proof_bundle.rs index 48632d0..47d62ad 100644 --- a/locks-service/src/application/use_cases/submit_proof_bundle.rs +++ b/locks-service/src/application/use_cases/submit_proof_bundle.rs @@ -70,20 +70,7 @@ impl<'a> SubmitProofBundleUseCase<'a> { if let Some(existing) = self.find_existing(&submitted_proof_bundle).await? { return Ok(existing); } - let creator = submitted_proof_bundle.pubky_lock_resource.creator().clone(); - - let task_id = self.task_ids.generate_task_id().await?; - let submitted_at = self.clock.now(); - let task = VerificationTaskRecord { - task_id, - creator, - submitted_proof_bundle, - status: VerificationTaskStatus::Pending, - submitted_at, - started_at: None, - completed_at: None, - failure_message: None, - }; + let task = self.prepare_task(submitted_proof_bundle).await?; match self.tasks.insert_verification_task(task.clone()).await { Ok(()) => Ok(VerificationTaskLifecycleView::from(task)), @@ -109,6 +96,23 @@ impl<'a> SubmitProofBundleUseCase<'a> { Err(error) => Err(error), } } + + /// Builds a new pending task without persisting it. + pub async fn prepare_task( + &self, + submitted_proof_bundle: SubmittedProofBundle, + ) -> Result { + Ok(VerificationTaskRecord { + task_id: self.task_ids.generate_task_id().await?, + creator: submitted_proof_bundle.pubky_lock_resource.creator().clone(), + submitted_proof_bundle, + status: VerificationTaskStatus::Pending, + submitted_at: self.clock.now(), + started_at: None, + completed_at: None, + failure_message: None, + }) + } } #[cfg(test)] diff --git a/locks-service/src/application/use_cases/validate_paykit_payment_submission.rs b/locks-service/src/application/use_cases/validate_paykit_payment_submission.rs index 01794a3..7498097 100644 --- a/locks-service/src/application/use_cases/validate_paykit_payment_submission.rs +++ b/locks-service/src/application/use_cases/validate_paykit_payment_submission.rs @@ -12,6 +12,12 @@ pub struct ValidatePaykitPaymentSubmissionRequest { pub submitted_proof_bundle: SubmittedProofBundle, } +/// Immutable canonical payment facts required before Paykit invoice side effects. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ValidatedPaykitPaymentSubmission { + pub payment_in: u64, +} + /// Validates payment proof identity before invoice side effects are allowed. pub struct ValidatePaykitPaymentSubmissionUseCase<'a> { content_locks: &'a dyn ContentLockRepository, @@ -27,7 +33,7 @@ impl<'a> ValidatePaykitPaymentSubmissionUseCase<'a> { pub async fn execute( &self, request: ValidatePaykitPaymentSubmissionRequest, - ) -> Result<(), ApplicationError> { + ) -> Result { let submitted = request.submitted_proof_bundle; let content_lock = self .content_locks @@ -53,15 +59,22 @@ impl<'a> ValidatePaykitPaymentSubmissionUseCase<'a> { if proof.verifier_type != VerifierType::PaykitPayment { return Err(ApplicationError::InvalidPaykitPaymentSubmission); } - let criterion_matches = content_lock.criteria.iter().any(|criterion| { - criterion.criterion_id == proof.criterion_id - && criterion.verifier_type == VerifierType::PaykitPayment - }); - if !criterion_matches { - return Err(ApplicationError::InvalidPaykitPaymentSubmission); - } - - Ok(()) + let criterion = content_lock + .criteria + .iter() + .find(|criterion| { + criterion.criterion_id == proof.criterion_id + && criterion.verifier_type == VerifierType::PaykitPayment + }) + .ok_or(ApplicationError::InvalidPaykitPaymentSubmission)?; + let params = criterion + .paykit_payment_params() + .map_err(|_| ApplicationError::InvalidPaykitPaymentSubmission)? + .ok_or(ApplicationError::InvalidPaykitPaymentSubmission)?; + + Ok(ValidatedPaykitPaymentSubmission { + payment_in: params.payment_in(), + }) } } @@ -84,7 +97,10 @@ mod tests { }; use locks_core::verification::{Proof, SUBMITTED_PROOF_BUNDLE_VERSION, SubmittedProofBundle}; - use super::{ValidatePaykitPaymentSubmissionRequest, ValidatePaykitPaymentSubmissionUseCase}; + use super::{ + ValidatePaykitPaymentSubmissionRequest, ValidatePaykitPaymentSubmissionUseCase, + ValidatedPaykitPaymentSubmission, + }; use crate::application::errors::ApplicationError; use crate::application::ports::ContentLockRepository; @@ -177,7 +193,10 @@ mod tests { }) .await; - assert_eq!(result, Ok(())); + assert_eq!( + result, + Ok(ValidatedPaykitPaymentSubmission { payment_in: 24 }) + ); } #[tokio::test] @@ -245,6 +264,14 @@ mod tests { ) -> Result, ApplicationError> { Ok(self.0.clone()) } + + async fn delete_content_lock( + &self, + _creator: &CreatorPubky, + _content_lock_path: &ContentLockPath, + ) -> Result { + unreachable!("validation must not delete content locks") + } } fn content_lock() -> ContentLock { @@ -264,7 +291,8 @@ mod tests { params: json!({ "recipient_pubky": CREATOR, "amount": "50000", - "asset": "BTC" + "asset": "BTC", + "payment_in": 24 }), }], lock_logic: LockLogic::All { diff --git a/locks-service/src/infrastructure/final_credentials.rs b/locks-service/src/infrastructure/final_credentials.rs new file mode 100644 index 0000000..2692db6 --- /dev/null +++ b/locks-service/src/infrastructure/final_credentials.rs @@ -0,0 +1,234 @@ +use std::fmt; + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use chacha20poly1305::aead::{Aead, KeyInit, Payload}; +use chacha20poly1305::{Key, XChaCha20Poly1305, XNonce}; +use rand::RngCore; +use rand::rngs::OsRng; + +use crate::application::errors::ApplicationError; +use crate::application::models::{ + AccessCredential, EncryptedFinalCredential, FinalCredentialContext, +}; + +const ENVELOPE_PREFIX: &str = "v1.xchacha20poly1305:"; +const AAD_DOMAIN: &[u8] = b"pubky-locks-final-credential-aad"; + +#[derive(Clone)] +pub struct FinalCredentialCipher { + key: [u8; 32], +} + +impl fmt::Debug for FinalCredentialCipher { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("FinalCredentialCipher") + .field(&"") + .finish() + } +} + +impl FinalCredentialCipher { + pub fn new(key: [u8; 32]) -> Self { + Self { key } + } + + pub fn encrypt( + &self, + context: &FinalCredentialContext, + credential: &AccessCredential, + ) -> Result { + let cipher = XChaCha20Poly1305::new(Key::from_slice(&self.key)); + let mut nonce_bytes = [0u8; 24]; + OsRng.fill_bytes(&mut nonce_bytes); + let ciphertext = cipher + .encrypt( + XNonce::from_slice(&nonce_bytes), + Payload { + msg: credential.as_str().as_bytes(), + aad: &associated_data(context), + }, + ) + .map_err(|_| encrypt_error())?; + Ok(EncryptedFinalCredential::new(format!( + "{ENVELOPE_PREFIX}{}:{}", + URL_SAFE_NO_PAD.encode(nonce_bytes), + URL_SAFE_NO_PAD.encode(ciphertext) + ))) + } + + pub fn decrypt( + &self, + context: &FinalCredentialContext, + envelope: &EncryptedFinalCredential, + ) -> Result { + let rest = envelope + .as_str() + .strip_prefix(ENVELOPE_PREFIX) + .ok_or_else(decrypt_error)?; + let (nonce, ciphertext) = rest.split_once(':').ok_or_else(decrypt_error)?; + let nonce: [u8; 24] = URL_SAFE_NO_PAD + .decode(nonce) + .map_err(|_| decrypt_error())? + .try_into() + .map_err(|_| decrypt_error())?; + let ciphertext = URL_SAFE_NO_PAD + .decode(ciphertext) + .map_err(|_| decrypt_error())?; + let cipher = XChaCha20Poly1305::new(Key::from_slice(&self.key)); + let plaintext = cipher + .decrypt( + XNonce::from_slice(&nonce), + Payload { + msg: &ciphertext, + aad: &associated_data(context), + }, + ) + .map_err(|_| decrypt_error())?; + String::from_utf8(plaintext) + .map(AccessCredential::new) + .map_err(|_| decrypt_error()) + } +} + +fn associated_data(context: &FinalCredentialContext) -> Vec { + let creator = context.creator.to_string(); + let bundle_id = context.bundle_id.to_string(); + let mut aad = Vec::with_capacity(AAD_DOMAIN.len() + creator.len() + bundle_id.len() + 32); + append_field(&mut aad, AAD_DOMAIN); + append_field(&mut aad, &[1]); + append_field(&mut aad, context.deletion_job_id.as_bytes()); + append_field(&mut aad, creator.as_bytes()); + append_field(&mut aad, bundle_id.as_bytes()); + aad +} + +fn append_field(output: &mut Vec, field: &[u8]) { + let length = u32::try_from(field.len()).expect("credential AAD fields fit in u32"); + output.extend_from_slice(&length.to_be_bytes()); + output.extend_from_slice(field); +} + +fn encrypt_error() -> ApplicationError { + ApplicationError::FinalCredentialSecret { + message: "failed to encrypt final credential".to_owned(), + } +} + +fn decrypt_error() -> ApplicationError { + ApplicationError::FinalCredentialSecret { + message: "invalid final credential envelope".to_owned(), + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use locks_core::ids::{BundleId, CreatorPubky}; + use uuid::Uuid; + + use super::*; + + #[test] + fn encrypted_final_credential_round_trips_under_exact_context() { + let cipher = FinalCredentialCipher::new([7; 32]); + let context = context(); + let credential = AccessCredential::new("secret-final-bearer"); + + let envelope = cipher.encrypt(&context, &credential).unwrap(); + let decrypted = cipher.decrypt(&context, &envelope).unwrap(); + + assert_eq!(decrypted, credential); + assert!(!envelope.as_str().contains(credential.as_str())); + assert!(!format!("{envelope:?}").contains(credential.as_str())); + assert!(!format!("{cipher:?}").contains('7')); + } + + #[test] + fn wrong_key_or_any_context_change_fails_closed() { + let cipher = FinalCredentialCipher::new([7; 32]); + let context = context(); + let envelope = cipher + .encrypt(&context, &AccessCredential::new("secret-final-bearer")) + .unwrap(); + + assert!( + FinalCredentialCipher::new([8; 32]) + .decrypt(&context, &envelope) + .is_err() + ); + for (field, changed) in [ + ( + "job", + FinalCredentialContext { + deletion_job_id: Uuid::new_v4(), + ..context.clone() + }, + ), + ( + "creator", + FinalCredentialContext { + creator: CreatorPubky::from_str( + &pubky_common::crypto::Keypair::from_secret(&[2; 32]) + .public_key() + .to_string(), + ) + .unwrap(), + ..context.clone() + }, + ), + ( + "bundle", + FinalCredentialContext { + bundle_id: BundleId::from_str("000G40R40M30E209185GR38E1V").unwrap(), + ..context.clone() + }, + ), + ] { + assert_ne!(changed, context, "{field} mutation must change context"); + assert!( + cipher.decrypt(&changed, &envelope).is_err(), + "altered {field} authenticated successfully" + ); + } + } + + #[test] + fn wrong_version_and_corrupt_envelopes_fail_without_secret_output() { + let cipher = FinalCredentialCipher::new([7; 32]); + let context = context(); + let bearer = "secret-final-bearer"; + let valid = cipher + .encrypt(&context, &AccessCredential::new(bearer)) + .unwrap(); + let corrupt = [ + EncryptedFinalCredential::new(valid.as_str().replacen("v1.", "v2.", 1)), + EncryptedFinalCredential::new("v1.xchacha20poly1305:not-base64:not-base64"), + EncryptedFinalCredential::new("v1.xchacha20poly1305:"), + ]; + + for envelope in corrupt { + let error = cipher.decrypt(&context, &envelope).unwrap_err(); + assert_eq!( + error, + ApplicationError::FinalCredentialSecret { + message: "invalid final credential envelope".to_owned() + } + ); + assert!(!format!("{error:?}").contains(bearer)); + assert!(!error.to_string().contains(envelope.as_str())); + } + } + + fn context() -> FinalCredentialContext { + FinalCredentialContext { + deletion_job_id: Uuid::from_u128(1), + creator: CreatorPubky::from_str( + "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy", + ) + .unwrap(), + bundle_id: BundleId::from_str("000G40R40M30E209185GR38E1W").unwrap(), + } + } +} diff --git a/locks-service/src/infrastructure/memory/access_credentials.rs b/locks-service/src/infrastructure/memory/access_credentials.rs index c03d9cd..cfdb897 100644 --- a/locks-service/src/infrastructure/memory/access_credentials.rs +++ b/locks-service/src/infrastructure/memory/access_credentials.rs @@ -1,39 +1,652 @@ -use std::collections::HashMap; +use std::{ + collections::{HashMap, HashSet}, + fmt, + sync::Arc, +}; use async_trait::async_trait; +use locks_core::{ + ids::{BundleId, CreatorPubky, LockId, TaskId}, + lock_policy::ContentLock, +}; +use rand::{RngCore, rngs::OsRng}; +use time::{Duration, OffsetDateTime}; use tokio::sync::RwLock; +use uuid::Uuid; -use crate::application::errors::ApplicationError; -use crate::application::models::{AccessCredentialLookupKey, AccessCredentialRecord}; -use crate::application::ports::AccessCredentialStore; +use crate::application::{ + errors::ApplicationError, + models::{ + AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, + ContentLockDeletionJob, ContentLockDeletionPhase, ContentLockDeletionState, + DeletionReadAuthorization, EncryptedFinalCredential, FinalAccessWindows, + FinalCredentialContext, FinalCredentialMaterialization, InitializeFinalAccessWindowsResult, + IssuedDeletionCredential, VerificationTaskStatus, + }, + ports::{AccessCredentialStore, FinalCredentialWorkerIssueRequest, VerificationTaskRepository}, +}; +use crate::infrastructure::{ + final_credentials::FinalCredentialCipher, + memory::verification_task_deletion_fence::InMemoryVerificationTaskDeletionFence, +}; + +type JobKey = (CreatorPubky, LockId); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AccessPhaseAdvanceStatus { + Ready, + ObligationsPending, + FinalCredentialIssuanceMissed, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DrainCredentialKind { + Ordinary, + Final, +} + +#[derive(Debug, Clone)] +struct StoredCredential { + record: AccessCredentialRecord, + lock_id: LockId, + deletion: Option<(Uuid, DrainCredentialKind)>, +} + +#[derive(Debug, Clone)] +struct DeletionAccessState { + creator: CreatorPubky, + lock_id: LockId, + frozen_content_lock: ContentLock, + state: ContentLockDeletionState, + phase: ContentLockDeletionPhase, + force_requested: bool, + claimed_by: Option, + claim_token: Option, + claim_expires_at: Option, + issuance_started_at: Option, + issuance_deadline: Option, + read_deadline: Option, + payment_aggregate: Option, + bundle_snapshots: HashMap, +} + +#[derive(Debug, Clone, Copy)] +struct DeletionPaymentAggregate { + completed: bool, + accepted_count: u64, +} + +#[derive(Debug, Clone, Copy)] +struct DeletionBundleSnapshot { + task_id: TaskId, + paykit_admission_required: bool, + had_active_credential_at_cutoff: bool, + status_at_cutoff: VerificationTaskStatus, + resolved_status: Option, + resolved_at: Option, + final_credential_eligible_at: Option, + final_credential_issued: bool, +} + +impl DeletionBundleSnapshot { + fn permits_final_credential(self) -> bool { + self.paykit_admission_required + && !self.had_active_credential_at_cutoff + && self.resolved_status.unwrap_or(self.status_at_cutoff) + == VerificationTaskStatus::Completed + && self.final_credential_eligible_at.is_some() + } +} + +#[derive(Debug, Clone)] +struct FinalCredentialRecord { + lookup_key: AccessCredentialLookupKey, + encrypted_bearer: EncryptedFinalCredential, + expires_at: OffsetDateTime, + reads: HashMap, +} + +#[derive(Debug, Clone, Default)] +struct FinalReadState { + claim_token: Option, + claim_expires_at: Option, + consumed_at: Option, +} -/// In-memory access credential store keyed by non-secret lookup key. #[derive(Debug, Default)] +struct StoreState { + records: HashMap, + deletions: HashMap, + deletion_jobs_by_key: HashMap, + blocked_keys: HashSet, + final_credentials: HashMap<(Uuid, BundleId), FinalCredentialRecord>, +} + +/// In-memory access credential store with deletion-drain parity. pub struct InMemoryAccessCredentialStore { - records: RwLock>, + state: RwLock, + final_credential_cipher: FinalCredentialCipher, + verification_tasks: Option>, + verification_task_deletion_fence: Option>, +} + +impl fmt::Debug for InMemoryAccessCredentialStore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InMemoryAccessCredentialStore") + .field("state", &self.state) + .field("final_credential_cipher", &self.final_credential_cipher) + .field( + "verification_tasks", + &self.verification_tasks.as_ref().map(|_| ""), + ) + .field( + "verification_task_deletion_fence", + &self + .verification_task_deletion_fence + .as_ref() + .map(|_| ""), + ) + .finish() + } +} + +impl Default for InMemoryAccessCredentialStore { + fn default() -> Self { + Self::new() + } } impl InMemoryAccessCredentialStore { - /// Creates an empty store. + /// Creates an empty standalone store. Final credentials are unavailable until + /// verification dependencies are supplied. pub fn new() -> Self { - Self::default() + Self::build(None, None) + } + + pub fn with_verification_task_repository_and_deletion_fence( + verification_tasks: Arc, + verification_task_deletion_fence: Arc, + ) -> Self { + Self::build( + Some(verification_tasks), + Some(verification_task_deletion_fence), + ) + } + + fn build( + verification_tasks: Option>, + verification_task_deletion_fence: Option>, + ) -> Self { + let mut key = [0_u8; 32]; + OsRng.fill_bytes(&mut key); + Self { + state: RwLock::new(StoreState::default()), + final_credential_cipher: FinalCredentialCipher::new(key), + verification_tasks, + verification_task_deletion_fence, + } + } + + fn authoritative_winner_time(&self) -> OffsetDateTime { + self.verification_task_deletion_fence + .as_ref() + .map_or_else(OffsetDateTime::now_utc, |fence| { + fence.authoritative_cutoff() + }) + } + + pub(crate) async fn register_deletion( + &self, + job: &ContentLockDeletionJob, + snapshot_bundles: &HashMap, + ) -> Result<(), ApplicationError> { + let key = (job.creator.clone(), job.lock_id.clone()); + let mut state = self.state.write().await; + if state.blocked_keys.contains(&key) || state.deletion_jobs_by_key.contains_key(&key) { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + + let bundle_snapshots = snapshot_bundles + .iter() + .map( + |(bundle_id, (task_id, paykit_admission_required, status_at_cutoff))| { + let had_active_credential_at_cutoff = state.records.values().any(|stored| { + stored.record.creator == job.creator + && stored.lock_id == job.lock_id + && stored.record.bundle_id == *bundle_id + && stored.record.expires_at > job.deletion_started_at + }); + ( + bundle_id.clone(), + DeletionBundleSnapshot { + task_id: *task_id, + paykit_admission_required: *paykit_admission_required, + had_active_credential_at_cutoff, + status_at_cutoff: *status_at_cutoff, + resolved_status: matches!( + status_at_cutoff, + VerificationTaskStatus::Completed + | VerificationTaskStatus::Failed + | VerificationTaskStatus::Expired + ) + .then_some(*status_at_cutoff), + resolved_at: matches!( + status_at_cutoff, + VerificationTaskStatus::Completed + | VerificationTaskStatus::Failed + | VerificationTaskStatus::Expired + ) + .then_some(job.deletion_started_at), + final_credential_eligible_at: (*paykit_admission_required + && *status_at_cutoff == VerificationTaskStatus::Completed + && !had_active_credential_at_cutoff) + .then_some(job.deletion_started_at), + final_credential_issued: false, + }, + ) + }, + ) + .collect(); + state.deletions.insert( + job.job_id, + DeletionAccessState { + creator: job.creator.clone(), + lock_id: job.lock_id.clone(), + frozen_content_lock: job.frozen_content_lock.clone(), + state: job.state, + phase: job.phase, + force_requested: job.force_requested_at.is_some(), + claimed_by: None, + claim_token: None, + claim_expires_at: None, + issuance_started_at: None, + issuance_deadline: None, + read_deadline: None, + payment_aggregate: None, + bundle_snapshots, + }, + ); + state.deletion_jobs_by_key.insert(key.clone(), job.job_id); + state.blocked_keys.insert(key); + + for stored in state.records.values_mut() { + if stored.record.creator == job.creator + && stored.lock_id == job.lock_id + && snapshot_bundles.contains_key(&stored.record.bundle_id) + && stored.record.expires_at > job.deletion_started_at + { + stored.deletion = Some((job.job_id, DrainCredentialKind::Ordinary)); + } + } + Ok(()) + } + + pub(crate) async fn synchronize_job( + &self, + job: &ContentLockDeletionJob, + claimed_by: Option<&str>, + claim_token: Option, + claim_expires_at: Option, + ) { + let mut state = self.state.write().await; + if let Some(deletion) = state.deletions.get_mut(&job.job_id) { + deletion.state = job.state; + deletion.phase = job.phase; + deletion.force_requested = job.force_requested_at.is_some(); + deletion.claimed_by = claimed_by.map(str::to_owned); + deletion.claim_token = claim_token; + deletion.claim_expires_at = claim_expires_at; + } + if job.force_requested_at.is_some() + || matches!( + job.state, + ContentLockDeletionState::Completed | ContentLockDeletionState::Failed + ) + { + disable_job_access(&mut state, job.job_id); + } else if job.phase == ContentLockDeletionPhase::DeleteContent { + disable_final_access(&mut state, job.job_id); + } + } + + pub(crate) async fn block_key_and_disable_job( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + job_id: Option, + ) { + let mut state = self.state.write().await; + state + .blocked_keys + .insert((creator.clone(), lock_id.clone())); + if let Some(job_id) = job_id { + disable_job_access(&mut state, job_id); + } + } + + /// Records a terminal payment result only while deletion owns a live drain claim. + pub async fn resolve_deletion_payment( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + task_id: &TaskId, + status: VerificationTaskStatus, + ) -> Result { + if !matches!( + status, + VerificationTaskStatus::Completed | VerificationTaskStatus::Expired + ) { + return Err(ApplicationError::InvalidVerificationTaskState { + message: "payment drain transition must be completed or expired".to_owned(), + }); + } + let mut state = self.state.write().await; + let Some(deletion) = state.deletions.get_mut(&deletion_job_id) else { + return Ok(false); + }; + let owns_live_claim = deletion.state == ContentLockDeletionState::Running + && deletion.phase == ContentLockDeletionPhase::DrainPayments + && !deletion.force_requested + && deletion.claimed_by.as_deref() == Some(worker_id) + && deletion.claim_token == Some(claim_token) + && deletion + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at > now); + if !owns_live_claim { + return Ok(false); + } + let Some(snapshot) = deletion + .bundle_snapshots + .values_mut() + .find(|snapshot| snapshot.task_id == *task_id) + else { + return Ok(false); + }; + if !snapshot.paykit_admission_required || snapshot.resolved_status.is_some() { + return Ok(false); + } + snapshot.resolved_status = Some(status); + snapshot.resolved_at = Some(now); + snapshot.final_credential_eligible_at = (status == VerificationTaskStatus::Completed + && !snapshot.had_active_credential_at_cutoff) + .then_some(now); + Ok(true) + } + + /// Marks the deletion-owned payment aggregate terminal only under the exact live drain claim. + pub async fn complete_deletion_payment_aggregate( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + ) -> Result { + let mut state = self.state.write().await; + let Some(deletion) = state.deletions.get_mut(&deletion_job_id) else { + return Ok(false); + }; + if !owns_live_payment_drain_claim(deletion, worker_id, claim_token, now) { + return Ok(false); + } + deletion.payment_aggregate = Some(DeletionPaymentAggregate { + completed: true, + accepted_count: 0, + }); + Ok(true) + } + + pub(crate) async fn expire_unresolved_non_paykit_tasks( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + ) -> Result { + let task_ids = { + let state = self.state.read().await; + let Some(deletion) = state.deletions.get(&deletion_job_id) else { + return Ok(false); + }; + if !owns_live_no_paykit_claim(deletion, worker_id, claim_token, now) { + return Ok(false); + } + if deletion + .bundle_snapshots + .values() + .any(|snapshot| snapshot.paykit_admission_required) + { + return Err(invalid_deletion_state( + "non-Paykit deletion drain cannot process a Paykit snapshot", + )); + } + deletion + .bundle_snapshots + .values() + .filter(|snapshot| snapshot.resolved_status.is_none()) + .map(|snapshot| snapshot.task_id) + .collect::>() + }; + + if let Some(tasks) = &self.verification_tasks { + let mut updates = Vec::with_capacity(task_ids.len()); + for task_id in &task_ids { + let Some(task) = tasks.get_verification_task(task_id).await? else { + return Err(invalid_deletion_state( + "frozen non-Paykit verification task is missing", + )); + }; + if matches!( + task.status, + VerificationTaskStatus::Pending | VerificationTaskStatus::InProgress + ) { + updates.push(task.transition_to(VerificationTaskStatus::Expired, now, None)?); + } + } + tasks.update_verification_tasks_atomically(updates).await?; + } + + let mut state = self.state.write().await; + let Some(deletion) = state.deletions.get_mut(&deletion_job_id) else { + return Ok(false); + }; + if !owns_live_no_paykit_claim(deletion, worker_id, claim_token, now) { + return Ok(false); + } + for snapshot in deletion.bundle_snapshots.values_mut() { + if snapshot.resolved_status.is_none() { + snapshot.resolved_status = Some(VerificationTaskStatus::Expired); + snapshot.resolved_at = Some(now); + } + } + Ok(true) + } + + pub(crate) async fn check_phase_advance( + &self, + deletion_job_id: Uuid, + current_phase: ContentLockDeletionPhase, + next_phase: ContentLockDeletionPhase, + now: OffsetDateTime, + ) -> Result { + let state = self.state.read().await; + let Some(deletion) = state.deletions.get(&deletion_job_id) else { + return Ok(AccessPhaseAdvanceStatus::Ready); + }; + if current_phase == ContentLockDeletionPhase::DrainPayments + && next_phase == ContentLockDeletionPhase::DrainExistingCredentials + { + if deletion + .bundle_snapshots + .values() + .any(|snapshot| snapshot.resolved_status.is_none()) + { + return Err(invalid_deletion_state( + "every frozen deletion obligation must be terminal before credential draining", + )); + } + if !payment_aggregate_completed(deletion) { + return Err(invalid_deletion_state( + "payment drain aggregate must be durably completed before credential draining", + )); + } + } + if current_phase == ContentLockDeletionPhase::DrainExistingCredentials + && next_phase == ContentLockDeletionPhase::IssueFinalCredentials + && state.records.values().any(|stored| { + stored.deletion == Some((deletion_job_id, DrainCredentialKind::Ordinary)) + && stored.record.expires_at > now + }) + { + return Ok(AccessPhaseAdvanceStatus::ObligationsPending); + } + if current_phase == ContentLockDeletionPhase::IssueFinalCredentials + && next_phase == ContentLockDeletionPhase::DrainFinalReads + && deletion.bundle_snapshots.values().any(|snapshot| { + snapshot.permits_final_credential() && !snapshot.final_credential_issued + }) + { + return Ok( + if deletion + .issuance_deadline + .is_some_and(|deadline| now >= deadline) + { + AccessPhaseAdvanceStatus::FinalCredentialIssuanceMissed + } else { + AccessPhaseAdvanceStatus::ObligationsPending + }, + ); + } + if current_phase == ContentLockDeletionPhase::DrainFinalReads + && next_phase == ContentLockDeletionPhase::DeleteContent + && has_live_access_obligation(&state, deletion_job_id, now) + { + return Ok(AccessPhaseAdvanceStatus::ObligationsPending); + } + Ok(AccessPhaseAdvanceStatus::Ready) } + + pub(crate) async fn check_successful_finish( + &self, + deletion_job_id: Uuid, + phase: ContentLockDeletionPhase, + now: OffsetDateTime, + ) -> Result<(), ApplicationError> { + let state = self.state.read().await; + let Some(deletion) = state.deletions.get(&deletion_job_id) else { + return Ok(()); + }; + if phase != ContentLockDeletionPhase::PurgeOperationalState { + return Err(invalid_deletion_state( + "successful completion requires the final operational-cleanup phase", + )); + } + if deletion + .bundle_snapshots + .values() + .any(|snapshot| snapshot.resolved_status.is_none()) + { + return Err(invalid_deletion_state( + "every frozen deletion obligation must be terminal before credential draining", + )); + } + if !payment_aggregate_completed(deletion) { + return Err(invalid_deletion_state( + "payment drain aggregate must be durably completed before credential draining", + )); + } + if has_live_access_obligation(&state, deletion_job_id, now) + || deletion.bundle_snapshots.values().any(|snapshot| { + snapshot.permits_final_credential() && !snapshot.final_credential_issued + }) + { + return Err(invalid_deletion_state( + "successful completion cannot bypass deletion access obligations", + )); + } + Ok(()) + } +} + +fn owns_live_payment_drain_claim( + deletion: &DeletionAccessState, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, +) -> bool { + deletion.state == ContentLockDeletionState::Running + && deletion.phase == ContentLockDeletionPhase::DrainPayments + && !deletion.force_requested + && deletion.claimed_by.as_deref() == Some(worker_id) + && deletion.claim_token == Some(claim_token) + && deletion + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at > now) +} + +fn owns_live_no_paykit_claim( + deletion: &DeletionAccessState, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, +) -> bool { + deletion.state == ContentLockDeletionState::Running + && matches!( + deletion.phase, + ContentLockDeletionPhase::StartPaymentDrain | ContentLockDeletionPhase::DrainPayments + ) + && !deletion.force_requested + && deletion.claimed_by.as_deref() == Some(worker_id) + && deletion.claim_token == Some(claim_token) + && deletion + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at > now) +} + +fn payment_aggregate_completed(deletion: &DeletionAccessState) -> bool { + deletion.payment_aggregate.map_or_else( + || { + !deletion + .bundle_snapshots + .values() + .any(|snapshot| snapshot.paykit_admission_required) + }, + |aggregate| aggregate.completed && aggregate.accepted_count == 0, + ) } #[async_trait] impl AccessCredentialStore for InMemoryAccessCredentialStore { async fn insert_access_credential( &self, + lock_id: &LockId, lookup_key: AccessCredentialLookupKey, record: AccessCredentialRecord, ) -> Result<(), ApplicationError> { - let mut records = self.records.write().await; - if records.contains_key(&lookup_key) { + let _admission = if let Some(fence) = &self.verification_task_deletion_fence { + Some(fence.acquire_lock_admission(&record.creator, lock_id).await) + } else { + None + }; + let mut state = self.state.write().await; + let key = (record.creator.clone(), lock_id.clone()); + if state.blocked_keys.contains(&key) || state.deletion_jobs_by_key.contains_key(&key) { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + if state.records.contains_key(&lookup_key) { return Err(ApplicationError::DuplicateRecord { record: "access_credential", }); } - records.insert(lookup_key, record); + state.records.insert( + lookup_key, + StoredCredential { + record, + lock_id: lock_id.clone(), + deletion: None, + }, + ); Ok(()) } @@ -41,28 +654,749 @@ impl AccessCredentialStore for InMemoryAccessCredentialStore { &self, lookup_key: &AccessCredentialLookupKey, ) -> Result, ApplicationError> { - Ok(self.records.read().await.get(lookup_key).cloned()) + Ok(self + .state + .read() + .await + .records + .get(lookup_key) + .map(|stored| stored.record.clone())) } async fn delete_access_credential( &self, lookup_key: &AccessCredentialLookupKey, ) -> Result<(), ApplicationError> { - self.records.write().await.remove(lookup_key); + let mut state = self.state.write().await; + state.records.remove(lookup_key); + state + .final_credentials + .retain(|_, credential| &credential.lookup_key != lookup_key); Ok(()) } + + async fn initialize_final_access_windows( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + issuance_window: Duration, + read_window: Duration, + ) -> Result { + if issuance_window <= Duration::ZERO || read_window <= Duration::ZERO { + return Err(ApplicationError::Storage { + message: "final access window durations must be positive".to_owned(), + }); + } + let mut state = self.state.write().await; + let now = self.authoritative_winner_time(); + let Some(deletion) = state.deletions.get_mut(&deletion_job_id) else { + return Ok(InitializeFinalAccessWindowsResult::ClaimLost); + }; + let owns_live_claim = deletion.state == ContentLockDeletionState::Running + && deletion.phase == ContentLockDeletionPhase::IssueFinalCredentials + && !deletion.force_requested + && deletion.claimed_by.as_deref() == Some(worker_id) + && deletion.claim_token == Some(claim_token) + && deletion + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at > now); + if !owns_live_claim { + return Ok(InitializeFinalAccessWindowsResult::ClaimLost); + } + match ( + deletion.issuance_started_at, + deletion.issuance_deadline, + deletion.read_deadline, + ) { + ( + Some(issuance_started_at), + Some(credential_issuance_deadline), + Some(read_deadline), + ) => { + return Ok(InitializeFinalAccessWindowsResult::Initialized( + FinalAccessWindows { + issuance_started_at, + credential_issuance_deadline, + read_deadline, + }, + )); + } + (None, None, None) => {} + _ => { + return Err(ApplicationError::Storage { + message: "incomplete final access windows in memory".to_owned(), + }); + } + } + let credential_issuance_deadline = + now.checked_add(issuance_window) + .ok_or_else(|| ApplicationError::Storage { + message: "final credential issuance deadline overflow".to_owned(), + })?; + let read_deadline = credential_issuance_deadline + .checked_add(read_window) + .ok_or_else(|| ApplicationError::Storage { + message: "final read deadline overflow".to_owned(), + })?; + deletion.issuance_started_at = Some(now); + deletion.issuance_deadline = Some(credential_issuance_deadline); + deletion.read_deadline = Some(read_deadline); + Ok(InitializeFinalAccessWindowsResult::Initialized( + FinalAccessWindows { + issuance_started_at: now, + credential_issuance_deadline, + read_deadline, + }, + )) + } + + async fn final_credentials_to_materialize( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + limit: usize, + ) -> Result, ApplicationError> { + if limit == 0 { + return Ok(Vec::new()); + } + let state = self.state.read().await; + let now = self.authoritative_winner_time(); + let Some(deletion) = state.deletions.get(&deletion_job_id) else { + return Ok(Vec::new()); + }; + let owns_live_issue_claim = deletion.state == ContentLockDeletionState::Running + && deletion.phase == ContentLockDeletionPhase::IssueFinalCredentials + && !deletion.force_requested + && deletion.claimed_by.as_deref() == Some(worker_id) + && deletion.claim_token == Some(claim_token) + && deletion + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at > now) + && deletion + .issuance_deadline + .is_some_and(|issuance_deadline| issuance_deadline > now); + if !owns_live_issue_claim { + return Ok(Vec::new()); + } + let mut pending: Vec<_> = deletion + .bundle_snapshots + .iter() + .filter(|(_, snapshot)| { + snapshot.permits_final_credential() && !snapshot.final_credential_issued + }) + .map(|(bundle_id, _)| FinalCredentialMaterialization { + creator: deletion.creator.clone(), + bundle_id: bundle_id.clone(), + }) + .collect(); + pending.sort_by(|left, right| left.bundle_id.as_str().cmp(right.bundle_id.as_str())); + pending.truncate(limit); + Ok(pending) + } + + async fn issue_or_replay_final_credential( + &self, + creator: &CreatorPubky, + bundle_id: &BundleId, + _caller_now: OffsetDateTime, + candidate: AccessCredential, + ) -> Result, ApplicationError> { + let mut state = self.state.write().await; + let now = self.authoritative_winner_time(); + let Some((deletion_job_id, _)) = state.deletions.iter().find(|(_, deletion)| { + deletion.creator == *creator && deletion.bundle_snapshots.contains_key(bundle_id) + }) else { + return Ok(None); + }; + let deletion_job_id = *deletion_job_id; + let Some(deletion) = state.deletions.get(&deletion_job_id) else { + return Ok(None); + }; + if deletion.creator != *creator + || deletion.force_requested + || !matches!( + deletion.state, + ContentLockDeletionState::Queued | ContentLockDeletionState::Running + ) + || !matches!( + deletion.phase, + ContentLockDeletionPhase::IssueFinalCredentials + | ContentLockDeletionPhase::DrainFinalReads + ) + || deletion + .read_deadline + .is_none_or(|deadline| deadline <= now) + { + return Ok(None); + } + + let context = FinalCredentialContext { + deletion_job_id, + creator: creator.clone(), + bundle_id: bundle_id.clone(), + }; + if let Some(existing) = state + .final_credentials + .get(&(deletion_job_id, bundle_id.clone())) + { + let credential = self + .final_credential_cipher + .decrypt(&context, &existing.encrypted_bearer)?; + return Ok(Some(IssuedDeletionCredential { + credential, + expires_at: existing.expires_at, + })); + } + + if deletion.phase != ContentLockDeletionPhase::IssueFinalCredentials + || deletion + .issuance_deadline + .is_none_or(|deadline| now >= deadline) + || !deletion + .bundle_snapshots + .get(bundle_id) + .copied() + .is_some_and(DeletionBundleSnapshot::permits_final_credential) + { + return Ok(None); + } + + let expires_at = deletion + .read_deadline + .expect("final issuance requires an initialized read deadline"); + let frozen_content_lock = deletion.frozen_content_lock.clone(); + let lock_id = deletion.lock_id.clone(); + let encrypted_bearer = self.final_credential_cipher.encrypt(&context, &candidate)?; + let lookup_key = AccessCredentialLookupKey::derive(&candidate); + if state.records.contains_key(&lookup_key) { + return Err(ApplicationError::DuplicateRecord { + record: "access_credential", + }); + } + let mut reads = HashMap::new(); + if let Some(resource) = frozen_content_lock.primary_resource { + reads.insert(resource.path, FinalReadState::default()); + } + for path in frozen_content_lock.secondary_resources.keys() { + reads.insert(path.clone(), FinalReadState::default()); + } + state.records.insert( + lookup_key.clone(), + StoredCredential { + record: AccessCredentialRecord { + creator: creator.clone(), + bundle_id: bundle_id.clone(), + expires_at, + }, + lock_id, + deletion: Some((deletion_job_id, DrainCredentialKind::Final)), + }, + ); + state.final_credentials.insert( + (deletion_job_id, bundle_id.clone()), + FinalCredentialRecord { + lookup_key, + encrypted_bearer, + expires_at, + reads, + }, + ); + state + .deletions + .get_mut(&deletion_job_id) + .and_then(|deletion| deletion.bundle_snapshots.get_mut(bundle_id)) + .expect("issued final credential must retain its immutable snapshot") + .final_credential_issued = true; + Ok(Some(IssuedDeletionCredential { + credential: candidate, + expires_at, + })) + } + + async fn issue_or_replay_final_credential_for_worker( + &self, + request: FinalCredentialWorkerIssueRequest<'_>, + ) -> Result, ApplicationError> { + let FinalCredentialWorkerIssueRequest { + deletion_job_id, + worker_id, + claim_token, + creator, + bundle_id, + now: _caller_now, + candidate, + } = request; + let mut state = self.state.write().await; + let now = self.authoritative_winner_time(); + let Some(deletion) = state.deletions.get(&deletion_job_id) else { + return Ok(None); + }; + let owns_live_issue_claim = deletion.creator == *creator + && deletion.state == ContentLockDeletionState::Running + && deletion.phase == ContentLockDeletionPhase::IssueFinalCredentials + && !deletion.force_requested + && deletion.claimed_by.as_deref() == Some(worker_id) + && deletion.claim_token == Some(claim_token) + && deletion + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at > now) + && deletion + .issuance_deadline + .is_some_and(|issuance_deadline| issuance_deadline > now) + && deletion + .read_deadline + .is_some_and(|read_deadline| read_deadline > now) + && deletion + .bundle_snapshots + .get(bundle_id) + .copied() + .is_some_and(DeletionBundleSnapshot::permits_final_credential); + if !owns_live_issue_claim { + return Ok(None); + } + + let context = FinalCredentialContext { + deletion_job_id, + creator: creator.clone(), + bundle_id: bundle_id.clone(), + }; + if let Some(existing) = state + .final_credentials + .get(&(deletion_job_id, bundle_id.clone())) + { + let credential = self + .final_credential_cipher + .decrypt(&context, &existing.encrypted_bearer)?; + return Ok(Some(IssuedDeletionCredential { + credential, + expires_at: existing.expires_at, + })); + } + + let deletion = state + .deletions + .get(&deletion_job_id) + .expect("worker-fenced deletion was validated under the write lock"); + let expires_at = deletion + .read_deadline + .expect("worker-fenced final issuance requires an initialized read deadline"); + let frozen_content_lock = deletion.frozen_content_lock.clone(); + let lock_id = deletion.lock_id.clone(); + let encrypted_bearer = self.final_credential_cipher.encrypt(&context, &candidate)?; + let lookup_key = AccessCredentialLookupKey::derive(&candidate); + if state.records.contains_key(&lookup_key) { + return Err(ApplicationError::DuplicateRecord { + record: "access_credential", + }); + } + let mut reads = HashMap::new(); + if let Some(resource) = frozen_content_lock.primary_resource { + reads.insert(resource.path, FinalReadState::default()); + } + for path in frozen_content_lock.secondary_resources.keys() { + reads.insert(path.clone(), FinalReadState::default()); + } + state.records.insert( + lookup_key.clone(), + StoredCredential { + record: AccessCredentialRecord { + creator: creator.clone(), + bundle_id: bundle_id.clone(), + expires_at, + }, + lock_id, + deletion: Some((deletion_job_id, DrainCredentialKind::Final)), + }, + ); + state.final_credentials.insert( + (deletion_job_id, bundle_id.clone()), + FinalCredentialRecord { + lookup_key, + encrypted_bearer, + expires_at, + reads, + }, + ); + state + .deletions + .get_mut(&deletion_job_id) + .and_then(|deletion| deletion.bundle_snapshots.get_mut(bundle_id)) + .expect("issued final credential must retain its immutable snapshot") + .final_credential_issued = true; + Ok(Some(IssuedDeletionCredential { + credential: candidate, + expires_at, + })) + } + + async fn final_credential_available( + &self, + creator: &CreatorPubky, + bundle_id: &BundleId, + now: OffsetDateTime, + ) -> Result { + let state = self.state.read().await; + let Some((deletion_job_id, deletion)) = state.deletions.iter().find(|(_, deletion)| { + deletion.creator == *creator && deletion.bundle_snapshots.contains_key(bundle_id) + }) else { + return Ok(false); + }; + let lifecycle_allows_access = !deletion.force_requested + && matches!( + deletion.state, + ContentLockDeletionState::Queued | ContentLockDeletionState::Running + ) + && matches!( + deletion.phase, + ContentLockDeletionPhase::IssueFinalCredentials + | ContentLockDeletionPhase::DrainFinalReads + ) + && deletion + .read_deadline + .is_some_and(|deadline| deadline > now); + if !lifecycle_allows_access { + return Ok(false); + } + if state + .final_credentials + .contains_key(&(*deletion_job_id, bundle_id.clone())) + { + return Ok(true); + } + Ok( + deletion.phase == ContentLockDeletionPhase::IssueFinalCredentials + && deletion + .issuance_deadline + .is_some_and(|deadline| now < deadline) + && deletion + .bundle_snapshots + .get(bundle_id) + .copied() + .is_some_and(DeletionBundleSnapshot::permits_final_credential), + ) + } + + async fn prepare_deletion_read( + &self, + lookup_key: &AccessCredentialLookupKey, + path: &str, + claim_duration: Duration, + ) -> Result, ApplicationError> { + let mut state = self.state.write().await; + let now = self.authoritative_winner_time(); + let Some(stored) = state.records.get(lookup_key).cloned() else { + return Ok(None); + }; + let Some((deletion_job_id, kind)) = stored.deletion else { + return Ok(None); + }; + let Some(deletion) = state.deletions.get(&deletion_job_id) else { + return Ok(None); + }; + if stored.record.expires_at <= now + || deletion.force_requested + || !matches!( + deletion.state, + ContentLockDeletionState::Queued | ContentLockDeletionState::Running + ) + || !matches!( + deletion.phase, + ContentLockDeletionPhase::Withdraw + | ContentLockDeletionPhase::StartPaymentDrain + | ContentLockDeletionPhase::DrainPayments + | ContentLockDeletionPhase::DrainExistingCredentials + | ContentLockDeletionPhase::IssueFinalCredentials + | ContentLockDeletionPhase::DrainFinalReads + ) + { + return Ok(None); + } + let Some(resource) = deletion.frozen_content_lock.resource_for_path(path) else { + return Ok(None); + }; + let creator = deletion.creator.clone(); + if kind == DrainCredentialKind::Ordinary { + return Ok(Some(DeletionReadAuthorization { + claim_token: None, + creator, + resource, + })); + } + if !matches!( + deletion.phase, + ContentLockDeletionPhase::IssueFinalCredentials + | ContentLockDeletionPhase::DrainFinalReads + ) || deletion + .read_deadline + .is_none_or(|deadline| deadline <= now) + { + return Ok(None); + } + let read_deadline = deletion + .read_deadline + .expect("final credential requires a read deadline"); + let Some(final_credential) = state + .final_credentials + .get_mut(&(deletion_job_id, stored.record.bundle_id.clone())) + else { + return Ok(None); + }; + let Some(read) = final_credential.reads.get_mut(path) else { + return Ok(None); + }; + if read.consumed_at.is_some() + || (read.claim_token.is_some() + && read + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at > now)) + { + return Ok(None); + } + let bounded_expiry = now + .checked_add(claim_duration) + .ok_or_else(|| ApplicationError::Storage { + message: "final read claim expiry overflow".to_owned(), + })? + .min(now + Duration::seconds(30)) + .min(final_credential.expires_at) + .min(read_deadline); + if bounded_expiry <= now { + return Ok(None); + } + let claim_token = Uuid::new_v4(); + read.claim_token = Some(claim_token); + read.claim_expires_at = Some(bounded_expiry); + Ok(Some(DeletionReadAuthorization { + claim_token: Some(claim_token), + creator, + resource, + })) + } + + async fn deletion_credential_enrolled( + &self, + lookup_key: &AccessCredentialLookupKey, + ) -> Result { + let state = self.state.read().await; + Ok(state + .records + .get(lookup_key) + .is_some_and(|stored| stored.deletion.is_some())) + } + + async fn release_deletion_read( + &self, + lookup_key: &AccessCredentialLookupKey, + path: &str, + claim_token: Uuid, + _now: OffsetDateTime, + ) -> Result { + let mut state = self.state.write().await; + let Some((deletion_job_id, bundle_id)) = final_credential_identity(&state, lookup_key) + else { + return Ok(false); + }; + let Some(read) = state + .final_credentials + .get_mut(&(deletion_job_id, bundle_id)) + .and_then(|credential| credential.reads.get_mut(path)) + else { + return Ok(false); + }; + if read.consumed_at.is_some() || read.claim_token != Some(claim_token) { + return Ok(false); + } + read.claim_token = None; + read.claim_expires_at = None; + Ok(true) + } + + async fn consume_deletion_read( + &self, + lookup_key: &AccessCredentialLookupKey, + path: &str, + claim_token: Uuid, + ) -> Result { + let mut state = self.state.write().await; + let now = self.authoritative_winner_time(); + let Some((deletion_job_id, bundle_id)) = final_credential_identity(&state, lookup_key) + else { + return Ok(false); + }; + let Some(read) = state + .final_credentials + .get_mut(&(deletion_job_id, bundle_id)) + .and_then(|credential| credential.reads.get_mut(path)) + else { + return Ok(false); + }; + if read.consumed_at.is_some() + || read.claim_token != Some(claim_token) + || read + .claim_expires_at + .is_none_or(|claim_expires_at| claim_expires_at <= now) + { + return Ok(false); + } + read.claim_token = None; + read.claim_expires_at = None; + read.consumed_at = Some(now); + Ok(true) + } +} + +fn has_live_access_obligation( + state: &StoreState, + deletion_job_id: Uuid, + now: OffsetDateTime, +) -> bool { + state.records.values().any(|stored| { + stored.deletion == Some((deletion_job_id, DrainCredentialKind::Ordinary)) + && stored.record.expires_at > now + }) || state + .final_credentials + .iter() + .any(|((job_id, _), credential)| { + *job_id == deletion_job_id + && credential.expires_at > now + && credential + .reads + .values() + .any(|read| read.consumed_at.is_none()) + }) +} + +fn invalid_deletion_state(message: &str) -> ApplicationError { + ApplicationError::InvalidContentLockDeletionState { + message: message.to_owned(), + } +} + +fn final_credential_identity( + state: &StoreState, + lookup_key: &AccessCredentialLookupKey, +) -> Option<(Uuid, BundleId)> { + let stored = state.records.get(lookup_key)?; + let (job_id, kind) = stored.deletion?; + (kind == DrainCredentialKind::Final).then(|| (job_id, stored.record.bundle_id.clone())) +} + +fn disable_final_access(state: &mut StoreState, job_id: Uuid) { + revoke_job_read_claims(state, job_id); +} + +fn disable_job_access(state: &mut StoreState, job_id: Uuid) { + revoke_job_read_claims(state, job_id); +} + +fn revoke_job_read_claims(state: &mut StoreState, job_id: Uuid) { + for ((deletion_job_id, _), credential) in &mut state.final_credentials { + if *deletion_job_id == job_id { + for read in credential.reads.values_mut() { + read.claim_token = None; + read.claim_expires_at = None; + } + } + } } #[cfg(test)] mod tests { - use std::str::FromStr; + use std::{collections::BTreeMap, str::FromStr, sync::Mutex as StdMutex}; + use serde_json::json; use time::macros::datetime; + use tokio::sync::RwLock; - use locks_core::ids::{BundleId, CreatorPubky}; + use locks_core::{ + ids::{BundleId, CreatorPubky, GuardedResourceHash, LockId, PubkyLockResource, TaskId}, + lock_policy::{ + AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, GuardedResource, LockLogic, + LockServerConfig, VerifierType, + }, + verification::{Proof, SUBMITTED_PROOF_BUNDLE_VERSION, SubmittedProofBundle}, + }; use super::*; - use crate::application::models::AccessCredential; + use crate::application::models::VerificationTaskRecord; + + #[derive(Debug, Default)] + struct FailingVerificationTaskRepository { + records: RwLock>, + fail_update_for: RwLock>, + } + + #[async_trait] + impl VerificationTaskRepository for FailingVerificationTaskRepository { + async fn insert_verification_task( + &self, + task: VerificationTaskRecord, + ) -> Result<(), ApplicationError> { + self.records.write().await.insert(task.task_id, task); + Ok(()) + } + + async fn update_verification_task( + &self, + task: VerificationTaskRecord, + ) -> Result<(), ApplicationError> { + if *self.fail_update_for.read().await == Some(task.task_id) { + return Err(ApplicationError::Storage { + message: "injected verification task update failure".to_owned(), + }); + } + let mut records = self.records.write().await; + if !records.contains_key(&task.task_id) { + return Err(ApplicationError::MissingRecord { + record: "verification_task", + }); + } + records.insert(task.task_id, task); + Ok(()) + } + + async fn update_verification_tasks_atomically( + &self, + tasks: Vec, + ) -> Result<(), ApplicationError> { + let mut records = self.records.write().await; + let fail_update_for = *self.fail_update_for.read().await; + if tasks + .iter() + .any(|task| !records.contains_key(&task.task_id)) + { + return Err(ApplicationError::MissingRecord { + record: "verification_task", + }); + } + if tasks + .iter() + .any(|task| fail_update_for == Some(task.task_id)) + { + return Err(ApplicationError::Storage { + message: "injected verification task update failure".to_owned(), + }); + } + for task in tasks { + records.insert(task.task_id, task); + } + Ok(()) + } + + async fn get_verification_task( + &self, + task_id: &TaskId, + ) -> Result, ApplicationError> { + Ok(self.records.read().await.get(task_id).cloned()) + } + + async fn delete_verification_task(&self, task_id: &TaskId) -> Result<(), ApplicationError> { + self.records.write().await.remove(task_id); + Ok(()) + } + } #[tokio::test] async fn insert_rejects_duplicate_read_miss_is_none_delete_is_ensure_absent() { @@ -70,13 +1404,15 @@ mod tests { let credential = AccessCredential::new("raw-bearer-credential"); let lookup_key = AccessCredentialLookupKey::derive(&credential); let record = record(); + let lock_id = + LockId::from_str("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG").unwrap(); assert_eq!( store.get_access_credential(&lookup_key).await.unwrap(), None ); store - .insert_access_credential(lookup_key.clone(), record.clone()) + .insert_access_credential(&lock_id, lookup_key.clone(), record.clone()) .await .unwrap(); assert_eq!( @@ -85,7 +1421,7 @@ mod tests { ); assert_eq!( store - .insert_access_credential(lookup_key.clone(), record) + .insert_access_credential(&lock_id, lookup_key.clone(), record) .await, Err(ApplicationError::DuplicateRecord { record: "access_credential", @@ -100,6 +1436,711 @@ mod tests { ); } + #[tokio::test] + async fn non_paykit_expiry_missing_later_task_mutates_neither_tasks_nor_snapshots() { + let tasks = Arc::new(FailingVerificationTaskRepository::default()); + let store = non_paykit_expiry_store(tasks.clone()).await; + let task_ids = unresolved_task_ids(&store).await; + tasks + .insert_verification_task(non_paykit_task(task_ids[0])) + .await + .unwrap(); + + assert!(expire_non_paykit(&store).await.is_err()); + assert_eq!( + tasks + .get_verification_task(&task_ids[0]) + .await + .unwrap() + .unwrap() + .status, + VerificationTaskStatus::Pending + ); + assert_snapshots_unresolved(&store).await; + } + + #[tokio::test] + async fn non_paykit_expiry_later_update_failure_mutates_neither_tasks_nor_snapshots() { + let tasks = Arc::new(FailingVerificationTaskRepository::default()); + let store = non_paykit_expiry_store(tasks.clone()).await; + let task_ids = unresolved_task_ids(&store).await; + for task_id in task_ids.iter().copied() { + tasks + .insert_verification_task(non_paykit_task(task_id)) + .await + .unwrap(); + } + *tasks.fail_update_for.write().await = Some(task_ids[1]); + + assert!(expire_non_paykit(&store).await.is_err()); + for task_id in task_ids { + assert_eq!( + tasks + .get_verification_task(&task_id) + .await + .unwrap() + .unwrap() + .status, + VerificationTaskStatus::Pending + ); + } + assert_snapshots_unresolved(&store).await; + } + + async fn expire_non_paykit( + store: &InMemoryAccessCredentialStore, + ) -> Result { + store + .expire_unresolved_non_paykit_tasks( + Uuid::from_u128(7), + "worker-drain", + Uuid::from_u128(8), + datetime!(2026-08-17 12:00:00 UTC), + ) + .await + } + + async fn non_paykit_expiry_store( + tasks: Arc, + ) -> InMemoryAccessCredentialStore { + let store = InMemoryAccessCredentialStore::build(Some(tasks), None); + let snapshots = [ + ( + "000G40R40M30E209185GR38E1W", + "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d10", + ), + ( + "000G40R40M30E209185GR38E1V", + "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d11", + ), + ] + .into_iter() + .map(|(bundle_id, task_id)| { + ( + BundleId::from_str(bundle_id).unwrap(), + DeletionBundleSnapshot { + task_id: TaskId::from_str(task_id).unwrap(), + paykit_admission_required: false, + had_active_credential_at_cutoff: false, + status_at_cutoff: VerificationTaskStatus::Pending, + resolved_status: None, + resolved_at: None, + final_credential_eligible_at: None, + final_credential_issued: false, + }, + ) + }) + .collect(); + store.state.write().await.deletions.insert( + Uuid::from_u128(7), + DeletionAccessState { + creator: record().creator, + lock_id: LockId::from_str("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG") + .unwrap(), + frozen_content_lock: test_content_lock(), + state: ContentLockDeletionState::Running, + phase: ContentLockDeletionPhase::StartPaymentDrain, + force_requested: false, + claimed_by: Some("worker-drain".to_owned()), + claim_token: Some(Uuid::from_u128(8)), + claim_expires_at: Some(datetime!(2026-08-17 12:05:00 UTC)), + issuance_started_at: None, + issuance_deadline: None, + read_deadline: None, + payment_aggregate: None, + bundle_snapshots: snapshots, + }, + ); + store + } + + async fn unresolved_task_ids(store: &InMemoryAccessCredentialStore) -> Vec { + store + .state + .read() + .await + .deletions + .get(&Uuid::from_u128(7)) + .unwrap() + .bundle_snapshots + .values() + .map(|snapshot| snapshot.task_id) + .collect() + } + + async fn assert_snapshots_unresolved(store: &InMemoryAccessCredentialStore) { + assert!( + store + .state + .read() + .await + .deletions + .get(&Uuid::from_u128(7)) + .unwrap() + .bundle_snapshots + .values() + .all( + |snapshot| snapshot.resolved_status.is_none() && snapshot.resolved_at.is_none() + ) + ); + } + + fn non_paykit_task(task_id: TaskId) -> VerificationTaskRecord { + VerificationTaskRecord { + task_id, + creator: record().creator, + submitted_proof_bundle: SubmittedProofBundle { + version: SUBMITTED_PROOF_BUNDLE_VERSION, + bundle_id: BundleId::from_str("000G40R40M30E209185GR38E1W").unwrap(), + pubky_lock_resource: PubkyLockResource::from_str( + "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy/pub/locks.app/000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG.json", + ) + .unwrap(), + reader_public_key: None, + proofs: vec![Proof { + criterion_id: "criterion-1".to_owned(), + verifier_type: VerifierType::DevStatic, + payload: json!({}), + }], + }, + status: VerificationTaskStatus::Pending, + submitted_at: datetime!(2026-08-17 11:00:00 UTC), + started_at: None, + completed_at: None, + failure_message: None, + } + } + + #[tokio::test] + async fn final_materialization_enumeration_is_ordered_eligible_bounded_and_claim_fenced() { + let store = final_materialization_store().await; + let job_id = Uuid::from_u128(7); + let claim_token = Uuid::from_u128(8); + + let selected = store + .final_credentials_to_materialize(job_id, "worker-final", claim_token, 1) + .await + .unwrap(); + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].bundle_id.as_str(), "000G40R40M30E209185GR38E1R"); + + for (worker, token) in [ + ("wrong-worker", claim_token), + ("worker-final", Uuid::from_u128(9)), + ] { + assert!( + store + .final_credentials_to_materialize(job_id, worker, token, 10) + .await + .unwrap() + .is_empty() + ); + } + + { + let mut state = store.state.write().await; + state.deletions.get_mut(&job_id).unwrap().force_requested = true; + } + assert!( + store + .final_credentials_to_materialize(job_id, "worker-final", claim_token, 10) + .await + .unwrap() + .is_empty() + ); + { + let mut state = store.state.write().await; + let deletion = state.deletions.get_mut(&job_id).unwrap(); + deletion.force_requested = false; + deletion.phase = ContentLockDeletionPhase::DrainFinalReads; + } + assert!( + store + .final_credentials_to_materialize(job_id, "worker-final", claim_token, 10) + .await + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn final_materialization_enumeration_samples_time_after_state_fence() { + let initial = datetime!(2026-08-17 12:00:00 UTC); + let claim_expiry = datetime!(2026-08-17 12:05:00 UTC); + let clock = Arc::new(MutableClock::new(initial)); + let store = Arc::new(final_materialization_store_with_clock(clock.clone()).await); + let state_guard = store.state.write().await; + let waiting_store = Arc::clone(&store); + + let enumeration = tokio::spawn(async move { + waiting_store + .final_credentials_to_materialize( + Uuid::from_u128(7), + "worker-final", + Uuid::from_u128(8), + 10, + ) + .await + }); + tokio::task::yield_now().await; + assert!(!enumeration.is_finished()); + clock.set(claim_expiry); + drop(state_guard); + + assert!(enumeration.await.unwrap().unwrap().is_empty()); + } + + #[tokio::test] + async fn worker_final_issuance_revalidates_exact_live_claim_and_fresh_deadlines() { + let job_id = Uuid::from_u128(7); + let claim_token = Uuid::from_u128(8); + let now = datetime!(2026-08-17 12:00:00 UTC); + let bundle_id = BundleId::from_str("000G40R40M30E209185GR38E1R").unwrap(); + let creator = record().creator; + + for (worker, token, at) in [ + ("wrong-worker", claim_token, now), + ("worker-final", Uuid::from_u128(9), now), + ( + "worker-final", + claim_token, + datetime!(2026-08-17 12:05:00 UTC), + ), + ( + "worker-final", + claim_token, + datetime!(2026-08-17 12:15:00 UTC), + ), + ] { + let store = final_materialization_store_at(at).await; + let candidate = AccessCredential::new(format!("denied-{worker}-{at}")); + let lookup_key = AccessCredentialLookupKey::derive(&candidate); + assert!( + store + .issue_or_replay_final_credential_for_worker( + FinalCredentialWorkerIssueRequest { + deletion_job_id: job_id, + worker_id: worker, + claim_token: token, + creator: &creator, + bundle_id: &bundle_id, + now: at, + candidate, + }, + ) + .await + .unwrap() + .is_none() + ); + assert!( + store + .get_access_credential(&lookup_key) + .await + .unwrap() + .is_none() + ); + } + + let forced = final_materialization_store().await; + forced + .state + .write() + .await + .deletions + .get_mut(&job_id) + .unwrap() + .force_requested = true; + assert!( + forced + .issue_or_replay_final_credential_for_worker(FinalCredentialWorkerIssueRequest { + deletion_job_id: job_id, + worker_id: "worker-final", + claim_token, + creator: &creator, + bundle_id: &bundle_id, + now, + candidate: AccessCredential::new("force-loser"), + },) + .await + .unwrap() + .is_none() + ); + + let store = final_materialization_store().await; + let winner = AccessCredential::new("worker-winner"); + let issued = store + .issue_or_replay_final_credential_for_worker(FinalCredentialWorkerIssueRequest { + deletion_job_id: job_id, + worker_id: "worker-final", + claim_token, + creator: &creator, + bundle_id: &bundle_id, + now, + candidate: winner.clone(), + }) + .await + .unwrap() + .unwrap(); + let replay = store + .issue_or_replay_final_credential_for_worker(FinalCredentialWorkerIssueRequest { + deletion_job_id: job_id, + worker_id: "worker-final", + claim_token, + creator: &creator, + bundle_id: &bundle_id, + now, + candidate: AccessCredential::new("worker-loser"), + }) + .await + .unwrap() + .unwrap(); + assert_eq!(issued.credential, winner); + assert_eq!(replay, issued); + assert_eq!(store.state.read().await.final_credentials.len(), 1); + } + + #[tokio::test] + async fn final_winner_paths_sample_time_after_waiting_for_state_fence() { + let initial = datetime!(2026-08-17 12:00:00 UTC); + let clock = Arc::new(MutableClock::new(initial)); + let job_id = Uuid::from_u128(7); + let claim_token = Uuid::from_u128(8); + let bundle_id = BundleId::from_str("000G40R40M30E209185GR38E1R").unwrap(); + let creator = record().creator; + + let public_store = Arc::new(final_materialization_store_with_clock(clock.clone()).await); + let public_guard = public_store.state.write().await; + let issuing_store = Arc::clone(&public_store); + let issuing_creator = creator.clone(); + let issuing_bundle = bundle_id.clone(); + let public_issue = tokio::spawn(async move { + issuing_store + .issue_or_replay_final_credential( + &issuing_creator, + &issuing_bundle, + initial, + AccessCredential::new("stale-public-candidate"), + ) + .await + }); + tokio::task::yield_now().await; + assert!(!public_issue.is_finished()); + clock.set(datetime!(2026-08-17 12:15:00 UTC)); + drop(public_guard); + assert!(public_issue.await.unwrap().unwrap().is_none()); + assert!(public_store.state.read().await.final_credentials.is_empty()); + + clock.set(initial); + let worker_store = Arc::new(final_materialization_store_with_clock(clock.clone()).await); + let worker_guard = worker_store.state.write().await; + let issuing_store = Arc::clone(&worker_store); + let worker_issue = tokio::spawn(async move { + issuing_store + .issue_or_replay_final_credential_for_worker(FinalCredentialWorkerIssueRequest { + deletion_job_id: job_id, + worker_id: "worker-final", + claim_token, + creator: &creator, + bundle_id: &bundle_id, + now: initial, + candidate: AccessCredential::new("stale-worker-candidate"), + }) + .await + }); + tokio::task::yield_now().await; + assert!(!worker_issue.is_finished()); + clock.set(datetime!(2026-08-17 12:05:00 UTC)); + drop(worker_guard); + assert!(worker_issue.await.unwrap().unwrap().is_none()); + assert!(worker_store.state.read().await.final_credentials.is_empty()); + } + + #[tokio::test] + async fn final_access_window_initialization_samples_time_after_waiting_for_state_fence() { + let initial = datetime!(2026-08-17 12:00:00 UTC); + let claim_expiry = datetime!(2026-08-17 12:05:00 UTC); + let clock = Arc::new(MutableClock::new(initial)); + let store = Arc::new(final_materialization_store_with_clock(clock.clone()).await); + { + let mut state = store.state.write().await; + let deletion = state.deletions.get_mut(&Uuid::from_u128(7)).unwrap(); + deletion.issuance_deadline = None; + deletion.read_deadline = None; + } + + let state_guard = store.state.write().await; + let initializing_store = Arc::clone(&store); + let initialization = tokio::spawn(async move { + initializing_store + .initialize_final_access_windows( + Uuid::from_u128(7), + "worker-final", + Uuid::from_u128(8), + Duration::minutes(15), + Duration::minutes(15), + ) + .await + }); + tokio::task::yield_now().await; + assert!(!initialization.is_finished()); + clock.set(claim_expiry); + drop(state_guard); + + assert_eq!( + initialization.await.unwrap().unwrap(), + InitializeFinalAccessWindowsResult::ClaimLost + ); + let state = store.state.read().await; + let deletion = state.deletions.get(&Uuid::from_u128(7)).unwrap(); + assert!(deletion.issuance_deadline.is_none()); + assert!(deletion.read_deadline.is_none()); + } + + #[tokio::test] + async fn final_read_prepare_samples_time_after_state_fence_and_rejects_deadline_equality() { + let initial = datetime!(2026-08-17 12:00:00 UTC); + let deadline = datetime!(2026-08-17 12:30:00 UTC); + let clock = Arc::new(MutableClock::new(initial)); + let store = Arc::new(final_materialization_store_with_clock(clock.clone()).await); + let credential = AccessCredential::new("blocked-final-read"); + let lookup = AccessCredentialLookupKey::derive(&credential); + let creator = record().creator; + let bundle = BundleId::from_str("000G40R40M30E209185GR38E1R").unwrap(); + store + .issue_or_replay_final_credential(&creator, &bundle, initial, credential) + .await + .unwrap() + .unwrap(); + + let state_guard = store.state.write().await; + let waiting_store = Arc::clone(&store); + let waiting_lookup = lookup.clone(); + let prepare = tokio::spawn(async move { + waiting_store + .prepare_deletion_read( + &waiting_lookup, + "/priv/locks.app/content/post.json", + Duration::seconds(30), + ) + .await + }); + tokio::task::yield_now().await; + assert!(!prepare.is_finished()); + clock.set(deadline); + drop(state_guard); + + assert!(prepare.await.unwrap().unwrap().is_none()); + let state = store.state.read().await; + let read = &state + .final_credentials + .get(&(Uuid::from_u128(7), bundle)) + .unwrap() + .reads["/priv/locks.app/content/post.json"]; + assert!(read.claim_token.is_none()); + assert!(read.consumed_at.is_none()); + } + + #[tokio::test] + async fn final_read_consume_samples_time_after_state_fence_and_rejects_claim_expiry_equality() { + let initial = datetime!(2026-08-17 12:00:00 UTC); + let claim_expiry = initial + Duration::seconds(30); + let clock = Arc::new(MutableClock::new(initial)); + let store = Arc::new(final_materialization_store_with_clock(clock.clone()).await); + let credential = AccessCredential::new("blocked-final-consume"); + let lookup = AccessCredentialLookupKey::derive(&credential); + let creator = record().creator; + let bundle = BundleId::from_str("000G40R40M30E209185GR38E1R").unwrap(); + store + .issue_or_replay_final_credential(&creator, &bundle, initial, credential) + .await + .unwrap() + .unwrap(); + let claim_token = store + .prepare_deletion_read( + &lookup, + "/priv/locks.app/content/post.json", + Duration::seconds(30), + ) + .await + .unwrap() + .unwrap() + .claim_token + .unwrap(); + + let state_guard = store.state.write().await; + let waiting_store = Arc::clone(&store); + let waiting_lookup = lookup.clone(); + let consume = tokio::spawn(async move { + waiting_store + .consume_deletion_read( + &waiting_lookup, + "/priv/locks.app/content/post.json", + claim_token, + ) + .await + }); + tokio::task::yield_now().await; + assert!(!consume.is_finished()); + clock.set(claim_expiry); + drop(state_guard); + + assert!(!consume.await.unwrap().unwrap()); + let state = store.state.read().await; + let read = &state + .final_credentials + .get(&(Uuid::from_u128(7), bundle)) + .unwrap() + .reads["/priv/locks.app/content/post.json"]; + assert!(read.consumed_at.is_none()); + } + + #[tokio::test] + async fn final_materialization_enumeration_does_not_create_eligibility() { + let store = final_materialization_store().await; + let job_id = Uuid::from_u128(7); + let ineligible = BundleId::from_str("000G40R40M30E209185GR38E1X").unwrap(); + { + let mut state = store.state.write().await; + let snapshot = state + .deletions + .get_mut(&job_id) + .unwrap() + .bundle_snapshots + .get_mut(&ineligible) + .unwrap(); + snapshot.resolved_status = Some(VerificationTaskStatus::Completed); + assert!(snapshot.final_credential_eligible_at.is_none()); + } + + let selected = store + .final_credentials_to_materialize(job_id, "worker-final", Uuid::from_u128(8), 10) + .await + .unwrap(); + assert!(!selected.iter().any(|item| item.bundle_id == ineligible)); + assert!( + store + .state + .read() + .await + .deletions + .get(&job_id) + .unwrap() + .bundle_snapshots + .get(&ineligible) + .unwrap() + .final_credential_eligible_at + .is_none() + ); + } + + struct MutableClock(StdMutex); + + impl MutableClock { + fn new(now: OffsetDateTime) -> Self { + Self(StdMutex::new(now)) + } + + fn set(&self, now: OffsetDateTime) { + *self.0.lock().unwrap() = now; + } + } + + impl crate::application::ports::Clock for MutableClock { + fn now(&self) -> OffsetDateTime { + *self.0.lock().unwrap() + } + } + + async fn final_materialization_store() -> InMemoryAccessCredentialStore { + final_materialization_store_at(datetime!(2026-08-17 12:00:00 UTC)).await + } + + async fn final_materialization_store_at(now: OffsetDateTime) -> InMemoryAccessCredentialStore { + final_materialization_store_with_clock(Arc::new(MutableClock::new(now))).await + } + + async fn final_materialization_store_with_clock( + clock: Arc, + ) -> InMemoryAccessCredentialStore { + let fence = Arc::new(InMemoryVerificationTaskDeletionFence::with_clock(clock)); + let store = InMemoryAccessCredentialStore::build(None, Some(fence)); + let creator = + CreatorPubky::from_str("pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy") + .unwrap(); + let eligible_at = datetime!(2026-08-17 11:00:00 UTC); + let snapshots = [ + ("000G40R40M30E209185GR38E1W", true), + ("000G40R40M30E209185GR38E1V", true), + ("000G40R40M30E209185GR38E1X", false), + ] + .into_iter() + .enumerate() + .map(|(index, (bundle, eligible))| { + ( + BundleId::from_str(bundle).unwrap(), + DeletionBundleSnapshot { + task_id: TaskId::from_str(&format!( + "550e8400-e29b-41d4-a716-44665544000{index}" + )) + .unwrap(), + paykit_admission_required: true, + had_active_credential_at_cutoff: false, + status_at_cutoff: VerificationTaskStatus::Completed, + resolved_status: Some(VerificationTaskStatus::Completed), + resolved_at: Some(eligible_at), + final_credential_eligible_at: eligible.then_some(eligible_at), + final_credential_issued: false, + }, + ) + }) + .collect(); + store.state.write().await.deletions.insert( + Uuid::from_u128(7), + DeletionAccessState { + creator, + lock_id: LockId::from_str("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG") + .unwrap(), + frozen_content_lock: test_content_lock(), + state: ContentLockDeletionState::Running, + phase: ContentLockDeletionPhase::IssueFinalCredentials, + force_requested: false, + claimed_by: Some("worker-final".to_owned()), + claim_token: Some(Uuid::from_u128(8)), + claim_expires_at: Some(datetime!(2026-08-17 12:05:00 UTC)), + issuance_started_at: Some(datetime!(2026-08-17 12:00:00 UTC)), + issuance_deadline: Some(datetime!(2026-08-17 12:15:00 UTC)), + read_deadline: Some(datetime!(2026-08-17 12:30:00 UTC)), + payment_aggregate: None, + bundle_snapshots: snapshots, + }, + ); + store + } + + fn test_content_lock() -> ContentLock { + ContentLock { + version: CONTENT_LOCK_VERSION, + creator: record().creator, + primary_resource: Some( + GuardedResource::new( + "/priv/locks.app/content/post.json".to_owned(), + GuardedResourceHash::from_bytes([7; 32]), + "application/json".to_owned(), + 42, + ) + .unwrap(), + ), + secondary_resources: BTreeMap::new(), + criteria: vec![], + lock_logic: LockLogic::All { criteria: vec![] }, + access_policy: AccessPolicy { + requested_credential_ttl_seconds: 900, + }, + lock_server: LockServerConfig { override_: None }, + created_at: datetime!(2026-08-17 11:00:00 UTC), + } + } + fn record() -> AccessCredentialRecord { AccessCredentialRecord { creator: CreatorPubky::from_str( diff --git a/locks-service/src/infrastructure/memory/content_lock_deletion_action_ownership.rs b/locks-service/src/infrastructure/memory/content_lock_deletion_action_ownership.rs new file mode 100644 index 0000000..a0fd6fa --- /dev/null +++ b/locks-service/src/infrastructure/memory/content_lock_deletion_action_ownership.rs @@ -0,0 +1,101 @@ +use std::{ + collections::HashSet, + sync::{Arc, Mutex}, +}; + +use async_trait::async_trait; +use uuid::Uuid; + +use crate::{ + application::{ + errors::ApplicationError, + ports::{ + ContentLockDeletionActionAcquireResult, ContentLockDeletionActionClaim, + ContentLockDeletionActionGuard, ContentLockDeletionActionOwnership, + }, + }, + infrastructure::memory::content_lock_deletions::InMemoryContentLockDeletionRepository, +}; + +/// Process-local parity adapter for per-job external action ownership. +#[derive(Debug, Clone)] +pub struct InMemoryContentLockDeletionActionOwnership { + deletions: Arc, + owned_jobs: Arc>>, +} + +impl InMemoryContentLockDeletionActionOwnership { + pub fn new(deletions: Arc) -> Self { + Self { + deletions, + owned_jobs: Arc::new(Mutex::new(HashSet::new())), + } + } +} + +#[async_trait] +impl ContentLockDeletionActionOwnership for InMemoryContentLockDeletionActionOwnership { + async fn try_acquire( + &self, + claim: ContentLockDeletionActionClaim<'_>, + ) -> Result { + { + let mut owned_jobs = self + .owned_jobs + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !owned_jobs.insert(claim.job_id) { + return Ok(ContentLockDeletionActionAcquireResult::Busy); + } + } + + if !self.deletions.action_claim_is_live(claim).await { + self.owned_jobs + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&claim.job_id); + return Ok(ContentLockDeletionActionAcquireResult::ClaimLost); + } + + Ok(ContentLockDeletionActionAcquireResult::Acquired(Box::new( + InMemoryContentLockDeletionActionGuard { + job_id: claim.job_id, + owned_jobs: Arc::clone(&self.owned_jobs), + released: false, + }, + ))) + } +} + +struct InMemoryContentLockDeletionActionGuard { + job_id: Uuid, + owned_jobs: Arc>>, + released: bool, +} + +impl InMemoryContentLockDeletionActionGuard { + fn release_inner(&mut self) { + if self.released { + return; + } + self.owned_jobs + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&self.job_id); + self.released = true; + } +} + +#[async_trait] +impl ContentLockDeletionActionGuard for InMemoryContentLockDeletionActionGuard { + async fn release(mut self: Box) -> Result<(), ApplicationError> { + self.release_inner(); + Ok(()) + } +} + +impl Drop for InMemoryContentLockDeletionActionGuard { + fn drop(&mut self) { + self.release_inner(); + } +} diff --git a/locks-service/src/infrastructure/memory/content_lock_deletions.rs b/locks-service/src/infrastructure/memory/content_lock_deletions.rs new file mode 100644 index 0000000..69c599e --- /dev/null +++ b/locks-service/src/infrastructure/memory/content_lock_deletions.rs @@ -0,0 +1,629 @@ +use std::{collections::HashMap, sync::Arc}; + +use async_trait::async_trait; +use locks_core::ids::{CreatorPubky, LockId}; +use time::OffsetDateTime; +use tokio::sync::{Mutex, RwLock}; +use uuid::Uuid; + +use crate::application::{ + errors::ApplicationError, + models::{ + AdvanceContentLockDeletionPhaseResult, ClaimedContentLockDeletionJob, + ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionPhase, + ContentLockDeletionState, PrepareForceDeletionResult, + }, + ports::{ContentLockDeletionActionClaim, ContentLockDeletionRepository}, +}; +use crate::infrastructure::memory::{ + access_credentials::{AccessPhaseAdvanceStatus, InMemoryAccessCredentialStore}, + verification_task_deletion_fence::InMemoryVerificationTaskDeletionFence, +}; + +type JobKey = (CreatorPubky, LockId); + +#[derive(Debug, Clone)] +struct StoredJob { + job: ContentLockDeletionJob, + claimed_by: Option, + claim_token: Option, + claim_expires_at: Option, +} + +/// In-memory deletion repository with the same lease-fencing semantics as PostgreSQL. +#[derive(Debug)] +pub struct InMemoryContentLockDeletionRepository { + jobs: RwLock>, + force_receipts: RwLock>, + publication_intents: RwLock>, + claim_transition_gate: Mutex<()>, + verification_task_fence: Arc, + access_credentials: Arc, +} + +impl Default for InMemoryContentLockDeletionRepository { + fn default() -> Self { + Self::with_access_credentials_and_verification_task_fence( + Arc::new(InMemoryAccessCredentialStore::new()), + Arc::new(InMemoryVerificationTaskDeletionFence::new()), + ) + } +} + +impl InMemoryContentLockDeletionRepository { + pub fn new() -> Self { + Self::default() + } + + pub fn with_verification_task_fence( + verification_task_fence: Arc, + ) -> Self { + Self::with_access_credentials_and_verification_task_fence( + Arc::new(InMemoryAccessCredentialStore::new()), + verification_task_fence, + ) + } + + pub fn with_access_credentials_and_verification_task_fence( + access_credentials: Arc, + verification_task_fence: Arc, + ) -> Self { + Self { + jobs: RwLock::new(HashMap::new()), + force_receipts: RwLock::new(HashMap::new()), + publication_intents: RwLock::new(HashMap::new()), + claim_transition_gate: Mutex::new(()), + verification_task_fence, + access_credentials, + } + } + + pub(super) async fn action_claim_is_live( + &self, + claim: ContentLockDeletionActionClaim<'_>, + ) -> bool { + let now = self.verification_task_fence.authoritative_cutoff(); + self.jobs.read().await.values().any(|stored| { + stored.job.job_id == claim.job_id + && stored.job.state == ContentLockDeletionState::Running + && stored.job.phase == claim.expected_phase + && stored.claimed_by.as_deref() == Some(claim.worker_id) + && stored.claim_token == Some(claim.claim_token) + && stored + .claim_expires_at + .is_some_and(|expires_at| expires_at > now) + && (stored.job.force_requested_at.is_some() == claim.force) + }) + } +} + +#[async_trait] +impl ContentLockDeletionRepository for InMemoryContentLockDeletionRepository { + async fn begin_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result<(), ApplicationError> { + let key = (creator.clone(), lock_id.clone()); + let mut intents = self.publication_intents.write().await; + let jobs = self.jobs.read().await; + let receipts = self.force_receipts.read().await; + if jobs.contains_key(&key) || receipts.contains_key(&key) { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + if intents.contains_key(&key) { + return Err(ApplicationError::ContentLockPathConflict { + guarded_path: "content lock publication in progress".to_owned(), + }); + } + intents.insert(key, publication_token); + Ok(()) + } + + async fn finish_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result { + remove_publication_intent( + &self.publication_intents, + creator, + lock_id, + publication_token, + ) + .await + } + + async fn abandon_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result { + remove_publication_intent( + &self.publication_intents, + creator, + lock_id, + publication_token, + ) + .await + } + + async fn publication_in_progress( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result { + Ok(self + .publication_intents + .read() + .await + .contains_key(&(creator.clone(), lock_id.clone()))) + } + + async fn insert_job(&self, mut job: ContentLockDeletionJob) -> Result<(), ApplicationError> { + job.validate_frozen_identity()?; + job.validate_state(false)?; + let key = (job.creator.clone(), job.lock_id.clone()); + let _admission = self + .verification_task_fence + .acquire_lock_admission(&job.creator, &job.lock_id) + .await; + job.deletion_started_at = self.verification_task_fence.authoritative_cutoff(); + let mut verification_tasks = self.verification_task_fence.records.write().await; + let intents = self.publication_intents.read().await; + let mut jobs = self.jobs.write().await; + let receipts = self.force_receipts.read().await; + if intents.contains_key(&key) || receipts.contains_key(&key) { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + if jobs.contains_key(&key) || jobs.values().any(|stored| stored.job.job_id == job.job_id) { + return Err(ApplicationError::DuplicateRecord { + record: "content_lock_deletion_job", + }); + } + let matching_task_ids = verification_tasks + .iter() + .filter_map(|(task_id, task)| { + (task.creator == job.creator && task.lock_id == job.lock_id).then_some(*task_id) + }) + .collect::>(); + let snapshot_bundles = matching_task_ids + .iter() + .filter_map(|task_id| { + verification_tasks.get(task_id).map(|task| { + ( + task.bundle_id.clone(), + (*task_id, task.paykit_admission_required, task.status), + ) + }) + }) + .collect::>(); + if matching_task_ids.iter().any(|task_id| { + verification_tasks + .get(task_id) + .is_some_and(|task| task.entitlement_publication_claim_token.is_some()) + }) { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + self.access_credentials + .register_deletion(&job, &snapshot_bundles) + .await?; + for task_id in matching_task_ids { + if let Some(task) = verification_tasks.get_mut(&task_id) { + task.deletion_job_id = Some(job.job_id); + } + } + jobs.insert( + key, + StoredJob { + job, + claimed_by: None, + claim_token: None, + claim_expires_at: None, + }, + ); + Ok(()) + } + + async fn get_job( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result, ApplicationError> { + let stored = self + .jobs + .read() + .await + .get(&(creator.clone(), lock_id.clone())) + .cloned(); + if let Some(stored) = stored { + stored.job.validate_frozen_identity()?; + let has_active_lease = stored.claimed_by.is_some() + && stored.claim_token.is_some() + && stored.claim_expires_at.is_some(); + stored.job.validate_state(has_active_lease)?; + Ok(Some(stored.job)) + } else { + Ok(None) + } + } + + async fn claim_next( + &self, + worker_id: &str, + claim_ttl: time::Duration, + ) -> Result, ApplicationError> { + let _claim_transition = self.claim_transition_gate.lock().await; + let mut jobs = self.jobs.write().await; + let now = self.verification_task_fence.authoritative_cutoff(); + let claim_expires_at = now + claim_ttl; + let Some(stored) = jobs + .values_mut() + .filter(|stored| is_claimable(stored, now)) + .min_by_key(|stored| stored.job.deletion_started_at) + else { + return Ok(None); + }; + let claim_token = Uuid::new_v4(); + stored.job.state = ContentLockDeletionState::Running; + stored.job.attempt_count = stored.job.attempt_count.saturating_add(1); + stored.job.next_attempt_at = None; + stored.claimed_by = Some(worker_id.to_owned()); + stored.claim_token = Some(claim_token); + stored.claim_expires_at = Some(claim_expires_at); + self.access_credentials + .synchronize_job( + &stored.job, + stored.claimed_by.as_deref(), + stored.claim_token, + stored.claim_expires_at, + ) + .await; + Ok(Some(ClaimedContentLockDeletionJob { + job: stored.job.clone(), + claim_token, + })) + } + + async fn schedule_retry( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + retry_after: time::Duration, + ) -> Result, ApplicationError> { + let _claim_transition = self.claim_transition_gate.lock().await; + let mut jobs = self.jobs.write().await; + let now = self.verification_task_fence.authoritative_cutoff(); + let next_attempt_at = now + retry_after; + let Some(stored) = jobs + .values_mut() + .find(|stored| owns_claim(stored, job_id, worker_id, claim_token, now)) + else { + return Ok(None); + }; + stored.job.state = ContentLockDeletionState::Queued; + stored.job.next_attempt_at = Some(next_attempt_at); + clear_claim(stored); + self.access_credentials + .synchronize_job(&stored.job, None, None, None) + .await; + Ok(Some(stored.job.clone())) + } + + async fn defer( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + defer_for: time::Duration, + ) -> Result, ApplicationError> { + let _claim_transition = self.claim_transition_gate.lock().await; + let mut jobs = self.jobs.write().await; + let now = self.verification_task_fence.authoritative_cutoff(); + let next_attempt_at = now + defer_for; + let Some(stored) = jobs + .values_mut() + .find(|stored| owns_claim(stored, job_id, worker_id, claim_token, now)) + else { + return Ok(None); + }; + stored.job.attempt_count = stored.job.attempt_count.saturating_sub(1); + stored.job.state = ContentLockDeletionState::Queued; + stored.job.next_attempt_at = Some(next_attempt_at); + clear_claim(stored); + self.access_credentials + .synchronize_job(&stored.job, None, None, None) + .await; + Ok(Some(stored.job.clone())) + } + + async fn advance_phase( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + next_phase: ContentLockDeletionPhase, + ) -> Result { + let _claim_transition = self.claim_transition_gate.lock().await; + let mut jobs = self.jobs.write().await; + let now = self.verification_task_fence.authoritative_cutoff(); + let Some(stored) = jobs + .values_mut() + .find(|stored| owns_claim(stored, job_id, worker_id, claim_token, now)) + else { + return Ok(AdvanceContentLockDeletionPhaseResult::ClaimLost); + }; + if !stored.job.phase.permits(next_phase) { + return Err(ApplicationError::InvalidContentLockDeletionState { + message: "deletion phase must advance to its immediate successor".to_owned(), + }); + } + let access_status = self + .access_credentials + .check_phase_advance(job_id, stored.job.phase, next_phase, now) + .await?; + match access_status { + AccessPhaseAdvanceStatus::Ready => {} + AccessPhaseAdvanceStatus::ObligationsPending => { + return Ok(AdvanceContentLockDeletionPhaseResult::ObligationsPending); + } + AccessPhaseAdvanceStatus::FinalCredentialIssuanceMissed => { + return Ok(AdvanceContentLockDeletionPhaseResult::TerminalFailure( + ContentLockDeletionFailureCode::StateCorrupt, + )); + } + } + stored.job.phase = next_phase; + stored.job.state = ContentLockDeletionState::Queued; + stored.job.attempt_count = 0; + stored.job.next_attempt_at = None; + stored.job.failure_code = None; + clear_claim(stored); + self.access_credentials + .synchronize_job(&stored.job, None, None, None) + .await; + Ok(AdvanceContentLockDeletionPhaseResult::Advanced(Box::new( + stored.job.clone(), + ))) + } + + async fn expire_unresolved_non_paykit_tasks( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + ) -> Result { + let _claim_transition = self.claim_transition_gate.lock().await; + let now = self.verification_task_fence.authoritative_cutoff(); + self.access_credentials + .expire_unresolved_non_paykit_tasks(job_id, worker_id, claim_token, now) + .await + } + + async fn finish( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + failure_code: Option, + ) -> Result, ApplicationError> { + let _claim_transition = self.claim_transition_gate.lock().await; + let mut jobs = self.jobs.write().await; + let now = self.verification_task_fence.authoritative_cutoff(); + let Some(stored) = jobs + .values_mut() + .find(|stored| owns_claim(stored, job_id, worker_id, claim_token, now)) + else { + return Ok(None); + }; + if failure_code.is_none() { + self.access_credentials + .check_successful_finish(job_id, stored.job.phase, now) + .await?; + } + stored.job.state = if failure_code.is_some() { + ContentLockDeletionState::Failed + } else { + ContentLockDeletionState::Completed + }; + stored.job.failure_code = failure_code; + stored.job.next_attempt_at = None; + clear_claim(stored); + self.access_credentials + .synchronize_job(&stored.job, None, None, None) + .await; + Ok(Some(stored.job.clone())) + } + + async fn resume_failed_job( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + _resumed_at: OffsetDateTime, + ) -> Result, ApplicationError> { + let _claim_transition = self.claim_transition_gate.lock().await; + let mut jobs = self.jobs.write().await; + let receipts = self.force_receipts.read().await; + if receipts.contains_key(&(creator.clone(), lock_id.clone())) { + return Ok(None); + } + let Some(stored) = jobs.get_mut(&(creator.clone(), lock_id.clone())) else { + return Ok(None); + }; + if stored.job.state == ContentLockDeletionState::Failed { + stored.job.state = ContentLockDeletionState::Queued; + stored.job.attempt_count = 0; + stored.job.next_attempt_at = None; + stored.job.failure_code = None; + clear_claim(stored); + } + self.access_credentials + .synchronize_job(&stored.job, None, None, None) + .await; + Ok(Some(stored.job.clone())) + } + + async fn prepare_force_deletion( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result { + let _claim_transition = self.claim_transition_gate.lock().await; + let forced_at = self.verification_task_fence.authoritative_cutoff(); + let key = (creator.clone(), lock_id.clone()); + let intents = self.publication_intents.read().await; + if intents.contains_key(&key) { + return Ok(PrepareForceDeletionResult::PublicationInProgress); + } + let verification_tasks = self.verification_task_fence.records.read().await; + if verification_tasks.values().any(|task| { + task.creator == *creator + && task.lock_id == *lock_id + && task.deletion_job_id.is_some() + && task.entitlement_publication_claim_token.is_some() + }) { + return Ok(PrepareForceDeletionResult::PublicationInProgress); + } + drop(verification_tasks); + let mut jobs = self.jobs.write().await; + let mut receipts = self.force_receipts.write().await; + if receipts.contains_key(&key) { + return Ok(PrepareForceDeletionResult::Synchronous( + jobs.get(&key).map(|stored| stored.job.clone()), + )); + } + if let Some(stored) = jobs.get_mut(&key) { + if matches!( + stored.job.state, + ContentLockDeletionState::Queued | ContentLockDeletionState::Running + ) { + stored.job.force_requested_at.get_or_insert(forced_at); + stored.job.state = ContentLockDeletionState::Queued; + stored.job.next_attempt_at = None; + clear_claim(stored); + self.access_credentials + .synchronize_job(&stored.job, None, None, None) + .await; + return Ok(PrepareForceDeletionResult::Active(stored.job.clone())); + } + let job = stored.job.clone(); + self.access_credentials + .block_key_and_disable_job(creator, lock_id, Some(job.job_id)) + .await; + jobs.remove(&key); + receipts.insert(key, forced_at); + return Ok(PrepareForceDeletionResult::Synchronous(Some(job))); + } + self.access_credentials + .block_key_and_disable_job(creator, lock_id, None) + .await; + receipts.insert(key, forced_at); + Ok(PrepareForceDeletionResult::Synchronous(None)) + } + + async fn complete_force_deletion( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + ) -> Result { + let _claim_transition = self.claim_transition_gate.lock().await; + let now = self.verification_task_fence.authoritative_cutoff(); + let key = self + .jobs + .read() + .await + .iter() + .find_map(|(key, stored)| (stored.job.job_id == job_id).then(|| key.clone())); + let Some(key) = key else { + return Ok(false); + }; + let _admission = self + .verification_task_fence + .acquire_lock_admission(&key.0, &key.1) + .await; + let mut jobs = self.jobs.write().await; + let mut receipts = self.force_receipts.write().await; + let Some(stored) = jobs.get(&key) else { + return Ok(false); + }; + if !owns_claim(stored, job_id, worker_id, claim_token, now) { + return Ok(false); + } + let Some(forced_at) = stored.job.force_requested_at else { + return Ok(false); + }; + self.access_credentials + .block_key_and_disable_job(&key.0, &key.1, Some(job_id)) + .await; + receipts.insert(key.clone(), forced_at); + jobs.remove(&key); + Ok(true) + } + + async fn has_force_receipt( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result { + Ok(self + .force_receipts + .read() + .await + .contains_key(&(creator.clone(), lock_id.clone()))) + } +} + +async fn remove_publication_intent( + intents: &RwLock>, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, +) -> Result { + let key = (creator.clone(), lock_id.clone()); + let mut intents = intents.write().await; + if intents.get(&key) != Some(&publication_token) { + return Ok(false); + } + intents.remove(&key); + Ok(true) +} + +fn is_claimable(stored: &StoredJob, now: OffsetDateTime) -> bool { + match stored.job.state { + ContentLockDeletionState::Queued => stored + .job + .next_attempt_at + .is_none_or(|next_attempt_at| next_attempt_at <= now), + ContentLockDeletionState::Running => stored + .claim_expires_at + .is_some_and(|claim_expires_at| claim_expires_at <= now), + ContentLockDeletionState::Completed | ContentLockDeletionState::Failed => false, + } +} + +fn owns_claim( + stored: &StoredJob, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, +) -> bool { + stored.job.job_id == job_id + && stored.job.state == ContentLockDeletionState::Running + && stored.claimed_by.as_deref() == Some(worker_id) + && stored.claim_token == Some(claim_token) + && stored + .claim_expires_at + .is_some_and(|claim_expires_at| now < claim_expires_at) +} + +fn clear_claim(stored: &mut StoredJob) { + stored.claimed_by = None; + stored.claim_token = None; + stored.claim_expires_at = None; +} diff --git a/locks-service/src/infrastructure/memory/content_lock_ownership.rs b/locks-service/src/infrastructure/memory/content_lock_ownership.rs new file mode 100644 index 0000000..6b0f35e --- /dev/null +++ b/locks-service/src/infrastructure/memory/content_lock_ownership.rs @@ -0,0 +1,119 @@ +use std::collections::HashMap; + +use async_trait::async_trait; +use locks_core::ids::{CreatorPubky, LockId}; +use tokio::sync::RwLock; + +use crate::application::errors::ApplicationError; +use crate::application::models::{ContentLockOwnership, ContentLockOwnershipStatus}; +use crate::application::ports::ContentLockOwnershipRepository; + +type OwnershipKey = (CreatorPubky, String); + +/// In-memory exclusive guarded-path ownership repository for tests and ephemeral runtime. +#[derive(Debug, Default)] +pub struct InMemoryContentLockOwnershipRepository { + records: RwLock>, +} + +impl InMemoryContentLockOwnershipRepository { + /// Creates an empty repository. + pub fn new() -> Self { + Self::default() + } +} + +#[async_trait] +impl ContentLockOwnershipRepository for InMemoryContentLockOwnershipRepository { + async fn reserve_paths( + &self, + creator: &CreatorPubky, + guarded_paths: &[String], + lock_id: &LockId, + ) -> Result<(), ApplicationError> { + let mut records = self.records.write().await; + for guarded_path in guarded_paths { + if let Some(existing) = records.get(&(creator.clone(), guarded_path.clone())) + && (existing.lock_id != *lock_id + || existing.status == ContentLockOwnershipStatus::Reserved) + { + return Err(ApplicationError::ContentLockPathConflict { + guarded_path: guarded_path.clone(), + }); + } + } + + for guarded_path in guarded_paths { + records + .entry((creator.clone(), guarded_path.clone())) + .or_insert_with(|| ContentLockOwnership { + creator: creator.clone(), + guarded_path: guarded_path.clone(), + lock_id: lock_id.clone(), + status: ContentLockOwnershipStatus::Reserved, + }); + } + Ok(()) + } + + async fn mark_paths_published( + &self, + creator: &CreatorPubky, + guarded_paths: &[String], + lock_id: &LockId, + ) -> Result<(), ApplicationError> { + let mut records = self.records.write().await; + for guarded_path in guarded_paths { + let Some(ownership) = records.get(&(creator.clone(), guarded_path.clone())) else { + return Err(ApplicationError::MissingRecord { + record: "content_lock_ownership", + }); + }; + if ownership.lock_id != *lock_id { + return Err(ApplicationError::ContentLockPathConflict { + guarded_path: guarded_path.clone(), + }); + } + } + for guarded_path in guarded_paths { + let ownership = records + .get_mut(&(creator.clone(), guarded_path.clone())) + .expect("ownership set was validated while holding the write lock"); + ownership.status = ContentLockOwnershipStatus::Published; + } + Ok(()) + } + + async fn compensate_reserved_paths( + &self, + creator: &CreatorPubky, + guarded_paths: &[String], + lock_id: &LockId, + ) -> Result<(), ApplicationError> { + let mut records = self.records.write().await; + for guarded_path in guarded_paths { + let key = (creator.clone(), guarded_path.clone()); + let remove = records.get(&key).is_some_and(|ownership| { + ownership.lock_id == *lock_id + && ownership.status == ContentLockOwnershipStatus::Reserved + }); + if remove { + records.remove(&key); + } + } + Ok(()) + } + + async fn get_path_ownership( + &self, + creator: &CreatorPubky, + guarded_path: &str, + ) -> Result, ApplicationError> { + Ok(self + .records + .read() + .await + .get(&(creator.clone(), guarded_path.to_owned())) + .cloned()) + } +} diff --git a/locks-service/src/infrastructure/memory/content_lock_tombstones.rs b/locks-service/src/infrastructure/memory/content_lock_tombstones.rs new file mode 100644 index 0000000..0b61a99 --- /dev/null +++ b/locks-service/src/infrastructure/memory/content_lock_tombstones.rs @@ -0,0 +1,98 @@ +use async_trait::async_trait; +use locks_core::content_lock_deletion::ContentLockDeletionTombstone; +use locks_core::ids::{ContentLockPath, CreatorPubky}; +use locks_core::lock_policy::ContentLock; + +use crate::application::errors::ApplicationError; +use crate::application::ports::content_lock_tombstone::{ + canonical_tombstone_bytes, classify_tombstone_bytes, +}; +use crate::application::ports::{ContentLockTombstoneRepository, TombstoneReadback}; +use crate::infrastructure::memory::public_content_locks::InMemoryPublicContentLockStore; + +/// In-memory exact-byte public tombstone adapter. +#[derive(Debug, Default)] +pub struct InMemoryContentLockTombstoneRepository { + public_store: InMemoryPublicContentLockStore, +} + +impl InMemoryContentLockTombstoneRepository { + pub fn new() -> Self { + Self::default() + } + + /// Creates a raw tombstone adapter over shared canonical public-path storage. + pub fn with_public_store(public_store: InMemoryPublicContentLockStore) -> Self { + Self { public_store } + } +} + +#[async_trait] +impl ContentLockTombstoneRepository for InMemoryContentLockTombstoneRepository { + async fn withdraw_content_lock( + &self, + creator: CreatorPubky, + content_lock_path: ContentLockPath, + frozen_original: &ContentLock, + tombstone: &ContentLockDeletionTombstone, + ) -> Result { + let expected = canonical_tombstone_bytes(tombstone)?; + let original = + frozen_original + .canonical_json_bytes() + .map_err(|error| ApplicationError::Storage { + message: format!("failed to serialize frozen content lock: {error}"), + })?; + match self + .public_store + .get(&creator, &content_lock_path) + .await + .as_deref() + { + Some(actual) if actual == expected => return Ok(TombstoneReadback::Exact), + Some(actual) if actual == original => {} + None => return Ok(TombstoneReadback::Missing), + Some(_) => return Ok(TombstoneReadback::Replaced), + } + self.public_store + .put(creator.clone(), content_lock_path.clone(), expected) + .await; + self.read_tombstone(&creator, &content_lock_path, tombstone) + .await + } + + async fn read_tombstone( + &self, + creator: &CreatorPubky, + content_lock_path: &ContentLockPath, + expected: &ContentLockDeletionTombstone, + ) -> Result { + let expected = canonical_tombstone_bytes(expected)?; + Ok(classify_tombstone_bytes( + self.public_store + .get(creator, content_lock_path) + .await + .as_deref(), + &expected, + )) + } + + async fn force_delete_content_lock_and_verify_absent( + &self, + creator: &CreatorPubky, + content_lock_path: &ContentLockPath, + ) -> Result<(), ApplicationError> { + self.public_store.remove(creator, content_lock_path).await; + if self + .public_store + .get(creator, content_lock_path) + .await + .is_some() + { + return Err(ApplicationError::Storage { + message: "forced public content lock deletion did not reach absence".to_owned(), + }); + } + Ok(()) + } +} diff --git a/locks-service/src/infrastructure/memory/content_locks.rs b/locks-service/src/infrastructure/memory/content_locks.rs index bbe9229..fb0fb38 100644 --- a/locks-service/src/infrastructure/memory/content_locks.rs +++ b/locks-service/src/infrastructure/memory/content_locks.rs @@ -1,20 +1,16 @@ -use std::collections::HashMap; - use async_trait::async_trait; -use tokio::sync::RwLock; use locks_core::ids::{ContentLockPath, CreatorPubky}; use locks_core::lock_policy::ContentLock; use crate::application::errors::ApplicationError; use crate::application::ports::ContentLockRepository; - -type ContentLockKey = (CreatorPubky, ContentLockPath); +use crate::infrastructure::memory::public_content_locks::InMemoryPublicContentLockStore; /// In-memory content lock repository for the first retrieval/access slice. #[derive(Debug, Default)] pub struct InMemoryContentLockRepository { - records: RwLock>, + public_store: InMemoryPublicContentLockStore, } impl InMemoryContentLockRepository { @@ -22,6 +18,11 @@ impl InMemoryContentLockRepository { pub fn new() -> Self { Self::default() } + + /// Creates a typed adapter over shared canonical public-path storage. + pub fn with_public_store(public_store: InMemoryPublicContentLockStore) -> Self { + Self { public_store } + } } #[async_trait] @@ -32,10 +33,15 @@ impl ContentLockRepository for InMemoryContentLockRepository { content_lock_path: ContentLockPath, content_lock: ContentLock, ) -> Result<(), ApplicationError> { - self.records - .write() - .await - .insert((creator, content_lock_path), content_lock); + let bytes = + content_lock + .canonical_json_bytes() + .map_err(|error| ApplicationError::Storage { + message: format!("failed to serialize in-memory content lock: {error}"), + })?; + self.public_store + .put(creator, content_lock_path, bytes) + .await; Ok(()) } @@ -44,12 +50,27 @@ impl ContentLockRepository for InMemoryContentLockRepository { creator: &CreatorPubky, content_lock_path: &ContentLockPath, ) -> Result, ApplicationError> { + self.public_store + .get(creator, content_lock_path) + .await + .map(|bytes| { + serde_json::from_slice(&bytes).map_err(|error| ApplicationError::Storage { + message: format!("failed to deserialize in-memory content lock: {error}"), + }) + }) + .transpose() + } + + async fn delete_content_lock( + &self, + creator: &CreatorPubky, + content_lock_path: &ContentLockPath, + ) -> Result { Ok(self - .records - .read() + .public_store + .remove(creator, content_lock_path) .await - .get(&(creator.clone(), content_lock_path.clone())) - .cloned()) + .is_some()) } } diff --git a/locks-service/src/infrastructure/memory/mod.rs b/locks-service/src/infrastructure/memory/mod.rs index 437553b..3351b28 100644 --- a/locks-service/src/infrastructure/memory/mod.rs +++ b/locks-service/src/infrastructure/memory/mod.rs @@ -1,7 +1,13 @@ pub mod access_credentials; +pub mod content_lock_deletion_action_ownership; +pub mod content_lock_deletions; +pub mod content_lock_ownership; +pub mod content_lock_tombstones; pub mod content_locks; pub mod entitlements; pub mod guarded_resources; pub mod lock_service_pointers; +pub mod public_content_locks; pub mod verification_task_claims; +pub mod verification_task_deletion_fence; pub mod verification_tasks; diff --git a/locks-service/src/infrastructure/memory/public_content_locks.rs b/locks-service/src/infrastructure/memory/public_content_locks.rs new file mode 100644 index 0000000..006c35f --- /dev/null +++ b/locks-service/src/infrastructure/memory/public_content_locks.rs @@ -0,0 +1,45 @@ +use std::{collections::HashMap, sync::Arc}; + +use locks_core::ids::{ContentLockPath, CreatorPubky}; +use tokio::sync::RwLock; + +pub(crate) type PublicContentLockKey = (CreatorPubky, ContentLockPath); + +/// Shared in-memory backing for the one canonical public content-lock path. +#[derive(Debug, Clone, Default)] +pub struct InMemoryPublicContentLockStore { + records: Arc>>>, +} + +impl InMemoryPublicContentLockStore { + pub fn new() -> Self { + Self::default() + } + + pub(crate) async fn put(&self, creator: CreatorPubky, path: ContentLockPath, bytes: Vec) { + self.records.write().await.insert((creator, path), bytes); + } + + pub(crate) async fn get( + &self, + creator: &CreatorPubky, + path: &ContentLockPath, + ) -> Option> { + self.records + .read() + .await + .get(&(creator.clone(), path.clone())) + .cloned() + } + + pub(crate) async fn remove( + &self, + creator: &CreatorPubky, + path: &ContentLockPath, + ) -> Option> { + self.records + .write() + .await + .remove(&(creator.clone(), path.clone())) + } +} diff --git a/locks-service/src/infrastructure/memory/verification_task_claims.rs b/locks-service/src/infrastructure/memory/verification_task_claims.rs index 377a826..c6f4745 100644 --- a/locks-service/src/infrastructure/memory/verification_task_claims.rs +++ b/locks-service/src/infrastructure/memory/verification_task_claims.rs @@ -9,12 +9,14 @@ use crate::application::models::{ ClaimedVerificationTask, VerificationTaskRecord, VerificationTaskStatus, }; use crate::application::ports::{VerificationTaskClaimer, VerificationTaskRepository}; +use crate::infrastructure::memory::verification_task_deletion_fence::InMemoryVerificationTaskDeletionFence; /// In-memory verification task claimer used to model worker lease semantics. #[derive(Default)] pub struct InMemoryVerificationTaskClaimer { records: RwLock>, task_repository: Option>, + deletion_fence: Arc, } #[derive(Debug, Clone)] @@ -29,6 +31,14 @@ struct ClaimableVerificationTask { impl InMemoryVerificationTaskClaimer { /// Creates a claimer seeded with unclaimed task records. pub fn new(records: Vec) -> Self { + let deletion_fence = Arc::new(InMemoryVerificationTaskDeletionFence::from_tasks(&records)); + Self::with_deletion_fence(records, deletion_fence) + } + + pub fn with_deletion_fence( + records: Vec, + deletion_fence: Arc, + ) -> Self { Self { records: RwLock::new( records @@ -43,6 +53,7 @@ impl InMemoryVerificationTaskClaimer { .collect(), ), task_repository: None, + deletion_fence, } } @@ -56,10 +67,34 @@ impl InMemoryVerificationTaskClaimer { claimer } + pub fn with_task_repository_and_deletion_fence( + records: Vec, + task_repository: Arc, + deletion_fence: Arc, + ) -> Self { + let mut claimer = Self::with_deletion_fence(records, deletion_fence); + claimer.task_repository = Some(task_repository); + claimer + } + /// Creates a claimer seeded with already-claimed task records. pub fn with_claimed_tasks( records: Vec<(VerificationTaskRecord, String, time::OffsetDateTime)>, ) -> Self { + Self::with_claimed_tasks_and_clock( + records, + Arc::new(crate::infrastructure::memory::verification_task_deletion_fence::SystemClock), + ) + } + + fn with_claimed_tasks_and_clock( + records: Vec<(VerificationTaskRecord, String, time::OffsetDateTime)>, + clock: Arc, + ) -> Self { + let tasks = records + .iter() + .map(|(task, _, _)| task.clone()) + .collect::>(); Self { records: RwLock::new( records @@ -76,23 +111,60 @@ impl InMemoryVerificationTaskClaimer { .collect(), ), task_repository: None, + deletion_fence: Arc::new( + InMemoryVerificationTaskDeletionFence::from_tasks_with_clock(&tasks, clock), + ), } } } #[async_trait] impl VerificationTaskClaimer for InMemoryVerificationTaskClaimer { + async fn begin_claimed_entitlement_publication( + &self, + task_id: &TaskId, + worker_id: &str, + claim_token: &uuid::Uuid, + ) -> Result { + let mut fence_records = self.deletion_fence.records.write().await; + let records = self.records.read().await; + let now = self.deletion_fence.authoritative_cutoff(); + let owned = records.iter().any(|record| { + record.task.task_id == *task_id + && record.task.status == VerificationTaskStatus::InProgress + && record.claimed_by.as_deref() == Some(worker_id) + && record.claim_token.as_ref() == Some(claim_token) + && record.claim_expires_at.is_some_and(|expires| now < expires) + }); + let Some(fence) = fence_records.get_mut(task_id) else { + return Ok(false); + }; + if !owned || fence.deletion_job_id.is_some() { + return Ok(false); + } + fence.entitlement_publication_claim_token = Some(*claim_token); + Ok(true) + } + async fn claim_next_verification_task( &self, worker_id: &str, - now: time::OffsetDateTime, - claim_expires_at: time::OffsetDateTime, + claim_ttl: time::Duration, ) -> Result, ApplicationError> { + let fence_records = self.deletion_fence.records.read().await; let mut records = self.records.write().await; - let Some(index) = records - .iter() - .position(|record| record.is_claimable_at(now)) - else { + let now = self.deletion_fence.authoritative_cutoff(); + let claim_expires_at = + now.checked_add(claim_ttl) + .ok_or_else(|| ApplicationError::Storage { + message: "verification task claim expiry overflow".to_owned(), + })?; + let Some(index) = records.iter().position(|record| { + record.is_claimable_at(now) + && fence_records + .get(&record.task.task_id) + .is_some_and(|fence| fence.deletion_job_id.is_none()) + }) else { return Ok(None); }; @@ -126,10 +198,22 @@ impl VerificationTaskClaimer for InMemoryVerificationTaskClaimer { task_id: &TaskId, worker_id: &str, claim_token: &uuid::Uuid, - now: time::OffsetDateTime, - next_attempt_at: time::OffsetDateTime, + retry_after: time::Duration, ) -> Result, ApplicationError> { + let fence_records = self.deletion_fence.records.read().await; + if fence_records + .get(task_id) + .is_none_or(|fence| fence.deletion_job_id.is_some()) + { + return Ok(None); + } let mut records = self.records.write().await; + let now = self.deletion_fence.authoritative_cutoff(); + let next_attempt_at = + now.checked_add(retry_after) + .ok_or_else(|| ApplicationError::Storage { + message: "verification task retry time overflow".to_owned(), + })?; let Some(record) = records.iter_mut().find(|record| { record.task.task_id == *task_id && record.task.status == VerificationTaskStatus::InProgress @@ -137,7 +221,7 @@ impl VerificationTaskClaimer for InMemoryVerificationTaskClaimer { && record.claim_token.as_ref() == Some(claim_token) && record .claim_expires_at - .is_some_and(|claim_expires_at| claim_expires_at >= now) + .is_some_and(|claim_expires_at| now < claim_expires_at) }) else { return Ok(None); }; @@ -163,7 +247,6 @@ impl VerificationTaskClaimer for InMemoryVerificationTaskClaimer { task: VerificationTaskRecord, worker_id: &str, claim_token: &uuid::Uuid, - now: time::OffsetDateTime, ) -> Result, ApplicationError> { if !matches!( task.status, @@ -175,7 +258,15 @@ impl VerificationTaskClaimer for InMemoryVerificationTaskClaimer { message: "claimed task transition must be terminal".to_owned(), }); } + let mut fence_records = self.deletion_fence.records.write().await; + if fence_records + .get(&task.task_id) + .is_none_or(|fence| fence.deletion_job_id.is_some()) + { + return Ok(None); + } let mut records = self.records.write().await; + let now = self.deletion_fence.authoritative_cutoff(); let Some(record) = records.iter_mut().find(|record| { record.task.task_id == task.task_id && record.task.status == VerificationTaskStatus::InProgress @@ -183,7 +274,7 @@ impl VerificationTaskClaimer for InMemoryVerificationTaskClaimer { && record.claim_token.as_ref() == Some(claim_token) && record .claim_expires_at - .is_some_and(|claim_expires_at| claim_expires_at >= now) + .is_some_and(|claim_expires_at| now < claim_expires_at) }) else { return Ok(None); }; @@ -197,6 +288,9 @@ impl VerificationTaskClaimer for InMemoryVerificationTaskClaimer { record.claim_token = None; record.claim_expires_at = None; record.next_attempt_at = None; + if let Some(fence) = fence_records.get_mut(&record.task.task_id) { + fence.entitlement_publication_claim_token = None; + } Ok(Some(record.task.clone())) } } @@ -209,7 +303,7 @@ impl ClaimableVerificationTask { .is_none_or(|next_attempt_at| next_attempt_at <= now), VerificationTaskStatus::InProgress => self .claim_expires_at - .is_some_and(|claim_expires_at| claim_expires_at < now), + .is_some_and(|claim_expires_at| claim_expires_at <= now), VerificationTaskStatus::Completed | VerificationTaskStatus::Failed | VerificationTaskStatus::Expired => false, @@ -219,47 +313,310 @@ impl ClaimableVerificationTask { #[cfg(test)] mod tests { - use std::{str::FromStr, sync::Arc}; + use std::{ + collections::BTreeMap, + str::FromStr, + sync::{Arc, Mutex}, + }; use serde_json::json; use time::macros::datetime; - use locks_core::ids::{BundleId, CreatorPubky, PubkyLockResource, TaskId}; - use locks_core::lock_policy::VerifierType; + use locks_core::ids::{ + BundleId, CreatorPubky, GuardedResourceHash, LockId, PubkyLockResource, TaskId, + }; + use locks_core::lock_policy::{ + AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, GuardedResource, LockLogic, + LockServerConfig, VerifierType, + }; use locks_core::verification::{Proof, SUBMITTED_PROOF_BUNDLE_VERSION, SubmittedProofBundle}; use super::InMemoryVerificationTaskClaimer; - use crate::application::models::{VerificationTaskRecord, VerificationTaskStatus}; - use crate::application::ports::{VerificationTaskClaimer, VerificationTaskRepository}; + use crate::application::errors::ApplicationError; + use crate::application::models::{ + ContentLockDeletionJob, PrepareForceDeletionResult, VerificationTaskRecord, + VerificationTaskStatus, + }; + use crate::application::ports::{ + Clock, ContentLockDeletionRepository, VerificationTaskClaimer, VerificationTaskRepository, + }; + use crate::infrastructure::memory::content_lock_deletions::InMemoryContentLockDeletionRepository; + use crate::infrastructure::memory::verification_task_deletion_fence::InMemoryVerificationTaskDeletionFence; use crate::infrastructure::memory::verification_tasks::InMemoryVerificationTaskRepository; const LOCK_ID: &str = "000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG"; const NOW: time::OffsetDateTime = datetime!(2026-05-29 12:10:00 UTC); const CLAIM_EXPIRES_AT: time::OffsetDateTime = datetime!(2026-05-29 12:15:00 UTC); + #[derive(Debug)] + struct MutableClock(Mutex); + + impl MutableClock { + fn new(now: time::OffsetDateTime) -> Self { + Self(Mutex::new(now)) + } + + fn set(&self, now: time::OffsetDateTime) { + *self.0.lock().unwrap() = now; + } + } + + impl Clock for MutableClock { + fn now(&self) -> time::OffsetDateTime { + *self.0.lock().unwrap() + } + } + + fn claimer_with_clock( + records: Vec, + ) -> (InMemoryVerificationTaskClaimer, Arc) { + let clock = Arc::new(MutableClock::new(NOW)); + let fence = Arc::new( + InMemoryVerificationTaskDeletionFence::from_tasks_with_clock(&records, clock.clone()), + ); + ( + InMemoryVerificationTaskClaimer::with_deletion_fence(records, fence), + clock, + ) + } + + fn claimer_at_now(records: Vec) -> InMemoryVerificationTaskClaimer { + claimer_with_clock(records).0 + } + #[tokio::test] async fn no_pending_or_expired_in_progress_task_returns_none() { - let claimer = InMemoryVerificationTaskClaimer::new(vec![]); + let claimer = claimer_at_now(vec![]); assert_eq!( claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap(), None ); } + #[tokio::test] + async fn publication_first_blocks_in_memory_deletion_admission() { + let job = deletion_job(); + let task = task_for_lock( + "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d20", + VerificationTaskStatus::Pending, + &job.lock_id, + ); + let fence = Arc::new(InMemoryVerificationTaskDeletionFence::from_tasks( + std::slice::from_ref(&task), + )); + let claimer = + InMemoryVerificationTaskClaimer::with_deletion_fence(vec![task], Arc::clone(&fence)); + let deletions = InMemoryContentLockDeletionRepository::with_verification_task_fence(fence); + let claimed = claimer + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) + .await + .unwrap() + .unwrap(); + assert!( + claimer + .begin_claimed_entitlement_publication( + &claimed.task.task_id, + "worker-a", + &claimed.claim_token, + ) + .await + .unwrap() + ); + + assert_eq!( + deletions.insert_job(job).await, + Err(ApplicationError::ContentLockDeletionInProgress) + ); + } + + #[tokio::test] + async fn deletion_first_blocks_in_memory_publication_and_terminal_transition() { + let job = deletion_job(); + let task = task_for_lock( + "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d21", + VerificationTaskStatus::Pending, + &job.lock_id, + ); + let fence = Arc::new(InMemoryVerificationTaskDeletionFence::from_tasks( + std::slice::from_ref(&task), + )); + let claimer = + InMemoryVerificationTaskClaimer::with_deletion_fence(vec![task], Arc::clone(&fence)); + let deletions = InMemoryContentLockDeletionRepository::with_verification_task_fence(fence); + let claimed = claimer + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) + .await + .unwrap() + .unwrap(); + deletions.insert_job(job).await.unwrap(); + + assert!( + !claimer + .begin_claimed_entitlement_publication( + &claimed.task.task_id, + "worker-a", + &claimed.claim_token, + ) + .await + .unwrap() + ); + let completed = claimed + .task + .transition_to(VerificationTaskStatus::Completed, NOW, None) + .unwrap(); + assert_eq!( + claimer + .persist_claimed_verification_task_transition( + completed, + "worker-a", + &claimed.claim_token, + ) + .await + .unwrap(), + None + ); + } + + #[tokio::test] + async fn deletion_owned_publication_marker_blocks_in_memory_force_escalation() { + let job = deletion_job(); + let task = task_for_lock( + "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d24", + VerificationTaskStatus::Pending, + &job.lock_id, + ); + let task_id = task.task_id; + let fence = Arc::new(InMemoryVerificationTaskDeletionFence::from_tasks( + std::slice::from_ref(&task), + )); + let deletions = + InMemoryContentLockDeletionRepository::with_verification_task_fence(Arc::clone(&fence)); + deletions.insert_job(job.clone()).await.unwrap(); + { + let mut records = fence.records.write().await; + let record = records.get_mut(&task_id).unwrap(); + assert_eq!(record.deletion_job_id, Some(job.job_id)); + record.entitlement_publication_claim_token = Some(uuid::Uuid::new_v4()); + } + + assert_eq!( + deletions + .prepare_force_deletion(&job.creator, &job.lock_id) + .await + .unwrap(), + PrepareForceDeletionResult::PublicationInProgress + ); + assert!( + deletions + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .unwrap() + .force_requested_at + .is_none() + ); + } + + #[tokio::test] + async fn retry_retains_publication_fence_until_reconciled_terminal_transition() { + let job = deletion_job(); + let task = task_for_lock( + "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d22", + VerificationTaskStatus::Pending, + &job.lock_id, + ); + let clock = Arc::new(MutableClock::new(NOW)); + let fence = Arc::new( + InMemoryVerificationTaskDeletionFence::from_tasks_with_clock( + std::slice::from_ref(&task), + clock.clone(), + ), + ); + let claimer = + InMemoryVerificationTaskClaimer::with_deletion_fence(vec![task], Arc::clone(&fence)); + let deletions = InMemoryContentLockDeletionRepository::with_verification_task_fence(fence); + let first = claimer + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) + .await + .unwrap() + .unwrap(); + assert!( + claimer + .begin_claimed_entitlement_publication( + &first.task.task_id, + "worker-a", + &first.claim_token, + ) + .await + .unwrap() + ); + let retry_at = NOW + time::Duration::seconds(10); + claimer + .schedule_verification_task_retry( + &first.task.task_id, + "worker-a", + &first.claim_token, + (retry_at) - (NOW), + ) + .await + .unwrap() + .unwrap(); + + assert_eq!( + deletions.insert_job(job.clone()).await, + Err(ApplicationError::ContentLockDeletionInProgress) + ); + + clock.set(retry_at); + let second = claimer + .claim_next_verification_task( + "worker-b", + (retry_at + time::Duration::minutes(5)) - (retry_at), + ) + .await + .unwrap() + .unwrap(); + assert!( + claimer + .begin_claimed_entitlement_publication( + &second.task.task_id, + "worker-b", + &second.claim_token, + ) + .await + .unwrap() + ); + let completed = second + .task + .transition_to(VerificationTaskStatus::Completed, retry_at, None) + .unwrap(); + claimer + .persist_claimed_verification_task_transition( + completed, + "worker-b", + &second.claim_token, + ) + .await + .unwrap() + .unwrap(); + + deletions.insert_job(job).await.unwrap(); + } + #[tokio::test] async fn pending_task_can_be_claimed_and_transitions_to_in_progress() { let pending = task( "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d10", VerificationTaskStatus::Pending, ); - let claimer = InMemoryVerificationTaskClaimer::new(vec![pending.clone()]); + let claimer = claimer_at_now(vec![pending.clone()]); let claimed = claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap() .expect("pending task is claimed"); @@ -271,7 +628,7 @@ mod tests { assert_eq!(claimed.task.failure_message, None); assert_eq!( claimer - .claim_next_verification_task("worker-b", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-b", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap(), None @@ -295,7 +652,7 @@ mod tests { ); claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap() .unwrap(); @@ -325,7 +682,7 @@ mod tests { assert!( claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .is_err() ); @@ -333,7 +690,7 @@ mod tests { assert!( claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap() .is_some() @@ -347,18 +704,18 @@ mod tests { VerificationTaskStatus::Pending, ); let task_id = pending.task_id; - let claimer = InMemoryVerificationTaskClaimer::new(vec![pending]); + let (claimer, clock) = claimer_with_clock(vec![pending]); let first = claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap() .unwrap(); let reclaimed_at = CLAIM_EXPIRES_AT + time::Duration::milliseconds(1); + clock.set(reclaimed_at); let second = claimer .claim_next_verification_task( "worker-a", - reclaimed_at, - reclaimed_at + time::Duration::minutes(5), + (reclaimed_at + time::Duration::minutes(5)) - (reclaimed_at), ) .await .unwrap() @@ -371,8 +728,7 @@ mod tests { &task_id, "worker-a", &first.claim_token, - reclaimed_at, - reclaimed_at + time::Duration::seconds(10), + (reclaimed_at + time::Duration::seconds(10)) - (reclaimed_at), ) .await .unwrap(), @@ -384,8 +740,7 @@ mod tests { &task_id, "worker-a", &second.claim_token, - reclaimed_at, - reclaimed_at + time::Duration::seconds(10), + (reclaimed_at + time::Duration::seconds(10)) - (reclaimed_at), ) .await .unwrap() @@ -399,18 +754,18 @@ mod tests { "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d10", VerificationTaskStatus::Pending, ); - let claimer = InMemoryVerificationTaskClaimer::new(vec![pending]); + let (claimer, clock) = claimer_with_clock(vec![pending]); let first = claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap() .unwrap(); let reclaimed_at = CLAIM_EXPIRES_AT + time::Duration::milliseconds(1); + clock.set(reclaimed_at); let second = claimer .claim_next_verification_task( "worker-a", - reclaimed_at, - reclaimed_at + time::Duration::minutes(5), + (reclaimed_at + time::Duration::minutes(5)) - (reclaimed_at), ) .await .unwrap() @@ -441,7 +796,6 @@ mod tests { terminal, "worker-a", &first.claim_token, - reclaimed_at, ) .await .unwrap(), @@ -454,7 +808,6 @@ mod tests { completed.clone(), "worker-a", &second.claim_token, - reclaimed_at, ) .await .unwrap(), @@ -469,9 +822,9 @@ mod tests { VerificationTaskStatus::Pending, ); let task_id = pending.task_id; - let claimer = InMemoryVerificationTaskClaimer::new(vec![pending]); + let (claimer, clock) = claimer_with_clock(vec![pending]); let claim = claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap() .unwrap(); @@ -483,21 +836,7 @@ mod tests { &task_id, "worker-b", &claim.claim_token, - NOW, - next_attempt_at, - ) - .await - .unwrap(), - None - ); - assert_eq!( - claimer - .schedule_verification_task_retry( - &task_id, - "worker-a", - &claim.claim_token, - CLAIM_EXPIRES_AT + time::Duration::nanoseconds(1), - next_attempt_at, + (next_attempt_at) - (NOW), ) .await .unwrap(), @@ -508,8 +847,7 @@ mod tests { &task_id, "worker-a", &claim.claim_token, - NOW, - next_attempt_at, + (next_attempt_at) - (NOW), ) .await .unwrap() @@ -523,19 +861,18 @@ mod tests { claimer .claim_next_verification_task( "worker-b", - NOW + time::Duration::seconds(9), - CLAIM_EXPIRES_AT, + (CLAIM_EXPIRES_AT) - (NOW + time::Duration::seconds(9)), ) .await .unwrap(), None ); + clock.set(next_attempt_at); assert!( claimer .claim_next_verification_task( "worker-b", - next_attempt_at, - CLAIM_EXPIRES_AT + time::Duration::seconds(10), + (CLAIM_EXPIRES_AT + time::Duration::seconds(10)) - (next_attempt_at), ) .await .unwrap() @@ -575,11 +912,11 @@ mod tests { ) .transition_to(VerificationTaskStatus::Expired, NOW, None) .unwrap(); - let claimer = InMemoryVerificationTaskClaimer::new(vec![completed, failed, expired]); + let claimer = claimer_at_now(vec![completed, failed, expired]); assert_eq!( claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap(), None @@ -598,14 +935,17 @@ mod tests { None, ) .unwrap(); - let claimer = InMemoryVerificationTaskClaimer::with_claimed_tasks(vec![( - in_progress.clone(), - "worker-a".to_owned(), - datetime!(2026-05-29 12:05:00 UTC), - )]); + let claimer = InMemoryVerificationTaskClaimer::with_claimed_tasks_and_clock( + vec![( + in_progress.clone(), + "worker-a".to_owned(), + datetime!(2026-05-29 12:05:00 UTC), + )], + Arc::new(MutableClock::new(NOW)), + ); let reclaimed = claimer - .claim_next_verification_task("worker-b", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-b", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap() .expect("expired in-progress claim is reclaimed"); @@ -627,15 +967,18 @@ mod tests { None, ) .unwrap(); - let claimer = InMemoryVerificationTaskClaimer::with_claimed_tasks(vec![( - in_progress, - "worker-a".to_owned(), - datetime!(2026-05-29 12:11:00 UTC), - )]); + let claimer = InMemoryVerificationTaskClaimer::with_claimed_tasks_and_clock( + vec![( + in_progress, + "worker-a".to_owned(), + datetime!(2026-05-29 12:11:00 UTC), + )], + Arc::new(MutableClock::new(NOW)), + ); assert_eq!( claimer - .claim_next_verification_task("worker-b", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-b", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap(), None @@ -643,6 +986,14 @@ mod tests { } fn task(task_id: &str, status: VerificationTaskStatus) -> VerificationTaskRecord { + task_for_lock(task_id, status, &LockId::from_str(LOCK_ID).unwrap()) + } + + fn task_for_lock( + task_id: &str, + status: VerificationTaskStatus, + lock_id: &LockId, + ) -> VerificationTaskRecord { VerificationTaskRecord { task_id: TaskId::from_str(task_id).unwrap(), creator: CreatorPubky::from_str("pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy").unwrap(), @@ -650,7 +1001,7 @@ mod tests { version: SUBMITTED_PROOF_BUNDLE_VERSION, bundle_id: BundleId::from_str("000G40R40M30E209185GR38E1W").unwrap(), pubky_lock_resource: PubkyLockResource::from_str(&format!( - "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy/pub/locks.app/{LOCK_ID}.json" + "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy/pub/locks.app/{lock_id}.json" )) .unwrap(), reader_public_key: None, @@ -667,4 +1018,36 @@ mod tests { failure_message: None, } } + + fn deletion_job() -> ContentLockDeletionJob { + ContentLockDeletionJob::new( + uuid::Uuid::new_v4(), + ContentLock { + version: CONTENT_LOCK_VERSION, + creator: CreatorPubky::from_str( + "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy", + ) + .unwrap(), + primary_resource: Some( + GuardedResource::new( + "/priv/locks.app/content/post.json".to_owned(), + GuardedResourceHash::from_bytes([7; 32]), + "application/json".to_owned(), + 42, + ) + .unwrap(), + ), + secondary_resources: BTreeMap::new(), + criteria: vec![], + lock_logic: LockLogic::All { criteria: vec![] }, + access_policy: AccessPolicy { + requested_credential_ttl_seconds: 900, + }, + lock_server: LockServerConfig { override_: None }, + created_at: datetime!(2026-08-12 04:00:00 UTC), + }, + datetime!(2026-08-12 05:00:00 UTC), + ) + .unwrap() + } } diff --git a/locks-service/src/infrastructure/memory/verification_task_deletion_fence.rs b/locks-service/src/infrastructure/memory/verification_task_deletion_fence.rs new file mode 100644 index 0000000..a7bad85 --- /dev/null +++ b/locks-service/src/infrastructure/memory/verification_task_deletion_fence.rs @@ -0,0 +1,137 @@ +use std::{collections::HashMap, fmt, sync::Arc}; + +use locks_core::ids::{BundleId, CreatorPubky, LockId, TaskId}; +use locks_core::lock_policy::VerifierType; +use time::OffsetDateTime; +use tokio::sync::{Mutex, OwnedMutexGuard, RwLock}; +use uuid::Uuid; + +use crate::application::{ + models::{VerificationTaskRecord, VerificationTaskStatus}, + ports::Clock, +}; + +type LockKey = (CreatorPubky, LockId); + +#[derive(Debug)] +pub(crate) struct SystemClock; + +impl Clock for SystemClock { + fn now(&self) -> OffsetDateTime { + OffsetDateTime::now_utc() + } +} + +#[derive(Debug, Clone)] +pub(crate) struct InMemoryVerificationTaskFenceRecord { + pub(crate) creator: CreatorPubky, + pub(crate) lock_id: LockId, + pub(crate) bundle_id: BundleId, + pub(crate) paykit_admission_required: bool, + pub(crate) status: VerificationTaskStatus, + pub(crate) entitlement_publication_claim_token: Option, + pub(crate) deletion_job_id: Option, +} + +/// Shared in-memory serialization state for all admission decisions for one content lock. +pub struct InMemoryVerificationTaskDeletionFence { + pub(crate) records: RwLock>, + lock_admissions: Mutex>>>, + clock: Arc, +} + +impl fmt::Debug for InMemoryVerificationTaskDeletionFence { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("InMemoryVerificationTaskDeletionFence") + .field("records", &self.records) + .field("lock_admissions", &self.lock_admissions) + .field("clock", &"") + .finish() + } +} + +impl Default for InMemoryVerificationTaskDeletionFence { + fn default() -> Self { + Self::with_clock(Arc::new(SystemClock)) + } +} + +impl InMemoryVerificationTaskDeletionFence { + pub fn new() -> Self { + Self::default() + } + + /// Creates a canonical in-memory admission fence with an injected cutoff clock. + pub fn with_clock(clock: Arc) -> Self { + Self { + records: RwLock::new(HashMap::new()), + lock_admissions: Mutex::new(HashMap::new()), + clock, + } + } + + pub(crate) async fn acquire_lock_admission( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> OwnedMutexGuard<()> { + let key = (creator.clone(), lock_id.clone()); + let admission = { + let mut admissions = self.lock_admissions.lock().await; + Arc::clone( + admissions + .entry(key) + .or_insert_with(|| Arc::new(Mutex::new(()))), + ) + }; + admission.lock_owned().await + } + + pub(crate) fn authoritative_cutoff(&self) -> OffsetDateTime { + self.clock.now() + } + + pub(crate) fn from_tasks(tasks: &[VerificationTaskRecord]) -> Self { + Self::from_tasks_with_clock(tasks, Arc::new(SystemClock)) + } + + pub(crate) fn from_tasks_with_clock( + tasks: &[VerificationTaskRecord], + clock: Arc, + ) -> Self { + Self { + records: RwLock::new( + tasks + .iter() + .map(|task| { + ( + task.task_id, + InMemoryVerificationTaskFenceRecord { + creator: task.creator.clone(), + lock_id: task + .submitted_proof_bundle + .pubky_lock_resource + .lock_id() + .clone(), + bundle_id: task.submitted_proof_bundle.bundle_id.clone(), + paykit_admission_required: task + .submitted_proof_bundle + .proofs + .iter() + .any(|proof| { + proof.verifier_type == VerifierType::PaykitPayment + }), + status: task.status, + entitlement_publication_claim_token: None, + deletion_job_id: None, + }, + ) + }) + .collect(), + ), + lock_admissions: Mutex::new(HashMap::new()), + clock, + } + } +} diff --git a/locks-service/src/infrastructure/memory/verification_tasks.rs b/locks-service/src/infrastructure/memory/verification_tasks.rs index 45aa255..bd2a899 100644 --- a/locks-service/src/infrastructure/memory/verification_tasks.rs +++ b/locks-service/src/infrastructure/memory/verification_tasks.rs @@ -1,18 +1,29 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; use async_trait::async_trait; use tokio::sync::RwLock; use locks_core::ids::{BundleId, CreatorPubky, TaskId}; +use locks_core::lock_policy::VerifierType; use crate::application::errors::ApplicationError; use crate::application::models::VerificationTaskRecord; use crate::application::ports::VerificationTaskRepository; +use crate::infrastructure::memory::verification_task_deletion_fence::{ + InMemoryVerificationTaskDeletionFence, InMemoryVerificationTaskFenceRecord, +}; /// In-memory verification task repository. -#[derive(Debug, Default)] +#[derive(Debug)] pub struct InMemoryVerificationTaskRepository { records: RwLock>, + deletion_fence: Arc, +} + +impl Default for InMemoryVerificationTaskRepository { + fn default() -> Self { + Self::with_deletion_fence(Arc::new(InMemoryVerificationTaskDeletionFence::new())) + } } impl InMemoryVerificationTaskRepository { @@ -20,6 +31,13 @@ impl InMemoryVerificationTaskRepository { pub fn new() -> Self { Self::default() } + + pub fn with_deletion_fence(deletion_fence: Arc) -> Self { + Self { + records: RwLock::new(HashMap::new()), + deletion_fence, + } + } } #[async_trait] @@ -28,6 +46,7 @@ impl VerificationTaskRepository for InMemoryVerificationTaskRepository { &self, task: VerificationTaskRecord, ) -> Result<(), ApplicationError> { + let mut fence_records = self.deletion_fence.records.write().await; let mut records = self.records.write().await; if records.contains_key(&task.task_id) || records.values().any(|existing| { @@ -40,6 +59,26 @@ impl VerificationTaskRepository for InMemoryVerificationTaskRepository { record: "verification_task", }); } + fence_records.insert( + task.task_id, + InMemoryVerificationTaskFenceRecord { + creator: task.creator.clone(), + lock_id: task + .submitted_proof_bundle + .pubky_lock_resource + .lock_id() + .clone(), + bundle_id: task.submitted_proof_bundle.bundle_id.clone(), + paykit_admission_required: task + .submitted_proof_bundle + .proofs + .iter() + .any(|proof| proof.verifier_type == VerifierType::PaykitPayment), + status: task.status, + entitlement_publication_claim_token: None, + deletion_job_id: None, + }, + ); records.insert(task.task_id, task); Ok(()) } @@ -48,16 +87,43 @@ impl VerificationTaskRepository for InMemoryVerificationTaskRepository { &self, task: VerificationTaskRecord, ) -> Result<(), ApplicationError> { + let mut fence_records = self.deletion_fence.records.write().await; let mut records = self.records.write().await; if !records.contains_key(&task.task_id) { return Err(ApplicationError::MissingRecord { record: "verification_task", }); } + if let Some(fence_record) = fence_records.get_mut(&task.task_id) { + fence_record.status = task.status; + } records.insert(task.task_id, task); Ok(()) } + async fn update_verification_tasks_atomically( + &self, + tasks: Vec, + ) -> Result<(), ApplicationError> { + let mut fence_records = self.deletion_fence.records.write().await; + let mut records = self.records.write().await; + if tasks + .iter() + .any(|task| !records.contains_key(&task.task_id)) + { + return Err(ApplicationError::MissingRecord { + record: "verification_task", + }); + } + for task in tasks { + if let Some(fence_record) = fence_records.get_mut(&task.task_id) { + fence_record.status = task.status; + } + records.insert(task.task_id, task); + } + Ok(()) + } + async fn get_verification_task( &self, task_id: &TaskId, @@ -82,7 +148,9 @@ impl VerificationTaskRepository for InMemoryVerificationTaskRepository { } async fn delete_verification_task(&self, task_id: &TaskId) -> Result<(), ApplicationError> { + let mut fence_records = self.deletion_fence.records.write().await; self.records.write().await.remove(task_id); + fence_records.remove(task_id); Ok(()) } } diff --git a/locks-service/src/infrastructure/mod.rs b/locks-service/src/infrastructure/mod.rs index d0b718f..2acc2b3 100644 --- a/locks-service/src/infrastructure/mod.rs +++ b/locks-service/src/infrastructure/mod.rs @@ -1,4 +1,6 @@ +pub mod final_credentials; pub mod memory; pub mod postgres; pub mod pubky; +pub mod runtime_master_key; pub mod verifiers; diff --git a/locks-service/src/infrastructure/postgres/access_credentials.rs b/locks-service/src/infrastructure/postgres/access_credentials.rs index ab17cbe..0bd5695 100644 --- a/locks-service/src/infrastructure/postgres/access_credentials.rs +++ b/locks-service/src/infrastructure/postgres/access_credentials.rs @@ -1,24 +1,50 @@ use std::str::FromStr; use async_trait::async_trait; -use sqlx::{PgPool, Row}; +use sqlx::{PgPool, Postgres, Row, Transaction}; +use time::{Duration, OffsetDateTime}; +use uuid::Uuid; -use locks_core::ids::{BundleId, CreatorPubky}; +use locks_core::{ + ids::{BundleId, CreatorPubky, LockId}, + lock_policy::{ContentLock, GuardedResource}, +}; use crate::application::errors::ApplicationError; -use crate::application::models::{AccessCredentialLookupKey, AccessCredentialRecord}; -use crate::application::ports::AccessCredentialStore; +use crate::application::models::{ + AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, DeletionReadAuthorization, + EncryptedFinalCredential, FinalAccessWindows, FinalCredentialContext, + FinalCredentialMaterialization, InitializeFinalAccessWindowsResult, IssuedDeletionCredential, +}; +use crate::application::ports::{AccessCredentialStore, FinalCredentialWorkerIssueRequest}; +use crate::infrastructure::final_credentials::FinalCredentialCipher; + +use super::proof_admission::lock_proof_admission; /// Postgres-backed store for issued access credential lookup records. #[derive(Debug, Clone)] pub struct PostgresAccessCredentialStore { pool: PgPool, + final_credential_cipher: Option, } impl PostgresAccessCredentialStore { /// Creates a store backed by the provided migrated Postgres pool. pub fn new(pool: PgPool) -> Self { - Self { pool } + Self { + pool, + final_credential_cipher: None, + } + } + + pub fn with_final_credential_cipher( + pool: PgPool, + final_credential_cipher: FinalCredentialCipher, + ) -> Self { + Self { + pool, + final_credential_cipher: Some(final_credential_cipher), + } } } @@ -26,9 +52,28 @@ impl PostgresAccessCredentialStore { impl AccessCredentialStore for PostgresAccessCredentialStore { async fn insert_access_credential( &self, + lock_id: &LockId, lookup_key: AccessCredentialLookupKey, record: AccessCredentialRecord, ) -> Result<(), ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, &record.creator, lock_id).await?; + let deletion_exists: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 + FROM content_lock_deletion_jobs + WHERE creator = $1 AND lock_id = $2 + )", + ) + .bind(record.creator.to_string()) + .bind(lock_id.to_string()) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if deletion_exists { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + let result = sqlx::query( "INSERT INTO access_credentials (lookup_key, creator, bundle_id, expires_at) VALUES ($1, $2, $3, $4) @@ -38,7 +83,7 @@ impl AccessCredentialStore for PostgresAccessCredentialStore { .bind(record.creator.to_string()) .bind(record.bundle_id.to_string()) .bind(record.expires_at) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(storage_error)?; @@ -48,6 +93,7 @@ impl AccessCredentialStore for PostgresAccessCredentialStore { }); } + transaction.commit().await.map_err(storage_error)?; Ok(()) } @@ -79,6 +125,973 @@ impl AccessCredentialStore for PostgresAccessCredentialStore { .map_err(storage_error)?; Ok(()) } + + async fn initialize_final_access_windows( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + issuance_window: Duration, + read_window: Duration, + ) -> Result { + if issuance_window <= Duration::ZERO || read_window <= Duration::ZERO { + return Err(ApplicationError::Storage { + message: "final access window durations must be positive".to_owned(), + }); + } + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let job = sqlx::query( + "SELECT state, phase, force_requested_at, claimed_by, claim_token, claim_expires_at, + final_issuance_started_at, final_credential_issuance_deadline, + final_read_deadline + FROM content_lock_deletion_jobs + WHERE job_id = $1 + FOR UPDATE", + ) + .bind(deletion_job_id) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let Some(job) = job else { + transaction.commit().await.map_err(storage_error)?; + return Ok(InitializeFinalAccessWindowsResult::ClaimLost); + }; + let now: OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + let owns_live_claim = job.try_get::("state").map_err(storage_error)? + == "running" + && job.try_get::("phase").map_err(storage_error)? + == "issue_final_credentials" + && job + .try_get::, _>("force_requested_at") + .map_err(storage_error)? + .is_none() + && job + .try_get::, _>("claimed_by") + .map_err(storage_error)? + .as_deref() + == Some(worker_id) + && job + .try_get::, _>("claim_token") + .map_err(storage_error)? + == Some(claim_token) + && job + .try_get::, _>("claim_expires_at") + .map_err(storage_error)? + .is_some_and(|expires_at| expires_at > now); + if !owns_live_claim { + transaction.commit().await.map_err(storage_error)?; + return Ok(InitializeFinalAccessWindowsResult::ClaimLost); + } + let existing = ( + job.try_get::, _>("final_issuance_started_at") + .map_err(storage_error)?, + job.try_get::, _>("final_credential_issuance_deadline") + .map_err(storage_error)?, + job.try_get::, _>("final_read_deadline") + .map_err(storage_error)?, + ); + let windows = match existing { + ( + Some(issuance_started_at), + Some(credential_issuance_deadline), + Some(read_deadline), + ) => FinalAccessWindows { + issuance_started_at, + credential_issuance_deadline, + read_deadline, + }, + (None, None, None) => { + let credential_issuance_deadline = + now.checked_add(issuance_window) + .ok_or_else(|| ApplicationError::Storage { + message: "final credential issuance deadline overflow".to_owned(), + })?; + let read_deadline = credential_issuance_deadline + .checked_add(read_window) + .ok_or_else(|| ApplicationError::Storage { + message: "final read deadline overflow".to_owned(), + })?; + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET final_issuance_started_at = $2, + final_credential_issuance_deadline = $3, + final_read_deadline = $4 + WHERE job_id = $1", + ) + .bind(deletion_job_id) + .bind(now) + .bind(credential_issuance_deadline) + .bind(read_deadline) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + FinalAccessWindows { + issuance_started_at: now, + credential_issuance_deadline, + read_deadline, + } + } + _ => { + return Err(ApplicationError::Storage { + message: "incomplete final access windows in Postgres".to_owned(), + }); + } + }; + transaction.commit().await.map_err(storage_error)?; + Ok(InitializeFinalAccessWindowsResult::Initialized(windows)) + } + + async fn final_credentials_to_materialize( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + limit: usize, + ) -> Result, ApplicationError> { + if limit == 0 { + return Ok(Vec::new()); + } + let limit = i64::try_from(limit).map_err(|_| ApplicationError::Storage { + message: "final credential materialization limit exceeds Postgres BIGINT".to_owned(), + })?; + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let job = sqlx::query( + "SELECT creator, state, phase, force_requested_at, claimed_by, claim_token, + claim_expires_at, final_credential_issuance_deadline + FROM content_lock_deletion_jobs + WHERE job_id = $1 + FOR UPDATE", + ) + .bind(deletion_job_id) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let Some(job) = job else { + transaction.commit().await.map_err(storage_error)?; + return Ok(Vec::new()); + }; + let now: OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + let owns_live_issue_claim = job.try_get::("state").map_err(storage_error)? + == "running" + && job.try_get::("phase").map_err(storage_error)? + == "issue_final_credentials" + && job + .try_get::, _>("force_requested_at") + .map_err(storage_error)? + .is_none() + && job + .try_get::, _>("claimed_by") + .map_err(storage_error)? + .as_deref() + == Some(worker_id) + && job + .try_get::, _>("claim_token") + .map_err(storage_error)? + == Some(claim_token) + && job + .try_get::, _>("claim_expires_at") + .map_err(storage_error)? + .is_some_and(|deadline| now < deadline) + && job + .try_get::, _>("final_credential_issuance_deadline") + .map_err(storage_error)? + .is_some_and(|deadline| now < deadline); + if !owns_live_issue_claim { + transaction.commit().await.map_err(storage_error)?; + return Ok(Vec::new()); + } + let creator: String = job.try_get("creator").map_err(storage_error)?; + let creator = + CreatorPubky::from_str(&creator).map_err(|error| ApplicationError::Storage { + message: format!("invalid deletion job creator stored in Postgres: {error}"), + })?; + let bundle_ids: Vec = sqlx::query_scalar( + "SELECT bundle_id + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 + AND resolved_status = 'completed' + AND final_credential_eligible_at IS NOT NULL + AND final_credential_issued_at IS NULL + ORDER BY bundle_id ASC + LIMIT $2", + ) + .bind(deletion_job_id) + .bind(limit) + .fetch_all(&mut *transaction) + .await + .map_err(storage_error)?; + let pending = bundle_ids + .into_iter() + .map(|bundle_id| { + BundleId::from_str(&bundle_id) + .map(|bundle_id| FinalCredentialMaterialization { + creator: creator.clone(), + bundle_id, + }) + .map_err(|error| ApplicationError::Storage { + message: format!( + "invalid final credential bundle_id stored in Postgres: {error}" + ), + }) + }) + .collect::, _>>()?; + transaction.commit().await.map_err(storage_error)?; + Ok(pending) + } + + async fn issue_or_replay_final_credential( + &self, + creator: &CreatorPubky, + bundle_id: &BundleId, + _caller_now: OffsetDateTime, + candidate: AccessCredential, + ) -> Result, ApplicationError> { + let cipher = match &self.final_credential_cipher { + Some(cipher) => cipher, + None => return Ok(None), + }; + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let job = sqlx::query( + "SELECT job.job_id, job.phase, job.final_credential_issuance_deadline, + job.final_read_deadline, job.frozen_content_lock + FROM content_lock_deletion_jobs AS job + WHERE job.creator = $1 + AND job.state IN ('queued', 'running') + AND job.force_requested_at IS NULL + AND job.phase IN ('issue_final_credentials', 'drain_final_reads') + AND EXISTS ( + SELECT 1 + FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.deletion_job_id = job.job_id + AND snapshot.bundle_id = $2 + AND snapshot.resolved_status = 'completed' + AND snapshot.final_credential_eligible_at IS NOT NULL + ) + FOR UPDATE OF job", + ) + .bind(creator.to_string()) + .bind(bundle_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let Some(job) = job else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + // `transaction_timestamp()` is fixed at BEGIN; sample the wall clock only after the + // job-row lock has serialized this winner with force/phase/deadline transitions. + let now: OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + let deletion_job_id: Uuid = job.try_get("job_id").map_err(storage_error)?; + let phase: String = job.try_get("phase").map_err(storage_error)?; + let issuance_deadline: OffsetDateTime = job + .try_get("final_credential_issuance_deadline") + .map_err(storage_error)?; + let expires_at: OffsetDateTime = + job.try_get("final_read_deadline").map_err(storage_error)?; + if now >= expires_at { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let snapshot_exists = sqlx::query_scalar::<_, bool>( + "SELECT TRUE + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 AND bundle_id = $2 + AND resolved_status = 'completed' + AND final_credential_eligible_at IS NOT NULL + FOR UPDATE", + ) + .bind(deletion_job_id) + .bind(bundle_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)? + .unwrap_or(false); + if !snapshot_exists { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let existing_encrypted: Option = sqlx::query_scalar( + "SELECT encrypted_bearer + FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND creator = $2 AND bundle_id = $3 + AND credential_kind = 'final' + FOR UPDATE", + ) + .bind(deletion_job_id) + .bind(creator.to_string()) + .bind(bundle_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)? + .flatten(); + let context = FinalCredentialContext { + deletion_job_id, + creator: creator.clone(), + bundle_id: bundle_id.clone(), + }; + if let Some(encrypted) = existing_encrypted { + let credential = cipher.decrypt(&context, &EncryptedFinalCredential::new(encrypted))?; + transaction.commit().await.map_err(storage_error)?; + return Ok(Some(IssuedDeletionCredential { + credential, + expires_at, + })); + } + if phase != "issue_final_credentials" || now >= issuance_deadline { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let frozen: serde_json::Value = + job.try_get("frozen_content_lock").map_err(storage_error)?; + let frozen: ContentLock = + serde_json::from_value(frozen).map_err(|error| ApplicationError::Storage { + message: format!("invalid frozen content lock stored in Postgres: {error}"), + })?; + let encrypted = cipher.encrypt(&context, &candidate)?; + let lookup_key = AccessCredentialLookupKey::derive(&candidate); + let credential_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO access_credentials ( + lookup_key, creator, bundle_id, expires_at, deletion_job_id + ) VALUES ($1, $2, $3, $4, $5)", + ) + .bind(lookup_key.as_bytes().as_slice()) + .bind(creator.to_string()) + .bind(bundle_id.to_string()) + .bind(expires_at) + .bind(deletion_job_id) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + sqlx::query( + "INSERT INTO content_lock_access_drain_credentials ( + credential_id, deletion_job_id, lookup_key, creator, bundle_id, + credential_kind, issued_at, expires_at, encrypted_bearer + ) VALUES ($1, $2, $3, $4, $5, 'final', $6, $7, $8)", + ) + .bind(credential_id) + .bind(deletion_job_id) + .bind(lookup_key.as_bytes().as_slice()) + .bind(creator.to_string()) + .bind(bundle_id.to_string()) + .bind(now) + .bind(expires_at) + .bind(encrypted.as_str()) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + let mut resources: Vec = frozen.primary_resource.into_iter().collect(); + resources.extend( + frozen + .secondary_resources + .into_iter() + .map(|(path, resource)| { + GuardedResource::new(path, resource.hash, resource.content_type, resource.size) + .expect("persisted frozen manifest was validated at deletion admission") + }), + ); + for resource in resources { + sqlx::query( + "INSERT INTO content_lock_access_drain_reads (credential_id, guarded_path) + VALUES ($1, $2)", + ) + .bind(credential_id) + .bind(resource.path) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + } + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET final_credential_issued_at = $3 + WHERE deletion_job_id = $1 AND bundle_id = $2 + AND final_credential_issued_at IS NULL", + ) + .bind(deletion_job_id) + .bind(bundle_id.to_string()) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(Some(IssuedDeletionCredential { + credential: candidate, + expires_at, + })) + } + + async fn issue_or_replay_final_credential_for_worker( + &self, + request: FinalCredentialWorkerIssueRequest<'_>, + ) -> Result, ApplicationError> { + let FinalCredentialWorkerIssueRequest { + deletion_job_id, + worker_id, + claim_token, + creator, + bundle_id, + now: _caller_now, + candidate, + } = request; + let cipher = match &self.final_credential_cipher { + Some(cipher) => cipher, + None => return Ok(None), + }; + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let job = sqlx::query( + "SELECT job.job_id, job.creator, job.state, job.phase, job.force_requested_at, + job.claimed_by, job.claim_token, job.claim_expires_at, + job.final_credential_issuance_deadline, job.final_read_deadline, + job.frozen_content_lock + FROM content_lock_deletion_jobs AS job + WHERE job.job_id = $1 + FOR UPDATE OF job", + ) + .bind(deletion_job_id) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let Some(job) = job else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + // PostgreSQL transaction time is fixed at BEGIN, so only clock_timestamp() sampled after + // the row lock is authoritative for a statement that may have waited behind another owner. + let now: OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + let owns_live_issue_claim = job.try_get::("creator").map_err(storage_error)? + == creator.to_string() + && job.try_get::("state").map_err(storage_error)? == "running" + && job.try_get::("phase").map_err(storage_error)? + == "issue_final_credentials" + && job + .try_get::, _>("force_requested_at") + .map_err(storage_error)? + .is_none() + && job + .try_get::, _>("claimed_by") + .map_err(storage_error)? + .as_deref() + == Some(worker_id) + && job + .try_get::, _>("claim_token") + .map_err(storage_error)? + == Some(claim_token) + && job + .try_get::, _>("claim_expires_at") + .map_err(storage_error)? + .is_some_and(|deadline| deadline > now) + && job + .try_get::, _>("final_credential_issuance_deadline") + .map_err(storage_error)? + .is_some_and(|deadline| deadline > now) + && job + .try_get::, _>("final_read_deadline") + .map_err(storage_error)? + .is_some_and(|deadline| deadline > now); + if !owns_live_issue_claim { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let expires_at: OffsetDateTime = + job.try_get("final_read_deadline").map_err(storage_error)?; + let snapshot_exists = sqlx::query_scalar::<_, bool>( + "SELECT TRUE + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 AND bundle_id = $2 + AND resolved_status = 'completed' + AND final_credential_eligible_at IS NOT NULL + FOR UPDATE", + ) + .bind(deletion_job_id) + .bind(bundle_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)? + .unwrap_or(false); + if !snapshot_exists { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let existing_encrypted: Option = sqlx::query_scalar( + "SELECT encrypted_bearer + FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND creator = $2 AND bundle_id = $3 + AND credential_kind = 'final' + FOR UPDATE", + ) + .bind(deletion_job_id) + .bind(creator.to_string()) + .bind(bundle_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)? + .flatten(); + let context = FinalCredentialContext { + deletion_job_id, + creator: creator.clone(), + bundle_id: bundle_id.clone(), + }; + if let Some(encrypted) = existing_encrypted { + let credential = cipher.decrypt(&context, &EncryptedFinalCredential::new(encrypted))?; + transaction.commit().await.map_err(storage_error)?; + return Ok(Some(IssuedDeletionCredential { + credential, + expires_at, + })); + } + let frozen: serde_json::Value = + job.try_get("frozen_content_lock").map_err(storage_error)?; + let frozen: ContentLock = + serde_json::from_value(frozen).map_err(|error| ApplicationError::Storage { + message: format!("invalid frozen content lock stored in Postgres: {error}"), + })?; + let encrypted = cipher.encrypt(&context, &candidate)?; + let lookup_key = AccessCredentialLookupKey::derive(&candidate); + let credential_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO access_credentials ( + lookup_key, creator, bundle_id, expires_at, deletion_job_id + ) VALUES ($1, $2, $3, $4, $5)", + ) + .bind(lookup_key.as_bytes().as_slice()) + .bind(creator.to_string()) + .bind(bundle_id.to_string()) + .bind(expires_at) + .bind(deletion_job_id) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + sqlx::query( + "INSERT INTO content_lock_access_drain_credentials ( + credential_id, deletion_job_id, lookup_key, creator, bundle_id, + credential_kind, issued_at, expires_at, encrypted_bearer + ) VALUES ($1, $2, $3, $4, $5, 'final', $6, $7, $8)", + ) + .bind(credential_id) + .bind(deletion_job_id) + .bind(lookup_key.as_bytes().as_slice()) + .bind(creator.to_string()) + .bind(bundle_id.to_string()) + .bind(now) + .bind(expires_at) + .bind(encrypted.as_str()) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + let mut resources: Vec = frozen.primary_resource.into_iter().collect(); + resources.extend( + frozen + .secondary_resources + .into_iter() + .map(|(path, resource)| { + GuardedResource::new(path, resource.hash, resource.content_type, resource.size) + .expect("persisted frozen manifest was validated at deletion admission") + }), + ); + for resource in resources { + sqlx::query( + "INSERT INTO content_lock_access_drain_reads (credential_id, guarded_path) + VALUES ($1, $2)", + ) + .bind(credential_id) + .bind(resource.path) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + } + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET final_credential_issued_at = $3 + WHERE deletion_job_id = $1 AND bundle_id = $2 + AND final_credential_issued_at IS NULL", + ) + .bind(deletion_job_id) + .bind(bundle_id.to_string()) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(Some(IssuedDeletionCredential { + credential: candidate, + expires_at, + })) + } + + async fn final_credential_available( + &self, + creator: &CreatorPubky, + bundle_id: &BundleId, + now: OffsetDateTime, + ) -> Result { + if self.final_credential_cipher.is_none() { + return Ok(false); + } + sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 + FROM content_lock_deletion_jobs AS job + JOIN content_lock_deletion_task_snapshot AS snapshot + ON snapshot.deletion_job_id = job.job_id + WHERE job.creator = $1 AND snapshot.bundle_id = $2 + AND job.state IN ('queued', 'running') + AND job.force_requested_at IS NULL + AND job.phase IN ('issue_final_credentials', 'drain_final_reads') + AND snapshot.resolved_status = 'completed' + AND snapshot.final_credential_eligible_at IS NOT NULL + AND job.final_read_deadline > $3 + AND ( + EXISTS ( + SELECT 1 + FROM content_lock_access_drain_credentials AS credential + WHERE credential.deletion_job_id = job.job_id + AND credential.creator = $1 + AND credential.bundle_id = $2 + AND credential.credential_kind = 'final' + ) + OR ( + job.phase = 'issue_final_credentials' + AND job.final_credential_issuance_deadline > $3 + ) + ) + )", + ) + .bind(creator.to_string()) + .bind(bundle_id.to_string()) + .bind(now) + .fetch_one(&self.pool) + .await + .map_err(storage_error) + } + + async fn prepare_deletion_read( + &self, + lookup_key: &AccessCredentialLookupKey, + path: &str, + claim_duration: Duration, + ) -> Result, ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let Some(deletion_job_id) = lookup_deletion_job_id(&mut transaction, lookup_key).await? + else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let Some(job) = lock_active_drain_job(&mut transaction, deletion_job_id).await? else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + // Transaction time is fixed at BEGIN. Sample wall-clock time only after the exact job-row + // fence, so a wait cannot revive an expired credential, deadline, or read claim. + let now: OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + let credential = sqlx::query( + "SELECT credential_id, credential_kind, creator, expires_at + FROM content_lock_access_drain_credentials + WHERE lookup_key = $1 AND deletion_job_id = $2 + FOR UPDATE", + ) + .bind(lookup_key.as_bytes().as_slice()) + .bind(deletion_job_id) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let Some(credential) = credential else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let kind: String = credential + .try_get("credential_kind") + .map_err(storage_error)?; + let credential_expiry: OffsetDateTime = + credential.try_get("expires_at").map_err(storage_error)?; + if credential_expiry <= now || !phase_allows_credential_access(&job.phase, &kind) { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let frozen: ContentLock = + serde_json::from_value(job.frozen_content_lock).map_err(|error| { + ApplicationError::Storage { + message: format!("invalid frozen content lock stored in Postgres: {error}"), + } + })?; + let Some(resource) = frozen.resource_for_path(path) else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let creator = CreatorPubky::from_str( + &credential + .try_get::("creator") + .map_err(storage_error)?, + ) + .map_err(|error| ApplicationError::Storage { + message: format!("invalid drain credential creator stored in Postgres: {error}"), + })?; + if kind == "ordinary" { + transaction.commit().await.map_err(storage_error)?; + return Ok(Some(DeletionReadAuthorization { + claim_token: None, + creator, + resource, + })); + } + let credential_id: Uuid = credential.try_get("credential_id").map_err(storage_error)?; + let read = sqlx::query( + "SELECT claim_token, claim_expires_at, consumed_at + FROM content_lock_access_drain_reads + WHERE credential_id = $1 AND guarded_path = $2 + FOR UPDATE", + ) + .bind(credential_id) + .bind(path) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let Some(read) = read else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + if read + .try_get::, _>("consumed_at") + .map_err(storage_error)? + .is_some() + { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let existing_claim: Option = read.try_get("claim_token").map_err(storage_error)?; + let existing_expiry: Option = + read.try_get("claim_expires_at").map_err(storage_error)?; + if existing_claim.is_some() && existing_expiry.is_some_and(|expiry| expiry > now) { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let Some(read_deadline) = job.final_read_deadline else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; + let bounded_expiry = now + .checked_add(claim_duration) + .ok_or_else(|| ApplicationError::Storage { + message: "final read claim expiry overflow".to_owned(), + })? + .min(now + time::Duration::seconds(30)) + .min(credential_expiry) + .min(read_deadline); + if read_deadline <= now || bounded_expiry <= now { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let claim_token = Uuid::new_v4(); + let updated = sqlx::query( + "UPDATE content_lock_access_drain_reads + SET claim_token = $3, claim_expires_at = $4 + WHERE credential_id = $1 AND guarded_path = $2 + AND consumed_at IS NULL + AND (claim_token IS NULL OR claim_expires_at <= $5)", + ) + .bind(credential_id) + .bind(path) + .bind(claim_token) + .bind(bounded_expiry) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + if updated.rows_affected() != 1 { + transaction.rollback().await.map_err(storage_error)?; + return Ok(None); + } + transaction.commit().await.map_err(storage_error)?; + Ok(Some(DeletionReadAuthorization { + claim_token: Some(claim_token), + creator, + resource, + })) + } + + async fn deletion_credential_enrolled( + &self, + lookup_key: &AccessCredentialLookupKey, + ) -> Result { + sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 + FROM content_lock_access_drain_credentials + WHERE lookup_key = $1 + )", + ) + .bind(lookup_key.as_bytes().as_slice()) + .fetch_one(&self.pool) + .await + .map_err(storage_error) + } + + async fn release_deletion_read( + &self, + lookup_key: &AccessCredentialLookupKey, + path: &str, + claim_token: Uuid, + _now: OffsetDateTime, + ) -> Result { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let Some(deletion_job_id) = lookup_deletion_job_id(&mut transaction, lookup_key).await? + else { + transaction.commit().await.map_err(storage_error)?; + return Ok(false); + }; + let Some(job) = lock_active_drain_job(&mut transaction, deletion_job_id).await? else { + transaction.commit().await.map_err(storage_error)?; + return Ok(false); + }; + if !phase_allows_credential_access(&job.phase, "final") { + transaction.commit().await.map_err(storage_error)?; + return Ok(false); + } + let updated = sqlx::query( + "UPDATE content_lock_access_drain_reads AS read + SET claim_token = NULL, claim_expires_at = NULL + FROM content_lock_access_drain_credentials AS credential + WHERE read.credential_id = credential.credential_id + AND credential.deletion_job_id = $4 + AND credential.lookup_key = $1 AND read.guarded_path = $2 + AND read.claim_token = $3 AND read.consumed_at IS NULL", + ) + .bind(lookup_key.as_bytes().as_slice()) + .bind(path) + .bind(claim_token) + .bind(deletion_job_id) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(updated.rows_affected() == 1) + } + + async fn consume_deletion_read( + &self, + lookup_key: &AccessCredentialLookupKey, + path: &str, + claim_token: Uuid, + ) -> Result { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let Some(deletion_job_id) = lookup_deletion_job_id(&mut transaction, lookup_key).await? + else { + transaction.commit().await.map_err(storage_error)?; + return Ok(false); + }; + let Some(job) = lock_active_drain_job(&mut transaction, deletion_job_id).await? else { + transaction.commit().await.map_err(storage_error)?; + return Ok(false); + }; + let now: OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if !phase_allows_credential_access(&job.phase, "final") + || job + .final_read_deadline + .is_none_or(|deadline| deadline <= now) + { + transaction.commit().await.map_err(storage_error)?; + return Ok(false); + } + let updated = sqlx::query( + "UPDATE content_lock_access_drain_reads AS read + SET claim_token = NULL, claim_expires_at = NULL, consumed_at = $4 + FROM content_lock_access_drain_credentials AS credential + WHERE read.credential_id = credential.credential_id + AND credential.deletion_job_id = $5 + AND credential.lookup_key = $1 AND read.guarded_path = $2 + AND read.claim_token = $3 AND read.claim_expires_at > $4 + AND read.consumed_at IS NULL", + ) + .bind(lookup_key.as_bytes().as_slice()) + .bind(path) + .bind(claim_token) + .bind(now) + .bind(deletion_job_id) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(updated.rows_affected() == 1) + } +} + +struct LockedDrainJob { + phase: String, + final_read_deadline: Option, + frozen_content_lock: serde_json::Value, +} + +async fn lookup_deletion_job_id( + transaction: &mut Transaction<'_, Postgres>, + lookup_key: &AccessCredentialLookupKey, +) -> Result, ApplicationError> { + sqlx::query_scalar( + "SELECT deletion_job_id + FROM content_lock_access_drain_credentials + WHERE lookup_key = $1", + ) + .bind(lookup_key.as_bytes().as_slice()) + .fetch_optional(&mut **transaction) + .await + .map_err(storage_error) +} + +async fn lock_active_drain_job( + transaction: &mut Transaction<'_, Postgres>, + deletion_job_id: Uuid, +) -> Result, ApplicationError> { + let row = sqlx::query( + "SELECT phase, final_read_deadline, frozen_content_lock + FROM content_lock_deletion_jobs + WHERE job_id = $1 AND state IN ('queued', 'running') + AND force_requested_at IS NULL + FOR UPDATE", + ) + .bind(deletion_job_id) + .fetch_optional(&mut **transaction) + .await + .map_err(storage_error)?; + row.map(|row| { + Ok(LockedDrainJob { + phase: row.try_get("phase").map_err(storage_error)?, + final_read_deadline: row.try_get("final_read_deadline").map_err(storage_error)?, + frozen_content_lock: row.try_get("frozen_content_lock").map_err(storage_error)?, + }) + }) + .transpose() +} + +fn phase_allows_credential_access(phase: &str, credential_kind: &str) -> bool { + match credential_kind { + "ordinary" => matches!( + phase, + "withdraw" + | "start_payment_drain" + | "drain_payments" + | "drain_existing_credentials" + | "issue_final_credentials" + | "drain_final_reads" + ), + "final" => matches!(phase, "issue_final_credentials" | "drain_final_reads"), + _ => false, + } } fn row_to_record(row: sqlx::postgres::PgRow) -> Result { @@ -110,19 +1123,28 @@ fn storage_error(error: sqlx::Error) -> ApplicationError { #[cfg(test)] mod tests { - use std::str::FromStr; + use std::{collections::BTreeMap, str::FromStr}; use sqlx::Row; use time::macros::datetime; + use uuid::Uuid; - use locks_core::ids::{BundleId, CreatorPubky}; + use locks_core::{ + ids::{BundleId, CreatorPubky, GuardedResourceHash, LockId}, + lock_policy::{ + AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, GuardedResource, LockLogic, + LockServerConfig, + }, + }; use super::PostgresAccessCredentialStore; use crate::application::errors::ApplicationError; use crate::application::models::{ AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, + InitializeFinalAccessWindowsResult, }; - use crate::application::ports::AccessCredentialStore; + use crate::application::ports::{AccessCredentialStore, FinalCredentialWorkerIssueRequest}; + use crate::infrastructure::final_credentials::FinalCredentialCipher; use crate::infrastructure::postgres::testing::TestDatabase; #[tokio::test] @@ -132,6 +1154,8 @@ mod tests { let credential = AccessCredential::new("raw-bearer-credential"); let lookup_key = AccessCredentialLookupKey::derive(&credential); let record = access_credential_record(); + let lock_id = + LockId::from_str("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG").unwrap(); assert_eq!( store.get_access_credential(&lookup_key).await.unwrap(), @@ -139,7 +1163,7 @@ mod tests { ); store - .insert_access_credential(lookup_key.clone(), record.clone()) + .insert_access_credential(&lock_id, lookup_key.clone(), record.clone()) .await .unwrap(); assert_eq!( @@ -148,7 +1172,7 @@ mod tests { ); assert_eq!( store - .insert_access_credential(lookup_key.clone(), record) + .insert_access_credential(&lock_id, lookup_key.clone(), record) .await, Err(ApplicationError::DuplicateRecord { record: "access_credential", @@ -175,9 +1199,11 @@ mod tests { let credential = AccessCredential::new(raw_credential); let lookup_key = AccessCredentialLookupKey::derive(&credential); let record = access_credential_record(); + let lock_id = + LockId::from_str("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG").unwrap(); original_store - .insert_access_credential(lookup_key.clone(), record.clone()) + .insert_access_credential(&lock_id, lookup_key.clone(), record.clone()) .await .unwrap(); @@ -194,6 +1220,660 @@ mod tests { database.cleanup().await; } + #[tokio::test] + async fn committed_deletion_rejects_ordinary_credential_without_inserting() { + let database = TestDatabase::create().await; + let store = PostgresAccessCredentialStore::new(database.pool().clone()); + let lock_id = + LockId::from_str("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG").unwrap(); + let record = access_credential_record(); + let lookup_key = AccessCredentialLookupKey::derive(&AccessCredential::new("rejected")); + sqlx::query( + "INSERT INTO content_lock_deletion_jobs ( + job_id, creator, lock_id, deletion_started_at, frozen_content_lock + ) VALUES ($1, $2, $3, $4, $5)", + ) + .bind(uuid::Uuid::new_v4()) + .bind(record.creator.to_string()) + .bind(lock_id.to_string()) + .bind(datetime!(2026-05-29 12:00:00 UTC)) + .bind(serde_json::json!({"version": "1"})) + .execute(database.pool()) + .await + .unwrap(); + + assert_eq!( + store + .insert_access_credential(&lock_id, lookup_key, record) + .await, + Err(ApplicationError::ContentLockDeletionInProgress) + ); + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM access_credentials") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(count, 0); + + database.cleanup().await; + } + + #[tokio::test] + async fn final_credentials_to_materialize_returns_eligible_unissued_rows_in_order_with_limit() { + let database = TestDatabase::create().await; + let store = PostgresAccessCredentialStore::new(database.pool().clone()); + let now = sqlx::query_scalar::<_, time::OffsetDateTime>("SELECT clock_timestamp()") + .fetch_one(database.pool()) + .await + .unwrap(); + let claim_token = Uuid::new_v4(); + let job_id = insert_final_issuance_job(database.pool(), now, claim_token).await; + insert_final_snapshot( + database.pool(), + job_id, + "000G40R40M30E209185GR38E1W", + true, + false, + ) + .await; + insert_final_snapshot( + database.pool(), + job_id, + "000G40R40M30E209185GR38E1R", + true, + false, + ) + .await; + insert_final_snapshot( + database.pool(), + job_id, + "000G40R40M30E209185GR38E1M", + false, + false, + ) + .await; + insert_final_snapshot( + database.pool(), + job_id, + "000G40R40M30E209185GR38E1G", + true, + true, + ) + .await; + + let bounded = store + .final_credentials_to_materialize(job_id, "worker", claim_token, 1) + .await + .unwrap(); + assert_eq!(bounded.len(), 1); + assert_eq!(bounded[0].bundle_id.as_str(), "000G40R40M30E209185GR38E1R"); + assert_eq!(bounded[0].creator, creator()); + + let all = store + .final_credentials_to_materialize(job_id, "worker", claim_token, 10) + .await + .unwrap(); + assert_eq!( + all.iter() + .map(|pending| pending.bundle_id.as_str()) + .collect::>(), + vec!["000G40R40M30E209185GR38E1R", "000G40R40M30E209185GR38E1W"] + ); + assert!( + store + .final_credentials_to_materialize(job_id, "worker", claim_token, 0) + .await + .unwrap() + .is_empty() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn final_credentials_to_materialize_revalidates_exact_live_issue_claim_and_deadline() { + let database = TestDatabase::create().await; + let store = PostgresAccessCredentialStore::new(database.pool().clone()); + let now = sqlx::query_scalar::<_, time::OffsetDateTime>("SELECT clock_timestamp()") + .fetch_one(database.pool()) + .await + .unwrap(); + let claim_token = Uuid::new_v4(); + let job_id = insert_final_issuance_job(database.pool(), now, claim_token).await; + insert_final_snapshot( + database.pool(), + job_id, + "000G40R40M30E209185GR38E1W", + true, + false, + ) + .await; + + assert!( + store + .final_credentials_to_materialize(job_id, "worker", Uuid::new_v4(), 10) + .await + .unwrap() + .is_empty() + ); + + sqlx::query( + "UPDATE content_lock_deletion_jobs SET force_requested_at = $2 WHERE job_id = $1", + ) + .bind(job_id) + .bind(now) + .execute(database.pool()) + .await + .unwrap(); + assert!( + store + .final_credentials_to_materialize(job_id, "worker", claim_token, 10) + .await + .unwrap() + .is_empty() + ); + + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET force_requested_at = NULL, phase = 'drain_final_reads' + WHERE job_id = $1", + ) + .bind(job_id) + .execute(database.pool()) + .await + .unwrap(); + assert!( + store + .final_credentials_to_materialize(job_id, "worker", claim_token, 10) + .await + .unwrap() + .is_empty() + ); + + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET phase = 'issue_final_credentials', final_credential_issuance_deadline = $2 + WHERE job_id = $1", + ) + .bind(job_id) + .bind(now) + .execute(database.pool()) + .await + .unwrap(); + assert!( + store + .final_credentials_to_materialize(job_id, "worker", claim_token, 10) + .await + .unwrap() + .is_empty() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn final_credentials_to_materialize_samples_time_after_job_lock_and_rejects_equality() { + let database = TestDatabase::create().await; + let store = PostgresAccessCredentialStore::new(database.pool().clone()); + let now = sqlx::query_scalar::<_, time::OffsetDateTime>("SELECT clock_timestamp()") + .fetch_one(database.pool()) + .await + .unwrap(); + let claim_token = Uuid::new_v4(); + let job_id = insert_final_issuance_job(database.pool(), now, claim_token).await; + insert_final_snapshot( + database.pool(), + job_id, + "000G40R40M30E209185GR38E1W", + true, + false, + ) + .await; + + let mut blocker = database.pool().begin().await.unwrap(); + sqlx::query("SELECT job_id FROM content_lock_deletion_jobs WHERE job_id = $1 FOR UPDATE") + .bind(job_id) + .fetch_one(&mut *blocker) + .await + .unwrap(); + let waiting_store = store.clone(); + let enumeration = tokio::spawn(async move { + waiting_store + .final_credentials_to_materialize(job_id, "worker", claim_token, 10) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + assert!(!enumeration.is_finished()); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET claim_expires_at = clock_timestamp(), + final_credential_issuance_deadline = clock_timestamp() + WHERE job_id = $1", + ) + .bind(job_id) + .execute(&mut *blocker) + .await + .unwrap(); + blocker.commit().await.unwrap(); + + assert!(enumeration.await.unwrap().unwrap().is_empty()); + database.cleanup().await; + } + + #[tokio::test] + async fn worker_final_issuance_is_exact_claim_fenced_in_winner_transaction() { + let database = TestDatabase::create().await; + let store = PostgresAccessCredentialStore::with_final_credential_cipher( + database.pool().clone(), + FinalCredentialCipher::new([41; 32]), + ); + let now = sqlx::query_scalar::<_, time::OffsetDateTime>("SELECT CURRENT_TIMESTAMP") + .fetch_one(database.pool()) + .await + .unwrap(); + let claim_token = Uuid::new_v4(); + let job_id = insert_final_issuance_job(database.pool(), now, claim_token).await; + let bundle_id = BundleId::from_str("000G40R40M30E209185GR38E1R").unwrap(); + let creator = creator(); + insert_final_snapshot(database.pool(), job_id, bundle_id.as_str(), true, false).await; + + for (job, worker, token, at) in [ + (Uuid::new_v4(), "worker", claim_token, now), + (job_id, "reclaimer", claim_token, now), + (job_id, "worker", Uuid::new_v4(), now), + ] { + let candidate = AccessCredential::new(format!("denied-{job}-{worker}-{at}")); + assert!( + store + .issue_or_replay_final_credential_for_worker( + FinalCredentialWorkerIssueRequest { + deletion_job_id: job, + worker_id: worker, + claim_token: token, + creator: &creator, + bundle_id: &bundle_id, + now: at, + candidate, + }, + ) + .await + .unwrap() + .is_none() + ); + } + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM access_credentials") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(count, 0); + + sqlx::query( + "UPDATE content_lock_deletion_jobs SET force_requested_at = $2 WHERE job_id = $1", + ) + .bind(job_id) + .bind(now) + .execute(database.pool()) + .await + .unwrap(); + assert!( + store + .issue_or_replay_final_credential_for_worker(FinalCredentialWorkerIssueRequest { + deletion_job_id: job_id, + worker_id: "worker", + claim_token, + creator: &creator, + bundle_id: &bundle_id, + now, + candidate: AccessCredential::new("force-loser"), + },) + .await + .unwrap() + .is_none() + ); + + let reclaimed_token = Uuid::new_v4(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET force_requested_at = NULL, claimed_by = 'reclaimer', claim_token = $2, + claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job_id) + .bind(reclaimed_token) + .bind(now + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + assert!( + store + .issue_or_replay_final_credential_for_worker(FinalCredentialWorkerIssueRequest { + deletion_job_id: job_id, + worker_id: "worker", + claim_token, + creator: &creator, + bundle_id: &bundle_id, + now, + candidate: AccessCredential::new("stale-reclaimed-loser"), + },) + .await + .unwrap() + .is_none() + ); + + let winner = AccessCredential::new("postgres-worker-winner"); + let issued = store + .issue_or_replay_final_credential_for_worker(FinalCredentialWorkerIssueRequest { + deletion_job_id: job_id, + worker_id: "reclaimer", + claim_token: reclaimed_token, + creator: &creator, + bundle_id: &bundle_id, + now, + candidate: winner.clone(), + }) + .await + .unwrap() + .unwrap(); + let replay = store + .issue_or_replay_final_credential_for_worker(FinalCredentialWorkerIssueRequest { + deletion_job_id: job_id, + worker_id: "reclaimer", + claim_token: reclaimed_token, + creator: &creator, + bundle_id: &bundle_id, + now, + candidate: AccessCredential::new("postgres-worker-loser"), + }) + .await + .unwrap() + .unwrap(); + assert_eq!(issued.credential, winner); + assert_eq!(replay, issued); + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM access_credentials") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(count, 1); + + database.cleanup().await; + } + + #[tokio::test] + async fn public_final_issuance_rechecks_database_time_after_waiting_for_job_lock() { + let database = TestDatabase::create().await; + let store = PostgresAccessCredentialStore::with_final_credential_cipher( + database.pool().clone(), + FinalCredentialCipher::new([43; 32]), + ); + let caller_now = datetime!(2026-05-29 12:00:00 UTC); + let claim_token = Uuid::new_v4(); + let job_id = insert_final_issuance_job(database.pool(), caller_now, claim_token).await; + let bundle_id = BundleId::from_str("000G40R40M30E209185GR38E1R").unwrap(); + let creator = creator(); + insert_final_snapshot(database.pool(), job_id, bundle_id.as_str(), true, false).await; + + let mut blocker = database.pool().begin().await.unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET final_credential_issuance_deadline = clock_timestamp() + interval '500 milliseconds', + final_read_deadline = clock_timestamp() + interval '10 minutes' + WHERE job_id = $1", + ) + .bind(job_id) + .execute(&mut *blocker) + .await + .unwrap(); + + let issuance = tokio::spawn(async move { + store + .issue_or_replay_final_credential( + &creator, + &bundle_id, + caller_now, + AccessCredential::new("must-not-persist-after-public-lock-wait"), + ) + .await + }); + sqlx::query("SELECT pg_sleep(1)") + .execute(&mut *blocker) + .await + .unwrap(); + blocker.commit().await.unwrap(); + + assert!(issuance.await.unwrap().unwrap().is_none()); + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM access_credentials") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(count, 0); + + database.cleanup().await; + } + + #[tokio::test] + async fn worker_final_issuance_rechecks_database_time_after_waiting_for_job_lock() { + let database = TestDatabase::create().await; + let store = PostgresAccessCredentialStore::with_final_credential_cipher( + database.pool().clone(), + FinalCredentialCipher::new([42; 32]), + ); + let caller_now = datetime!(2026-05-29 12:00:00 UTC); + let claim_token = Uuid::new_v4(); + let job_id = insert_final_issuance_job(database.pool(), caller_now, claim_token).await; + let bundle_id = BundleId::from_str("000G40R40M30E209185GR38E1R").unwrap(); + let creator = creator(); + insert_final_snapshot(database.pool(), job_id, bundle_id.as_str(), true, false).await; + + let mut blocker = database.pool().begin().await.unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET claim_expires_at = clock_timestamp() + interval '500 milliseconds', + final_credential_issuance_deadline = clock_timestamp() + interval '500 milliseconds', + final_read_deadline = clock_timestamp() + interval '10 minutes' + WHERE job_id = $1", + ) + .bind(job_id) + .execute(&mut *blocker) + .await + .unwrap(); + + let issuance = tokio::spawn(async move { + store + .issue_or_replay_final_credential_for_worker(FinalCredentialWorkerIssueRequest { + deletion_job_id: job_id, + worker_id: "worker", + claim_token, + creator: &creator, + bundle_id: &bundle_id, + now: caller_now, + candidate: AccessCredential::new("must-not-persist-after-lock-wait"), + }) + .await + }); + sqlx::query("SELECT pg_sleep(1)") + .execute(&mut *blocker) + .await + .unwrap(); + blocker.commit().await.unwrap(); + + assert!(issuance.await.unwrap().unwrap().is_none()); + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM access_credentials") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(count, 0); + + database.cleanup().await; + } + + #[tokio::test] + async fn final_access_window_initialization_rechecks_database_time_after_job_lock() { + let database = TestDatabase::create().await; + let store = PostgresAccessCredentialStore::new(database.pool().clone()); + let fixture_now: time::OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(database.pool()) + .await + .unwrap(); + let claim_token = Uuid::new_v4(); + let job_id = insert_final_issuance_job(database.pool(), fixture_now, claim_token).await; + + let mut blocker = database.pool().begin().await.unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET claim_expires_at = clock_timestamp() + interval '500 milliseconds', + final_issuance_started_at = NULL, + final_credential_issuance_deadline = NULL, + final_read_deadline = NULL + WHERE job_id = $1", + ) + .bind(job_id) + .execute(&mut *blocker) + .await + .unwrap(); + + let initialization = tokio::spawn(async move { + store + .initialize_final_access_windows( + job_id, + "worker", + claim_token, + time::Duration::minutes(15), + time::Duration::minutes(15), + ) + .await + }); + sqlx::query("SELECT pg_sleep(1)") + .execute(&mut *blocker) + .await + .unwrap(); + blocker.commit().await.unwrap(); + + assert_eq!( + initialization.await.unwrap().unwrap(), + InitializeFinalAccessWindowsResult::ClaimLost + ); + let windows: ( + Option, + Option, + Option, + ) = sqlx::query_as( + "SELECT final_issuance_started_at, final_credential_issuance_deadline, + final_read_deadline + FROM content_lock_deletion_jobs WHERE job_id = $1", + ) + .bind(job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(windows, (None, None, None)); + + database.cleanup().await; + } + + async fn insert_final_issuance_job( + pool: &sqlx::PgPool, + now: time::OffsetDateTime, + claim_token: Uuid, + ) -> Uuid { + let job_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO content_lock_deletion_jobs ( + job_id, creator, lock_id, frozen_content_lock, deletion_started_at, + state, phase, claimed_by, claim_token, claim_expires_at, + final_issuance_started_at, final_credential_issuance_deadline, + final_read_deadline + ) VALUES ($1, $2, $3, $4, $5, 'running', + 'issue_final_credentials', 'worker', $6, $7, $8, $9, $10)", + ) + .bind(job_id) + .bind(creator().to_string()) + .bind("000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG") + .bind(serde_json::to_value(test_content_lock()).unwrap()) + .bind(now) + .bind(claim_token) + .bind(now + time::Duration::minutes(5)) + .bind(now - time::Duration::minutes(1)) + .bind(now + time::Duration::minutes(10)) + .bind(now + time::Duration::minutes(20)) + .execute(pool) + .await + .unwrap(); + job_id + } + + async fn insert_final_snapshot( + pool: &sqlx::PgPool, + job_id: Uuid, + bundle_id: &str, + eligible: bool, + issued: bool, + ) { + let task_id = Uuid::new_v4(); + let now = datetime!(2026-05-29 12:00:00 UTC); + sqlx::query( + "INSERT INTO verification_tasks ( + task_id, status, submitted_proof_bundle, submitted_at, creator, bundle_id, + deletion_job_id + ) VALUES ($1, 'completed', '{}'::jsonb, $2, $3, $4, $5)", + ) + .bind(task_id) + .bind(now) + .bind(creator().to_string()) + .bind(bundle_id) + .bind(job_id) + .execute(pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO content_lock_deletion_task_snapshot ( + deletion_job_id, verification_task_id, creator, bundle_id, + pubky_lock_resource, criterion_id, status_at_cutoff, + paykit_admission_required, resolved_status, resolved_at, + final_credential_eligible_at, final_credential_issued_at + ) VALUES ($1, $2, $3, $4, 'pubkycreator/pub/locks.app/lock.json', + 'payment', 'completed', TRUE, 'completed', $5, + CASE WHEN $6 THEN $5 END, + CASE WHEN $7 THEN $5 END)", + ) + .bind(job_id) + .bind(task_id) + .bind(creator().to_string()) + .bind(bundle_id) + .bind(now) + .bind(eligible) + .bind(issued) + .execute(pool) + .await + .unwrap(); + } + + fn creator() -> CreatorPubky { + CreatorPubky::from_str("pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy").unwrap() + } + + fn test_content_lock() -> ContentLock { + ContentLock { + version: CONTENT_LOCK_VERSION, + creator: creator(), + primary_resource: Some( + GuardedResource::new( + "/priv/locks.app/content/post.json".to_owned(), + GuardedResourceHash::from_bytes([7; 32]), + "application/json".to_owned(), + 42, + ) + .unwrap(), + ), + secondary_resources: BTreeMap::new(), + criteria: vec![], + lock_logic: LockLogic::All { criteria: vec![] }, + access_policy: AccessPolicy { + requested_credential_ttl_seconds: 900, + }, + lock_server: LockServerConfig { override_: None }, + created_at: datetime!(2026-05-29 11:00:00 UTC), + } + } + async fn assert_stored_lookup_key_is_exact( pool: &sqlx::PgPool, lookup_key: &AccessCredentialLookupKey, diff --git a/locks-service/src/infrastructure/postgres/content_lock_deletion_action_ownership.rs b/locks-service/src/infrastructure/postgres/content_lock_deletion_action_ownership.rs new file mode 100644 index 0000000..21f6136 --- /dev/null +++ b/locks-service/src/infrastructure/postgres/content_lock_deletion_action_ownership.rs @@ -0,0 +1,152 @@ +use async_trait::async_trait; +use sqlx::{Connection, PgConnection, PgPool}; +use uuid::Uuid; + +use crate::application::{ + errors::ApplicationError, + models::ContentLockDeletionPhase, + ports::{ + ContentLockDeletionActionAcquireResult, ContentLockDeletionActionClaim, + ContentLockDeletionActionGuard, ContentLockDeletionActionOwnership, + }, +}; + +/// PostgreSQL session-advisory-lock ownership for deletion external actions. +#[derive(Debug, Clone)] +pub struct PostgresContentLockDeletionActionOwnership { + pool: PgPool, +} + +impl PostgresContentLockDeletionActionOwnership { + pub fn new(pool: PgPool) -> Self { + Self { pool } + } +} + +#[async_trait] +impl ContentLockDeletionActionOwnership for PostgresContentLockDeletionActionOwnership { + async fn try_acquire( + &self, + claim: ContentLockDeletionActionClaim<'_>, + ) -> Result { + let lock_key = action_lock_key(claim.job_id); + let mut pooled = self.pool.acquire().await.map_err(storage_error)?; + let acquired: bool = sqlx::query_scalar("SELECT pg_try_advisory_lock($1)") + .bind(lock_key) + .fetch_one(&mut *pooled) + .await + .map_err(storage_error)?; + if !acquired { + return Ok(ContentLockDeletionActionAcquireResult::Busy); + } + + // Detach immediately after locking. Any validation error then closes the + // session on drop instead of returning a locked connection to the pool. + let mut connection = pooled.detach(); + let live: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 + FROM content_lock_deletion_jobs + WHERE job_id = $1 + AND state = 'running' + AND claimed_by = $2 + AND claim_token = $3 + AND claim_expires_at > clock_timestamp() + AND phase = $4 + AND (($5 AND force_requested_at IS NOT NULL) + OR (NOT $5 AND force_requested_at IS NULL)) + )", + ) + .bind(claim.job_id) + .bind(claim.worker_id) + .bind(claim.claim_token) + .bind(phase_to_database(claim.expected_phase)) + .bind(claim.force) + .fetch_one(&mut connection) + .await + .map_err(storage_error)?; + + if !live { + unlock_and_close(connection, lock_key).await?; + return Ok(ContentLockDeletionActionAcquireResult::ClaimLost); + } + + Ok(ContentLockDeletionActionAcquireResult::Acquired(Box::new( + PostgresContentLockDeletionActionGuard { + connection: Some(connection), + lock_key, + }, + ))) + } +} + +struct PostgresContentLockDeletionActionGuard { + connection: Option, + lock_key: i64, +} + +#[async_trait] +impl ContentLockDeletionActionGuard for PostgresContentLockDeletionActionGuard { + async fn release(mut self: Box) -> Result<(), ApplicationError> { + let connection = self + .connection + .take() + .ok_or_else(|| ApplicationError::Storage { + message: "content lock deletion action ownership was already released".to_owned(), + })?; + unlock_and_close(connection, self.lock_key).await + } +} + +impl Drop for PostgresContentLockDeletionActionGuard { + fn drop(&mut self) { + drop(self.connection.take()); + } +} + +async fn unlock_and_close( + mut connection: PgConnection, + lock_key: i64, +) -> Result<(), ApplicationError> { + let unlock_result = sqlx::query_scalar::<_, bool>("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .fetch_one(&mut connection) + .await; + let close_result = connection.close().await; + let unlocked = unlock_result.map_err(storage_error)?; + close_result.map_err(storage_error)?; + if !unlocked { + return Err(ApplicationError::Storage { + message: "content lock deletion action ownership was not held".to_owned(), + }); + } + Ok(()) +} + +fn phase_to_database(phase: ContentLockDeletionPhase) -> &'static str { + match phase { + ContentLockDeletionPhase::Withdraw => "withdraw", + ContentLockDeletionPhase::StartPaymentDrain => "start_payment_drain", + ContentLockDeletionPhase::DrainPayments => "drain_payments", + ContentLockDeletionPhase::DrainExistingCredentials => "drain_existing_credentials", + ContentLockDeletionPhase::IssueFinalCredentials => "issue_final_credentials", + ContentLockDeletionPhase::DrainFinalReads => "drain_final_reads", + ContentLockDeletionPhase::DeleteContent => "delete_content", + ContentLockDeletionPhase::DeleteTombstone => "delete_tombstone", + ContentLockDeletionPhase::PurgeOperationalState => "purge_operational_state", + } +} + +fn action_lock_key(job_id: Uuid) -> i64 { + let digest = blake3::derive_key( + "pubky-locks content-lock deletion external action ownership v1", + job_id.as_bytes(), + ); + i64::from_be_bytes(digest[..8].try_into().expect("eight-byte digest prefix")) +} + +fn storage_error(error: sqlx::Error) -> ApplicationError { + ApplicationError::Storage { + message: error.to_string(), + } +} diff --git a/locks-service/src/infrastructure/postgres/content_lock_deletions.rs b/locks-service/src/infrastructure/postgres/content_lock_deletions.rs new file mode 100644 index 0000000..2988ef8 --- /dev/null +++ b/locks-service/src/infrastructure/postgres/content_lock_deletions.rs @@ -0,0 +1,4525 @@ +use std::str::FromStr; + +use async_trait::async_trait; +use locks_core::{ + ids::{CreatorPubky, LockId}, + lock_policy::ContentLock, +}; +use sqlx::{FromRow, PgPool, Postgres, Row, Transaction}; +use time::OffsetDateTime; +use uuid::Uuid; + +use crate::application::{ + errors::ApplicationError, + models::{ + AdvanceContentLockDeletionPhaseResult, ClaimedContentLockDeletionJob, + ContentLockDeletionFailureCode, ContentLockDeletionJob, ContentLockDeletionPhase, + ContentLockDeletionState, PrepareForceDeletionResult, + }, + ports::ContentLockDeletionRepository, +}; +use crate::infrastructure::postgres::proof_admission::lock_proof_admission; + +const ROW_COLUMNS: &str = "job_id, creator, lock_id, frozen_content_lock, deletion_started_at, state, phase, attempt_count, next_attempt_at, force_requested_at, failure_code, claimed_by, claim_token, claim_expires_at"; +const CLAIMED_ROW_COLUMNS: &str = "job.job_id, job.creator, job.lock_id, job.frozen_content_lock, job.deletion_started_at, job.state, job.phase, job.attempt_count, job.next_attempt_at, job.force_requested_at, job.failure_code, job.claimed_by, job.claim_token, job.claim_expires_at"; + +#[derive(Debug, FromRow)] +struct DeletionJobRow { + job_id: Uuid, + creator: String, + lock_id: String, + frozen_content_lock: serde_json::Value, + deletion_started_at: OffsetDateTime, + state: String, + phase: String, + attempt_count: i64, + next_attempt_at: Option, + force_requested_at: Option, + failure_code: Option, + claimed_by: Option, + claim_token: Option, + claim_expires_at: Option, +} + +/// PostgreSQL-backed durable deletion job queue and permanent force receipt store. +#[derive(Debug, Clone)] +pub struct PostgresContentLockDeletionRepository { + pool: PgPool, +} + +impl PostgresContentLockDeletionRepository { + pub fn new(pool: PgPool) -> Self { + Self { pool } + } +} + +#[async_trait] +impl ContentLockDeletionRepository for PostgresContentLockDeletionRepository { + async fn begin_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result<(), ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, creator, lock_id).await?; + let deletion_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM content_lock_force_deletion_receipts WHERE creator = $1 AND lock_id = $2) + OR EXISTS (SELECT 1 FROM content_lock_deletion_jobs WHERE creator = $1 AND lock_id = $2)", + ) + .bind(creator.to_string()).bind(lock_id.to_string()) + .fetch_one(&mut *transaction).await.map_err(storage_error)?; + if deletion_exists { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + sqlx::query("INSERT INTO content_lock_publication_intents (creator, lock_id, publication_token) VALUES ($1, $2, $3)") + .bind(creator.to_string()).bind(lock_id.to_string()).bind(publication_token) + .execute(&mut *transaction).await.map_err(map_publication_insert_error)?; + transaction.commit().await.map_err(storage_error) + } + + async fn finish_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result { + delete_publication_intent(&self.pool, creator, lock_id, publication_token).await + } + + async fn abandon_publication( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, + ) -> Result { + delete_publication_intent(&self.pool, creator, lock_id, publication_token).await + } + + async fn publication_in_progress( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, creator, lock_id).await?; + let exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM content_lock_publication_intents WHERE creator = $1 AND lock_id = $2)", + ) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(exists) + } + + async fn insert_job(&self, job: ContentLockDeletionJob) -> Result<(), ApplicationError> { + job.validate_frozen_identity()?; + job.validate_state(false)?; + let frozen = serde_json::to_value(&job.frozen_content_lock).map_err(storage_display)?; + let lock_resource = format!( + "{}/pub/locks.app/{}.json", + job.creator, + job.lock_id.as_str() + ); + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, &job.creator, &job.lock_id).await?; + let admission_cutoff: OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + let deletion_cutoff_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM content_lock_force_deletion_receipts + WHERE creator = $1 AND lock_id = $2) + OR EXISTS (SELECT 1 FROM content_lock_publication_intents + WHERE creator = $1 AND lock_id = $2)", + ) + .bind(job.creator.to_string()) + .bind(job.lock_id.to_string()) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if deletion_cutoff_exists { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + sqlx::query( + "SELECT task_id FROM verification_tasks + WHERE creator = $1 + AND submitted_proof_bundle->>'pubky_lock_resource' = $2 + FOR UPDATE", + ) + .bind(job.creator.to_string()) + .bind(&lock_resource) + .fetch_all(&mut *transaction) + .await + .map_err(storage_error)?; + let publication_in_progress: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM verification_tasks + WHERE creator = $1 + AND submitted_proof_bundle->>'pubky_lock_resource' = $2 + AND entitlement_publication_claim_token IS NOT NULL + )", + ) + .bind(job.creator.to_string()) + .bind(&lock_resource) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if publication_in_progress { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + sqlx::query( + "INSERT INTO content_lock_deletion_jobs + (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, state, phase, + attempt_count, next_attempt_at, force_requested_at, failure_code) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", + ) + .bind(job.job_id) + .bind(job.creator.to_string()) + .bind(job.lock_id.to_string()) + .bind(frozen) + .bind(admission_cutoff) + .bind(state_to_database(job.state)) + .bind(phase_to_database(job.phase)) + .bind(i64::from(job.attempt_count)) + .bind(job.next_attempt_at) + .bind(job.force_requested_at) + .bind(job.failure_code.map(ContentLockDeletionFailureCode::as_str)) + .execute(&mut *transaction) + .await + .map_err(map_insert_error)?; + sqlx::query( + "INSERT INTO content_lock_deletion_task_snapshot + (deletion_job_id, verification_task_id, creator, bundle_id, + pubky_lock_resource, criterion_id, status_at_cutoff, + paykit_admission_required) + SELECT $1, task_id, creator, bundle_id, + submitted_proof_bundle->>'pubky_lock_resource', + ( + SELECT proof->>'criterion_id' + FROM jsonb_array_elements(submitted_proof_bundle->'proofs') AS proof + WHERE proof->>'verifier_type' = 'paykit-payment' + LIMIT 1 + ), + status, + ( + EXISTS ( + SELECT 1 + FROM jsonb_array_elements(submitted_proof_bundle->'proofs') AS proof + WHERE proof->>'verifier_type' = 'paykit-payment' + ) + OR EXISTS ( + SELECT 1 FROM paykit_task_admissions AS admission + WHERE admission.verification_task_id = verification_tasks.task_id + ) + ) + FROM verification_tasks + WHERE creator = $2 + AND submitted_proof_bundle->>'pubky_lock_resource' = $3", + ) + .bind(job.job_id) + .bind(job.creator.to_string()) + .bind(lock_resource) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot AS snapshot + SET had_active_credential_at_cutoff = EXISTS ( + SELECT 1 FROM access_credentials AS credential + WHERE credential.creator = snapshot.creator + AND credential.bundle_id = snapshot.bundle_id + AND credential.expires_at > $2 + ) + WHERE snapshot.deletion_job_id = $1", + ) + .bind(job.job_id) + .bind(admission_cutoff) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET resolved_status = status_at_cutoff, + resolved_at = $2, + final_credential_eligible_at = CASE + WHEN status_at_cutoff = 'completed' + AND paykit_admission_required + AND NOT had_active_credential_at_cutoff + THEN $2 + ELSE NULL + END + WHERE deletion_job_id = $1 + AND status_at_cutoff IN ('completed', 'failed', 'expired')", + ) + .bind(job.job_id) + .bind(admission_cutoff) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + sqlx::query( + "UPDATE access_credentials AS credential + SET deletion_job_id = $1 + FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.deletion_job_id = $1 + AND credential.creator = snapshot.creator + AND credential.bundle_id = snapshot.bundle_id + AND credential.expires_at > $2", + ) + .bind(job.job_id) + .bind(admission_cutoff) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + let attached_credentials = sqlx::query( + "SELECT lookup_key, creator, bundle_id, expires_at + FROM access_credentials + WHERE deletion_job_id = $1 + ORDER BY lookup_key", + ) + .bind(job.job_id) + .fetch_all(&mut *transaction) + .await + .map_err(storage_error)?; + for credential in attached_credentials { + let credential_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO content_lock_access_drain_credentials ( + credential_id, deletion_job_id, lookup_key, creator, bundle_id, + credential_kind, issued_at, expires_at + ) VALUES ($1, $2, $3, $4, $5, 'ordinary', $6, $7)", + ) + .bind(credential_id) + .bind(job.job_id) + .bind( + credential + .try_get::, _>("lookup_key") + .map_err(storage_error)?, + ) + .bind( + credential + .try_get::("creator") + .map_err(storage_error)?, + ) + .bind( + credential + .try_get::("bundle_id") + .map_err(storage_error)?, + ) + .bind(admission_cutoff) + .bind( + credential + .try_get::("expires_at") + .map_err(storage_error)?, + ) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + } + sqlx::query( + "UPDATE verification_tasks AS task + SET status = 'pending', started_at = NULL, claimed_by = NULL, + claim_token = NULL, claim_expires_at = NULL, + entitlement_publication_claim_token = NULL, deletion_job_id = $1, + next_attempt_at = NULL, + last_attempt_error = NULL, updated_at = $2 + FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.deletion_job_id = $1 + AND snapshot.verification_task_id = task.task_id + AND task.status IN ('pending', 'in_progress')", + ) + .bind(job.job_id) + .bind(admission_cutoff) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error) + } + + async fn get_job( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result, ApplicationError> { + let sql = format!( + "SELECT {ROW_COLUMNS} FROM content_lock_deletion_jobs WHERE creator = $1 AND lock_id = $2" + ); + sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .fetch_optional(&self.pool) + .await + .map_err(storage_error)? + .map(row_to_job) + .transpose() + } + + async fn claim_next( + &self, + worker_id: &str, + claim_ttl: time::Duration, + ) -> Result, ApplicationError> { + let claim_token = Uuid::new_v4(); + let claim_ttl_seconds = claim_ttl.as_seconds_f64(); + let sql = format!( + "WITH winner AS MATERIALIZED (SELECT clock_timestamp() AS at), + candidate AS MATERIALIZED ( + SELECT job_id FROM content_lock_deletion_jobs, winner + WHERE (state = 'queued' AND (next_attempt_at IS NULL OR next_attempt_at <= winner.at)) + OR (state = 'running' AND claim_expires_at <= winner.at) + ORDER BY deletion_started_at + FOR UPDATE OF content_lock_deletion_jobs SKIP LOCKED LIMIT 1 + ) + UPDATE content_lock_deletion_jobs AS job + SET state = 'running', claimed_by = $1, claim_token = $2, + claim_expires_at = winner.at + ($3 * interval '1 second'), + next_attempt_at = NULL, attempt_count = attempt_count + 1, + updated_at = winner.at + FROM candidate, winner + WHERE job.job_id = candidate.job_id + RETURNING {CLAIMED_ROW_COLUMNS}" + ); + let row = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(worker_id) + .bind(claim_token) + .bind(claim_ttl_seconds) + .fetch_optional(&self.pool) + .await + .map_err(storage_error)?; + row.map(row_to_job) + .transpose() + .map(|job| job.map(|job| ClaimedContentLockDeletionJob { job, claim_token })) + } + + async fn schedule_retry( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + retry_after: time::Duration, + ) -> Result, ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let Some((_current, winner_time)) = + load_owned_claim(&mut transaction, job_id, worker_id, claim_token).await? + else { + transaction.rollback().await.map_err(storage_error)?; + return Ok(None); + }; + let sql = format!( + "UPDATE content_lock_deletion_jobs + SET state = 'queued', next_attempt_at = $2, claimed_by = NULL, + claim_token = NULL, claim_expires_at = NULL, updated_at = $3 + WHERE job_id = $1 RETURNING {ROW_COLUMNS}" + ); + let row = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(job_id) + .bind(winner_time + retry_after) + .bind(winner_time) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + fetch_optional_job(Some(row)) + } + + async fn defer( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + defer_for: time::Duration, + ) -> Result, ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let Some((_current, winner_time)) = + load_owned_claim(&mut transaction, job_id, worker_id, claim_token).await? + else { + transaction.rollback().await.map_err(storage_error)?; + return Ok(None); + }; + let sql = format!( + "UPDATE content_lock_deletion_jobs + SET state = 'queued', attempt_count = GREATEST(attempt_count - 1, 0), + next_attempt_at = $2, claimed_by = NULL, claim_token = NULL, + claim_expires_at = NULL, updated_at = $3 + WHERE job_id = $1 RETURNING {ROW_COLUMNS}" + ); + let row = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(job_id) + .bind(winner_time + defer_for) + .bind(winner_time) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + fetch_optional_job(Some(row)) + } + + async fn advance_phase( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + next_phase: ContentLockDeletionPhase, + ) -> Result { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let current = load_owned_claim(&mut transaction, job_id, worker_id, claim_token).await?; + let Some((current, now)) = current else { + transaction.rollback().await.map_err(storage_error)?; + return Ok(AdvanceContentLockDeletionPhaseResult::ClaimLost); + }; + if !current.phase.permits(next_phase) { + return Err(ApplicationError::InvalidContentLockDeletionState { + message: "deletion phase must advance to its immediate successor".to_owned(), + }); + } + let access_status = check_access_obligations_for_phase( + &mut transaction, + job_id, + current.phase, + next_phase, + now, + ) + .await?; + match access_status { + AccessPhaseAdvanceStatus::Ready => {} + AccessPhaseAdvanceStatus::ObligationsPending => { + transaction.commit().await.map_err(storage_error)?; + return Ok(AdvanceContentLockDeletionPhaseResult::ObligationsPending); + } + AccessPhaseAdvanceStatus::FinalCredentialIssuanceMissed => { + transaction.commit().await.map_err(storage_error)?; + return Ok(AdvanceContentLockDeletionPhaseResult::TerminalFailure( + ContentLockDeletionFailureCode::StateCorrupt, + )); + } + } + if current.phase == ContentLockDeletionPhase::DrainPayments + && next_phase == ContentLockDeletionPhase::DrainExistingCredentials + { + ensure_all_frozen_snapshots_terminal(&mut transaction, job_id).await?; + ensure_payment_drain_completed(&mut transaction, job_id).await?; + } + if next_phase == ContentLockDeletionPhase::StartPaymentDrain { + let has_unready_paykit_admission = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS ( + SELECT 1 + FROM content_lock_deletion_task_snapshot AS snapshot + LEFT JOIN paykit_task_admissions AS admission + ON admission.verification_task_id = snapshot.verification_task_id + WHERE snapshot.deletion_job_id = $1 + AND ( + snapshot.paykit_admission_required IS NULL + OR ( + snapshot.paykit_admission_required = TRUE + AND ( + snapshot.criterion_id IS NULL + OR admission.verification_task_id IS NULL + OR admission.ready = FALSE + OR admission.payment_in_hours IS NULL + OR admission.payment_in_hours <= 0 + OR admission.invoice_created_at IS NULL + OR admission.payment_deadline IS NULL + OR admission.invoice_created_at > admission.payment_deadline + ) + ) + ) + )", + ) + .bind(job_id) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if has_unready_paykit_admission { + return Err(ApplicationError::InvalidContentLockDeletionState { + message: + "payment drain cannot start before reserved Paykit admissions are ready" + .to_owned(), + }); + } + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot AS snapshot + SET payment_in_hours = admission.payment_in_hours, + invoice_created_at = admission.invoice_created_at, + payment_deadline = admission.payment_deadline, + resolved_status = CASE + WHEN snapshot.status_at_cutoff IN ('completed', 'failed', 'expired') + THEN snapshot.status_at_cutoff + ELSE NULL + END, + resolved_at = CASE + WHEN snapshot.status_at_cutoff IN ('completed', 'failed', 'expired') + THEN $2 + ELSE NULL + END + FROM paykit_task_admissions AS admission + WHERE snapshot.deletion_job_id = $1 + AND snapshot.paykit_admission_required = TRUE + AND admission.verification_task_id = snapshot.verification_task_id", + ) + .bind(job_id) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + } + if next_phase == ContentLockDeletionPhase::DeleteContent { + revoke_read_claims(&mut transaction, job_id).await?; + } + let sql = format!( + "UPDATE content_lock_deletion_jobs + SET phase = $2, state = 'queued', attempt_count = 0, next_attempt_at = NULL, + failure_code = NULL, claimed_by = NULL, claim_token = NULL, + claim_expires_at = NULL, updated_at = $3 + WHERE job_id = $1 RETURNING {ROW_COLUMNS}" + ); + let updated = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(job_id) + .bind(phase_to_database(next_phase)) + .bind(now) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(AdvanceContentLockDeletionPhaseResult::Advanced(Box::new( + row_to_job(updated)?, + ))) + } + + async fn expire_unresolved_non_paykit_tasks( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + ) -> Result { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let Some((current, now)) = + load_owned_claim(&mut transaction, job_id, worker_id, claim_token).await? + else { + transaction.rollback().await.map_err(storage_error)?; + return Ok(false); + }; + if !matches!( + current.phase, + ContentLockDeletionPhase::StartPaymentDrain | ContentLockDeletionPhase::DrainPayments + ) { + return Err(invalid_state( + "non-Paykit deletion drain requires a payment drain phase", + )); + } + let has_paykit_or_unknown: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 + AND paykit_admission_required IS DISTINCT FROM FALSE + )", + ) + .bind(job_id) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if has_paykit_or_unknown { + return Err(invalid_state( + "non-Paykit deletion drain cannot process a Paykit snapshot", + )); + } + sqlx::query( + "UPDATE verification_tasks AS task + SET status = 'expired', completed_at = $2, failure_message = NULL, + claimed_by = NULL, claim_token = NULL, claim_expires_at = NULL, + entitlement_publication_claim_token = NULL, next_attempt_at = NULL, + last_attempt_error = NULL, updated_at = $2 + FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.deletion_job_id = $1 + AND snapshot.verification_task_id = task.task_id + AND snapshot.paykit_admission_required = FALSE + AND snapshot.resolved_status IS NULL + AND task.status IN ('pending', 'in_progress')", + ) + .bind(job_id) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET resolved_status = 'expired', resolved_at = $2 + WHERE deletion_job_id = $1 + AND paykit_admission_required = FALSE + AND resolved_status IS NULL", + ) + .bind(job_id) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(true) + } + + async fn finish( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + failure_code: Option, + ) -> Result, ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let current = load_owned_claim(&mut transaction, job_id, worker_id, claim_token).await?; + let Some((current, now)) = current else { + transaction.rollback().await.map_err(storage_error)?; + return Ok(None); + }; + if failure_code.is_none() { + if current.phase != ContentLockDeletionPhase::PurgeOperationalState { + return Err(invalid_state( + "successful completion requires the final operational-state cleanup phase", + )); + } + ensure_all_frozen_snapshots_terminal(&mut transaction, job_id).await?; + ensure_payment_drain_completed(&mut transaction, job_id).await?; + ensure_no_live_access_obligations(&mut transaction, job_id, now).await?; + let issuance_incomplete: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 + FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.deletion_job_id = $1 + AND snapshot.final_credential_eligible_at IS NOT NULL + AND snapshot.final_credential_issued_at IS NULL + )", + ) + .bind(job_id) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if issuance_incomplete { + return Err(invalid_state( + "successful completion cannot bypass final credential issuance", + )); + } + } + revoke_read_claims(&mut transaction, job_id).await?; + let state = if failure_code.is_some() { + "failed" + } else { + "completed" + }; + let sql = format!( + "UPDATE content_lock_deletion_jobs + SET state = $2, failure_code = $3, next_attempt_at = NULL, + claimed_by = NULL, claim_token = NULL, claim_expires_at = NULL, updated_at = $4 + WHERE job_id = $1 + RETURNING {ROW_COLUMNS}" + ); + let updated = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(job_id) + .bind(state) + .bind(failure_code.map(ContentLockDeletionFailureCode::as_str)) + .bind(now) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(Some(row_to_job(updated)?)) + } + + async fn resume_failed_job( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + resumed_at: OffsetDateTime, + ) -> Result, ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, creator, lock_id).await?; + let receipt_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM content_lock_force_deletion_receipts + WHERE creator = $1 AND lock_id = $2)", + ) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if receipt_exists { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + } + let sql = format!( + "UPDATE content_lock_deletion_jobs + SET state = 'queued', attempt_count = 0, next_attempt_at = NULL, + failure_code = NULL, claimed_by = NULL, claim_token = NULL, + claim_expires_at = NULL, updated_at = $3 + WHERE creator = $1 AND lock_id = $2 AND state = 'failed' + RETURNING {ROW_COLUMNS}" + ); + let resumed = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .bind(resumed_at) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let current = if resumed.is_some() { + resumed + } else { + let sql = format!( + "SELECT {ROW_COLUMNS} FROM content_lock_deletion_jobs + WHERE creator = $1 AND lock_id = $2" + ); + sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)? + }; + transaction.commit().await.map_err(storage_error)?; + fetch_optional_job(current) + } + + async fn prepare_force_deletion( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, creator, lock_id).await?; + let forced_at: OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + let publication_in_progress = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM content_lock_publication_intents WHERE creator = $1 AND lock_id = $2)", + ) + .bind(creator.to_string()).bind(lock_id.to_string()) + .fetch_one(&mut *transaction).await.map_err(storage_error)?; + if publication_in_progress { + transaction.commit().await.map_err(storage_error)?; + return Ok(PrepareForceDeletionResult::PublicationInProgress); + } + let sql = format!( + "SELECT {ROW_COLUMNS} FROM content_lock_deletion_jobs + WHERE creator = $1 AND lock_id = $2 FOR UPDATE" + ); + let existing = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + if let Some(row) = existing { + let job = row_to_job(row)?; + if matches!( + job.state, + ContentLockDeletionState::Queued | ContentLockDeletionState::Running + ) { + let entitlement_publication_in_progress: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM verification_tasks + WHERE deletion_job_id = $1 + AND entitlement_publication_claim_token IS NOT NULL + )", + ) + .bind(job.job_id) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if entitlement_publication_in_progress { + transaction.commit().await.map_err(storage_error)?; + return Ok(PrepareForceDeletionResult::PublicationInProgress); + } + revoke_read_claims(&mut transaction, job.job_id).await?; + let sql = format!( + "UPDATE content_lock_deletion_jobs + SET force_requested_at = COALESCE(force_requested_at, $3), + state = 'queued', next_attempt_at = NULL, + claimed_by = NULL, claim_token = NULL, claim_expires_at = NULL, + updated_at = $3 + WHERE creator = $1 AND lock_id = $2 RETURNING {ROW_COLUMNS}" + ); + let active = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .bind(forced_at) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + return Ok(PrepareForceDeletionResult::Active(row_to_job(active)?)); + } + sqlx::query( + "INSERT INTO content_lock_force_deletion_receipts (creator, lock_id, forced_at) + VALUES ($1, $2, $3) ON CONFLICT (creator, lock_id) DO NOTHING", + ) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .bind(forced_at) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + sqlx::query("DELETE FROM content_lock_deletion_jobs WHERE job_id = $1") + .bind(job.job_id) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + return Ok(PrepareForceDeletionResult::Synchronous(Some(job))); + } + sqlx::query( + "INSERT INTO content_lock_force_deletion_receipts (creator, lock_id, forced_at) + VALUES ($1, $2, $3) ON CONFLICT (creator, lock_id) DO NOTHING", + ) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .bind(forced_at) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(PrepareForceDeletionResult::Synchronous(None)) + } + + async fn complete_force_deletion( + &self, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + ) -> Result { + let key = sqlx::query_as::<_, (String, String)>( + "SELECT creator, lock_id FROM content_lock_deletion_jobs WHERE job_id = $1", + ) + .bind(job_id) + .fetch_optional(&self.pool) + .await + .map_err(storage_error)?; + let Some((creator, lock_id)) = key else { + return Ok(false); + }; + let creator = CreatorPubky::from_str(&creator).map_err(storage_display)?; + let lock_id = LockId::from_str(&lock_id).map_err(storage_display)?; + + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, &creator, &lock_id).await?; + let current = load_owned_claim(&mut transaction, job_id, worker_id, claim_token).await?; + let Some((current, _now)) = current else { + transaction.rollback().await.map_err(storage_error)?; + return Ok(false); + }; + let Some(forced_at) = current.force_requested_at else { + transaction.rollback().await.map_err(storage_error)?; + return Ok(false); + }; + + sqlx::query( + "INSERT INTO content_lock_force_deletion_receipts (creator, lock_id, forced_at) + VALUES ($1, $2, $3)", + ) + .bind(current.creator.to_string()) + .bind(current.lock_id.to_string()) + .bind(forced_at) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + sqlx::query( + "UPDATE verification_tasks SET deletion_job_id = NULL WHERE deletion_job_id = $1", + ) + .bind(job_id) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + let deleted = sqlx::query("DELETE FROM content_lock_deletion_jobs WHERE job_id = $1") + .bind(job_id) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + if deleted.rows_affected() != 1 { + return Err(invalid_state( + "force completion lost its locked deletion job", + )); + } + transaction.commit().await.map_err(storage_error)?; + Ok(true) + } + + async fn has_force_receipt( + &self, + creator: &CreatorPubky, + lock_id: &LockId, + ) -> Result { + sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM content_lock_force_deletion_receipts + WHERE creator = $1 AND lock_id = $2)", + ) + .bind(creator.to_string()) + .bind(lock_id.to_string()) + .fetch_one(&self.pool) + .await + .map_err(storage_error) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AccessPhaseAdvanceStatus { + Ready, + ObligationsPending, + FinalCredentialIssuanceMissed, +} + +async fn check_access_obligations_for_phase( + transaction: &mut Transaction<'_, Postgres>, + job_id: Uuid, + current_phase: ContentLockDeletionPhase, + next_phase: ContentLockDeletionPhase, + now: OffsetDateTime, +) -> Result { + if current_phase == ContentLockDeletionPhase::DrainExistingCredentials + && next_phase == ContentLockDeletionPhase::IssueFinalCredentials + { + let ordinary_active: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND credential_kind = 'ordinary' + AND expires_at > $2 + )", + ) + .bind(job_id) + .bind(now) + .fetch_one(&mut **transaction) + .await + .map_err(storage_error)?; + if ordinary_active { + return Ok(AccessPhaseAdvanceStatus::ObligationsPending); + } + } + + if current_phase == ContentLockDeletionPhase::IssueFinalCredentials + && next_phase == ContentLockDeletionPhase::DrainFinalReads + { + let has_unissued_eligible: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 + AND final_credential_eligible_at IS NOT NULL + AND final_credential_issued_at IS NULL + )", + ) + .bind(job_id) + .fetch_one(&mut **transaction) + .await + .map_err(storage_error)?; + if has_unissued_eligible { + let issuance_deadline: Option = sqlx::query_scalar( + "SELECT final_credential_issuance_deadline + FROM content_lock_deletion_jobs WHERE job_id = $1", + ) + .bind(job_id) + .fetch_one(&mut **transaction) + .await + .map_err(storage_error)?; + return Ok( + if issuance_deadline.is_some_and(|deadline| now >= deadline) { + AccessPhaseAdvanceStatus::FinalCredentialIssuanceMissed + } else { + AccessPhaseAdvanceStatus::ObligationsPending + }, + ); + } + } + + if current_phase == ContentLockDeletionPhase::DrainFinalReads + && next_phase == ContentLockDeletionPhase::DeleteContent + { + return Ok( + if has_live_access_obligations(transaction, job_id, now).await? { + AccessPhaseAdvanceStatus::ObligationsPending + } else { + AccessPhaseAdvanceStatus::Ready + }, + ); + } + Ok(AccessPhaseAdvanceStatus::Ready) +} + +async fn has_live_access_obligations( + transaction: &mut Transaction<'_, Postgres>, + job_id: Uuid, + now: OffsetDateTime, +) -> Result { + sqlx::query( + "UPDATE content_lock_access_drain_reads AS read + SET claim_token = NULL, claim_expires_at = NULL + FROM content_lock_access_drain_credentials AS credential + WHERE read.credential_id = credential.credential_id + AND credential.deletion_job_id = $1 + AND read.claim_token IS NOT NULL + AND read.claim_expires_at <= $2", + ) + .bind(job_id) + .bind(now) + .execute(&mut **transaction) + .await + .map_err(storage_error)?; + + let has_live_obligation: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 + FROM content_lock_access_drain_credentials AS credential + WHERE credential.deletion_job_id = $1 + AND credential.credential_kind = 'ordinary' + AND credential.expires_at > $2 + ) OR EXISTS ( + SELECT 1 + FROM content_lock_access_drain_credentials AS credential + JOIN content_lock_access_drain_reads AS read + ON read.credential_id = credential.credential_id + WHERE credential.deletion_job_id = $1 + AND credential.credential_kind = 'final' + AND credential.expires_at > $2 + AND read.consumed_at IS NULL + ) OR EXISTS ( + SELECT 1 + FROM content_lock_access_drain_credentials AS credential + JOIN content_lock_access_drain_reads AS read + ON read.credential_id = credential.credential_id + WHERE credential.deletion_job_id = $1 + AND read.claim_token IS NOT NULL + AND read.claim_expires_at > $2 + )", + ) + .bind(job_id) + .bind(now) + .fetch_one(&mut **transaction) + .await + .map_err(storage_error)?; + Ok(has_live_obligation) +} + +async fn ensure_no_live_access_obligations( + transaction: &mut Transaction<'_, Postgres>, + job_id: Uuid, + now: OffsetDateTime, +) -> Result<(), ApplicationError> { + if has_live_access_obligations(transaction, job_id, now).await? { + return Err(invalid_state( + "credential expiry and final-read obligations must drain before destructive deletion", + )); + } + Ok(()) +} + +async fn ensure_all_frozen_snapshots_terminal( + transaction: &mut Transaction<'_, Postgres>, + job_id: Uuid, +) -> Result<(), ApplicationError> { + let resolved_statuses = sqlx::query_scalar::<_, Option>( + "SELECT resolved_status + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 + ORDER BY verification_task_id + FOR UPDATE", + ) + .bind(job_id) + .fetch_all(&mut **transaction) + .await + .map_err(storage_error)?; + if resolved_statuses.iter().any(Option::is_none) { + return Err(invalid_state( + "every frozen deletion obligation must be terminal before credential draining", + )); + } + Ok(()) +} + +async fn ensure_payment_drain_completed( + transaction: &mut Transaction<'_, Postgres>, + job_id: Uuid, +) -> Result<(), ApplicationError> { + let aggregate: Option<(String, i64)> = sqlx::query_as( + "SELECT status, accepted_count + FROM content_lock_payment_drains + WHERE deletion_job_id = $1 + FOR UPDATE", + ) + .bind(job_id) + .fetch_optional(&mut **transaction) + .await + .map_err(storage_error)?; + let completed = aggregate + .as_ref() + .is_some_and(|(status, accepted_count)| status == "completed" && *accepted_count == 0); + let has_paykit_snapshot: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 + AND paykit_admission_required IS DISTINCT FROM FALSE + )", + ) + .bind(job_id) + .fetch_one(&mut **transaction) + .await + .map_err(storage_error)?; + if !completed && (aggregate.is_some() || has_paykit_snapshot) { + return Err(invalid_state( + "payment drain aggregate must be durably completed before credential draining", + )); + } + Ok(()) +} + +async fn revoke_read_claims( + transaction: &mut Transaction<'_, Postgres>, + job_id: Uuid, +) -> Result<(), ApplicationError> { + sqlx::query( + "UPDATE content_lock_access_drain_reads AS read + SET claim_token = NULL, claim_expires_at = NULL + FROM content_lock_access_drain_credentials AS credential + WHERE read.credential_id = credential.credential_id + AND credential.deletion_job_id = $1 + AND read.claim_token IS NOT NULL", + ) + .bind(job_id) + .execute(&mut **transaction) + .await + .map_err(storage_error)?; + Ok(()) +} + +async fn delete_publication_intent( + pool: &PgPool, + creator: &CreatorPubky, + lock_id: &LockId, + publication_token: Uuid, +) -> Result { + let mut transaction = pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, creator, lock_id).await?; + let result = sqlx::query("DELETE FROM content_lock_publication_intents WHERE creator = $1 AND lock_id = $2 AND publication_token = $3") + .bind(creator.to_string()).bind(lock_id.to_string()).bind(publication_token) + .execute(&mut *transaction).await.map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(result.rows_affected() == 1) +} + +fn map_publication_insert_error(error: sqlx::Error) -> ApplicationError { + if let sqlx::Error::Database(database_error) = &error + && database_error.is_unique_violation() + { + return ApplicationError::ContentLockPathConflict { + guarded_path: "content lock publication in progress".to_owned(), + }; + } + storage_error(error) +} + +async fn load_owned_claim( + transaction: &mut Transaction<'_, Postgres>, + job_id: Uuid, + worker_id: &str, + claim_token: Uuid, +) -> Result, ApplicationError> { + let sql = format!( + "SELECT {ROW_COLUMNS} FROM content_lock_deletion_jobs + WHERE job_id = $1 AND state = 'running' AND claimed_by = $2 + AND claim_token = $3 FOR UPDATE" + ); + let row = sqlx::query_as::<_, DeletionJobRow>(&sql) + .bind(job_id) + .bind(worker_id) + .bind(claim_token) + .fetch_optional(&mut **transaction) + .await + .map_err(storage_error)?; + let Some(row) = row else { + return Ok(None); + }; + let winner_time: OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut **transaction) + .await + .map_err(storage_error)?; + if row + .claim_expires_at + .is_none_or(|claim_expires_at| winner_time >= claim_expires_at) + { + return Ok(None); + } + Ok(Some((row_to_job(row)?, winner_time))) +} + +fn fetch_optional_job( + row: Option, +) -> Result, ApplicationError> { + row.map(row_to_job).transpose() +} + +fn row_to_job(row: DeletionJobRow) -> Result { + let has_active_lease = match ( + row.claimed_by.is_some(), + row.claim_token.is_some(), + row.claim_expires_at.is_some(), + ) { + (false, false, false) => false, + (true, true, true) => true, + _ => return Err(invalid_state("deletion lease fields are inconsistent")), + }; + let job = ContentLockDeletionJob { + job_id: row.job_id, + creator: CreatorPubky::from_str(&row.creator).map_err(storage_display)?, + lock_id: LockId::from_str(&row.lock_id).map_err(storage_display)?, + frozen_content_lock: serde_json::from_value::(row.frozen_content_lock) + .map_err(storage_display)?, + deletion_started_at: row.deletion_started_at, + state: state_from_database(&row.state)?, + phase: phase_from_database(&row.phase)?, + attempt_count: u32::try_from(row.attempt_count).map_err(storage_display)?, + next_attempt_at: row.next_attempt_at, + force_requested_at: row.force_requested_at, + failure_code: row + .failure_code + .map(|code| code.parse::()) + .transpose()?, + }; + job.validate_frozen_identity()?; + job.validate_state(has_active_lease)?; + Ok(job) +} + +fn state_to_database(state: ContentLockDeletionState) -> &'static str { + match state { + ContentLockDeletionState::Queued => "queued", + ContentLockDeletionState::Running => "running", + ContentLockDeletionState::Completed => "completed", + ContentLockDeletionState::Failed => "failed", + } +} + +fn state_from_database(value: &str) -> Result { + match value { + "queued" => Ok(ContentLockDeletionState::Queued), + "running" => Ok(ContentLockDeletionState::Running), + "completed" => Ok(ContentLockDeletionState::Completed), + "failed" => Ok(ContentLockDeletionState::Failed), + _ => Err(invalid_state("unknown deletion state")), + } +} + +fn phase_to_database(phase: ContentLockDeletionPhase) -> &'static str { + match phase { + ContentLockDeletionPhase::Withdraw => "withdraw", + ContentLockDeletionPhase::StartPaymentDrain => "start_payment_drain", + ContentLockDeletionPhase::DrainPayments => "drain_payments", + ContentLockDeletionPhase::DrainExistingCredentials => "drain_existing_credentials", + ContentLockDeletionPhase::IssueFinalCredentials => "issue_final_credentials", + ContentLockDeletionPhase::DrainFinalReads => "drain_final_reads", + ContentLockDeletionPhase::DeleteContent => "delete_content", + ContentLockDeletionPhase::DeleteTombstone => "delete_tombstone", + ContentLockDeletionPhase::PurgeOperationalState => "purge_operational_state", + } +} + +fn phase_from_database(value: &str) -> Result { + match value { + "withdraw" => Ok(ContentLockDeletionPhase::Withdraw), + "start_payment_drain" => Ok(ContentLockDeletionPhase::StartPaymentDrain), + "drain_payments" => Ok(ContentLockDeletionPhase::DrainPayments), + "drain_existing_credentials" => Ok(ContentLockDeletionPhase::DrainExistingCredentials), + "issue_final_credentials" => Ok(ContentLockDeletionPhase::IssueFinalCredentials), + "drain_final_reads" => Ok(ContentLockDeletionPhase::DrainFinalReads), + "delete_content" => Ok(ContentLockDeletionPhase::DeleteContent), + "delete_tombstone" => Ok(ContentLockDeletionPhase::DeleteTombstone), + "purge_operational_state" => Ok(ContentLockDeletionPhase::PurgeOperationalState), + _ => Err(invalid_state("unknown deletion phase")), + } +} + +fn invalid_state(message: &str) -> ApplicationError { + ApplicationError::InvalidContentLockDeletionState { + message: message.to_owned(), + } +} + +fn map_insert_error(error: sqlx::Error) -> ApplicationError { + if error + .as_database_error() + .is_some_and(|error| error.is_unique_violation()) + { + ApplicationError::DuplicateRecord { + record: "content_lock_deletion_job", + } + } else { + storage_error(error) + } +} + +fn storage_error(error: sqlx::Error) -> ApplicationError { + storage_display(error) +} + +fn storage_display(error: impl std::fmt::Display) -> ApplicationError { + ApplicationError::Storage { + message: error.to_string(), + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + str::FromStr, + sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, + }, + }; + + use async_trait::async_trait; + use locks_core::{ + ids::{ + BundleId, CreatorPubky, GuardedResourceHash, LockServerPubky, PubkyLockResource, TaskId, + }, + lock_policy::{ + AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, GuardedResource, LockLogic, + LockServerConfig, VerifierType, + }, + verification::{Proof, SUBMITTED_PROOF_BUNDLE_VERSION, SubmittedProofBundle}, + }; + use time::macros::datetime; + use uuid::Uuid; + + use super::PostgresContentLockDeletionRepository; + use crate::{ + application::{ + errors::ApplicationError, + models::{ + AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, + AdvanceContentLockDeletionPhaseResult, ContentLockDeletionFailureCode, + ContentLockDeletionJob, ContentLockDeletionPhase, ContentLockDeletionState, + InitializeFinalAccessWindowsResult, PrepareForceDeletionResult, + VerificationTaskRecord, VerificationTaskStatus, + }, + ports::{ + AccessCredentialStore, Clock, ContentLockDeletionActionAcquireResult, + ContentLockDeletionActionClaim, ContentLockDeletionActionOwnership, + ContentLockDeletionRepository, EntitlementRepository, PaymentDrainCleanupToken, + PaymentDrainClient, PaymentDrainClientError, PaymentDrainRepository, + PaymentDrainStatus, PaymentDrainSummary, PaymentDrainTerminalTransition, + PaymentRequestState, PaymentRequestStatus, PaymentState, VerificationTaskClaimer, + VerificationTaskRepository, + }, + use_cases::drain_lock_payments::DrainLockPaymentsUseCase, + }, + infrastructure::memory::entitlements::InMemoryEntitlementRepository, + infrastructure::postgres::{ + PostgresAccessCredentialStore, PostgresContentLockDeletionActionOwnership, + PostgresPaymentDrainRepository, PostgresVerificationTaskClaimer, + PostgresVerificationTaskRepository, testing::TestDatabase, + }, + }; + + const CREATOR: &str = "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy"; + const NOW: time::OffsetDateTime = datetime!(2026-08-12 05:00:00 UTC); + + #[tokio::test] + async fn healthy_defer_restores_the_postgres_attempt_budget_and_fences_stale_tokens() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + + let first = repository + .claim_next("worker", (NOW + time::Duration::minutes(5)) - (NOW)) + .await + .unwrap() + .unwrap(); + assert_eq!(first.job.attempt_count, 1); + assert!( + repository + .defer( + job.job_id, + "worker", + Uuid::new_v4(), + (NOW + time::Duration::minutes(1)) - (NOW), + ) + .await + .unwrap() + .is_none() + ); + + let deferred = repository + .defer( + job.job_id, + "worker", + first.claim_token, + time::Duration::minutes(1), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(deferred.attempt_count, 0); + let (first_due, first_updated_at): (time::OffsetDateTime, time::OffsetDateTime) = + sqlx::query_as( + "SELECT next_attempt_at, updated_at + FROM content_lock_deletion_jobs WHERE job_id = $1", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(deferred.next_attempt_at, Some(first_due)); + assert_eq!(first_due - first_updated_at, time::Duration::minutes(1)); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET next_attempt_at = clock_timestamp() + WHERE job_id = $1", + ) + .bind(job.job_id) + .execute(database.pool()) + .await + .unwrap(); + + let second = repository + .claim_next("worker", time::Duration::minutes(5)) + .await + .unwrap() + .unwrap(); + assert_eq!(second.job.attempt_count, 1); + let deferred = repository + .defer( + job.job_id, + "worker", + second.claim_token, + time::Duration::minutes(1), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(deferred.attempt_count, 0); + let (second_due, second_updated_at): (time::OffsetDateTime, time::OffsetDateTime) = + sqlx::query_as( + "SELECT next_attempt_at, updated_at + FROM content_lock_deletion_jobs WHERE job_id = $1", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(deferred.next_attempt_at, Some(second_due)); + assert_eq!(second_due - second_updated_at, time::Duration::minutes(1)); + + database.cleanup().await; + } + + #[tokio::test] + async fn active_force_completion_persists_original_receipt_and_removes_operational_job() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let lock = content_lock(); + let task = verification_task(&lock, BundleId::from_bytes([99; 16])); + PostgresVerificationTaskRepository::new(database.pool().clone()) + .insert_verification_task(task.clone()) + .await + .unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + let revoked = repository + .claim_next("worker-old", (NOW + time::Duration::minutes(1)) - (NOW)) + .await + .unwrap() + .unwrap(); + let forced_at = match repository + .prepare_force_deletion(&job.creator, &job.lock_id) + .await + .unwrap() + { + PrepareForceDeletionResult::Active(job) => job.force_requested_at.unwrap(), + result => panic!("expected active force deletion, got {result:?}"), + }; + assert!(matches!( + repository + .prepare_force_deletion(&job.creator, &job.lock_id,) + .await + .unwrap(), + PrepareForceDeletionResult::Active(_) + )); + assert!( + !repository + .complete_force_deletion(job.job_id, "worker-old", revoked.claim_token) + .await + .unwrap() + ); + + let live = repository + .claim_next("worker-live", time::Duration::minutes(5)) + .await + .unwrap() + .unwrap(); + assert!( + !repository + .complete_force_deletion(job.job_id, "worker-live", Uuid::new_v4()) + .await + .unwrap() + ); + assert!( + repository + .complete_force_deletion(job.job_id, "worker-live", live.claim_token) + .await + .unwrap() + ); + + let receipt: (String, String, time::OffsetDateTime) = sqlx::query_as( + "SELECT creator, lock_id, forced_at + FROM content_lock_force_deletion_receipts", + ) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(receipt.0, job.creator.to_string()); + assert_eq!(receipt.1, job.lock_id.to_string()); + assert_eq!(receipt.2, forced_at); + assert!( + repository + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .is_none() + ); + let retained_task_job: Option = + sqlx::query_scalar("SELECT deletion_job_id FROM verification_tasks WHERE task_id = $1") + .bind(task.task_id.as_uuid()) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(retained_task_job, None); + let snapshot_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM content_lock_deletion_task_snapshot WHERE deletion_job_id = $1", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(snapshot_count, 0); + assert!( + !repository + .complete_force_deletion(job.job_id, "worker-live", live.claim_token) + .await + .unwrap() + ); + assert_eq!( + repository + .begin_publication(&job.creator, &job.lock_id, Uuid::new_v4()) + .await, + Err(ApplicationError::ContentLockDeletionInProgress) + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn expired_or_unforced_postgres_claim_cannot_finalize_force_deletion() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let unforced = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(unforced.clone()).await.unwrap(); + let claim = repository + .claim_next("worker", (NOW + time::Duration::minutes(1)) - (NOW)) + .await + .unwrap() + .unwrap(); + assert!( + !repository + .complete_force_deletion(unforced.job_id, "worker", claim.claim_token) + .await + .unwrap() + ); + repository + .prepare_force_deletion(&unforced.creator, &unforced.lock_id) + .await + .unwrap(); + let expiring = repository + .claim_next("worker", time::Duration::minutes(1)) + .await + .unwrap() + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET claim_expires_at = clock_timestamp() + WHERE job_id = $1 AND claim_token = $2", + ) + .bind(unforced.job_id) + .bind(expiring.claim_token) + .execute(database.pool()) + .await + .unwrap(); + assert!( + !repository + .complete_force_deletion(unforced.job_id, "worker", expiring.claim_token,) + .await + .unwrap() + ); + assert!( + repository + .get_job(&unforced.creator, &unforced.lock_id) + .await + .unwrap() + .is_some() + ); + assert!( + !repository + .has_force_receipt(&unforced.creator, &unforced.lock_id) + .await + .unwrap() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn postgres_action_ownership_validates_exact_live_claim_after_locking() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let first_owner = PostgresContentLockDeletionActionOwnership::new(database.pool().clone()); + let second_owner = PostgresContentLockDeletionActionOwnership::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job).await.unwrap(); + let claimed = repository + .claim_next("worker", time::Duration::minutes(5)) + .await + .unwrap() + .unwrap(); + let request = || ContentLockDeletionActionClaim { + job_id: claimed.job.job_id, + worker_id: "worker", + claim_token: claimed.claim_token, + expected_phase: claimed.job.phase, + force: false, + }; + + let ContentLockDeletionActionAcquireResult::Acquired(first) = + first_owner.try_acquire(request()).await.unwrap() + else { + panic!("live claim must acquire") + }; + assert!(matches!( + second_owner.try_acquire(request()).await.unwrap(), + ContentLockDeletionActionAcquireResult::Busy + )); + + first.release().await.unwrap(); + let ContentLockDeletionActionAcquireResult::Acquired(reacquired) = + second_owner.try_acquire(request()).await.unwrap() + else { + panic!("released live claim must reacquire") + }; + reacquired.release().await.unwrap(); + + let stale = ContentLockDeletionActionClaim { + worker_id: "other-worker", + ..request() + }; + assert!(matches!( + second_owner.try_acquire(stale).await.unwrap(), + ContentLockDeletionActionAcquireResult::ClaimLost + )); + let wrong_phase = ContentLockDeletionActionClaim { + expected_phase: ContentLockDeletionPhase::DrainPayments, + ..request() + }; + assert!(matches!( + second_owner.try_acquire(wrong_phase).await.unwrap(), + ContentLockDeletionActionAcquireResult::ClaimLost + )); + let wrong_mode = ContentLockDeletionActionClaim { + force: true, + ..request() + }; + assert!(matches!( + second_owner.try_acquire(wrong_mode).await.unwrap(), + ContentLockDeletionActionAcquireResult::ClaimLost + )); + + database.cleanup().await; + } + + #[tokio::test] + async fn postgres_action_ownership_rejects_lease_expiry_equality_and_releases_session_lock() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let ownership = PostgresContentLockDeletionActionOwnership::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job).await.unwrap(); + let claimed = repository + .claim_next("worker", time::Duration::minutes(5)) + .await + .unwrap() + .unwrap(); + sqlx::query("UPDATE content_lock_deletion_jobs SET claim_expires_at = clock_timestamp() WHERE job_id = $1") + .bind(claimed.job.job_id) + .execute(database.pool()).await.unwrap(); + let request = ContentLockDeletionActionClaim { + job_id: claimed.job.job_id, + worker_id: "worker", + claim_token: claimed.claim_token, + expected_phase: claimed.job.phase, + force: false, + }; + assert!(matches!( + ownership.try_acquire(request).await.unwrap(), + ContentLockDeletionActionAcquireResult::ClaimLost + )); + + let value: i32 = sqlx::query_scalar("SELECT 1") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(value, 1); + + database.cleanup().await; + } + + #[tokio::test] + async fn admission_cutoff_timestamp_is_established_after_the_canonical_fence() { + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let credentials = PostgresAccessCredentialStore::new(database.pool().clone()); + let lock = content_lock(); + let lock_id = lock.lock_id().unwrap(); + let bundle_id = BundleId::from_bytes([91; 16]); + let mut task = verification_task(&lock, bundle_id.clone()); + task.status = VerificationTaskStatus::Completed; + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + tasks.insert_verification_task(task).await.unwrap(); + let lookup_key = AccessCredentialLookupKey::derive(&AccessCredential::new("pre-fence")); + let expires_at = NOW + time::Duration::hours(1); + credentials + .insert_access_credential( + &lock_id, + lookup_key.clone(), + AccessCredentialRecord { + creator: lock.creator.clone(), + bundle_id, + expires_at, + }, + ) + .await + .unwrap(); + + let mut blocker = database.pool().begin().await.unwrap(); + super::lock_proof_admission(&mut blocker, &lock.creator, &lock_id) + .await + .unwrap(); + let fence_held_at: time::OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *blocker) + .await + .unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + let inserting = tokio::spawn({ + let deletions = deletions.clone(); + let job = job.clone(); + async move { deletions.insert_job(job).await } + }); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + assert!(!inserting.is_finished()); + blocker.commit().await.unwrap(); + inserting.await.unwrap().unwrap(); + + let (cutoff, had_active, eligible_at): ( + time::OffsetDateTime, + bool, + Option, + ) = sqlx::query_as( + "SELECT job.deletion_started_at, snapshot.had_active_credential_at_cutoff, + snapshot.final_credential_eligible_at + FROM content_lock_deletion_jobs AS job + JOIN content_lock_deletion_task_snapshot AS snapshot + ON snapshot.deletion_job_id = job.job_id + WHERE job.job_id = $1", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert!(cutoff >= fence_held_at); + assert!(cutoff > expires_at); + assert!(!had_active); + assert_eq!(eligible_at, Some(cutoff)); + assert!( + !credentials + .deletion_credential_enrolled(&lookup_key) + .await + .unwrap() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn deletion_commit_order_is_the_authoritative_proof_admission_cutoff() { + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + + let admitted_before = verification_task(&lock, BundleId::from_bytes([1; 16])); + tasks + .insert_verification_task(admitted_before.clone()) + .await + .unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock.clone(), NOW).unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + + let snapshotted: Vec = sqlx::query_scalar( + "SELECT verification_task_id + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1", + ) + .bind(job.job_id) + .fetch_all(database.pool()) + .await + .unwrap(); + assert_eq!(snapshotted, vec![admitted_before.task_id.as_uuid()]); + assert_eq!( + tasks + .insert_verification_task(admitted_before.clone()) + .await, + Err(ApplicationError::DuplicateRecord { + record: "verification_task", + }) + ); + + let admitted_after = verification_task(&lock, BundleId::from_bytes([2; 16])); + assert_eq!( + tasks.insert_verification_task(admitted_after).await, + Err(ApplicationError::ContentLockDeletionInProgress) + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn credential_committed_before_deletion_is_classified_and_enrolled_at_cutoff() { + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let credentials = PostgresAccessCredentialStore::new(database.pool().clone()); + let lock = content_lock(); + let guarded_path = lock.primary_resource.as_ref().unwrap().path.clone(); + let lock_id = lock.lock_id().unwrap(); + let bundle_id = BundleId::from_bytes([3; 16]); + tasks + .insert_verification_task(verification_task(&lock, bundle_id.clone())) + .await + .unwrap(); + let bearer = AccessCredential::new("cutoff-active-credential"); + let lookup_key = AccessCredentialLookupKey::derive(&bearer); + let expires_at: time::OffsetDateTime = + sqlx::query_scalar("SELECT clock_timestamp() + INTERVAL '1 hour'") + .fetch_one(database.pool()) + .await + .unwrap(); + credentials + .insert_access_credential( + &lock_id, + lookup_key.clone(), + AccessCredentialRecord { + creator: lock.creator.clone(), + bundle_id, + expires_at, + }, + ) + .await + .unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + + deletions.insert_job(job.clone()).await.unwrap(); + + let attached_job: Option = sqlx::query_scalar( + "SELECT deletion_job_id FROM access_credentials WHERE lookup_key = $1", + ) + .bind(lookup_key.as_bytes().as_slice()) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(attached_job, Some(job.job_id)); + let had_active: bool = sqlx::query_scalar( + "SELECT had_active_credential_at_cutoff + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert!(had_active); + let enrolled_expiry: time::OffsetDateTime = sqlx::query_scalar( + "SELECT expires_at FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND lookup_key = $2 AND credential_kind = 'ordinary'", + ) + .bind(job.job_id) + .bind(lookup_key.as_bytes().as_slice()) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(enrolled_expiry, expires_at); + let final_read_allowances: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM content_lock_access_drain_reads") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(final_read_allowances, 0); + assert!( + credentials + .deletion_credential_enrolled(&lookup_key) + .await + .unwrap() + ); + let authorization = credentials + .prepare_deletion_read(&lookup_key, &guarded_path, time::Duration::seconds(30)) + .await + .unwrap() + .unwrap(); + assert_eq!(authorization.resource.path, guarded_path); + assert_eq!(authorization.claim_token, None); + assert!( + credentials + .prepare_deletion_read( + &lookup_key, + "/priv/locks.app/content/not-frozen.json", + time::Duration::seconds(30), + ) + .await + .unwrap() + .is_none() + ); + assert!( + credentials + .deletion_credential_enrolled(&lookup_key) + .await + .unwrap() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn concurrent_deletion_and_credential_issuance_have_one_serialized_cutoff_order() { + for iteration in 1..=20_u8 { + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let credentials = PostgresAccessCredentialStore::new(database.pool().clone()); + let lock = content_lock(); + let lock_id = lock.lock_id().unwrap(); + let bundle_id = BundleId::from_bytes([iteration; 16]); + tasks + .insert_verification_task(verification_task(&lock, bundle_id.clone())) + .await + .unwrap(); + let lookup_key = AccessCredentialLookupKey::derive(&AccessCredential::new(format!( + "concurrent-{iteration}" + ))); + let record = AccessCredentialRecord { + creator: lock.creator.clone(), + bundle_id, + expires_at: datetime!(2026-08-12 06:00:00 UTC), + }; + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + + let (deletion_result, credential_result) = tokio::join!( + deletions.insert_job(job.clone()), + credentials.insert_access_credential(&lock_id, lookup_key.clone(), record) + ); + deletion_result.unwrap(); + + let attached_job: Option> = sqlx::query_scalar( + "SELECT deletion_job_id FROM access_credentials WHERE lookup_key = $1", + ) + .bind(lookup_key.as_bytes().as_slice()) + .fetch_optional(database.pool()) + .await + .unwrap(); + let enrolled: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND lookup_key = $2 + )", + ) + .bind(job.job_id) + .bind(lookup_key.as_bytes().as_slice()) + .fetch_one(database.pool()) + .await + .unwrap(); + match credential_result { + Ok(()) => { + assert_eq!(attached_job, Some(Some(job.job_id))); + assert!(enrolled); + } + Err(ApplicationError::ContentLockDeletionInProgress) => { + assert_eq!(attached_job, None); + assert!(!enrolled); + } + other => panic!("unexpected concurrent credential result: {other:?}"), + } + + database.cleanup().await; + } + } + + #[tokio::test] + async fn concurrent_deletion_and_new_bundle_have_one_serialized_cutoff_order() { + for iteration in 0..20_u8 { + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + let task = verification_task(&lock, BundleId::from_bytes([iteration; 16])); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + + let (deletion_result, task_result) = tokio::join!( + deletions.insert_job(job.clone()), + tasks.insert_verification_task(task.clone()) + ); + deletion_result.unwrap(); + + let snapshotted = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 AND verification_task_id = $2 + )", + ) + .bind(job.job_id) + .bind(task.task_id.as_uuid()) + .fetch_one(database.pool()) + .await + .unwrap(); + match task_result { + Ok(()) => assert!(snapshotted), + Err(ApplicationError::ContentLockDeletionInProgress) => assert!(!snapshotted), + other => panic!("unexpected concurrent admission result: {other:?}"), + } + + database.cleanup().await; + } + } + + #[tokio::test] + async fn concurrent_graceful_start_and_force_prepare_leave_exactly_one_durable_mode() { + for _ in 0..20 { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + + let (graceful, force) = tokio::join!( + repository.insert_job(job.clone()), + repository.prepare_force_deletion(&job.creator, &job.lock_id) + ); + let persisted_job = repository + .get_job(&job.creator, &job.lock_id) + .await + .unwrap(); + let receipt = repository + .has_force_receipt(&job.creator, &job.lock_id) + .await + .unwrap(); + + match (graceful, force) { + (Ok(()), Ok(PrepareForceDeletionResult::Active(active))) => { + assert_eq!(active.job_id, job.job_id); + assert!(active.force_requested_at.is_some()); + assert_eq!(persisted_job, Some(active)); + assert!(!receipt); + } + ( + Err(ApplicationError::ContentLockDeletionInProgress), + Ok(PrepareForceDeletionResult::Synchronous(None)), + ) => { + assert!(persisted_job.is_none()); + assert!(receipt); + } + other => panic!("unexpected graceful/force race result: {other:?}"), + } + + database.cleanup().await; + } + } + + #[tokio::test] + async fn publication_intent_and_force_receipt_have_one_serialized_cutoff_order() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let lock = content_lock(); + let lock_id = lock.lock_id().unwrap(); + let token = Uuid::new_v4(); + + repository + .begin_publication(&lock.creator, &lock_id, token) + .await + .unwrap(); + assert_eq!( + repository + .prepare_force_deletion(&lock.creator, &lock_id) + .await + .unwrap(), + PrepareForceDeletionResult::PublicationInProgress + ); + assert!( + !repository + .has_force_receipt(&lock.creator, &lock_id) + .await + .unwrap() + ); + assert!( + repository + .finish_publication(&lock.creator, &lock_id, token) + .await + .unwrap() + ); + assert_eq!( + repository + .prepare_force_deletion(&lock.creator, &lock_id) + .await + .unwrap(), + PrepareForceDeletionResult::Synchronous(None) + ); + assert_eq!( + repository + .begin_publication(&lock.creator, &lock_id, Uuid::new_v4()) + .await, + Err(ApplicationError::ContentLockDeletionInProgress) + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn durable_paykit_reservation_commits_before_deletion_and_is_snapshotted() { + use crate::infrastructure::postgres::PostgresPaykitTaskAdmissionRepository; + + let database = TestDatabase::create().await; + let lock = content_lock(); + let task = verification_task(&lock, BundleId::from_bytes([3; 16])); + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let admission = admissions.reserve(task.clone(), 24).await.unwrap(); + assert!(admission.requires_paykit); + assert_eq!(admission.payment_in, 24); + assert_eq!(admission.invoice_window, None); + + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository + .insert_job(ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap()) + .await + .unwrap(); + + let snapshotted = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_deletion_task_snapshot + WHERE verification_task_id = $1 + )", + ) + .bind(task.task_id.as_uuid()) + .fetch_one(database.pool()) + .await + .unwrap(); + assert!(snapshotted); + + database.cleanup().await; + } + + #[tokio::test] + async fn deletion_first_rejects_durable_paykit_reservation_before_external_work() { + use crate::infrastructure::postgres::PostgresPaykitTaskAdmissionRepository; + + let database = TestDatabase::create().await; + let lock = content_lock(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository + .insert_job(ContentLockDeletionJob::new(Uuid::new_v4(), lock.clone(), NOW).unwrap()) + .await + .unwrap(); + + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let mut external_calls = 0; + let result = admissions + .reserve(verification_task(&lock, BundleId::from_bytes([4; 16])), 24) + .await; + if result.is_ok() { + external_calls += 1; + } + assert!(matches!( + result, + Err(ApplicationError::ContentLockDeletionInProgress) + )); + assert_eq!(external_calls, 0); + + database.cleanup().await; + } + + #[tokio::test] + async fn payment_drain_phase_waits_for_every_snapshotted_paykit_reservation() { + use crate::infrastructure::postgres::PostgresPaykitTaskAdmissionRepository; + + let database = TestDatabase::create().await; + let lock = content_lock(); + let mut task = verification_task(&lock, BundleId::from_bytes([6; 16])); + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let admission = admissions.reserve(task.clone(), 24).await.unwrap(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository + .insert_job(ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap()) + .await + .unwrap(); + let claim = repository + .claim_next("worker", (NOW + time::Duration::seconds(60)) - (NOW)) + .await + .unwrap() + .unwrap(); + + let blocked = repository + .advance_phase( + claim.job.job_id, + "worker", + claim.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await; + assert!(matches!( + blocked, + Err(ApplicationError::InvalidContentLockDeletionState { message }) + if message == "payment drain cannot start before reserved Paykit admissions are ready" + )); + let still_running = repository + .get_job(&claim.job.creator, &claim.job.lock_id) + .await + .unwrap() + .unwrap(); + assert_eq!(still_running.state, ContentLockDeletionState::Running); + assert_eq!(still_running.phase, ContentLockDeletionPhase::Withdraw); + + let invoice_window = crate::infrastructure::postgres::PaykitInvoiceWindow { + invoice_created_at: datetime!(2026-08-12 05:01:00 UTC), + payment_deadline: datetime!(2026-08-13 05:01:00 UTC), + }; + admissions + .mark_ready(&admission.task, invoice_window) + .await + .unwrap(); + let replay = admissions + .find_existing(&task.submitted_proof_bundle) + .await + .unwrap() + .unwrap(); + assert!(!replay.requires_paykit); + assert_eq!(replay.payment_in, 24); + assert_eq!(replay.invoice_window, Some(invoice_window)); + let advanced = repository + .advance_phase( + claim.job.job_id, + "worker", + claim.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + assert_eq!(advanced.phase, ContentLockDeletionPhase::StartPaymentDrain); + + database.cleanup().await; + } + + #[tokio::test] + async fn payment_drain_phase_rejects_snapshotted_legacy_admission_without_invoice_window() { + use crate::infrastructure::postgres::verification_tasks::PostgresVerificationTaskRepository; + + let database = TestDatabase::create().await; + let lock = content_lock(); + let task = verification_task(&lock, BundleId::from_bytes([7; 16])); + PostgresVerificationTaskRepository::new(database.pool().clone()) + .insert_verification_task(task.clone()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO paykit_task_admissions + (verification_task_id, ready, ready_at) + VALUES ($1::uuid, TRUE, now())", + ) + .bind(task.task_id.to_string()) + .execute(database.pool()) + .await + .unwrap(); + + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository + .insert_job(ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap()) + .await + .unwrap(); + let claim = repository + .claim_next("worker", (NOW + time::Duration::seconds(60)) - (NOW)) + .await + .unwrap() + .unwrap(); + + let result = repository + .advance_phase( + claim.job.job_id, + "worker", + claim.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await; + assert!(matches!( + result, + Err(ApplicationError::InvalidContentLockDeletionState { message }) + if message == "payment drain cannot start before reserved Paykit admissions are ready" + )); + + database.cleanup().await; + } + + #[tokio::test] + async fn paykit_admission_insert_failure_rolls_back_verification_task() { + use crate::infrastructure::postgres::PostgresPaykitTaskAdmissionRepository; + + let database = TestDatabase::create().await; + sqlx::query( + "CREATE FUNCTION reject_paykit_admission() RETURNS trigger AS $$ + BEGIN + RAISE EXCEPTION 'injected admission failure'; + END; + $$ LANGUAGE plpgsql", + ) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "CREATE TRIGGER reject_paykit_admission + BEFORE INSERT ON paykit_task_admissions + FOR EACH ROW EXECUTE FUNCTION reject_paykit_admission()", + ) + .execute(database.pool()) + .await + .unwrap(); + + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let task = verification_task(&content_lock(), BundleId::from_bytes([5; 16])); + assert!(admissions.reserve(task, 24).await.is_err()); + let task_count = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM verification_tasks") + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(task_count, 0); + + database.cleanup().await; + } + + #[tokio::test] + async fn persists_and_fences_the_full_job_lifecycle_across_repository_recreation() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + + let reopened = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let persisted = reopened + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .unwrap(); + assert!(persisted.deletion_started_at > job.deletion_started_at); + let mut expected = job.clone(); + expected.deletion_started_at = persisted.deletion_started_at; + assert_eq!(persisted, expected); + assert!(reopened.insert_job(job.clone()).await.is_err()); + let mut distinct_lock = content_lock(); + distinct_lock.access_policy.requested_credential_ttl_seconds = 901; + let mut distinct_job = + ContentLockDeletionJob::new(Uuid::new_v4(), distinct_lock, NOW).unwrap(); + distinct_job.job_id = job.job_id; + assert!(reopened.insert_job(distinct_job).await.is_err()); + + let first = reopened + .claim_next("worker-a", (datetime!(2026-08-12 05:05:00 UTC)) - (NOW)) + .await + .unwrap() + .unwrap(); + assert!( + reopened + .claim_next("worker-b", (datetime!(2026-08-12 05:05:00 UTC)) - (NOW)) + .await + .unwrap() + .is_none() + ); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET claim_expires_at = clock_timestamp() + WHERE job_id = $1 AND claim_token = $2", + ) + .bind(job.job_id) + .bind(first.claim_token) + .execute(database.pool()) + .await + .unwrap(); + let reclaimed = reopened + .claim_next("worker-b", time::Duration::minutes(5)) + .await + .unwrap() + .unwrap(); + assert_ne!(first.claim_token, reclaimed.claim_token); + assert!( + reopened + .schedule_retry( + job.job_id, + "worker-a", + first.claim_token, + (datetime!(2026-08-12 05:06:00 UTC)) - (datetime!(2026-08-12 05:05:01 UTC)), + ) + .await + .unwrap() + .is_none() + ); + assert_eq!( + reopened + .advance_phase( + job.job_id, + "worker-a", + first.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap(), + AdvanceContentLockDeletionPhaseResult::ClaimLost + ); + assert!( + reopened + .finish( + job.job_id, + "worker-a", + first.claim_token, + Some(ContentLockDeletionFailureCode::StateCorrupt), + ) + .await + .unwrap() + .is_none() + ); + + let advanced = reopened + .advance_phase( + job.job_id, + "worker-b", + reclaimed.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + assert_eq!(advanced.state, ContentLockDeletionState::Queued); + assert_eq!(advanced.attempt_count, 0); + + let final_claim = reopened + .claim_next( + "worker-c", + (datetime!(2026-08-12 05:11:00 UTC)) - (datetime!(2026-08-12 05:06:01 UTC)), + ) + .await + .unwrap() + .unwrap(); + let failed = reopened + .finish( + job.job_id, + "worker-c", + final_claim.claim_token, + Some(ContentLockDeletionFailureCode::TombstoneMissing), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(failed.state, ContentLockDeletionState::Failed); + assert_eq!( + failed.failure_code, + Some(ContentLockDeletionFailureCode::TombstoneMissing) + ); + + assert!(matches!( + reopened + .prepare_force_deletion(&job.creator, &job.lock_id) + .await + .unwrap(), + PrepareForceDeletionResult::Synchronous(Some(_)) + )); + assert!(matches!( + reopened + .prepare_force_deletion(&job.creator, &job.lock_id) + .await + .unwrap(), + PrepareForceDeletionResult::Synchronous(None) + )); + assert!( + reopened + .has_force_receipt(&job.creator, &job.lock_id) + .await + .unwrap() + ); + assert!( + reopened + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .is_none() + ); + assert!( + reopened + .has_force_receipt(&job.creator, &job.lock_id) + .await + .unwrap() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn concurrent_claims_return_a_job_once() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job).await.unwrap(); + + let (left, right) = tokio::join!( + repository.claim_next("worker-a", (datetime!(2026-08-12 05:05:00 UTC)) - (NOW)), + repository.claim_next("worker-b", (datetime!(2026-08-12 05:05:00 UTC)) - (NOW)), + ); + assert_eq!( + usize::from(left.unwrap().is_some()) + usize::from(right.unwrap().is_some()), + 1 + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn force_escalation_invalidates_the_active_claim_and_requeues_for_force_processing() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + let claimed = repository + .claim_next( + "graceful-worker", + (NOW + time::Duration::minutes(1)) - (NOW), + ) + .await + .unwrap() + .unwrap(); + + let escalated = repository + .prepare_force_deletion(&job.creator, &job.lock_id) + .await + .unwrap(); + let PrepareForceDeletionResult::Active(escalated) = escalated else { + panic!("active job must be escalated asynchronously"); + }; + assert_eq!(escalated.state, ContentLockDeletionState::Queued); + assert!(escalated.force_requested_at.is_some()); + + assert_eq!( + repository + .schedule_retry( + job.job_id, + "graceful-worker", + claimed.claim_token, + (NOW + time::Duration::seconds(1)) - (NOW), + ) + .await + .unwrap(), + None + ); + assert_eq!( + repository + .advance_phase( + job.job_id, + "graceful-worker", + claimed.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap(), + AdvanceContentLockDeletionPhaseResult::ClaimLost + ); + assert_eq!( + repository + .finish(job.job_id, "graceful-worker", claimed.claim_token, None,) + .await + .unwrap(), + None + ); + + let force_claim = repository + .claim_next("force-worker", (NOW + time::Duration::minutes(1)) - (NOW)) + .await + .unwrap() + .unwrap(); + assert_eq!(force_claim.job.job_id, job.job_id); + assert!(force_claim.job.force_requested_at.is_some()); + assert_ne!(force_claim.claim_token, claimed.claim_token); + + database.cleanup().await; + } + + #[tokio::test] + async fn deletion_lease_takes_exclusive_ownership_of_snapshotted_paykit_obligation() { + use crate::infrastructure::postgres::{ + PaykitInvoiceWindow, PostgresPaykitTaskAdmissionRepository, + }; + + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let drains = PostgresPaymentDrainRepository::new(database.pool().clone()); + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let ordinary = PostgresVerificationTaskClaimer::new(database.pool().clone()); + let lock = content_lock(); + let mut task = verification_task(&lock, BundleId::from_bytes([9; 16])); + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + admissions.reserve(task.clone(), 24).await.unwrap(); + let window = PaykitInvoiceWindow { + invoice_created_at: NOW, + payment_deadline: NOW + time::Duration::hours(24), + }; + admissions.mark_ready(&task, window).await.unwrap(); + + let ordinary_claim = ordinary + .claim_next_verification_task("ordinary", (NOW + time::Duration::minutes(5)) - (NOW)) + .await + .unwrap() + .unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + + assert!( + !ordinary + .begin_claimed_entitlement_publication( + &ordinary_claim.task.task_id, + "ordinary", + &ordinary_claim.claim_token, + ) + .await + .unwrap() + ); + + let ordinary_completed = ordinary_claim + .task + .clone() + .transition_to(VerificationTaskStatus::Completed, NOW, None) + .unwrap(); + assert!( + ordinary + .persist_claimed_verification_task_transition( + ordinary_completed, + "ordinary", + &ordinary_claim.claim_token, + ) + .await + .unwrap() + .is_none() + ); + + let withdraw_claim = deletions + .claim_next("deletion", (NOW + time::Duration::minutes(5)) - (NOW)) + .await + .unwrap() + .unwrap(); + deletions + .advance_phase( + job.job_id, + "deletion", + withdraw_claim.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let start_claim = deletions + .claim_next("deletion", (NOW + time::Duration::minutes(5)) - (NOW)) + .await + .unwrap() + .unwrap(); + let token = + PaymentDrainCleanupToken::parse("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA").unwrap(); + let summary = PaymentDrainSummary { + status: PaymentDrainStatus::Active, + accepted_count: 1, + terminal_count: 0, + cancellation_enqueued_count: 0, + cleanup_token: token, + }; + assert!( + drains + .store_payment_drain(job.job_id, "deletion", start_claim.claim_token, &summary,) + .await + .unwrap() + ); + assert!( + drains + .store_payment_drain(job.job_id, "deletion", start_claim.claim_token, &summary,) + .await + .unwrap() + ); + let divergent = PaymentDrainSummary { + accepted_count: 2, + ..summary.clone() + }; + assert!( + drains + .store_payment_drain(job.job_id, "deletion", start_claim.claim_token, &divergent,) + .await + .is_err() + ); + assert_eq!( + drains.get_payment_drain(job.job_id).await.unwrap(), + Some(summary) + ); + let obligations = drains.list_obligations(job.job_id).await.unwrap(); + assert_eq!(obligations.len(), 1); + assert_eq!(obligations[0].task_id, task.task_id); + assert_eq!(obligations[0].invoice_created_at, window.invoice_created_at); + assert_eq!(obligations[0].payment_deadline, window.payment_deadline); + assert!(!drains.all_obligations_terminal(job.job_id).await.unwrap()); + + deletions + .advance_phase( + job.job_id, + "deletion", + start_claim.claim_token, + ContentLockDeletionPhase::DrainPayments, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let drain_claim = deletions + .claim_next("deletion", (NOW + time::Duration::minutes(5)) - (NOW)) + .await + .unwrap() + .unwrap(); + let publication_token = drains + .begin_entitlement_publication( + job.job_id, + "deletion", + drain_claim.claim_token, + &task.task_id, + ) + .await + .unwrap() + .unwrap(); + assert_eq!( + drains + .begin_entitlement_publication( + job.job_id, + "deletion", + drain_claim.claim_token, + &task.task_id, + ) + .await + .unwrap(), + Some(publication_token) + ); + assert_eq!( + deletions + .prepare_force_deletion(&job.creator, &job.lock_id) + .await + .unwrap(), + PrepareForceDeletionResult::PublicationInProgress + ); + assert!( + !drains + .persist_terminal_obligation( + job.job_id, + "deletion", + drain_claim.claim_token, + &task.task_id, + PaymentDrainTerminalTransition { + status: VerificationTaskStatus::Completed, + entitlement_publication_token: Some(Uuid::new_v4()), + }, + ) + .await + .unwrap() + ); + assert!( + drains + .persist_terminal_obligation( + job.job_id, + "deletion", + drain_claim.claim_token, + &task.task_id, + PaymentDrainTerminalTransition { + status: VerificationTaskStatus::Completed, + entitlement_publication_token: Some(publication_token), + }, + ) + .await + .unwrap() + ); + assert!(drains.all_obligations_terminal(job.job_id).await.unwrap()); + let final_credential_eligible_at: Option = sqlx::query_scalar( + "SELECT final_credential_eligible_at + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 AND verification_task_id = $2", + ) + .bind(job.job_id) + .bind(task.task_id.as_uuid()) + .fetch_one(database.pool()) + .await + .unwrap(); + let resolved_at: Option = sqlx::query_scalar( + "SELECT resolved_at FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 AND verification_task_id = $2", + ) + .bind(job.job_id) + .bind(task.task_id.as_uuid()) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(final_credential_eligible_at, resolved_at); + let final_access_started_at: time::OffsetDateTime = + sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(database.pool()) + .await + .unwrap(); + let issuance_deadline = final_access_started_at + time::Duration::minutes(15); + let read_deadline = issuance_deadline + time::Duration::minutes(15); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET phase = 'issue_final_credentials', claim_expires_at = $2 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(read_deadline + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + let access = PostgresAccessCredentialStore::with_final_credential_cipher( + database.pool().clone(), + crate::infrastructure::final_credentials::FinalCredentialCipher::new([8; 32]), + ); + let initialized = access + .initialize_final_access_windows( + job.job_id, + "deletion", + drain_claim.claim_token, + time::Duration::minutes(15), + time::Duration::minutes(15), + ) + .await + .unwrap(); + assert!(matches!( + initialized, + InitializeFinalAccessWindowsResult::Initialized(_) + )); + assert_eq!( + access + .initialize_final_access_windows( + job.job_id, + "deletion", + drain_claim.claim_token, + time::Duration::hours(1), + time::Duration::hours(1), + ) + .await + .unwrap(), + initialized + ); + let persisted_windows: ( + Option, + Option, + Option, + ) = sqlx::query_as( + "SELECT final_issuance_started_at, + final_credential_issuance_deadline, + final_read_deadline + FROM content_lock_deletion_jobs WHERE job_id = $1", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + let InitializeFinalAccessWindowsResult::Initialized(windows) = initialized else { + unreachable!() + }; + assert_eq!( + persisted_windows, + ( + Some(windows.issuance_started_at), + Some(windows.credential_issuance_deadline), + Some(windows.read_deadline), + ) + ); + let first = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + final_access_started_at, + AccessCredential::new("first-final-bearer"), + ) + .await + .unwrap() + .unwrap(); + let replay = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + final_access_started_at + time::Duration::seconds(1), + AccessCredential::new("different-retry-candidate"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(first.credential, replay.credential); + assert_eq!(first.expires_at, windows.read_deadline); + assert_eq!(replay.expires_at, windows.read_deadline); + let late_replay = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + issuance_deadline + time::Duration::seconds(1), + AccessCredential::new("candidate-after-issuance-deadline"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(late_replay, first); + let encrypted: String = sqlx::query_scalar( + "SELECT encrypted_bearer + FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND credential_kind = 'final'", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert!(!encrypted.contains(first.credential.as_str())); + let final_rows: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND credential_kind = 'final'", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(final_rows, 1); + let read_rows: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM content_lock_access_drain_reads AS read + JOIN content_lock_access_drain_credentials AS credential + ON credential.credential_id = read.credential_id + WHERE credential.deletion_job_id = $1 AND credential.credential_kind = 'final'", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(read_rows, 1); + let final_lookup = AccessCredentialLookupKey::derive(&first.credential); + let guarded_path = "/priv/locks.app/content/post.json"; + let (first_attempt, second_attempt) = tokio::join!( + access.prepare_deletion_read(&final_lookup, guarded_path, time::Duration::seconds(30),), + access.prepare_deletion_read(&final_lookup, guarded_path, time::Duration::seconds(30),), + ); + let first_attempt = first_attempt.unwrap(); + let second_attempt = second_attempt.unwrap(); + assert_eq!( + usize::from(first_attempt.is_some()) + usize::from(second_attempt.is_some()), + 1 + ); + let first_claim = first_attempt.or(second_attempt).unwrap(); + let first_token = first_claim.claim_token.unwrap(); + assert_eq!(first_claim.resource.path, guarded_path); + assert!( + access + .prepare_deletion_read(&final_lookup, guarded_path, time::Duration::seconds(30),) + .await + .unwrap() + .is_none() + ); + assert!( + !access + .release_deletion_read( + &final_lookup, + guarded_path, + Uuid::new_v4(), + final_access_started_at + time::Duration::seconds(1), + ) + .await + .unwrap() + ); + assert!( + access + .release_deletion_read( + &final_lookup, + guarded_path, + first_token, + final_access_started_at + time::Duration::seconds(1), + ) + .await + .unwrap() + ); + let second_claim = access + .prepare_deletion_read(&final_lookup, guarded_path, time::Duration::seconds(30)) + .await + .unwrap() + .unwrap(); + let second_token = second_claim.claim_token.unwrap(); + assert_ne!(second_token, first_token); + assert!( + !access + .consume_deletion_read(&final_lookup, guarded_path, first_token,) + .await + .unwrap() + ); + assert!( + access + .consume_deletion_read(&final_lookup, guarded_path, second_token,) + .await + .unwrap() + ); + assert!( + access + .prepare_deletion_read(&final_lookup, guarded_path, time::Duration::seconds(30),) + .await + .unwrap() + .is_none() + ); + assert!( + access + .deletion_credential_enrolled(&final_lookup) + .await + .unwrap() + ); + let retained_marker: Option = sqlx::query_scalar( + "SELECT entitlement_publication_claim_token FROM verification_tasks WHERE task_id = $1", + ) + .bind(task.task_id.as_uuid()) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(retained_marker, None); + + database.cleanup().await; + } + + #[tokio::test] + async fn entitlement_publication_fence_committed_first_blocks_deletion_cutoff() { + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let ordinary = PostgresVerificationTaskClaimer::new(database.pool().clone()); + let lock = content_lock(); + let task = verification_task(&lock, BundleId::from_bytes([11; 16])); + tasks.insert_verification_task(task).await.unwrap(); + let claim = ordinary + .claim_next_verification_task("ordinary", (NOW + time::Duration::minutes(5)) - (NOW)) + .await + .unwrap() + .unwrap(); + assert!( + ordinary + .begin_claimed_entitlement_publication( + &claim.task.task_id, + "ordinary", + &claim.claim_token, + ) + .await + .unwrap() + ); + + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + assert_eq!( + deletions.insert_job(job.clone()).await, + Err(ApplicationError::ContentLockDeletionInProgress) + ); + assert!( + deletions + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .is_none() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn payment_drain_reclaim_reconciles_external_start_before_local_persistence() { + use crate::infrastructure::postgres::{ + PaykitInvoiceWindow, PostgresPaykitTaskAdmissionRepository, + }; + + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let drains = PostgresPaymentDrainRepository::new(database.pool().clone()); + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let entitlements = InMemoryEntitlementRepository::new(); + let lock = content_lock(); + let mut task = verification_task(&lock, BundleId::from_bytes([11; 16])); + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + admissions.reserve(task.clone(), 24).await.unwrap(); + admissions + .mark_ready( + &task, + PaykitInvoiceWindow { + invoice_created_at: NOW, + payment_deadline: NOW + time::Duration::hours(24), + }, + ) + .await + .unwrap(); + + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let withdraw = deletions + .claim_next("worker-a", (NOW + time::Duration::minutes(5)) - (NOW)) + .await + .unwrap() + .unwrap(); + deletions + .advance_phase( + job.job_id, + "worker-a", + withdraw.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let reclaimed = deletions + .claim_next("worker-b", (NOW + time::Duration::minutes(5)) - (NOW)) + .await + .unwrap() + .unwrap(); + + let summary = PaymentDrainSummary { + status: PaymentDrainStatus::Active, + accepted_count: 1, + terminal_count: 0, + cancellation_enqueued_count: 0, + cleanup_token: PaymentDrainCleanupToken::parse( + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ) + .unwrap(), + }; + let paykit = ReclaimingPaymentDrainClient { + summary, + start_calls: AtomicUsize::new(0), + }; + let lock_resource = PubkyLockResource::new( + job.creator.clone(), + locks_core::ids::ContentLockPath::from_lock_id(job.lock_id.clone()), + ); + paykit.start_payment_drain(&lock_resource).await.unwrap(); + + let use_case = DrainLockPaymentsUseCase::new( + &deletions, + &drains, + &paykit, + &entitlements, + &FixedClock(NOW), + LockServerPubky::from_str("pubky7ir1ttte48bcp4zjychjyscicrwi1j34mtt91ptsafdbjmr8g9eo") + .unwrap(), + 6, + ); + assert!( + use_case + .execute_claimed(reclaimed, "worker-b") + .await + .unwrap() + ); + assert_eq!(paykit.start_calls.load(Ordering::SeqCst), 2); + assert!( + drains + .get_payment_drain(job.job_id) + .await + .unwrap() + .is_some() + ); + assert_eq!( + deletions + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .unwrap() + .phase, + ContentLockDeletionPhase::DrainPayments + ); + + database.cleanup().await; + } + + struct ReclaimingPaymentDrainClient { + summary: PaymentDrainSummary, + start_calls: AtomicUsize, + } + + #[async_trait] + impl PaymentDrainClient for ReclaimingPaymentDrainClient { + async fn start_payment_drain( + &self, + _lock_resource: &PubkyLockResource, + ) -> Result { + if self.start_calls.fetch_add(1, Ordering::SeqCst) == 0 { + Ok(self.summary.clone()) + } else { + Err(PaymentDrainClientError::Conflict) + } + } + + async fn lookup_payment_drain( + &self, + _lock_resource: &PubkyLockResource, + ) -> Result, PaymentDrainClientError> { + Ok(Some(self.summary.clone())) + } + + async fn payment_request_status( + &self, + _creator: &CreatorPubky, + _bundle_id: &BundleId, + ) -> Result, PaymentDrainClientError> { + unreachable!() + } + } + + #[tokio::test] + async fn completed_paykit_aggregate_waits_for_locks_confirmation_and_local_terminal_state() { + use crate::infrastructure::postgres::{ + PaykitInvoiceWindow, PostgresPaykitTaskAdmissionRepository, + }; + + let database = TestDatabase::create().await; + let deletions = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let drains = PostgresPaymentDrainRepository::new(database.pool().clone()); + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let entitlements = InMemoryEntitlementRepository::new(); + let lock = content_lock(); + let mut task = verification_task(&lock, BundleId::from_bytes([10; 16])); + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + admissions.reserve(task.clone(), 24).await.unwrap(); + let window = PaykitInvoiceWindow { + invoice_created_at: NOW, + payment_deadline: NOW + time::Duration::hours(24), + }; + admissions.mark_ready(&task, window).await.unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let withdraw = deletions + .claim_next("deletion", (NOW + time::Duration::minutes(5)) - (NOW)) + .await + .unwrap() + .unwrap(); + deletions + .advance_phase( + job.job_id, + "deletion", + withdraw.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let start = deletions + .claim_next("deletion", (NOW + time::Duration::minutes(5)) - (NOW)) + .await + .unwrap() + .unwrap(); + let token = + PaymentDrainCleanupToken::parse("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA").unwrap(); + let initial_summary = PaymentDrainSummary { + status: PaymentDrainStatus::Active, + accepted_count: 1, + terminal_count: 0, + cancellation_enqueued_count: 0, + cleanup_token: token.clone(), + }; + drains + .store_payment_drain(job.job_id, "deletion", start.claim_token, &initial_summary) + .await + .unwrap(); + deletions + .advance_phase( + job.job_id, + "deletion", + start.claim_token, + ContentLockDeletionPhase::DrainPayments, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let claim = deletions + .claim_next("deletion", (NOW + time::Duration::minutes(5)) - (NOW)) + .await + .unwrap() + .unwrap(); + let paykit = MutablePaymentDrainClient { + summary: PaymentDrainSummary { + status: PaymentDrainStatus::Completed, + accepted_count: 0, + terminal_count: 1, + cancellation_enqueued_count: 0, + cleanup_token: token, + }, + status: Mutex::new(PaymentRequestStatus { + request_state: PaymentRequestState::Accepted, + payment_state: PaymentState::Detected, + invoice_created_at: window.invoice_created_at, + payment_deadline: window.payment_deadline, + confirmations: 0, + amount_matched: true, + }), + }; + let clock = FixedClock(NOW); + let use_case = DrainLockPaymentsUseCase::new( + &deletions, + &drains, + &paykit, + &entitlements, + &clock, + LockServerPubky::from_str("pubky7ir1ttte48bcp4zjychjyscicrwi1j34mtt91ptsafdbjmr8g9eo") + .unwrap(), + 6, + ); + assert!( + !use_case + .execute_claimed(claim.clone(), "deletion") + .await + .unwrap() + ); + assert!(!drains.all_obligations_terminal(job.job_id).await.unwrap()); + assert!( + entitlements + .get_verified_proof_bundle(&task.creator, &task.submitted_proof_bundle.bundle_id,) + .await + .unwrap() + .is_none() + ); + + { + let mut status = paykit.status.lock().unwrap(); + status.payment_state = PaymentState::Confirmed; + status.confirmations = 6; + } + assert!(use_case.execute_claimed(claim, "deletion").await.unwrap()); + assert!(drains.all_obligations_terminal(job.job_id).await.unwrap()); + assert!( + entitlements + .get_verified_proof_bundle(&task.creator, &task.submitted_proof_bundle.bundle_id,) + .await + .unwrap() + .is_some() + ); + assert_eq!( + deletions + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .unwrap() + .phase, + ContentLockDeletionPhase::DrainExistingCredentials + ); + + database.cleanup().await; + } + + struct FixedClock(time::OffsetDateTime); + + impl Clock for FixedClock { + fn now(&self) -> time::OffsetDateTime { + self.0 + } + } + + struct MutablePaymentDrainClient { + summary: PaymentDrainSummary, + status: Mutex, + } + + #[tokio::test] + async fn concurrent_final_issuers_replay_one_winner() { + let database = TestDatabase::create().await; + let (job, task, access, final_access_started_at) = + eligible_final_credential_fixture(&database).await; + + let (first, second) = tokio::join!( + access.issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + final_access_started_at, + AccessCredential::new("concurrent-final-one"), + ), + access.issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + final_access_started_at, + AccessCredential::new("concurrent-final-two"), + ), + ); + let first = first.unwrap().unwrap(); + let second = second.unwrap().unwrap(); + assert_eq!(first.credential, second.credential); + let final_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND credential_kind = 'final'", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(final_count, 1); + + database.cleanup().await; + } + + #[tokio::test] + async fn force_revokes_live_final_read_claim() { + let database = TestDatabase::create().await; + let (job, task, access, final_access_started_at) = + eligible_final_credential_fixture(&database).await; + let issued = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + final_access_started_at, + AccessCredential::new("final-before-force"), + ) + .await + .unwrap() + .unwrap(); + let lookup = AccessCredentialLookupKey::derive(&issued.credential); + let path = job + .frozen_content_lock + .primary_resource + .as_ref() + .unwrap() + .path + .clone(); + let prepared = access + .prepare_deletion_read(&lookup, &path, time::Duration::seconds(30)) + .await + .unwrap() + .unwrap(); + let read_token = prepared.claim_token.unwrap(); + + let force = PostgresContentLockDeletionRepository::new(database.pool().clone()); + assert!(matches!( + force + .prepare_force_deletion(&job.creator, &job.lock_id,) + .await + .unwrap(), + PrepareForceDeletionResult::Active(_) + )); + assert!( + !access + .consume_deletion_read(&lookup, &path, read_token,) + .await + .unwrap() + ); + assert!( + access + .prepare_deletion_read(&lookup, &path, time::Duration::seconds(28),) + .await + .unwrap() + .is_none() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn final_issuance_waiting_on_snapshot_observes_force_winner() { + let database = TestDatabase::create().await; + let (job, task, access, final_access_started_at) = + eligible_final_credential_fixture(&database).await; + let mut blocker = database.pool().begin().await.unwrap(); + sqlx::query( + "SELECT job_id FROM content_lock_deletion_jobs + WHERE job_id = $1 FOR UPDATE", + ) + .bind(job.job_id) + .fetch_one(&mut *blocker) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET force_requested_at = $2 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(final_access_started_at) + .execute(&mut *blocker) + .await + .unwrap(); + + let issuing_access = access.clone(); + let issuing_creator = job.creator.clone(); + let issuing_bundle = task.submitted_proof_bundle.bundle_id.clone(); + let issuing = tokio::spawn(async move { + issuing_access + .issue_or_replay_final_credential( + &issuing_creator, + &issuing_bundle, + final_access_started_at, + AccessCredential::new("must-not-escape-force"), + ) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + assert!(!issuing.is_finished()); + + blocker.commit().await.unwrap(); + assert!(issuing.await.unwrap().unwrap().is_none()); + + database.cleanup().await; + } + + #[tokio::test] + async fn phase_advancement_and_successful_finish_cannot_bypass_access_obligations() { + let database = TestDatabase::create().await; + let (job, task, access, final_access_started_at) = + eligible_final_credential_fixture(&database).await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let claim_token = Uuid::new_v4(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET state = 'running', claimed_by = 'worker', claim_token = $2, + claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(claim_token) + .bind(final_access_started_at + time::Duration::hours(1)) + .execute(database.pool()) + .await + .unwrap(); + + assert!(matches!( + repository + .advance_phase( + job.job_id, + "worker", + claim_token, + ContentLockDeletionPhase::DrainFinalReads, + ) + .await, + Ok(AdvanceContentLockDeletionPhaseResult::ObligationsPending) + )); + assert!(matches!( + repository + .finish(job.job_id, "worker", claim_token, None,) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + + let issued = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + final_access_started_at, + AccessCredential::new("phase-obligation-final"), + ) + .await + .unwrap() + .unwrap(); + assert!( + repository + .advance_phase( + job.job_id, + "worker", + claim_token, + ContentLockDeletionPhase::DrainFinalReads, + ) + .await + .unwrap() + .advanced() + .is_some() + ); + let drain_claim = repository + .claim_next( + "worker", + (final_access_started_at + time::Duration::hours(1)) - (final_access_started_at), + ) + .await + .unwrap() + .unwrap(); + assert!(matches!( + repository + .advance_phase( + job.job_id, + "worker", + drain_claim.claim_token, + ContentLockDeletionPhase::DeleteContent, + ) + .await, + Ok(AdvanceContentLockDeletionPhaseResult::ObligationsPending) + )); + let lookup = AccessCredentialLookupKey::derive(&issued.credential); + let path = job + .frozen_content_lock + .primary_resource + .as_ref() + .unwrap() + .path + .clone(); + let read = access + .prepare_deletion_read(&lookup, &path, time::Duration::seconds(30)) + .await + .unwrap() + .unwrap(); + assert!( + access + .consume_deletion_read(&lookup, &path, read.claim_token.unwrap(),) + .await + .unwrap() + ); + assert!( + repository + .advance_phase( + job.job_id, + "worker", + drain_claim.claim_token, + ContentLockDeletionPhase::DeleteContent, + ) + .await + .unwrap() + .advanced() + .is_some() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn no_paykit_drain_expires_pending_without_creating_paykit_state() { + let database = TestDatabase::create().await; + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + let task = verification_task(&lock, BundleId::from_bytes([60; 16])); + let task_id = task.task_id; + tasks.insert_verification_task(task).await.unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository.insert_job(job.clone()).await.unwrap(); + let claim_token = Uuid::new_v4(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET phase = 'drain_payments', state = 'running', claimed_by = 'worker', + claim_token = $2, claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(claim_token) + .bind(database_now(&database).await + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + + assert!( + !repository + .expire_unresolved_non_paykit_tasks(job.job_id, "worker", Uuid::new_v4()) + .await + .unwrap() + ); + assert!( + repository + .expire_unresolved_non_paykit_tasks(job.job_id, "worker", claim_token) + .await + .unwrap() + ); + assert!( + repository + .advance_phase( + job.job_id, + "worker", + claim_token, + ContentLockDeletionPhase::DrainExistingCredentials, + ) + .await + .unwrap() + .advanced() + .is_some() + ); + + let task_status: String = + sqlx::query_scalar("SELECT status FROM verification_tasks WHERE task_id = $1") + .bind(task_id.as_uuid()) + .fetch_one(database.pool()) + .await + .unwrap(); + let snapshot_status: Option = sqlx::query_scalar("SELECT resolved_status FROM content_lock_deletion_task_snapshot WHERE deletion_job_id = $1") + .bind(job.job_id).fetch_one(database.pool()).await.unwrap(); + let drain_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM content_lock_payment_drains WHERE deletion_job_id = $1", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(task_status, "expired"); + assert_eq!(snapshot_status.as_deref(), Some("expired")); + assert_eq!(drain_count, 0); + database.cleanup().await; + } + + #[tokio::test] + async fn no_paykit_drain_rejects_paykit_and_missing_paykit_aggregate_still_blocks() { + let database = TestDatabase::create().await; + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + let mut task = verification_task(&lock, BundleId::from_bytes([59; 16])); + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + tasks.insert_verification_task(task).await.unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository.insert_job(job.clone()).await.unwrap(); + let claim_token = Uuid::new_v4(); + sqlx::query("UPDATE content_lock_deletion_jobs SET phase = 'drain_payments', state = 'running', claimed_by = 'worker', claim_token = $2, claim_expires_at = $3 WHERE job_id = $1") + .bind(job.job_id) + .bind(claim_token) + .bind(database_now(&database).await + time::Duration::minutes(5)) + .execute(database.pool()).await.unwrap(); + assert!(matches!( + repository + .expire_unresolved_non_paykit_tasks(job.job_id, "worker", claim_token) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + sqlx::query("UPDATE content_lock_deletion_task_snapshot SET resolved_status = 'expired', resolved_at = $2 WHERE deletion_job_id = $1") + .bind(job.job_id).bind(NOW).execute(database.pool()).await.unwrap(); + assert!(matches!( + repository + .advance_phase( + job.job_id, + "worker", + claim_token, + ContentLockDeletionPhase::DrainExistingCredentials, + ) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + database.cleanup().await; + } + + #[tokio::test] + async fn drain_payments_phase_allows_missing_aggregate_for_non_paykit_snapshots() { + let database = TestDatabase::create().await; + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + let mut task = verification_task(&lock, BundleId::from_bytes([61; 16])); + task.status = VerificationTaskStatus::Completed; + task.started_at = Some(NOW); + task.completed_at = Some(NOW); + tasks.insert_verification_task(task).await.unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository.insert_job(job.clone()).await.unwrap(); + let claim_token = Uuid::new_v4(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET phase = 'drain_payments', state = 'running', claimed_by = 'worker', + claim_token = $2, claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(claim_token) + .bind(database_now(&database).await + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + + assert!( + repository + .advance_phase( + job.job_id, + "worker", + claim_token, + ContentLockDeletionPhase::DrainExistingCredentials, + ) + .await + .unwrap() + .advanced() + .is_some() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn drain_payments_phase_requires_every_frozen_snapshot_terminal() { + let database = TestDatabase::create().await; + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + let mut paykit = verification_task(&lock, BundleId::from_bytes([62; 16])); + paykit.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + let local = verification_task(&lock, BundleId::from_bytes([63; 16])); + tasks + .insert_verification_task(paykit.clone()) + .await + .unwrap(); + tasks.insert_verification_task(local).await.unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository.insert_job(job.clone()).await.unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET resolved_status = 'expired', resolved_at = $3 + WHERE deletion_job_id = $1 AND verification_task_id = $2", + ) + .bind(job.job_id) + .bind(paykit.task_id.as_uuid()) + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO content_lock_payment_drains ( + deletion_job_id, status, accepted_count, terminal_count, + cancellation_enqueued_count, cleanup_token, created_at, updated_at + ) VALUES ($1, 'completed', 0, 1, 0, $2, $3, $3)", + ) + .bind(job.job_id) + .bind("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB") + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + let claim_token = Uuid::new_v4(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET phase = 'drain_payments', state = 'running', claimed_by = 'worker', + claim_token = $2, claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(claim_token) + .bind(database_now(&database).await + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + + assert!(matches!( + repository + .advance_phase( + job.job_id, + "worker", + claim_token, + ContentLockDeletionPhase::DrainExistingCredentials, + ) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + + database.cleanup().await; + } + + #[tokio::test] + async fn expired_final_read_claim_does_not_wedge_destructive_phase_and_cannot_consume() { + let database = TestDatabase::create().await; + let (job, task, access, final_access_started_at) = + eligible_final_credential_fixture(&database).await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let issued = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + final_access_started_at, + AccessCredential::new("expires-with-final-read-window"), + ) + .await + .unwrap() + .unwrap(); + let lookup = AccessCredentialLookupKey::derive(&issued.credential); + let path = job + .frozen_content_lock + .primary_resource + .as_ref() + .unwrap() + .path + .clone(); + let issue_claim = Uuid::new_v4(); + let read_deadline = final_access_started_at + time::Duration::minutes(30); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET state = 'running', claimed_by = 'worker', claim_token = $2, + claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(issue_claim) + .bind(read_deadline + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + repository + .advance_phase( + job.job_id, + "worker", + issue_claim, + ContentLockDeletionPhase::DrainFinalReads, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let drain_claim = repository + .claim_next( + "worker", + (read_deadline + time::Duration::minutes(5)) + - (read_deadline - time::Duration::seconds(10)), + ) + .await + .unwrap() + .unwrap(); + let read = access + .prepare_deletion_read(&lookup, &path, time::Duration::seconds(70)) + .await + .unwrap() + .unwrap(); + let stale_read_token = read.claim_token.unwrap(); + sqlx::query( + "WITH anchor AS (SELECT clock_timestamp() AS at) + UPDATE content_lock_deletion_jobs + SET final_credential_issuance_deadline = final_issuance_started_at + + ((anchor.at - final_issuance_started_at) / 2), + final_read_deadline = anchor.at + FROM anchor + WHERE job_id = $1", + ) + .bind(job.job_id) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_access_drain_credentials + SET expires_at = clock_timestamp() + WHERE deletion_job_id = $1 AND credential_kind = 'final'", + ) + .bind(job.job_id) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_access_drain_reads AS read + SET claim_expires_at = clock_timestamp() + FROM content_lock_access_drain_credentials AS credential + WHERE read.credential_id = credential.credential_id + AND credential.deletion_job_id = $1", + ) + .bind(job.job_id) + .execute(database.pool()) + .await + .unwrap(); + + assert!( + repository + .advance_phase( + job.job_id, + "worker", + drain_claim.claim_token, + ContentLockDeletionPhase::DeleteContent, + ) + .await + .unwrap() + .advanced() + .is_some() + ); + assert!( + !access + .consume_deletion_read(&lookup, &path, stale_read_token) + .await + .unwrap() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn unissued_eligible_snapshot_reports_irrecoverable_miss_after_issuance_deadline() { + let database = TestDatabase::create().await; + let (job, _task, _access, _final_access_started_at) = + eligible_final_credential_fixture(&database).await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let claim_token = Uuid::new_v4(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET state = 'running', claimed_by = 'worker', claim_token = $2, + claim_expires_at = clock_timestamp() + INTERVAL '5 minutes', + final_credential_issuance_deadline = clock_timestamp() + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(claim_token) + .execute(database.pool()) + .await + .unwrap(); + + assert!(matches!( + repository + .advance_phase( + job.job_id, + "worker", + claim_token, + ContentLockDeletionPhase::DrainFinalReads, + ) + .await, + Ok(AdvanceContentLockDeletionPhaseResult::TerminalFailure( + ContentLockDeletionFailureCode::StateCorrupt + )) + )); + + database.cleanup().await; + } + + #[tokio::test] + async fn successful_finish_rechecks_paykit_and_non_paykit_frozen_obligations() { + let database = TestDatabase::create().await; + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + let mut paykit = verification_task(&lock, BundleId::from_bytes([64; 16])); + paykit.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + let local = verification_task(&lock, BundleId::from_bytes([65; 16])); + tasks + .insert_verification_task(paykit.clone()) + .await + .unwrap(); + tasks.insert_verification_task(local).await.unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + repository.insert_job(job.clone()).await.unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET resolved_status = 'expired', resolved_at = $3 + WHERE deletion_job_id = $1 AND verification_task_id = $2", + ) + .bind(job.job_id) + .bind(paykit.task_id.as_uuid()) + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO content_lock_payment_drains ( + deletion_job_id, status, accepted_count, terminal_count, + cancellation_enqueued_count, cleanup_token, created_at, updated_at + ) VALUES ($1, 'completed', 0, 1, 0, $2, $3, $3)", + ) + .bind(job.job_id) + .bind("CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC") + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + let claim_token = Uuid::new_v4(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET phase = 'purge_operational_state', state = 'running', claimed_by = 'worker', + claim_token = $2, claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(claim_token) + .bind(database_now(&database).await + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + + assert!(matches!( + repository + .finish(job.job_id, "worker", claim_token, None) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET resolved_status = 'failed', resolved_at = $2 + WHERE deletion_job_id = $1 AND resolved_status IS NULL", + ) + .bind(job.job_id) + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_payment_drains + SET status = 'active', accepted_count = 1, updated_at = $2 + WHERE deletion_job_id = $1", + ) + .bind(job.job_id) + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + assert!(matches!( + repository + .finish(job.job_id, "worker", claim_token, None) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + sqlx::query( + "UPDATE content_lock_payment_drains + SET status = 'completed', accepted_count = 0, updated_at = $2 + WHERE deletion_job_id = $1", + ) + .bind(job.job_id) + .bind(NOW) + .execute(database.pool()) + .await + .unwrap(); + repository + .finish(job.job_id, "worker", claim_token, None) + .await + .unwrap(); + + database.cleanup().await; + } + + #[tokio::test] + async fn successful_finish_requires_exact_final_cleanup_phase() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + let claimed = repository + .claim_next("worker", (NOW + time::Duration::minutes(5)) - (NOW)) + .await + .unwrap() + .unwrap(); + + assert!(matches!( + repository + .finish(job.job_id, "worker", claimed.claim_token, None) + .await, + Err(ApplicationError::InvalidContentLockDeletionState { .. }) + )); + + database.cleanup().await; + } + + #[tokio::test] + async fn issuance_deadline_is_half_open_and_transition_preserves_only_exact_replay() { + let database = TestDatabase::create().await; + let (job, task, access, _final_access_started_at) = + eligible_final_credential_fixture(&database).await; + let issuance_deadline: time::OffsetDateTime = sqlx::query_scalar( + "UPDATE content_lock_deletion_jobs + SET final_credential_issuance_deadline = clock_timestamp() + WHERE job_id = $1 + RETURNING final_credential_issuance_deadline", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + + assert!( + access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + issuance_deadline, + AccessCredential::new("must-not-insert-at-deadline"), + ) + .await + .unwrap() + .is_none() + ); + let final_count: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM content_lock_access_drain_credentials + WHERE deletion_job_id = $1 AND credential_kind = 'final'", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(final_count, 0); + + let issuance_deadline: time::OffsetDateTime = sqlx::query_scalar( + "UPDATE content_lock_deletion_jobs + SET final_credential_issuance_deadline = clock_timestamp() + INTERVAL '15 minutes' + WHERE job_id = $1 + RETURNING final_credential_issuance_deadline", + ) + .bind(job.job_id) + .fetch_one(database.pool()) + .await + .unwrap(); + let issued = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + issuance_deadline - time::Duration::seconds(1), + AccessCredential::new("persisted-before-transition"), + ) + .await + .unwrap() + .unwrap(); + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let claim_token = Uuid::new_v4(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET state = 'running', claimed_by = 'worker', claim_token = $2, + claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(claim_token) + .bind(issuance_deadline + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + repository + .advance_phase( + job.job_id, + "worker", + claim_token, + ContentLockDeletionPhase::DrainFinalReads, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + + let replay = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + issuance_deadline + time::Duration::seconds(1), + AccessCredential::new("must-not-replace-persisted-winner"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(replay, issued); + + database.cleanup().await; + } + + #[tokio::test] + async fn final_read_claim_lease_is_capped_at_thirty_seconds_by_storage_time() { + let database = TestDatabase::create().await; + let (job, task, access, final_access_started_at) = + eligible_final_credential_fixture(&database).await; + let issued = access + .issue_or_replay_final_credential( + &job.creator, + &task.submitted_proof_bundle.bundle_id, + final_access_started_at, + AccessCredential::new("fixed-storage-lease"), + ) + .await + .unwrap() + .unwrap(); + let lookup = AccessCredentialLookupKey::derive(&issued.credential); + let path = job + .frozen_content_lock + .primary_resource + .as_ref() + .unwrap() + .path + .clone(); + + let before_claim = database_now(&database).await; + access + .prepare_deletion_read(&lookup, &path, time::Duration::seconds(70)) + .await + .unwrap() + .unwrap(); + let after_claim = database_now(&database).await; + let stored_expiry: time::OffsetDateTime = sqlx::query_scalar( + "SELECT read.claim_expires_at + FROM content_lock_access_drain_reads AS read + JOIN content_lock_access_drain_credentials AS credential + ON credential.credential_id = read.credential_id + WHERE credential.lookup_key = $1 AND read.guarded_path = $2", + ) + .bind(lookup.as_bytes().as_slice()) + .bind(&path) + .fetch_one(database.pool()) + .await + .unwrap(); + assert!(stored_expiry >= before_claim + time::Duration::seconds(30)); + assert!(stored_expiry <= after_claim + time::Duration::seconds(30)); + + database.cleanup().await; + } + + #[async_trait] + impl PaymentDrainClient for MutablePaymentDrainClient { + async fn start_payment_drain( + &self, + _lock_resource: &PubkyLockResource, + ) -> Result { + Ok(self.summary.clone()) + } + + async fn lookup_payment_drain( + &self, + _lock_resource: &PubkyLockResource, + ) -> Result, PaymentDrainClientError> { + Ok(Some(self.summary.clone())) + } + + async fn payment_request_status( + &self, + _creator: &CreatorPubky, + _bundle_id: &BundleId, + ) -> Result, PaymentDrainClientError> { + Ok(Some(*self.status.lock().unwrap())) + } + } + + #[tokio::test] + async fn read_rejects_corrupt_frozen_manifest_identity() { + let database = TestDatabase::create().await; + let repository = PostgresContentLockDeletionRepository::new(database.pool().clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET frozen_content_lock = jsonb_set( + frozen_content_lock, + '{access_policy,requested_credential_ttl_seconds}', + '901'::jsonb + ) + WHERE job_id = $1", + ) + .bind(job.job_id) + .execute(database.pool()) + .await + .unwrap(); + + assert!( + repository + .get_job(&job.creator, &job.lock_id) + .await + .is_err() + ); + + database.cleanup().await; + } + + async fn eligible_final_credential_fixture( + database: &TestDatabase, + ) -> ( + ContentLockDeletionJob, + VerificationTaskRecord, + PostgresAccessCredentialStore, + time::OffsetDateTime, + ) { + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let lock = content_lock(); + let mut task = verification_task(&lock, BundleId::from_bytes([42; 16])); + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::PaykitPayment; + tasks.insert_verification_task(task.clone()).await.unwrap(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + PostgresContentLockDeletionRepository::new(database.pool().clone()) + .insert_job(job.clone()) + .await + .unwrap(); + let final_access_started_at: time::OffsetDateTime = + sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(database.pool()) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET resolved_status = 'completed', resolved_at = $2, + final_credential_eligible_at = $2 + WHERE deletion_job_id = $1", + ) + .bind(job.job_id) + .bind(final_access_started_at) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET phase = 'issue_final_credentials', final_issuance_started_at = $2, + final_credential_issuance_deadline = $3, final_read_deadline = $4 + WHERE job_id = $1", + ) + .bind(job.job_id) + .bind(final_access_started_at) + .bind(final_access_started_at + time::Duration::minutes(15)) + .bind(final_access_started_at + time::Duration::minutes(30)) + .execute(database.pool()) + .await + .unwrap(); + let access = PostgresAccessCredentialStore::with_final_credential_cipher( + database.pool().clone(), + crate::infrastructure::final_credentials::FinalCredentialCipher::new([9; 32]), + ); + (job, task, access, final_access_started_at) + } + + async fn database_now(database: &TestDatabase) -> time::OffsetDateTime { + sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(database.pool()) + .await + .unwrap() + } + + fn verification_task(lock: &ContentLock, bundle_id: BundleId) -> VerificationTaskRecord { + let lock_resource = PubkyLockResource::from_str(&format!( + "{}/pub/locks.app/{}.json", + lock.creator, + lock.lock_id().unwrap() + )) + .unwrap(); + VerificationTaskRecord { + task_id: TaskId::from_str(&Uuid::new_v4().to_string()).unwrap(), + creator: lock.creator.clone(), + submitted_proof_bundle: SubmittedProofBundle { + version: SUBMITTED_PROOF_BUNDLE_VERSION, + bundle_id, + pubky_lock_resource: lock_resource, + reader_public_key: None, + proofs: vec![Proof { + criterion_id: "criterion-1".to_owned(), + verifier_type: VerifierType::DevStatic, + payload: serde_json::json!({"satisfied": true}), + }], + }, + status: VerificationTaskStatus::Pending, + submitted_at: NOW, + started_at: None, + completed_at: None, + failure_message: None, + } + } + + fn content_lock() -> ContentLock { + ContentLock { + version: CONTENT_LOCK_VERSION, + creator: CreatorPubky::from_str(CREATOR).unwrap(), + primary_resource: Some( + GuardedResource::new( + "/priv/locks.app/content/post.json".to_owned(), + GuardedResourceHash::from_bytes([7; 32]), + "application/json".to_owned(), + 42, + ) + .unwrap(), + ), + secondary_resources: BTreeMap::new(), + criteria: vec![], + lock_logic: LockLogic::All { criteria: vec![] }, + access_policy: AccessPolicy { + requested_credential_ttl_seconds: 900, + }, + lock_server: LockServerConfig { override_: None }, + created_at: datetime!(2026-08-12 04:00:00 UTC), + } + } +} diff --git a/locks-service/src/infrastructure/postgres/content_lock_ownership.rs b/locks-service/src/infrastructure/postgres/content_lock_ownership.rs new file mode 100644 index 0000000..9494a08 --- /dev/null +++ b/locks-service/src/infrastructure/postgres/content_lock_ownership.rs @@ -0,0 +1,442 @@ +use std::collections::BTreeSet; +use std::str::FromStr; + +use async_trait::async_trait; +use locks_core::ids::{CreatorPubky, LockId}; +use sqlx::PgPool; + +use crate::application::errors::ApplicationError; +use crate::application::models::{ContentLockOwnership, ContentLockOwnershipStatus}; +use crate::application::ports::ContentLockOwnershipRepository; + +/// PostgreSQL-backed exclusive guarded-path ownership repository. +#[derive(Debug, Clone)] +pub struct PostgresContentLockOwnershipRepository { + pool: PgPool, +} + +impl PostgresContentLockOwnershipRepository { + /// Creates an ownership repository backed by the supplied pool. + pub fn new(pool: PgPool) -> Self { + Self { pool } + } +} + +#[async_trait] +impl ContentLockOwnershipRepository for PostgresContentLockOwnershipRepository { + async fn reserve_paths( + &self, + creator: &CreatorPubky, + guarded_paths: &[String], + lock_id: &LockId, + ) -> Result<(), ApplicationError> { + let creator = creator.to_string(); + let lock_id = lock_id.to_string(); + let mut transaction = self.pool.begin().await.map_err(map_sqlx_error)?; + + for guarded_path in sorted_unique_paths(guarded_paths) { + let insert = sqlx::query( + r#" + INSERT INTO content_lock_ownership (creator, guarded_path, lock_id, status) + VALUES ($1, $2, $3, 'reserved') + ON CONFLICT (creator, guarded_path) DO NOTHING + "#, + ) + .bind(&creator) + .bind(guarded_path) + .bind(&lock_id) + .execute(&mut *transaction) + .await + .map_err(map_sqlx_error)?; + + let (existing_lock_id, existing_status) = sqlx::query_as::<_, (String, String)>( + r#" + SELECT lock_id, status + FROM content_lock_ownership + WHERE creator = $1 AND guarded_path = $2 + "#, + ) + .bind(&creator) + .bind(guarded_path) + .fetch_one(&mut *transaction) + .await + .map_err(map_sqlx_error)?; + if existing_lock_id != lock_id + || (insert.rows_affected() == 0 && existing_status == "reserved") + { + return Err(ApplicationError::ContentLockPathConflict { + guarded_path: guarded_path.to_owned(), + }); + } + } + + transaction.commit().await.map_err(map_sqlx_error) + } + + async fn mark_paths_published( + &self, + creator: &CreatorPubky, + guarded_paths: &[String], + lock_id: &LockId, + ) -> Result<(), ApplicationError> { + let creator = creator.to_string(); + let lock_id = lock_id.to_string(); + let mut transaction = self.pool.begin().await.map_err(map_sqlx_error)?; + + for guarded_path in sorted_unique_paths(guarded_paths) { + let result = sqlx::query( + r#" + UPDATE content_lock_ownership + SET status = 'published' + WHERE creator = $1 AND guarded_path = $2 AND lock_id = $3 + "#, + ) + .bind(&creator) + .bind(guarded_path) + .bind(&lock_id) + .execute(&mut *transaction) + .await + .map_err(map_sqlx_error)?; + if result.rows_affected() == 0 { + let existing_lock_id = sqlx::query_scalar::<_, String>( + r#" + SELECT lock_id + FROM content_lock_ownership + WHERE creator = $1 AND guarded_path = $2 + "#, + ) + .bind(&creator) + .bind(guarded_path) + .fetch_optional(&mut *transaction) + .await + .map_err(map_sqlx_error)?; + return match existing_lock_id { + Some(_) => Err(ApplicationError::ContentLockPathConflict { + guarded_path: guarded_path.to_owned(), + }), + None => Err(ApplicationError::MissingRecord { + record: "content_lock_ownership", + }), + }; + } + } + + transaction.commit().await.map_err(map_sqlx_error) + } + + async fn compensate_reserved_paths( + &self, + creator: &CreatorPubky, + guarded_paths: &[String], + lock_id: &LockId, + ) -> Result<(), ApplicationError> { + let creator = creator.to_string(); + let lock_id = lock_id.to_string(); + let mut transaction = self.pool.begin().await.map_err(map_sqlx_error)?; + + for guarded_path in sorted_unique_paths(guarded_paths) { + sqlx::query( + r#" + DELETE FROM content_lock_ownership + WHERE creator = $1 + AND guarded_path = $2 + AND lock_id = $3 + AND status = 'reserved' + "#, + ) + .bind(&creator) + .bind(guarded_path) + .bind(&lock_id) + .execute(&mut *transaction) + .await + .map_err(map_sqlx_error)?; + } + + transaction.commit().await.map_err(map_sqlx_error) + } + + async fn get_path_ownership( + &self, + creator: &CreatorPubky, + guarded_path: &str, + ) -> Result, ApplicationError> { + let row = sqlx::query_as::<_, (String, String)>( + r#" + SELECT lock_id, status + FROM content_lock_ownership + WHERE creator = $1 AND guarded_path = $2 + "#, + ) + .bind(creator.to_string()) + .bind(guarded_path) + .fetch_optional(&self.pool) + .await + .map_err(map_sqlx_error)?; + + row.map(|(lock_id, status)| { + Ok(ContentLockOwnership { + creator: creator.clone(), + guarded_path: guarded_path.to_owned(), + lock_id: LockId::from_str(&lock_id).map_err(|error| ApplicationError::Storage { + message: format!("invalid stored content lock ownership Lock ID: {error}"), + })?, + status: ContentLockOwnershipStatus::from_storage(&status)?, + }) + }) + .transpose() + } +} + +fn sorted_unique_paths(guarded_paths: &[String]) -> Vec<&str> { + guarded_paths + .iter() + .map(String::as_str) + .collect::>() + .into_iter() + .collect() +} + +fn map_sqlx_error(error: sqlx::Error) -> ApplicationError { + ApplicationError::Storage { + message: error.to_string(), + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + use std::sync::Arc; + + use locks_core::ids::{CreatorPubky, LockHash, LockId}; + use tokio::sync::Barrier; + + use super::PostgresContentLockOwnershipRepository; + use crate::application::errors::ApplicationError; + use crate::application::models::ContentLockOwnershipStatus; + use crate::application::ports::ContentLockOwnershipRepository; + use crate::infrastructure::postgres::testing::TestDatabase; + + #[tokio::test] + async fn reserved_path_blocks_retry_and_conflicting_multi_path_request_is_atomic() { + let database = TestDatabase::create().await; + let store = PostgresContentLockOwnershipRepository::new(database.pool().clone()); + let creator = creator(); + let first_lock = lock_id(1); + let second_lock = lock_id(2); + let owned_paths = paths(&["a.txt", "b.txt"]); + + store + .reserve_paths(&creator, &owned_paths, &first_lock) + .await + .unwrap(); + assert_eq!( + store + .reserve_paths(&creator, &owned_paths, &first_lock) + .await, + Err(ApplicationError::ContentLockPathConflict { + guarded_path: owned_paths[0].clone(), + }) + ); + + let conflicting_paths = paths(&["c.txt", "b.txt"]); + assert_eq!( + store + .reserve_paths(&creator, &conflicting_paths, &second_lock) + .await, + Err(ApplicationError::ContentLockPathConflict { + guarded_path: owned_paths[1].clone(), + }) + ); + assert_eq!( + store + .get_path_ownership(&creator, &conflicting_paths[0]) + .await + .unwrap(), + None + ); + for guarded_path in &owned_paths { + let ownership = store + .get_path_ownership(&creator, guarded_path) + .await + .unwrap() + .unwrap(); + assert_eq!(ownership.lock_id, first_lock); + assert_eq!(ownership.status, ContentLockOwnershipStatus::Reserved); + } + store + .reserve_paths(&second_creator(), &owned_paths, &second_lock) + .await + .unwrap(); + assert_eq!( + store + .get_path_ownership(&second_creator(), &owned_paths[0]) + .await + .unwrap() + .unwrap() + .lock_id, + second_lock + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn concurrent_competing_reservations_choose_one_owner_without_partial_rows() { + let database = TestDatabase::create().await; + let store = PostgresContentLockOwnershipRepository::new(database.pool().clone()); + let creator = creator(); + let first_lock = lock_id(1); + let second_lock = lock_id(2); + let shared_path = "/priv/locks.app/content/a-shared.txt".to_owned(); + let first_only_path = "/priv/locks.app/content/b-first.txt".to_owned(); + let second_only_path = "/priv/locks.app/content/c-second.txt".to_owned(); + let barrier = Arc::new(Barrier::new(3)); + + let first_store = store.clone(); + let first_creator = creator.clone(); + let first_paths = vec![shared_path.clone(), first_only_path.clone()]; + let first_task_lock = first_lock.clone(); + let first_barrier = barrier.clone(); + let first = tokio::spawn(async move { + first_barrier.wait().await; + first_store + .reserve_paths(&first_creator, &first_paths, &first_task_lock) + .await + }); + + let second_store = store.clone(); + let second_creator = creator.clone(); + let second_paths = vec![shared_path.clone(), second_only_path.clone()]; + let second_task_lock = second_lock.clone(); + let second_barrier = barrier.clone(); + let second = tokio::spawn(async move { + second_barrier.wait().await; + second_store + .reserve_paths(&second_creator, &second_paths, &second_task_lock) + .await + }); + + barrier.wait().await; + let first_result = first.await.unwrap(); + let second_result = second.await.unwrap(); + let shared_owner = store + .get_path_ownership(&creator, &shared_path) + .await + .unwrap() + .unwrap(); + + match (first_result, second_result, shared_owner.lock_id) { + (Ok(()), Err(ApplicationError::ContentLockPathConflict { guarded_path }), owner) + if guarded_path == shared_path && owner == first_lock => + { + assert!( + store + .get_path_ownership(&creator, &first_only_path) + .await + .unwrap() + .is_some() + ); + assert_eq!( + store + .get_path_ownership(&creator, &second_only_path) + .await + .unwrap(), + None + ); + } + (Err(ApplicationError::ContentLockPathConflict { guarded_path }), Ok(()), owner) + if guarded_path == shared_path && owner == second_lock => + { + assert_eq!( + store + .get_path_ownership(&creator, &first_only_path) + .await + .unwrap(), + None + ); + assert!( + store + .get_path_ownership(&creator, &second_only_path) + .await + .unwrap() + .is_some() + ); + } + results => panic!("expected exactly one complete reservation, got {results:?}"), + } + + database.cleanup().await; + } + + #[tokio::test] + async fn compensation_removes_only_reserved_rows_and_published_ownership_is_durable() { + let database = TestDatabase::create().await; + let store = PostgresContentLockOwnershipRepository::new(database.pool().clone()); + let recreated = PostgresContentLockOwnershipRepository::new(database.pool().clone()); + let creator = creator(); + let lock_id = lock_id(1); + let guarded_paths = paths(&["a.txt"]); + + store + .reserve_paths(&creator, &guarded_paths, &lock_id) + .await + .unwrap(); + store + .compensate_reserved_paths(&creator, &guarded_paths, &lock_id) + .await + .unwrap(); + assert_eq!( + store + .get_path_ownership(&creator, &guarded_paths[0]) + .await + .unwrap(), + None + ); + + store + .reserve_paths(&creator, &guarded_paths, &lock_id) + .await + .unwrap(); + store + .mark_paths_published(&creator, &guarded_paths, &lock_id) + .await + .unwrap(); + store + .reserve_paths(&creator, &guarded_paths, &lock_id) + .await + .unwrap(); + store + .compensate_reserved_paths(&creator, &guarded_paths, &lock_id) + .await + .unwrap(); + + let ownership = recreated + .get_path_ownership(&creator, &guarded_paths[0]) + .await + .unwrap() + .unwrap(); + assert_eq!(ownership.lock_id, lock_id); + assert_eq!(ownership.status, ContentLockOwnershipStatus::Published); + + database.cleanup().await; + } + + fn creator() -> CreatorPubky { + CreatorPubky::from_str("pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy").unwrap() + } + + fn second_creator() -> CreatorPubky { + CreatorPubky::from_str("pubky7ir1ttte48bcp4zjychjyscicrwi1j34mtt91ptsafdbjmr8g9eo").unwrap() + } + + fn lock_id(byte: u8) -> LockId { + LockId::from_hash(LockHash::from_bytes([byte; 32])) + } + + fn paths(names: &[&str]) -> Vec { + names + .iter() + .map(|name| format!("/priv/locks.app/content/{name}")) + .collect() + } +} diff --git a/locks-service/src/infrastructure/postgres/migrations.rs b/locks-service/src/infrastructure/postgres/migrations.rs index 22faed1..b7983e6 100644 --- a/locks-service/src/infrastructure/postgres/migrations.rs +++ b/locks-service/src/infrastructure/postgres/migrations.rs @@ -49,10 +49,67 @@ mod tests { assert_table_exists(&mut connection, "pending_creator_connect_flows").await; assert_table_exists(&mut connection, "frontend_session_codes").await; assert_table_exists(&mut connection, "frontend_sessions").await; + assert_table_exists(&mut connection, "content_lock_ownership").await; + assert_table_exists(&mut connection, "content_lock_deletion_jobs").await; + assert_table_exists(&mut connection, "content_lock_force_deletion_receipts").await; + assert_table_exists(&mut connection, "content_lock_publication_intents").await; + assert_table_exists(&mut connection, "content_lock_deletion_task_snapshot").await; + assert_table_exists(&mut connection, "content_lock_access_drain_credentials").await; + assert_table_exists(&mut connection, "content_lock_access_drain_reads").await; + assert_table_exists(&mut connection, "paykit_task_admissions").await; + assert_column_exists( + &mut connection, + "paykit_task_admissions", + "payment_in_hours", + ) + .await; + assert_column_exists( + &mut connection, + "paykit_task_admissions", + "invoice_created_at", + ) + .await; + assert_column_exists( + &mut connection, + "paykit_task_admissions", + "payment_deadline", + ) + .await; assert_column_exists(&mut connection, "verification_tasks", "creator").await; assert_column_exists(&mut connection, "verification_tasks", "bundle_id").await; assert_column_exists(&mut connection, "verification_tasks", "next_attempt_at").await; assert_column_exists(&mut connection, "verification_tasks", "claim_token").await; + assert_column_exists( + &mut connection, + "content_lock_deletion_jobs", + "final_issuance_started_at", + ) + .await; + assert_column_exists( + &mut connection, + "content_lock_deletion_jobs", + "final_credential_issuance_deadline", + ) + .await; + assert_column_exists( + &mut connection, + "content_lock_deletion_jobs", + "final_read_deadline", + ) + .await; + assert_column_exists( + &mut connection, + "content_lock_deletion_task_snapshot", + "had_active_credential_at_cutoff", + ) + .await; + assert_column_exists( + &mut connection, + "content_lock_deletion_task_snapshot", + "final_credential_eligible_at", + ) + .await; + assert_column_exists(&mut connection, "access_credentials", "deletion_job_id").await; assert_index_exists( &mut connection, "verification_tasks", @@ -70,17 +127,114 @@ mod tests { .await; assert_column_exists(&mut connection, "frontend_session_codes", "code_hash").await; assert_column_exists(&mut connection, "frontend_sessions", "token_hash").await; + assert_column_exists(&mut connection, "content_lock_ownership", "creator").await; + assert_column_exists(&mut connection, "content_lock_ownership", "guarded_path").await; + assert_column_exists(&mut connection, "content_lock_ownership", "lock_id").await; + assert_column_exists(&mut connection, "content_lock_ownership", "status").await; + assert_column_exists( + &mut connection, + "content_lock_deletion_jobs", + "frozen_content_lock", + ) + .await; + assert_column_exists(&mut connection, "content_lock_deletion_jobs", "claim_token").await; + assert_column_exists( + &mut connection, + "content_lock_deletion_jobs", + "force_requested_at", + ) + .await; assert_unique_constraint_exists( &mut connection, "verification_tasks", "verification_tasks_creator_bundle_unique", ) .await; + assert_unique_constraint_exists( + &mut connection, + "content_lock_ownership", + "content_lock_ownership_creator_path_unique", + ) + .await; + assert_unique_constraint_exists( + &mut connection, + "content_lock_deletion_jobs", + "content_lock_deletion_jobs_creator_lock_unique", + ) + .await; drop(connection); database.cleanup().await; } + #[tokio::test] + async fn migration_0016_rejects_every_preexisting_resumable_deletion_state() { + for (state, failure_code) in [ + ("queued", None), + ("running", None), + ("failed", Some("retry_exhausted")), + ] { + let database = TestDatabase::create().await; + sqlx::raw_sql( + "DROP TABLE content_lock_access_drain_reads; + DROP TABLE content_lock_access_drain_credentials; + ALTER TABLE access_credentials DROP COLUMN deletion_job_id; + ALTER TABLE content_lock_deletion_task_snapshot + DROP CONSTRAINT content_lock_deletion_task_snapshot_final_issuance_valid, + DROP CONSTRAINT content_lock_deletion_task_snapshot_final_eligibility_valid, + DROP COLUMN final_credential_issued_at, + DROP COLUMN final_credential_eligible_at, + DROP COLUMN had_active_credential_at_cutoff; + ALTER TABLE content_lock_deletion_jobs + DROP CONSTRAINT content_lock_deletion_jobs_final_window_shape, + DROP COLUMN final_read_deadline, + DROP COLUMN final_credential_issuance_deadline, + DROP COLUMN final_issuance_started_at;", + ) + .execute(database.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO content_lock_deletion_jobs + (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, + state, phase, failure_code, claimed_by, claim_token, claim_expires_at) + VALUES ( + $1, 'creator', 'lock', '{}'::jsonb, NOW(), + $2, 'withdraw', $3, + CASE WHEN $2 = 'running' THEN 'worker' ELSE NULL END, + CASE WHEN $2 = 'running' THEN $4::uuid ELSE NULL END, + CASE WHEN $2 = 'running' THEN NOW() + INTERVAL '1 minute' ELSE NULL END + )", + ) + .bind(uuid::Uuid::new_v4()) + .bind(state) + .bind(failure_code) + .bind(uuid::Uuid::new_v4()) + .execute(database.pool()) + .await + .unwrap(); + let migration = super::MIGRATOR + .iter() + .find(|migration| migration.version == 16) + .expect("migration 0016 exists"); + + let error = sqlx::raw_sql(migration.sql.as_ref()) + .execute(database.pool()) + .await + .expect_err( + "0016 must fail closed instead of misclassifying a resumable Task 7 job", + ); + assert!( + error + .to_string() + .contains("drain or explicitly reset pre-0016 deletion jobs"), + "unexpected migration error for {state}: {error}" + ); + + database.cleanup().await; + } + } + async fn assert_table_exists( connection: &mut sqlx::pool::PoolConnection, table_name: &str, diff --git a/locks-service/src/infrastructure/postgres/mod.rs b/locks-service/src/infrastructure/postgres/mod.rs index 1366910..5be1214 100644 --- a/locks-service/src/infrastructure/postgres/mod.rs +++ b/locks-service/src/infrastructure/postgres/mod.rs @@ -7,21 +7,31 @@ //! adapters or explicit production indexes are designed. pub mod access_credentials; +pub mod content_lock_deletion_action_ownership; +pub mod content_lock_deletions; +pub mod content_lock_ownership; pub mod creator_authority; pub mod creator_connect_flows; pub mod errors; pub mod frontend_sessions; pub mod migrations; +pub mod payment_drains; +mod proof_admission; +pub use proof_admission::{PaykitInvoiceWindow, PostgresPaykitTaskAdmissionRepository}; #[cfg(test)] pub(crate) mod testing; pub mod verification_task_claims; pub mod verification_tasks; pub use access_credentials::PostgresAccessCredentialStore; +pub use content_lock_deletion_action_ownership::PostgresContentLockDeletionActionOwnership; +pub use content_lock_deletions::PostgresContentLockDeletionRepository; +pub use content_lock_ownership::PostgresContentLockOwnershipRepository; pub use creator_authority::{CreatorAuthoritySecretCipher, PostgresCreatorAuthorityStore}; pub use creator_connect_flows::PostgresCreatorConnectFlowStore; pub use errors::PostgresError; pub use frontend_sessions::{PostgresFrontendSessionCodeStore, PostgresFrontendSessionStore}; pub use migrations::run_migrations; +pub use payment_drains::PostgresPaymentDrainRepository; pub use verification_task_claims::PostgresVerificationTaskClaimer; pub use verification_tasks::PostgresVerificationTaskRepository; diff --git a/locks-service/src/infrastructure/postgres/payment_drains.rs b/locks-service/src/infrastructure/postgres/payment_drains.rs new file mode 100644 index 0000000..99ae437 --- /dev/null +++ b/locks-service/src/infrastructure/postgres/payment_drains.rs @@ -0,0 +1,1084 @@ +use std::str::FromStr; + +use async_trait::async_trait; +use locks_core::ids::{BundleId, CreatorPubky, PubkyLockResource, TaskId}; +use sqlx::{FromRow, PgPool}; +use time::OffsetDateTime; +use uuid::Uuid; + +use crate::application::errors::ApplicationError; +use crate::application::models::VerificationTaskStatus; +use crate::application::ports::{ + PaymentDrainCleanupToken, PaymentDrainObligation, PaymentDrainRepository, PaymentDrainStatus, + PaymentDrainSummary, PaymentDrainTerminalTransition, +}; + +#[derive(Debug, Clone)] +pub struct PostgresPaymentDrainRepository { + pool: PgPool, +} + +impl PostgresPaymentDrainRepository { + pub fn new(pool: PgPool) -> Self { + Self { pool } + } +} + +#[derive(FromRow)] +struct ObligationRow { + task_id: Uuid, + creator: String, + bundle_id: String, + pubky_lock_resource: String, + criterion_id: String, + invoice_created_at: OffsetDateTime, + payment_deadline: OffsetDateTime, + status: String, +} + +#[derive(FromRow)] +struct DrainRow { + status: String, + accepted_count: i64, + terminal_count: i64, + cancellation_enqueued_count: i64, + cleanup_token: String, +} + +#[derive(FromRow)] +struct DeletionOwnershipRow { + state: String, + phase: String, + force_requested_at: Option, + claimed_by: Option, + claim_token: Option, + claim_expires_at: Option, +} + +#[derive(FromRow)] +struct ObligationFenceRow { + paykit_admission_required: Option, + resolved_status: Option, +} + +#[async_trait] +impl PaymentDrainRepository for PostgresPaymentDrainRepository { + async fn store_payment_drain( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + summary: &PaymentDrainSummary, + ) -> Result { + let counts = summary_counts(summary)?; + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let (ownership, now) = lock_deletion_ownership(&mut transaction, deletion_job_id).await?; + if !owns_live_drain_claim( + ownership.as_ref(), + worker_id, + claim_token, + now, + "start_payment_drain", + ) { + transaction.rollback().await.map_err(storage_error)?; + return Ok(false); + } + let existing = sqlx::query_as::<_, DrainRow>( + "SELECT status, accepted_count, terminal_count, + cancellation_enqueued_count, cleanup_token + FROM content_lock_payment_drains + WHERE deletion_job_id = $1 FOR UPDATE", + ) + .bind(deletion_job_id) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + if let Some(existing) = existing { + let existing = row_to_summary(existing)?; + if !valid_aggregate_progress(&existing, summary) { + return Err(invalid_deletion( + "Paykit payment drain aggregate changed for deletion job", + )); + } + sqlx::query( + "UPDATE content_lock_payment_drains + SET status = $2, accepted_count = $3, terminal_count = $4, updated_at = $5 + WHERE deletion_job_id = $1", + ) + .bind(deletion_job_id) + .bind(drain_status_to_database(summary.status)) + .bind(counts.0) + .bind(counts.1) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + } else { + sqlx::query( + "INSERT INTO content_lock_payment_drains + (deletion_job_id, status, accepted_count, terminal_count, + cancellation_enqueued_count, cleanup_token, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $7)", + ) + .bind(deletion_job_id) + .bind(drain_status_to_database(summary.status)) + .bind(counts.0) + .bind(counts.1) + .bind(counts.2) + .bind(summary.cleanup_token.as_str()) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + } + transaction.commit().await.map_err(storage_error)?; + Ok(true) + } + + async fn get_payment_drain( + &self, + deletion_job_id: Uuid, + ) -> Result, ApplicationError> { + sqlx::query_as::<_, DrainRow>( + "SELECT status, accepted_count, terminal_count, + cancellation_enqueued_count, cleanup_token + FROM content_lock_payment_drains WHERE deletion_job_id = $1", + ) + .bind(deletion_job_id) + .fetch_optional(&self.pool) + .await + .map_err(storage_error)? + .map(row_to_summary) + .transpose() + } + + async fn reconcile_payment_drain( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + summary: &PaymentDrainSummary, + ) -> Result { + let counts = summary_counts(summary)?; + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let (ownership, now) = lock_deletion_ownership(&mut transaction, deletion_job_id).await?; + if !owns_live_drain_claim( + ownership.as_ref(), + worker_id, + claim_token, + now, + "drain_payments", + ) { + transaction.rollback().await.map_err(storage_error)?; + return Ok(false); + } + let updated = sqlx::query( + "UPDATE content_lock_payment_drains + SET status = $3, accepted_count = $4, terminal_count = $5, updated_at = $2 + WHERE deletion_job_id = $1 + AND cleanup_token = $6 + AND cancellation_enqueued_count = $7 + AND accepted_count >= $4 + AND terminal_count <= $5 + AND accepted_count - $4 = $5 - terminal_count + AND NOT (status = 'completed' AND $3 <> 'completed') + AND (($3 = 'completed' AND $4 = 0) OR ($3 = 'active' AND $4 > 0))", + ) + .bind(deletion_job_id) + .bind(now) + .bind(drain_status_to_database(summary.status)) + .bind(counts.0) + .bind(counts.1) + .bind(summary.cleanup_token.as_str()) + .bind(counts.2) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(updated.rows_affected() == 1) + } + + async fn list_obligations( + &self, + deletion_job_id: Uuid, + ) -> Result, ApplicationError> { + let rows = sqlx::query_as::<_, ObligationRow>( + "SELECT snapshot.verification_task_id AS task_id, snapshot.creator, + snapshot.bundle_id, snapshot.pubky_lock_resource, + snapshot.criterion_id, + snapshot.invoice_created_at, snapshot.payment_deadline, + COALESCE(snapshot.resolved_status, snapshot.status_at_cutoff) AS status + FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.deletion_job_id = $1 + AND snapshot.paykit_admission_required = TRUE + ORDER BY snapshot.verification_task_id", + ) + .bind(deletion_job_id) + .fetch_all(&self.pool) + .await + .map_err(storage_error)?; + rows.into_iter().map(row_to_obligation).collect() + } + + async fn begin_entitlement_publication( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + task_id: &TaskId, + ) -> Result, ApplicationError> { + let publication_token = Uuid::new_v4(); + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let (ownership, now) = lock_deletion_ownership(&mut transaction, deletion_job_id).await?; + if !owns_live_drain_claim( + ownership.as_ref(), + worker_id, + claim_token, + now, + "drain_payments", + ) { + transaction.rollback().await.map_err(storage_error)?; + return Ok(None); + } + let admitted = sqlx::query_scalar( + "UPDATE verification_tasks AS task + SET entitlement_publication_claim_token = + COALESCE(task.entitlement_publication_claim_token, $6), + updated_at = $4 + FROM content_lock_deletion_jobs AS deletion, + content_lock_deletion_task_snapshot AS snapshot + WHERE deletion.job_id = $1 AND deletion.state = 'running' + AND deletion.claimed_by = $2 AND deletion.claim_token = $3 + AND deletion.claim_expires_at >= $4 AND deletion.phase = 'drain_payments' + AND deletion.force_requested_at IS NULL + AND snapshot.deletion_job_id = deletion.job_id + AND snapshot.verification_task_id = task.task_id + AND snapshot.paykit_admission_required = TRUE + AND snapshot.resolved_status IS NULL + AND task.task_id = $5::uuid + AND task.deletion_job_id = deletion.job_id + AND task.status IN ('pending', 'in_progress') + RETURNING task.entitlement_publication_claim_token", + ) + .bind(deletion_job_id) + .bind(worker_id) + .bind(claim_token) + .bind(now) + .bind(task_id.to_string()) + .bind(publication_token) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(admitted) + } + + async fn persist_terminal_obligation( + &self, + deletion_job_id: Uuid, + worker_id: &str, + claim_token: Uuid, + task_id: &TaskId, + transition: PaymentDrainTerminalTransition, + ) -> Result { + let PaymentDrainTerminalTransition { + status, + entitlement_publication_token, + } = transition; + if !matches!( + status, + VerificationTaskStatus::Completed | VerificationTaskStatus::Expired + ) { + return Err(ApplicationError::InvalidVerificationTaskState { + message: "payment drain transition must be completed or expired".to_owned(), + }); + } + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let (ownership, now) = lock_deletion_ownership(&mut transaction, deletion_job_id).await?; + if !owns_live_drain_claim( + ownership.as_ref(), + worker_id, + claim_token, + now, + "drain_payments", + ) { + transaction.rollback().await.map_err(storage_error)?; + return Ok(false); + } + let snapshot = sqlx::query_as::<_, ObligationFenceRow>( + "SELECT paykit_admission_required, resolved_status + FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 AND verification_task_id = $2::uuid + FOR UPDATE", + ) + .bind(deletion_job_id) + .bind(task_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + if !matches!( + snapshot, + Some(ObligationFenceRow { + paykit_admission_required: Some(true), + resolved_status: None, + }) + ) { + transaction.rollback().await.map_err(storage_error)?; + return Ok(false); + } + let updated = sqlx::query( + "UPDATE verification_tasks + SET status = $3, started_at = COALESCE(started_at, $2), completed_at = $2, + failure_message = NULL, claimed_by = NULL, claim_token = NULL, + claim_expires_at = NULL, next_attempt_at = NULL, + last_attempt_error = NULL, entitlement_publication_claim_token = NULL, + updated_at = $2 + WHERE task_id = $1::uuid + AND deletion_job_id = $4 + AND status IN ('pending', 'in_progress') + AND entitlement_publication_claim_token IS NOT DISTINCT FROM $5", + ) + .bind(task_id.to_string()) + .bind(now) + .bind(status_to_database(status)) + .bind(deletion_job_id) + .bind(entitlement_publication_token) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + if updated.rows_affected() != 1 { + transaction.rollback().await.map_err(storage_error)?; + return Ok(false); + } + let resolved = sqlx::query( + "UPDATE content_lock_deletion_task_snapshot + SET resolved_status = $3, resolved_at = $4, + final_credential_eligible_at = CASE + WHEN $3 = 'completed' + AND paykit_admission_required + AND NOT had_active_credential_at_cutoff + THEN $4 + ELSE NULL + END + WHERE deletion_job_id = $1 AND verification_task_id = $2::uuid + AND resolved_status IS NULL", + ) + .bind(deletion_job_id) + .bind(task_id.to_string()) + .bind(status_to_database(status)) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + if resolved.rows_affected() != 1 { + return Err(invalid_deletion( + "payment drain snapshot resolution lost its fence", + )); + } + transaction.commit().await.map_err(storage_error)?; + Ok(true) + } + + async fn all_obligations_terminal( + &self, + deletion_job_id: Uuid, + ) -> Result { + sqlx::query_scalar( + "SELECT NOT EXISTS ( + SELECT 1 FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 + AND paykit_admission_required = TRUE + AND COALESCE(resolved_status, status_at_cutoff) + NOT IN ('completed', 'failed', 'expired') + )", + ) + .bind(deletion_job_id) + .fetch_one(&self.pool) + .await + .map_err(storage_error) + } +} + +async fn lock_deletion_ownership( + transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>, + deletion_job_id: Uuid, +) -> Result<(Option, OffsetDateTime), ApplicationError> { + let ownership = sqlx::query_as::<_, DeletionOwnershipRow>( + "SELECT state, phase, force_requested_at, claimed_by, claim_token, claim_expires_at + FROM content_lock_deletion_jobs + WHERE job_id = $1 + FOR UPDATE", + ) + .bind(deletion_job_id) + .fetch_optional(&mut **transaction) + .await + .map_err(storage_error)?; + let winner_time = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut **transaction) + .await + .map_err(storage_error)?; + Ok((ownership, winner_time)) +} + +fn owns_live_drain_claim( + ownership: Option<&DeletionOwnershipRow>, + worker_id: &str, + claim_token: Uuid, + now: OffsetDateTime, + phase: &str, +) -> bool { + ownership.is_some_and(|ownership| { + ownership.state == "running" + && ownership.phase == phase + && ownership.force_requested_at.is_none() + && ownership.claimed_by.as_deref() == Some(worker_id) + && ownership.claim_token == Some(claim_token) + && ownership + .claim_expires_at + .is_some_and(|claim_expires_at| now < claim_expires_at) + }) +} + +fn row_to_obligation(row: ObligationRow) -> Result { + Ok(PaymentDrainObligation { + task_id: TaskId::from_str(&row.task_id.to_string()).map_err(storage_display)?, + creator: CreatorPubky::from_str(&row.creator).map_err(storage_display)?, + bundle_id: BundleId::from_str(&row.bundle_id).map_err(storage_display)?, + lock_resource: PubkyLockResource::from_str(&row.pubky_lock_resource) + .map_err(storage_display)?, + criterion_id: row.criterion_id, + invoice_created_at: row.invoice_created_at, + payment_deadline: row.payment_deadline, + status: status_from_database(&row.status)?, + }) +} + +fn row_to_summary(row: DrainRow) -> Result { + Ok(PaymentDrainSummary { + status: match row.status.as_str() { + "active" => PaymentDrainStatus::Active, + "completed" => PaymentDrainStatus::Completed, + _ => { + return Err(invalid_deletion( + "persisted Paykit payment drain status is invalid", + )); + } + }, + accepted_count: u64::try_from(row.accepted_count).map_err(storage_display)?, + terminal_count: u64::try_from(row.terminal_count).map_err(storage_display)?, + cancellation_enqueued_count: u64::try_from(row.cancellation_enqueued_count) + .map_err(storage_display)?, + cleanup_token: PaymentDrainCleanupToken::parse(&row.cleanup_token) + .ok_or_else(|| invalid_deletion("persisted Paykit cleanup token is invalid"))?, + }) +} + +fn summary_counts(summary: &PaymentDrainSummary) -> Result<(i64, i64, i64), ApplicationError> { + Ok(( + i64::try_from(summary.accepted_count).map_err(storage_display)?, + i64::try_from(summary.terminal_count).map_err(storage_display)?, + i64::try_from(summary.cancellation_enqueued_count).map_err(storage_display)?, + )) +} + +fn valid_aggregate_progress(previous: &PaymentDrainSummary, current: &PaymentDrainSummary) -> bool { + let accepted_delta = previous.accepted_count.checked_sub(current.accepted_count); + let terminal_delta = current.terminal_count.checked_sub(previous.terminal_count); + previous.cleanup_token == current.cleanup_token + && previous.cancellation_enqueued_count == current.cancellation_enqueued_count + && accepted_delta.is_some() + && accepted_delta == terminal_delta + && !(previous.status == PaymentDrainStatus::Completed + && current.status != PaymentDrainStatus::Completed) + && ((current.status == PaymentDrainStatus::Completed && current.accepted_count == 0) + || (current.status == PaymentDrainStatus::Active && current.accepted_count > 0)) +} + +fn drain_status_to_database(status: PaymentDrainStatus) -> &'static str { + match status { + PaymentDrainStatus::Active => "active", + PaymentDrainStatus::Completed => "completed", + } +} + +fn status_from_database(value: &str) -> Result { + match value { + "pending" => Ok(VerificationTaskStatus::Pending), + "in_progress" => Ok(VerificationTaskStatus::InProgress), + "completed" => Ok(VerificationTaskStatus::Completed), + "failed" => Ok(VerificationTaskStatus::Failed), + "expired" => Ok(VerificationTaskStatus::Expired), + _ => Err(ApplicationError::InvalidVerificationTaskState { + message: "unknown snapshotted verification task status".to_owned(), + }), + } +} + +fn status_to_database(status: VerificationTaskStatus) -> &'static str { + match status { + VerificationTaskStatus::Completed => "completed", + VerificationTaskStatus::Expired => "expired", + _ => unreachable!("validated terminal payment drain status"), + } +} + +fn invalid_deletion(message: &str) -> ApplicationError { + ApplicationError::InvalidContentLockDeletionState { + message: message.to_owned(), + } +} + +fn storage_error(error: sqlx::Error) -> ApplicationError { + storage_display(error) +} + +fn storage_display(error: impl std::fmt::Display) -> ApplicationError { + ApplicationError::Storage { + message: error.to_string(), + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use base64::Engine; + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use locks_core::ids::TaskId; + use time::OffsetDateTime; + use uuid::Uuid; + + use crate::application::models::VerificationTaskStatus; + use crate::application::ports::{ + PaymentDrainCleanupToken, PaymentDrainRepository, PaymentDrainStatus, PaymentDrainSummary, + PaymentDrainTerminalTransition, + }; + use crate::infrastructure::postgres::testing::TestDatabase; + + use super::PostgresPaymentDrainRepository; + + #[tokio::test] + async fn payment_drain_migration_creates_durable_token_table() { + let database = TestDatabase::create().await; + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = current_schema() + AND table_name = 'content_lock_payment_drains' + )", + ) + .fetch_one(database.pool()) + .await + .unwrap(); + assert!(exists); + database.cleanup().await; + } + + #[tokio::test] + async fn reconciliation_persists_only_monotonic_aggregate_progress_under_live_claim() { + let database = TestDatabase::create().await; + let repository = PostgresPaymentDrainRepository::new(database.pool().clone()); + let job_id = Uuid::new_v4(); + let claim_token = Uuid::new_v4(); + let now = OffsetDateTime::now_utc(); + sqlx::query( + "INSERT INTO content_lock_deletion_jobs + (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, + state, phase, claimed_by, claim_token, claim_expires_at) + VALUES ($1, 'creator', 'lock', '{}'::jsonb, $2, + 'running', 'drain_payments', 'worker', $3, $4)", + ) + .bind(job_id) + .bind(now) + .bind(claim_token) + .bind(now + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + let token = PaymentDrainCleanupToken::parse(&URL_SAFE_NO_PAD.encode([7_u8; 32])).unwrap(); + sqlx::query( + "INSERT INTO content_lock_payment_drains + (deletion_job_id, status, accepted_count, terminal_count, + cancellation_enqueued_count, cleanup_token, created_at, updated_at) + VALUES ($1, 'active', 2, 3, 1, $2, $3, $3)", + ) + .bind(job_id) + .bind(token.as_str()) + .bind(now) + .execute(database.pool()) + .await + .unwrap(); + + let completed = PaymentDrainSummary { + status: PaymentDrainStatus::Completed, + accepted_count: 0, + terminal_count: 5, + cancellation_enqueued_count: 1, + cleanup_token: token.clone(), + }; + assert!( + repository + .reconcile_payment_drain(job_id, "worker", claim_token, &completed) + .await + .unwrap() + ); + assert_eq!( + repository.get_payment_drain(job_id).await.unwrap(), + Some(completed) + ); + + let divergent = PaymentDrainSummary { + status: PaymentDrainStatus::Completed, + accepted_count: 0, + terminal_count: 6, + cancellation_enqueued_count: 1, + cleanup_token: token, + }; + assert!( + !repository + .reconcile_payment_drain(job_id, "worker", claim_token, &divergent) + .await + .unwrap() + ); + assert!( + !repository + .reconcile_payment_drain(job_id, "worker", Uuid::new_v4(), &divergent) + .await + .unwrap() + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn start_phase_replay_persists_monotonic_progress_after_crash_before_phase_advance() { + let database = TestDatabase::create().await; + let repository = PostgresPaymentDrainRepository::new(database.pool().clone()); + let job_id = Uuid::new_v4(); + let claim_token = Uuid::new_v4(); + let now = OffsetDateTime::now_utc(); + sqlx::query( + "INSERT INTO content_lock_deletion_jobs + (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, + state, phase, claimed_by, claim_token, claim_expires_at) + VALUES ($1, 'creator', 'lock', '{}'::jsonb, $2, + 'running', 'start_payment_drain', 'worker', $3, $4)", + ) + .bind(job_id) + .bind(now) + .bind(claim_token) + .bind(now + time::Duration::minutes(5)) + .execute(database.pool()) + .await + .unwrap(); + let token = PaymentDrainCleanupToken::parse(&URL_SAFE_NO_PAD.encode([8_u8; 32])).unwrap(); + let active = PaymentDrainSummary { + status: PaymentDrainStatus::Active, + accepted_count: 1, + terminal_count: 0, + cancellation_enqueued_count: 0, + cleanup_token: token.clone(), + }; + assert!( + repository + .store_payment_drain(job_id, "worker", claim_token, &active) + .await + .unwrap() + ); + + let completed = PaymentDrainSummary { + status: PaymentDrainStatus::Completed, + accepted_count: 0, + terminal_count: 1, + cancellation_enqueued_count: 0, + cleanup_token: token, + }; + assert!( + repository + .store_payment_drain(job_id, "worker", claim_token, &completed) + .await + .unwrap() + ); + assert_eq!( + repository.get_payment_drain(job_id).await.unwrap(), + Some(completed) + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn force_first_fences_stale_initial_payment_drain_store() { + let database = TestDatabase::create().await; + let repository = PostgresPaymentDrainRepository::new(database.pool().clone()); + let job_id = Uuid::new_v4(); + let stale_claim_token = Uuid::new_v4(); + let now = OffsetDateTime::now_utc(); + insert_start_drain_job(database.pool(), job_id, stale_claim_token, now).await; + + let mut force = database.pool().begin().await.unwrap(); + sqlx::query("SELECT job_id FROM content_lock_deletion_jobs WHERE job_id = $1 FOR UPDATE") + .bind(job_id) + .fetch_one(&mut *force) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET force_requested_at = $2, state = 'queued', claimed_by = NULL, + claim_token = NULL, claim_expires_at = NULL + WHERE job_id = $1", + ) + .bind(job_id) + .bind(now) + .execute(&mut *force) + .await + .unwrap(); + + let summary = PaymentDrainSummary { + status: PaymentDrainStatus::Active, + accepted_count: 1, + terminal_count: 0, + cancellation_enqueued_count: 0, + cleanup_token: PaymentDrainCleanupToken::parse(&URL_SAFE_NO_PAD.encode([10_u8; 32])) + .unwrap(), + }; + let stale = tokio::spawn(async move { + repository + .store_payment_drain(job_id, "worker", stale_claim_token, &summary) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!( + !stale.is_finished(), + "initial drain persistence must wait for the deletion-job ownership row" + ); + + force.commit().await.unwrap(); + assert!(!stale.await.unwrap().unwrap()); + assert_eq!( + PostgresPaymentDrainRepository::new(database.pool().clone()) + .get_payment_drain(job_id) + .await + .unwrap(), + None + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn reclaim_first_fences_stale_initial_payment_drain_store() { + let database = TestDatabase::create().await; + let repository = PostgresPaymentDrainRepository::new(database.pool().clone()); + let job_id = Uuid::new_v4(); + let stale_claim_token = Uuid::new_v4(); + let replacement_claim_token = Uuid::new_v4(); + let now = OffsetDateTime::now_utc(); + insert_start_drain_job(database.pool(), job_id, stale_claim_token, now).await; + + let mut reclaim = database.pool().begin().await.unwrap(); + sqlx::query("SELECT job_id FROM content_lock_deletion_jobs WHERE job_id = $1 FOR UPDATE") + .bind(job_id) + .fetch_one(&mut *reclaim) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET claimed_by = 'replacement-worker', claim_token = $2, + claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job_id) + .bind(replacement_claim_token) + .bind(now + time::Duration::minutes(10)) + .execute(&mut *reclaim) + .await + .unwrap(); + + let summary = PaymentDrainSummary { + status: PaymentDrainStatus::Active, + accepted_count: 1, + terminal_count: 0, + cancellation_enqueued_count: 0, + cleanup_token: PaymentDrainCleanupToken::parse(&URL_SAFE_NO_PAD.encode([11_u8; 32])) + .unwrap(), + }; + let stale = tokio::spawn(async move { + repository + .store_payment_drain(job_id, "worker", stale_claim_token, &summary) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!( + !stale.is_finished(), + "initial drain persistence must wait for the deletion-job ownership row" + ); + + reclaim.commit().await.unwrap(); + assert!(!stale.await.unwrap().unwrap()); + assert_eq!( + PostgresPaymentDrainRepository::new(database.pool().clone()) + .get_payment_drain(job_id) + .await + .unwrap(), + None + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn concurrent_force_winner_fences_stale_payment_drain_reconciliation() { + let database = TestDatabase::create().await; + let repository = PostgresPaymentDrainRepository::new(database.pool().clone()); + let job_id = Uuid::new_v4(); + let stale_claim_token = Uuid::new_v4(); + let now = OffsetDateTime::now_utc(); + let cleanup_token = + PaymentDrainCleanupToken::parse(&URL_SAFE_NO_PAD.encode([9_u8; 32])).unwrap(); + insert_drain_job( + database.pool(), + job_id, + stale_claim_token, + now, + cleanup_token.as_str(), + ) + .await; + + let mut force = database.pool().begin().await.unwrap(); + sqlx::query("SELECT job_id FROM content_lock_deletion_jobs WHERE job_id = $1 FOR UPDATE") + .bind(job_id) + .fetch_one(&mut *force) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET force_requested_at = $2, state = 'queued', claimed_by = NULL, + claim_token = NULL, claim_expires_at = NULL + WHERE job_id = $1", + ) + .bind(job_id) + .bind(now) + .execute(&mut *force) + .await + .unwrap(); + + let completed = PaymentDrainSummary { + status: PaymentDrainStatus::Completed, + accepted_count: 0, + terminal_count: 1, + cancellation_enqueued_count: 0, + cleanup_token: cleanup_token.clone(), + }; + let stale = tokio::spawn(async move { + repository + .reconcile_payment_drain(job_id, "worker", stale_claim_token, &completed) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!( + !stale.is_finished(), + "reconciliation must wait for the deletion-job ownership row" + ); + + force.commit().await.unwrap(); + assert!(!stale.await.unwrap().unwrap()); + assert_eq!( + PostgresPaymentDrainRepository::new(database.pool().clone()) + .get_payment_drain(job_id) + .await + .unwrap(), + Some(PaymentDrainSummary { + status: PaymentDrainStatus::Active, + accepted_count: 1, + terminal_count: 0, + cancellation_enqueued_count: 0, + cleanup_token, + }) + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn concurrent_reclaim_winner_fences_stale_terminal_obligation_persistence() { + let database = TestDatabase::create().await; + let repository = PostgresPaymentDrainRepository::new(database.pool().clone()); + let job_id = Uuid::new_v4(); + let task_uuid = Uuid::new_v4(); + let task_id = TaskId::from_str(&task_uuid.to_string()).unwrap(); + let stale_claim_token = Uuid::new_v4(); + let replacement_claim_token = Uuid::new_v4(); + let now = OffsetDateTime::now_utc(); + insert_terminal_obligation(database.pool(), job_id, task_uuid, stale_claim_token, now) + .await; + + let mut reclaim = database.pool().begin().await.unwrap(); + sqlx::query("SELECT job_id FROM content_lock_deletion_jobs WHERE job_id = $1 FOR UPDATE") + .bind(job_id) + .fetch_one(&mut *reclaim) + .await + .unwrap(); + sqlx::query( + "UPDATE content_lock_deletion_jobs + SET claimed_by = 'replacement-worker', claim_token = $2, + claim_expires_at = $3 + WHERE job_id = $1", + ) + .bind(job_id) + .bind(replacement_claim_token) + .bind(now + time::Duration::minutes(10)) + .execute(&mut *reclaim) + .await + .unwrap(); + + let stale = tokio::spawn(async move { + repository + .persist_terminal_obligation( + job_id, + "worker", + stale_claim_token, + &task_id, + PaymentDrainTerminalTransition { + status: VerificationTaskStatus::Completed, + entitlement_publication_token: None, + }, + ) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!( + !stale.is_finished(), + "terminal persistence must wait for the deletion-job ownership row" + ); + + reclaim.commit().await.unwrap(); + assert!(!stale.await.unwrap().unwrap()); + let task_status: String = + sqlx::query_scalar("SELECT status FROM verification_tasks WHERE task_id = $1") + .bind(task_uuid) + .fetch_one(database.pool()) + .await + .unwrap(); + let resolved_status: Option = sqlx::query_scalar( + "SELECT resolved_status FROM content_lock_deletion_task_snapshot + WHERE deletion_job_id = $1 AND verification_task_id = $2", + ) + .bind(job_id) + .bind(task_uuid) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(task_status, "pending"); + assert_eq!(resolved_status, None); + database.cleanup().await; + } + + async fn insert_start_drain_job( + pool: &sqlx::PgPool, + job_id: Uuid, + claim_token: Uuid, + now: OffsetDateTime, + ) { + sqlx::query( + "INSERT INTO content_lock_deletion_jobs + (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, + state, phase, claimed_by, claim_token, claim_expires_at) + VALUES ($1, 'creator', 'lock', '{}'::jsonb, $2, + 'running', 'start_payment_drain', 'worker', $3, $4)", + ) + .bind(job_id) + .bind(now) + .bind(claim_token) + .bind(now + time::Duration::minutes(5)) + .execute(pool) + .await + .unwrap(); + } + + async fn insert_drain_job( + pool: &sqlx::PgPool, + job_id: Uuid, + claim_token: Uuid, + now: OffsetDateTime, + cleanup_token: &str, + ) { + sqlx::query( + "INSERT INTO content_lock_deletion_jobs + (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, + state, phase, claimed_by, claim_token, claim_expires_at) + VALUES ($1, 'creator', 'lock', '{}'::jsonb, $2, + 'running', 'drain_payments', 'worker', $3, $4)", + ) + .bind(job_id) + .bind(now) + .bind(claim_token) + .bind(now + time::Duration::minutes(5)) + .execute(pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO content_lock_payment_drains + (deletion_job_id, status, accepted_count, terminal_count, + cancellation_enqueued_count, cleanup_token, created_at, updated_at) + VALUES ($1, 'active', 1, 0, 0, $2, $3, $3)", + ) + .bind(job_id) + .bind(cleanup_token) + .bind(now) + .execute(pool) + .await + .unwrap(); + } + + async fn insert_terminal_obligation( + pool: &sqlx::PgPool, + job_id: Uuid, + task_id: Uuid, + claim_token: Uuid, + now: OffsetDateTime, + ) { + sqlx::query( + "INSERT INTO content_lock_deletion_jobs + (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, + state, phase, claimed_by, claim_token, claim_expires_at) + VALUES ($1, 'creator', 'lock', '{}'::jsonb, $2, + 'running', 'drain_payments', 'worker', $3, $4)", + ) + .bind(job_id) + .bind(now) + .bind(claim_token) + .bind(now + time::Duration::minutes(5)) + .execute(pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO verification_tasks + (task_id, status, submitted_proof_bundle, submitted_at, creator, bundle_id, + deletion_job_id) + VALUES ($1, 'pending', '{}'::jsonb, $2, 'creator', 'bundle', $3)", + ) + .bind(task_id) + .bind(now) + .bind(job_id) + .execute(pool) + .await + .unwrap(); + sqlx::query( + "INSERT INTO content_lock_deletion_task_snapshot + (deletion_job_id, verification_task_id, creator, bundle_id, + pubky_lock_resource, criterion_id, status_at_cutoff, + paykit_admission_required, payment_in_hours, + invoice_created_at, payment_deadline) + VALUES ($1, $2, 'creator', 'bundle', 'pubkycreator/pub/locks.app/lock.json', + 'payment', 'pending', TRUE, 1, $3, $4)", + ) + .bind(job_id) + .bind(task_id) + .bind(now) + .bind(now + time::Duration::hours(1)) + .execute(pool) + .await + .unwrap(); + } +} diff --git a/locks-service/src/infrastructure/postgres/proof_admission.rs b/locks-service/src/infrastructure/postgres/proof_admission.rs new file mode 100644 index 0000000..79a7483 --- /dev/null +++ b/locks-service/src/infrastructure/postgres/proof_admission.rs @@ -0,0 +1,299 @@ +use locks_core::ids::{CreatorPubky, LockId}; +use sqlx::{PgPool, Postgres, Transaction}; + +use crate::application::errors::ApplicationError; +use crate::application::models::VerificationTaskRecord; +use locks_core::verification::SubmittedProofBundle; + +use super::verification_tasks::{ + VERIFICATION_TASK_ROW_COLUMNS, VerificationTaskRow, VerificationTaskWriteRow, row_to_task, +}; + +const PROOF_ADMISSION_LOCK_NAMESPACE: &str = "locks:proof-admission:v1"; + +/// Result of durably reserving one Paykit-backed proof admission. +#[derive(Debug)] +pub struct PaykitTaskAdmission { + /// The durable task associated with the public Bundle handle. + pub task: VerificationTaskRecord, + /// Immutable whole-hour payment window sent to Paykit for exact replay. + pub payment_in: u64, + /// Immutable timestamps returned by Paykit once the reservation is ready. + pub invoice_window: Option, + /// Whether the caller must create/reconcile the Paykit invoice before making the task claimable. + pub requires_paykit: bool, +} + +/// Immutable Paykit invoice timestamps bound to one admitted verification task. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PaykitInvoiceWindow { + pub invoice_created_at: time::OffsetDateTime, + pub payment_deadline: time::OffsetDateTime, +} + +/// PostgreSQL coordinator for durable persist-before-Paykit admission. +#[derive(Debug, Clone)] +pub struct PostgresPaykitTaskAdmissionRepository { + pool: PgPool, +} + +impl PostgresPaykitTaskAdmissionRepository { + /// Creates a coordinator backed by the migrated PostgreSQL pool. + pub fn new(pool: PgPool) -> Self { + Self { pool } + } + + /// Returns durable replay state without consulting mutable lock or reader discovery state. + pub async fn find_existing( + &self, + submitted: &SubmittedProofBundle, + ) -> Result, ApplicationError> { + let sql = format!( + "SELECT {VERIFICATION_TASK_ROW_COLUMNS}, + COALESCE(admission.ready, TRUE) AS paykit_ready, + admission.payment_in_hours, + admission.invoice_created_at, + admission.payment_deadline + FROM verification_tasks AS task + LEFT JOIN paykit_task_admissions AS admission + ON admission.verification_task_id = task.task_id + WHERE task.creator = $1 AND task.bundle_id = $2" + ); + let Some(existing) = sqlx::query_as::<_, PaykitAdmissionRow>(&sql) + .bind(submitted.pubky_lock_resource.creator().to_string()) + .bind(submitted.bundle_id.to_string()) + .fetch_optional(&self.pool) + .await + .map_err(storage_error)? + else { + return Ok(None); + }; + let ready = existing.paykit_ready; + let payment_in = payment_in_from_database(existing.payment_in_hours)?; + let invoice_window = invoice_window_from_row(&existing, ready)?; + let existing = row_to_task(existing.task)?; + if existing.submitted_proof_bundle != *submitted { + return Err(ApplicationError::VerificationTaskConflict); + } + Ok(Some(PaykitTaskAdmission { + task: existing, + payment_in, + invoice_window, + requires_paykit: !ready, + })) + } + + /// Reserves a task before Paykit mutation, serialized against deletion start. + pub async fn reserve( + &self, + task: VerificationTaskRecord, + payment_in: u64, + ) -> Result { + let payment_in_hours = payment_in_to_database(payment_in)?; + let row = VerificationTaskWriteRow::try_from(&task)?; + let lock_id = task.submitted_proof_bundle.pubky_lock_resource.lock_id(); + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, &task.creator, lock_id).await?; + + let existing_sql = format!( + "SELECT {VERIFICATION_TASK_ROW_COLUMNS}, + COALESCE(admission.ready, TRUE) AS paykit_ready, + admission.payment_in_hours, + admission.invoice_created_at, + admission.payment_deadline + FROM verification_tasks AS task + LEFT JOIN paykit_task_admissions AS admission + ON admission.verification_task_id = task.task_id + WHERE task.creator = $1 AND task.bundle_id = $2" + ); + if let Some(existing) = sqlx::query_as::<_, PaykitAdmissionRow>(&existing_sql) + .bind(&row.creator) + .bind(&row.bundle_id) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)? + { + let ready = existing.paykit_ready; + let stored_payment_in = payment_in_from_database(existing.payment_in_hours)?; + let invoice_window = invoice_window_from_row(&existing, ready)?; + let existing = row_to_task(existing.task)?; + if existing.submitted_proof_bundle != task.submitted_proof_bundle + || stored_payment_in != payment_in + { + return Err(ApplicationError::VerificationTaskConflict); + } + transaction.commit().await.map_err(storage_error)?; + return Ok(PaykitTaskAdmission { + task: existing, + payment_in: stored_payment_in, + invoice_window, + requires_paykit: !ready, + }); + } + + let deletion_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_deletion_jobs WHERE creator = $1 AND lock_id = $2 + )", + ) + .bind(&row.creator) + .bind(lock_id.to_string()) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if deletion_exists { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + + insert_task(&mut transaction, row).await?; + sqlx::query( + "INSERT INTO paykit_task_admissions + (verification_task_id, ready, payment_in_hours) + VALUES ($1::uuid, FALSE, $2)", + ) + .bind(task.task_id.to_string()) + .bind(payment_in_hours) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + + Ok(PaykitTaskAdmission { + task, + payment_in, + invoice_window: None, + requires_paykit: true, + }) + } + + /// Makes a reserved task claimable after Paykit confirms invoice creation/replay. + pub async fn mark_ready( + &self, + task: &VerificationTaskRecord, + invoice_window: PaykitInvoiceWindow, + ) -> Result<(), ApplicationError> { + if invoice_window.payment_deadline < invoice_window.invoice_created_at { + return Err(ApplicationError::VerificationTaskConflict); + } + let result = sqlx::query( + "UPDATE paykit_task_admissions + SET ready = TRUE, + ready_at = COALESCE(ready_at, now()), + invoice_created_at = COALESCE(invoice_created_at, $2), + payment_deadline = COALESCE(payment_deadline, $3) + WHERE verification_task_id = $1::uuid + AND ( + (ready = FALSE AND invoice_created_at IS NULL AND payment_deadline IS NULL) + OR + (ready = TRUE AND invoice_created_at = $2 AND payment_deadline = $3) + )", + ) + .bind(task.task_id.to_string()) + .bind(invoice_window.invoice_created_at) + .bind(invoice_window.payment_deadline) + .execute(&self.pool) + .await + .map_err(storage_error)?; + if result.rows_affected() == 0 { + return Err(ApplicationError::VerificationTaskConflict); + } + Ok(()) + } +} + +#[derive(sqlx::FromRow)] +struct PaykitAdmissionRow { + #[sqlx(flatten)] + task: VerificationTaskRow, + paykit_ready: bool, + payment_in_hours: Option, + invoice_created_at: Option, + payment_deadline: Option, +} + +fn payment_in_to_database(payment_in: u64) -> Result { + i64::try_from(payment_in) + .ok() + .filter(|value| *value > 0) + .ok_or(ApplicationError::VerificationTaskConflict) +} + +fn payment_in_from_database(payment_in: Option) -> Result { + payment_in + .and_then(|value| u64::try_from(value).ok()) + .filter(|value| *value > 0) + .ok_or(ApplicationError::Storage { + message: "invalid Paykit payment window stored in Postgres".to_owned(), + }) +} + +fn invoice_window_from_row( + row: &PaykitAdmissionRow, + ready: bool, +) -> Result, ApplicationError> { + match (ready, row.invoice_created_at, row.payment_deadline) { + (false, None, None) => Ok(None), + (true, Some(invoice_created_at), Some(payment_deadline)) + if invoice_created_at <= payment_deadline => + { + Ok(Some(PaykitInvoiceWindow { + invoice_created_at, + payment_deadline, + })) + } + _ => Err(ApplicationError::Storage { + message: "invalid Paykit invoice window stored in Postgres".to_owned(), + }), + } +} + +async fn insert_task( + transaction: &mut Transaction<'_, Postgres>, + row: VerificationTaskWriteRow, +) -> Result<(), ApplicationError> { + sqlx::query( + "INSERT INTO verification_tasks ( + task_id, creator, bundle_id, status, submitted_proof_bundle, + submitted_at, started_at, completed_at, failure_message + ) VALUES ($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9)", + ) + .bind(row.task_id) + .bind(row.creator) + .bind(row.bundle_id) + .bind(row.status) + .bind(row.submitted_proof_bundle) + .bind(row.submitted_at) + .bind(row.started_at) + .bind(row.completed_at) + .bind(row.failure_message) + .execute(&mut **transaction) + .await + .map_err(storage_error)?; + Ok(()) +} + +pub(super) async fn lock_proof_admission( + transaction: &mut Transaction<'_, Postgres>, + creator: &CreatorPubky, + lock_id: &LockId, +) -> Result<(), ApplicationError> { + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(proof_admission_lock_key(creator, lock_id)) + .execute(&mut **transaction) + .await + .map_err(storage_error)?; + Ok(()) +} + +fn proof_admission_lock_key(creator: &CreatorPubky, lock_id: &LockId) -> String { + format!( + "{PROOF_ADMISSION_LOCK_NAMESPACE}:{creator}:{}", + lock_id.as_str() + ) +} + +fn storage_error(error: sqlx::Error) -> ApplicationError { + ApplicationError::Storage { + message: error.to_string(), + } +} diff --git a/locks-service/src/infrastructure/postgres/verification_task_claims.rs b/locks-service/src/infrastructure/postgres/verification_task_claims.rs index cd874fa..971fbe7 100644 --- a/locks-service/src/infrastructure/postgres/verification_task_claims.rs +++ b/locks-service/src/infrastructure/postgres/verification_task_claims.rs @@ -26,50 +26,116 @@ impl PostgresVerificationTaskClaimer { #[async_trait] impl VerificationTaskClaimer for PostgresVerificationTaskClaimer { + async fn begin_claimed_entitlement_publication( + &self, + task_id: &TaskId, + worker_id: &str, + claim_token: &uuid::Uuid, + ) -> Result { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + sqlx::query("SELECT task_id FROM verification_tasks WHERE task_id = $1::uuid FOR UPDATE") + .bind(task_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let now: time::OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + let updated = sqlx::query( + "UPDATE verification_tasks + SET entitlement_publication_claim_token = $3, updated_at = $4 + WHERE task_id = $1::uuid AND status = 'in_progress' + AND claimed_by = $2 AND claim_token = $3 AND claim_expires_at > $4 + AND deletion_job_id IS NULL + AND NOT EXISTS ( + SELECT 1 FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.verification_task_id = verification_tasks.task_id + )", + ) + .bind(task_id.to_string()) + .bind(worker_id) + .bind(claim_token) + .bind(now) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + transaction.commit().await.map_err(storage_error)?; + Ok(updated.rows_affected() == 1) + } + async fn claim_next_verification_task( &self, worker_id: &str, - now: time::OffsetDateTime, - claim_expires_at: time::OffsetDateTime, + claim_ttl: time::Duration, ) -> Result, ApplicationError> { let claim_token = uuid::Uuid::new_v4(); + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + let candidates: Vec<( + uuid::Uuid, + String, + Option, + Option, + )> = sqlx::query_as( + "SELECT task_id, status, next_attempt_at, claim_expires_at + FROM verification_tasks + WHERE status IN ('pending', 'in_progress') + AND deletion_job_id IS NULL + AND NOT EXISTS ( + SELECT 1 FROM paykit_task_admissions + WHERE verification_task_id = verification_tasks.task_id + AND (ready = FALSE OR payment_in_hours IS NULL OR payment_in_hours <= 0 + OR invoice_created_at IS NULL OR payment_deadline IS NULL + OR invoice_created_at > payment_deadline) + ) + AND NOT EXISTS ( + SELECT 1 FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.verification_task_id = verification_tasks.task_id + ) + AND creator = split_part(submitted_proof_bundle->>'pubky_lock_resource', '/', 1) + AND bundle_id = submitted_proof_bundle->>'bundle_id' + ORDER BY submitted_at + FOR UPDATE SKIP LOCKED", + ) + .fetch_all(&mut *transaction) + .await + .map_err(storage_error)?; + let now: time::OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + let claim_expires_at = + now.checked_add(claim_ttl) + .ok_or_else(|| ApplicationError::Storage { + message: "verification task claim expiry overflow".to_owned(), + })?; + let Some((task_id, _, _, _)) = candidates.into_iter().find(|(_, status, next, expiry)| { + (status == "pending" && next.is_none_or(|due| due <= now)) + || (status == "in_progress" && expiry.is_some_and(|deadline| deadline <= now)) + }) else { + transaction.commit().await.map_err(storage_error)?; + return Ok(None); + }; let sql = format!( "UPDATE verification_tasks - SET - status = 'in_progress', - claimed_by = $1, - claim_expires_at = $2, - claim_token = $4, - next_attempt_at = NULL, - started_at = COALESCE(started_at, $3), - attempt_count = attempt_count + 1, - updated_at = $3 - WHERE task_id = ( - SELECT task_id - FROM verification_tasks - WHERE ((status = 'pending' - AND (next_attempt_at IS NULL OR next_attempt_at <= $3)) - OR (status = 'in_progress' AND claim_expires_at < $3)) - AND creator = split_part(submitted_proof_bundle->>'pubky_lock_resource', '/', 1) - AND bundle_id = submitted_proof_bundle->>'bundle_id' - ORDER BY submitted_at - FOR UPDATE SKIP LOCKED - LIMIT 1 - ) - RETURNING {VERIFICATION_TASK_ROW_COLUMNS}" + SET status = 'in_progress', claimed_by = $2, claim_expires_at = $3, + claim_token = $4, next_attempt_at = NULL, + started_at = COALESCE(started_at, $5), attempt_count = attempt_count + 1, + updated_at = $5 + WHERE task_id = $1 + RETURNING {VERIFICATION_TASK_ROW_COLUMNS}" ); let row = sqlx::query_as::<_, VerificationTaskRow>(&sql) + .bind(task_id) .bind(worker_id) .bind(claim_expires_at) - .bind(now) .bind(claim_token) - .fetch_optional(&self.pool) + .bind(now) + .fetch_one(&mut *transaction) .await .map_err(storage_error)?; - - row.map(row_to_task) - .transpose() - .map(|task| task.map(|task| ClaimedVerificationTask { task, claim_token })) + transaction.commit().await.map_err(storage_error)?; + row_to_task(row).map(|task| Some(ClaimedVerificationTask { task, claim_token })) } async fn schedule_verification_task_retry( @@ -77,9 +143,23 @@ impl VerificationTaskClaimer for PostgresVerificationTaskClaimer { task_id: &TaskId, worker_id: &str, claim_token: &uuid::Uuid, - now: time::OffsetDateTime, - next_attempt_at: time::OffsetDateTime, + retry_after: time::Duration, ) -> Result, ApplicationError> { + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + sqlx::query("SELECT task_id FROM verification_tasks WHERE task_id = $1::uuid FOR UPDATE") + .bind(task_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let now: time::OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + let next_attempt_at = + now.checked_add(retry_after) + .ok_or_else(|| ApplicationError::Storage { + message: "verification task retry time overflow".to_owned(), + })?; let sql = format!( "UPDATE verification_tasks SET @@ -97,7 +177,13 @@ impl VerificationTaskClaimer for PostgresVerificationTaskClaimer { AND status = 'in_progress' AND claimed_by = $2 AND claim_token = $3 - AND claim_expires_at >= $4 + AND claim_expires_at > $4 + AND deletion_job_id IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.verification_task_id = verification_tasks.task_id + ) RETURNING {VERIFICATION_TASK_ROW_COLUMNS}" ); let row = sqlx::query_as::<_, VerificationTaskRow>(&sql) @@ -106,11 +192,12 @@ impl VerificationTaskClaimer for PostgresVerificationTaskClaimer { .bind(claim_token) .bind(now) .bind(next_attempt_at) - .fetch_optional(&self.pool) + .fetch_optional(&mut *transaction) .await .map_err(storage_error)?; - - row.map(row_to_task).transpose() + let result = row.map(row_to_task).transpose()?; + transaction.commit().await.map_err(storage_error)?; + Ok(result) } async fn persist_claimed_verification_task_transition( @@ -118,7 +205,6 @@ impl VerificationTaskClaimer for PostgresVerificationTaskClaimer { task: VerificationTaskRecord, worker_id: &str, claim_token: &uuid::Uuid, - now: time::OffsetDateTime, ) -> Result, ApplicationError> { if !matches!( task.status, @@ -130,6 +216,16 @@ impl VerificationTaskClaimer for PostgresVerificationTaskClaimer { message: "claimed task transition must be terminal".to_owned(), }); } + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + sqlx::query("SELECT task_id FROM verification_tasks WHERE task_id = $1::uuid FOR UPDATE") + .bind(task.task_id.to_string()) + .fetch_optional(&mut *transaction) + .await + .map_err(storage_error)?; + let now: time::OffsetDateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; let sql = format!( "UPDATE verification_tasks SET status = $5, @@ -139,6 +235,7 @@ impl VerificationTaskClaimer for PostgresVerificationTaskClaimer { claimed_by = NULL, claim_token = NULL, claim_expires_at = NULL, + entitlement_publication_claim_token = NULL, next_attempt_at = NULL, last_attempt_error = NULL, updated_at = $4 @@ -146,7 +243,13 @@ impl VerificationTaskClaimer for PostgresVerificationTaskClaimer { AND status = 'in_progress' AND claimed_by = $2 AND claim_token = $3 - AND claim_expires_at >= $4 + AND claim_expires_at > $4 + AND deletion_job_id IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.verification_task_id = verification_tasks.task_id + ) RETURNING {VERIFICATION_TASK_ROW_COLUMNS}" ); let row = sqlx::query_as::<_, VerificationTaskRow>(&sql) @@ -158,11 +261,12 @@ impl VerificationTaskClaimer for PostgresVerificationTaskClaimer { .bind(task.started_at) .bind(task.completed_at) .bind(task.failure_message) - .fetch_optional(&self.pool) + .fetch_optional(&mut *transaction) .await .map_err(storage_error)?; - - row.map(row_to_task).transpose() + let result = row.map(row_to_task).transpose()?; + transaction.commit().await.map_err(storage_error)?; + Ok(result) } } @@ -196,6 +300,168 @@ mod tests { const NOW: time::OffsetDateTime = datetime!(2026-05-29 12:10:00 UTC); const CLAIM_EXPIRES_AT: time::OffsetDateTime = datetime!(2026-05-29 12:15:00 UTC); + #[tokio::test] + async fn waiting_publication_update_rechecks_task_row_deletion_fence() { + let database = TestDatabase::create().await; + let repository = PostgresVerificationTaskRepository::new(database.pool().clone()); + let claimer = PostgresVerificationTaskClaimer::new(database.pool().clone()); + let pending = task( + "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d13", + VerificationTaskStatus::Pending, + datetime!(2026-05-29 12:00:00 UTC), + ); + repository + .insert_verification_task(pending.clone()) + .await + .unwrap(); + let claim = claimer + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) + .await + .unwrap() + .unwrap(); + + let mut transaction = database.pool().begin().await.unwrap(); + sqlx::query("SELECT task_id FROM verification_tasks WHERE task_id = $1 FOR UPDATE") + .bind(pending.task_id.as_uuid()) + .fetch_one(&mut *transaction) + .await + .unwrap(); + + let waiting_claimer = claimer.clone(); + let task_id = pending.task_id; + let claim_token = claim.claim_token; + let publication = tokio::spawn(async move { + waiting_claimer + .begin_claimed_entitlement_publication(&task_id, "worker-a", &claim_token) + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + assert!(!publication.is_finished()); + + let deletion_job_id = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO content_lock_deletion_jobs + (job_id, creator, lock_id, frozen_content_lock, deletion_started_at, state, phase) + VALUES ($1, $2, $3, '{}'::jsonb, $4, 'queued', 'withdraw')", + ) + .bind(deletion_job_id) + .bind(pending.creator.to_string()) + .bind(LOCK_ID) + .bind(NOW) + .execute(&mut *transaction) + .await + .unwrap(); + sqlx::query("UPDATE verification_tasks SET deletion_job_id = $1 WHERE task_id = $2") + .bind(deletion_job_id) + .bind(pending.task_id.as_uuid()) + .execute(&mut *transaction) + .await + .unwrap(); + transaction.commit().await.unwrap(); + + assert!(!publication.await.unwrap().unwrap()); + database.cleanup().await; + } + + #[tokio::test] + async fn retry_retains_publication_marker_until_reconciled_terminal_transition() { + let database = TestDatabase::create().await; + let repository = PostgresVerificationTaskRepository::new(database.pool().clone()); + let claimer = PostgresVerificationTaskClaimer::new(database.pool().clone()); + let pending = task( + "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d17", + VerificationTaskStatus::Pending, + datetime!(2026-05-29 12:00:00 UTC), + ); + repository + .insert_verification_task(pending.clone()) + .await + .unwrap(); + let first = claimer + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) + .await + .unwrap() + .unwrap(); + assert!( + claimer + .begin_claimed_entitlement_publication( + &pending.task_id, + "worker-a", + &first.claim_token, + ) + .await + .unwrap() + ); + let retry_at = NOW + time::Duration::seconds(10); + claimer + .schedule_verification_task_retry( + &pending.task_id, + "worker-a", + &first.claim_token, + (retry_at) - (NOW), + ) + .await + .unwrap() + .unwrap(); + + let retained: Option = sqlx::query_scalar( + "SELECT entitlement_publication_claim_token + FROM verification_tasks + WHERE task_id = $1", + ) + .bind(pending.task_id.as_uuid()) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(retained, Some(first.claim_token)); + + mark_retry_due(database.pool(), &pending.task_id).await; + let second = claimer + .claim_next_verification_task( + "worker-b", + (retry_at + time::Duration::minutes(5)) - (retry_at), + ) + .await + .unwrap() + .unwrap(); + assert!( + claimer + .begin_claimed_entitlement_publication( + &pending.task_id, + "worker-b", + &second.claim_token, + ) + .await + .unwrap() + ); + let completed = second + .task + .transition_to(VerificationTaskStatus::Completed, retry_at, None) + .unwrap(); + claimer + .persist_claimed_verification_task_transition( + completed, + "worker-b", + &second.claim_token, + ) + .await + .unwrap() + .unwrap(); + + let cleared: Option = sqlx::query_scalar( + "SELECT entitlement_publication_claim_token + FROM verification_tasks + WHERE task_id = $1", + ) + .bind(pending.task_id.as_uuid()) + .fetch_one(database.pool()) + .await + .unwrap(); + assert_eq!(cleared, None); + + database.cleanup().await; + } + #[tokio::test] async fn claims_oldest_pending_task_first() { let database = TestDatabase::create().await; @@ -217,15 +483,129 @@ mod tests { .await .unwrap(); + let before_claim = database_time(database.pool()).await; let claimed = claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap() .expect("oldest pending task is claimed"); assert_eq!(claimed.task.task_id, older.task_id); assert_eq!(claimed.task.status, VerificationTaskStatus::InProgress); - assert_eq!(claimed.task.started_at, Some(NOW)); + let after_claim = database_time(database.pool()).await; + assert!( + claimed.task.started_at.is_some_and(|started_at| { + before_claim <= started_at && started_at <= after_claim + }) + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn paykit_reservation_is_not_claimable_until_marked_ready() { + use crate::infrastructure::postgres::PostgresPaykitTaskAdmissionRepository; + + let database = TestDatabase::create().await; + let claimer = PostgresVerificationTaskClaimer::new(database.pool().clone()); + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let pending = task( + "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d15", + VerificationTaskStatus::Pending, + datetime!(2026-05-29 12:00:00 UTC), + ); + + let first = admissions.reserve(pending.clone(), 24).await.unwrap(); + assert!(first.requires_paykit); + assert!( + claimer + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) + .await + .unwrap() + .is_none() + ); + + let replay = admissions.reserve(pending.clone(), 24).await.unwrap(); + assert!(replay.requires_paykit); + assert_eq!(replay.task, pending); + + let invoice_window = crate::infrastructure::postgres::PaykitInvoiceWindow { + invoice_created_at: datetime!(2026-05-29 12:00:00 UTC), + payment_deadline: datetime!(2026-05-30 12:00:00 UTC), + }; + admissions + .mark_ready(&pending, invoice_window) + .await + .unwrap(); + let divergent_window = crate::infrastructure::postgres::PaykitInvoiceWindow { + invoice_created_at: datetime!(2026-05-29 12:00:01 UTC), + payment_deadline: datetime!(2026-05-30 12:00:01 UTC), + }; + assert!( + admissions + .mark_ready(&pending, divergent_window) + .await + .is_err() + ); + let ready_replay = admissions.reserve(pending.clone(), 24).await.unwrap(); + assert!(!ready_replay.requires_paykit); + assert_eq!(ready_replay.task, pending); + assert_eq!(ready_replay.payment_in, 24); + assert_eq!(ready_replay.invoice_window, Some(invoice_window)); + assert_eq!( + claimer + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) + .await + .unwrap() + .unwrap() + .task + .task_id, + pending.task_id + ); + + database.cleanup().await; + } + + #[tokio::test] + async fn legacy_paykit_admission_without_authoritative_window_fails_closed() { + use crate::infrastructure::postgres::PostgresPaykitTaskAdmissionRepository; + + let database = TestDatabase::create().await; + let tasks = PostgresVerificationTaskRepository::new(database.pool().clone()); + let admissions = PostgresPaykitTaskAdmissionRepository::new(database.pool().clone()); + let pending = task( + "018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d16", + VerificationTaskStatus::Pending, + datetime!(2026-05-29 12:00:00 UTC), + ); + tasks + .insert_verification_task(pending.clone()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO paykit_task_admissions + (verification_task_id, ready, ready_at) + VALUES ($1::uuid, TRUE, now())", + ) + .bind(pending.task_id.to_string()) + .execute(database.pool()) + .await + .unwrap(); + + assert!( + admissions + .find_existing(&pending.submitted_proof_bundle) + .await + .is_err() + ); + let claimer = PostgresVerificationTaskClaimer::new(database.pool().clone()); + assert!( + claimer + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) + .await + .unwrap() + .is_none() + ); database.cleanup().await; } @@ -254,7 +634,7 @@ mod tests { assert_eq!( claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap(), None @@ -283,7 +663,7 @@ mod tests { mark_claim_expired(database.pool(), &in_progress.task_id).await; let reclaimed = claimer - .claim_next_verification_task("worker-b", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-b", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap() .expect("expired in-progress task is reclaimed"); @@ -316,7 +696,7 @@ mod tests { assert_eq!( claimer - .claim_next_verification_task("worker-b", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-b", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap(), None @@ -342,8 +722,8 @@ mod tests { .unwrap(); let (claim_a, claim_b) = tokio::join!( - claimer_a.claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT), - claimer_b.claim_next_verification_task("worker-b", NOW, CLAIM_EXPIRES_AT), + claimer_a.claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)), + claimer_b.claim_next_verification_task("worker-b", (CLAIM_EXPIRES_AT) - (NOW)), ); let claimed = [claim_a.unwrap(), claim_b.unwrap()]; @@ -375,16 +755,16 @@ mod tests { .await .unwrap(); let first = claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap() .unwrap(); - let reclaimed_at = CLAIM_EXPIRES_AT + time::Duration::milliseconds(1); + expire_claim(database.pool(), &first.task.task_id).await; + let reclaimed_at = database_time(database.pool()).await; let second = claimer .claim_next_verification_task( "worker-a", - reclaimed_at, - reclaimed_at + time::Duration::minutes(5), + (reclaimed_at + time::Duration::minutes(5)) - (reclaimed_at), ) .await .unwrap() @@ -397,8 +777,7 @@ mod tests { &pending.task_id, "worker-a", &first.claim_token, - reclaimed_at, - reclaimed_at + time::Duration::seconds(10), + (reclaimed_at + time::Duration::seconds(10)) - (reclaimed_at), ) .await .unwrap(), @@ -410,8 +789,7 @@ mod tests { &pending.task_id, "worker-a", &second.claim_token, - reclaimed_at, - reclaimed_at + time::Duration::seconds(10), + (reclaimed_at + time::Duration::seconds(10)) - (reclaimed_at), ) .await .unwrap() @@ -433,16 +811,16 @@ mod tests { ); repository.insert_verification_task(pending).await.unwrap(); let first = claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap() .unwrap(); - let reclaimed_at = CLAIM_EXPIRES_AT + time::Duration::milliseconds(1); + expire_claim(database.pool(), &first.task.task_id).await; + let reclaimed_at = database_time(database.pool()).await; let second = claimer .claim_next_verification_task( "worker-a", - reclaimed_at, - reclaimed_at + time::Duration::minutes(5), + (reclaimed_at + time::Duration::minutes(5)) - (reclaimed_at), ) .await .unwrap() @@ -473,7 +851,6 @@ mod tests { terminal, "worker-a", &first.claim_token, - reclaimed_at, ) .await .unwrap(), @@ -486,7 +863,6 @@ mod tests { completed.clone(), "worker-a", &second.claim_token, - reclaimed_at, ) .await .unwrap(), @@ -511,7 +887,7 @@ mod tests { .await .unwrap(); let claim = claimer - .claim_next_verification_task("worker-a", NOW, CLAIM_EXPIRES_AT) + .claim_next_verification_task("worker-a", (CLAIM_EXPIRES_AT) - (NOW)) .await .unwrap() .expect("pending task is claimed"); @@ -523,21 +899,7 @@ mod tests { &pending.task_id, "worker-b", &claim.claim_token, - NOW, - next_attempt_at, - ) - .await - .unwrap(), - None - ); - assert_eq!( - claimer - .schedule_verification_task_retry( - &pending.task_id, - "worker-a", - &claim.claim_token, - CLAIM_EXPIRES_AT + time::Duration::milliseconds(1), - next_attempt_at, + (next_attempt_at) - (NOW), ) .await .unwrap(), @@ -548,8 +910,7 @@ mod tests { &pending.task_id, "worker-a", &claim.claim_token, - NOW, - next_attempt_at, + (next_attempt_at) - (NOW), ) .await .unwrap() @@ -563,19 +924,18 @@ mod tests { claimer .claim_next_verification_task( "worker-b", - next_attempt_at - time::Duration::milliseconds(1), - CLAIM_EXPIRES_AT, + (CLAIM_EXPIRES_AT) - (next_attempt_at - time::Duration::milliseconds(1)), ) .await .unwrap(), None ); + mark_retry_due(database.pool(), &pending.task_id).await; assert!( claimer .claim_next_verification_task( "worker-b", - next_attempt_at, - CLAIM_EXPIRES_AT + time::Duration::seconds(10), + (CLAIM_EXPIRES_AT + time::Duration::seconds(10)) - (next_attempt_at), ) .await .unwrap() @@ -597,11 +957,10 @@ mod tests { async fn mark_claim_expired(pool: &sqlx::PgPool, task_id: &TaskId) { sqlx::query( "UPDATE verification_tasks - SET claimed_by = 'worker-a', claim_expires_at = $2 + SET claimed_by = 'worker-a', claim_expires_at = clock_timestamp() WHERE task_id = $1::uuid", ) .bind(task_id.to_string()) - .bind(datetime!(2026-05-29 12:05:00 UTC)) .execute(pool) .await .expect("mark claim expired"); @@ -610,16 +969,47 @@ mod tests { async fn mark_claim_active(pool: &sqlx::PgPool, task_id: &TaskId) { sqlx::query( "UPDATE verification_tasks - SET claimed_by = 'worker-a', claim_expires_at = $2 + SET claimed_by = 'worker-a', + claim_expires_at = clock_timestamp() + INTERVAL '5 minutes' WHERE task_id = $1::uuid", ) .bind(task_id.to_string()) - .bind(datetime!(2026-05-29 12:11:00 UTC)) .execute(pool) .await .expect("mark claim active"); } + async fn expire_claim(pool: &sqlx::PgPool, task_id: &TaskId) { + sqlx::query( + "UPDATE verification_tasks + SET claim_expires_at = clock_timestamp() + WHERE task_id = $1::uuid", + ) + .bind(task_id.to_string()) + .execute(pool) + .await + .expect("expire claim at the database clock boundary"); + } + + async fn mark_retry_due(pool: &sqlx::PgPool, task_id: &TaskId) { + sqlx::query( + "UPDATE verification_tasks + SET next_attempt_at = clock_timestamp() + WHERE task_id = $1::uuid", + ) + .bind(task_id.to_string()) + .execute(pool) + .await + .expect("make retry due at the database clock boundary"); + } + + async fn database_time(pool: &sqlx::PgPool) -> time::OffsetDateTime { + sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(pool) + .await + .expect("sample database clock") + } + fn terminal_task(task_id: &str, status: VerificationTaskStatus) -> VerificationTaskRecord { let started_at = datetime!(2026-05-29 12:00:00 UTC); let in_progress = task(task_id, VerificationTaskStatus::Pending, started_at) diff --git a/locks-service/src/infrastructure/postgres/verification_tasks.rs b/locks-service/src/infrastructure/postgres/verification_tasks.rs index f206ccb..cf0a68c 100644 --- a/locks-service/src/infrastructure/postgres/verification_tasks.rs +++ b/locks-service/src/infrastructure/postgres/verification_tasks.rs @@ -9,6 +9,7 @@ use locks_core::verification::SubmittedProofBundle; use crate::application::errors::ApplicationError; use crate::application::models::{VerificationTaskRecord, VerificationTaskStatus}; use crate::application::ports::VerificationTaskRepository; +use crate::infrastructure::postgres::proof_admission::lock_proof_admission; /// Postgres-backed repository for Lock Server private verification task state. #[derive(Debug, Clone)] @@ -18,27 +19,27 @@ pub struct PostgresVerificationTaskRepository { #[derive(Debug, FromRow)] pub(super) struct VerificationTaskRow { - task_id: String, - creator: String, - bundle_id: String, - status: String, - submitted_proof_bundle: serde_json::Value, - submitted_at: time::OffsetDateTime, - started_at: Option, - completed_at: Option, - failure_message: Option, + pub(super) task_id: String, + pub(super) creator: String, + pub(super) bundle_id: String, + pub(super) status: String, + pub(super) submitted_proof_bundle: serde_json::Value, + pub(super) submitted_at: time::OffsetDateTime, + pub(super) started_at: Option, + pub(super) completed_at: Option, + pub(super) failure_message: Option, } -struct VerificationTaskWriteRow { - task_id: String, - creator: String, - bundle_id: String, - status: &'static str, - submitted_proof_bundle: serde_json::Value, - submitted_at: time::OffsetDateTime, - started_at: Option, - completed_at: Option, - failure_message: Option, +pub(super) struct VerificationTaskWriteRow { + pub(super) task_id: String, + pub(super) creator: String, + pub(super) bundle_id: String, + pub(super) status: &'static str, + pub(super) submitted_proof_bundle: serde_json::Value, + pub(super) submitted_at: time::OffsetDateTime, + pub(super) started_at: Option, + pub(super) completed_at: Option, + pub(super) failure_message: Option, } pub(super) const VERIFICATION_TASK_ROW_COLUMNS: &str = " @@ -66,6 +67,40 @@ impl VerificationTaskRepository for PostgresVerificationTaskRepository { task: VerificationTaskRecord, ) -> Result<(), ApplicationError> { let row = VerificationTaskWriteRow::try_from(&task)?; + let lock_id = task.submitted_proof_bundle.pubky_lock_resource.lock_id(); + let mut transaction = self.pool.begin().await.map_err(storage_error)?; + lock_proof_admission(&mut transaction, &task.creator, lock_id).await?; + + let handle_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS ( + SELECT 1 FROM verification_tasks WHERE creator = $1 AND bundle_id = $2 + )", + ) + .bind(&row.creator) + .bind(&row.bundle_id) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if handle_exists { + return Err(ApplicationError::DuplicateRecord { + record: "verification_task", + }); + } + + let deletion_exists = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS ( + SELECT 1 FROM content_lock_deletion_jobs WHERE creator = $1 AND lock_id = $2 + )", + ) + .bind(&row.creator) + .bind(lock_id.to_string()) + .fetch_one(&mut *transaction) + .await + .map_err(storage_error)?; + if deletion_exists { + return Err(ApplicationError::ContentLockDeletionInProgress); + } + let result = sqlx::query( "INSERT INTO verification_tasks ( task_id, @@ -90,7 +125,7 @@ impl VerificationTaskRepository for PostgresVerificationTaskRepository { .bind(row.started_at) .bind(row.completed_at) .bind(row.failure_message) - .execute(&self.pool) + .execute(&mut *transaction) .await .map_err(storage_error)?; @@ -100,7 +135,7 @@ impl VerificationTaskRepository for PostgresVerificationTaskRepository { }); } - Ok(()) + transaction.commit().await.map_err(storage_error) } async fn update_verification_task( @@ -119,7 +154,11 @@ impl VerificationTaskRepository for PostgresVerificationTaskRepository { completed_at = $8, failure_message = $9, updated_at = now() - WHERE task_id = $1::uuid", + WHERE task_id = $1::uuid + AND NOT EXISTS ( + SELECT 1 FROM content_lock_deletion_task_snapshot AS snapshot + WHERE snapshot.verification_task_id = verification_tasks.task_id + )", ) .bind(row.task_id) .bind(row.creator) @@ -150,7 +189,19 @@ impl VerificationTaskRepository for PostgresVerificationTaskRepository { let sql = format!( "SELECT {VERIFICATION_TASK_ROW_COLUMNS} FROM verification_tasks - WHERE task_id = $1::uuid" + WHERE task_id = $1::uuid + AND NOT EXISTS ( + SELECT 1 FROM paykit_task_admissions + WHERE verification_task_id = verification_tasks.task_id + AND ( + ready = FALSE + OR payment_in_hours IS NULL + OR payment_in_hours <= 0 + OR invoice_created_at IS NULL + OR payment_deadline IS NULL + OR invoice_created_at > payment_deadline + ) + )" ); let row = sqlx::query_as::<_, VerificationTaskRow>(&sql) .bind(task_id.to_string()) @@ -169,7 +220,19 @@ impl VerificationTaskRepository for PostgresVerificationTaskRepository { let sql = format!( "SELECT {VERIFICATION_TASK_ROW_COLUMNS} FROM verification_tasks - WHERE creator = $1 AND bundle_id = $2" + WHERE creator = $1 AND bundle_id = $2 + AND NOT EXISTS ( + SELECT 1 FROM paykit_task_admissions + WHERE verification_task_id = verification_tasks.task_id + AND ( + ready = FALSE + OR payment_in_hours IS NULL + OR payment_in_hours <= 0 + OR invoice_created_at IS NULL + OR payment_deadline IS NULL + OR invoice_created_at > payment_deadline + ) + )" ); let row = sqlx::query_as::<_, VerificationTaskRow>(&sql) .bind(creator.to_string()) @@ -450,6 +513,36 @@ mod tests { database.cleanup().await; } + #[tokio::test] + async fn legacy_paykit_admission_without_authoritative_window_is_hidden_from_all_lookups() { + let database = TestDatabase::create().await; + let repo = PostgresVerificationTaskRepository::new(database.pool().clone()); + let pending = task(VerificationTaskStatus::Pending); + let task_id = pending.task_id; + let creator = pending.creator.clone(); + let bundle_id = pending.submitted_proof_bundle.bundle_id.clone(); + repo.insert_verification_task(pending).await.unwrap(); + sqlx::query( + "INSERT INTO paykit_task_admissions + (verification_task_id, ready, ready_at) + VALUES ($1::uuid, TRUE, now())", + ) + .bind(task_id.to_string()) + .execute(database.pool()) + .await + .unwrap(); + + assert_eq!(repo.get_verification_task(&task_id).await.unwrap(), None); + assert_eq!( + repo.get_verification_task_by_handle(&creator, &bundle_id) + .await + .unwrap(), + None + ); + + database.cleanup().await; + } + #[tokio::test] async fn insert_rejects_task_when_record_creator_diverges_from_submitted_bundle() { let database = TestDatabase::create().await; diff --git a/locks-service/src/infrastructure/pubky/content_lock_tombstones.rs b/locks-service/src/infrastructure/pubky/content_lock_tombstones.rs new file mode 100644 index 0000000..5115e9a --- /dev/null +++ b/locks-service/src/infrastructure/pubky/content_lock_tombstones.rs @@ -0,0 +1,111 @@ +use async_trait::async_trait; +use locks_core::content_lock_deletion::ContentLockDeletionTombstone; +use locks_core::ids::{ContentLockPath, CreatorPubky}; +use locks_core::lock_policy::ContentLock; + +use crate::application::errors::ApplicationError; +use crate::application::ports::content_lock_tombstone::{ + canonical_tombstone_bytes, classify_tombstone_bytes, +}; +use crate::application::ports::{ContentLockTombstoneRepository, TombstoneReadback}; +use crate::infrastructure::pubky::storage_client::PubkyHomeserverStorageClient; + +/// Pubky homeserver adapter for exact public content-lock tombstone bytes. +/// +/// Pubky 0.9.3 cannot condition a PUT on the bytes read below. The pre-write comparison protects +/// already-visible replacements and crash/reclaim replay, but an out-of-band replacement can race +/// between GET and PUT and be overwritten. This accepted limitation is documented in the active +/// graceful-deletion plan and public API reference; this adapter must not be described as CAS. +#[derive(Debug)] +pub struct PubkyContentLockTombstoneRepository { + client: C, +} + +impl PubkyContentLockTombstoneRepository { + pub fn new(client: C) -> Self { + Self { client } + } + + pub fn client(&self) -> &C { + &self.client + } +} + +#[async_trait] +impl ContentLockTombstoneRepository for PubkyContentLockTombstoneRepository +where + C: PubkyHomeserverStorageClient, +{ + async fn withdraw_content_lock( + &self, + creator: CreatorPubky, + content_lock_path: ContentLockPath, + frozen_original: &ContentLock, + tombstone: &ContentLockDeletionTombstone, + ) -> Result { + let tombstone_bytes = canonical_tombstone_bytes(tombstone)?; + let original_bytes = + frozen_original + .canonical_json_bytes() + .map_err(|error| ApplicationError::Storage { + message: format!("failed to serialize frozen content lock: {error}"), + })?; + let actual = self + .client + .get_bytes_as_creator(&creator, &content_lock_path.to_string()) + .await?; + match actual.as_ref().map(|resource| resource.bytes.as_slice()) { + Some(actual) if actual == tombstone_bytes => return Ok(TombstoneReadback::Exact), + Some(actual) if actual == original_bytes => {} + None => return Ok(TombstoneReadback::Missing), + Some(_) => return Ok(TombstoneReadback::Replaced), + } + self.client + .put_bytes_as_creator( + &creator, + &content_lock_path.to_string(), + tombstone_bytes, + "application/json", + ) + .await?; + self.read_tombstone(&creator, &content_lock_path, tombstone) + .await + } + + async fn read_tombstone( + &self, + creator: &CreatorPubky, + content_lock_path: &ContentLockPath, + expected: &ContentLockDeletionTombstone, + ) -> Result { + let expected = canonical_tombstone_bytes(expected)?; + let actual = self + .client + .get_bytes_as_creator(creator, &content_lock_path.to_string()) + .await?; + Ok(classify_tombstone_bytes( + actual.as_ref().map(|resource| resource.bytes.as_slice()), + &expected, + )) + } + + async fn force_delete_content_lock_and_verify_absent( + &self, + creator: &CreatorPubky, + content_lock_path: &ContentLockPath, + ) -> Result<(), ApplicationError> { + let path = content_lock_path.to_string(); + self.client.delete_as_creator(creator, &path).await?; + if self + .client + .get_bytes_as_creator(creator, &path) + .await? + .is_some() + { + return Err(ApplicationError::Storage { + message: "forced public content lock deletion did not reach absence".to_owned(), + }); + } + Ok(()) + } +} diff --git a/locks-service/src/infrastructure/pubky/content_locks.rs b/locks-service/src/infrastructure/pubky/content_locks.rs index 89d1a94..ebe0819 100644 --- a/locks-service/src/infrastructure/pubky/content_locks.rs +++ b/locks-service/src/infrastructure/pubky/content_locks.rs @@ -61,6 +61,21 @@ where }) .transpose() } + + async fn delete_content_lock( + &self, + creator: &CreatorPubky, + content_lock_path: &ContentLockPath, + ) -> Result { + let path = content_lock_path.to_string(); + let existed = self + .client + .get_json_value_as_creator(creator, &path) + .await? + .is_some(); + self.client.delete_as_creator(creator, &path).await?; + Ok(existed) + } } #[cfg(test)] @@ -144,6 +159,32 @@ mod tests { assert_eq!(loaded, None); } + #[tokio::test] + async fn delete_content_lock_reads_and_deletes_the_exact_canonical_path() { + let requested_path = content_lock(900).content_lock_path().unwrap(); + let repository = PubkyContentLockRepository::new( + FakeStorageClient::default().with_json_read(Some(json!({}))), + ); + + assert!( + repository + .delete_content_lock(&creator(), &requested_path) + .await + .unwrap() + ); + assert_eq!( + repository.client().operations(), + vec![ + format!( + "get_json pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy {requested_path}" + ), + format!( + "delete pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy {requested_path}" + ) + ] + ); + } + #[tokio::test] async fn storage_errors_are_propagated() { let repository = PubkyContentLockRepository::new(FakeStorageClient::default().with_error( @@ -253,10 +294,15 @@ mod tests { async fn delete_as_creator( &self, - _creator: &CreatorPubky, - _path: &str, + creator: &CreatorPubky, + path: &str, ) -> Result<(), ApplicationError> { - unimplemented!("not needed by content lock repository tests") + self.maybe_error()?; + self.operations + .lock() + .unwrap() + .push(format!("delete {creator} {path}")); + Ok(()) } } diff --git a/locks-service/src/infrastructure/pubky/mod.rs b/locks-service/src/infrastructure/pubky/mod.rs index 49c82c8..289d6d4 100644 --- a/locks-service/src/infrastructure/pubky/mod.rs +++ b/locks-service/src/infrastructure/pubky/mod.rs @@ -1,3 +1,4 @@ +pub mod content_lock_tombstones; pub mod content_locks; pub mod entitlements; pub mod legacy_connect_flow; @@ -6,6 +7,7 @@ pub mod lock_service_pointers; pub mod priv_resources; pub mod storage_client; +pub use content_lock_tombstones::PubkyContentLockTombstoneRepository; pub use content_locks::PubkyContentLockRepository; pub use entitlements::PubkyEntitlementRepository; pub use legacy_connect_flow::{ diff --git a/locks-service/src/infrastructure/pubky/priv_resources.rs b/locks-service/src/infrastructure/pubky/priv_resources.rs index 2e31ec3..803d021 100644 --- a/locks-service/src/infrastructure/pubky/priv_resources.rs +++ b/locks-service/src/infrastructure/pubky/priv_resources.rs @@ -138,7 +138,10 @@ fn record_from_resource( #[cfg(test)] mod tests { use std::str::FromStr; - use std::sync::Mutex; + use std::sync::{ + Mutex, + atomic::{AtomicBool, Ordering}, + }; use async_trait::async_trait; use locks_core::ids::{CreatorPubky, GuardedResourceHash}; @@ -334,6 +337,7 @@ mod tests { bytes_read: Mutex>, last_bytes: Mutex>>, operations: Mutex>, + fail_get: AtomicBool, } impl FakeStorageClient { @@ -394,6 +398,11 @@ mod tests { creator: &CreatorPubky, path: &str, ) -> Result, ApplicationError> { + if self.fail_get.load(Ordering::SeqCst) { + return Err(ApplicationError::InvalidGuardedResource { + message: "storage read failed".to_owned(), + }); + } self.operations .lock() .unwrap() @@ -410,6 +419,7 @@ mod tests { .lock() .unwrap() .push(format!("delete {creator} {path}")); + *self.bytes_read.lock().unwrap() = None; Ok(()) } } diff --git a/locks-service/src/infrastructure/runtime_master_key.rs b/locks-service/src/infrastructure/runtime_master_key.rs new file mode 100644 index 0000000..1f51158 --- /dev/null +++ b/locks-service/src/infrastructure/runtime_master_key.rs @@ -0,0 +1,85 @@ +use std::fmt; + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; + +const CREATOR_AUTHORITY_KEY_CONTEXT: &str = + "pubky-locks v1 runtime master key: creator authority secrets"; +const FINAL_CREDENTIAL_KEY_CONTEXT: &str = + "pubky-locks v1 runtime master key: final deletion credentials"; + +/// Root key for deriving independent runtime encryption keys. +#[derive(Clone)] +pub struct RuntimeMasterKey { + bytes: [u8; 32], +} + +impl fmt::Debug for RuntimeMasterKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("RuntimeMasterKey") + .field(&"") + .finish() + } +} + +#[derive(Debug, thiserror::Error)] +#[error("runtime master key must be an unpadded base64url-encoded 32-byte key")] +pub struct InvalidRuntimeMasterKey; + +impl RuntimeMasterKey { + pub fn from_base64url(value: &str) -> Result { + let bytes = URL_SAFE_NO_PAD + .decode(value) + .map_err(|_| InvalidRuntimeMasterKey)?; + let bytes = bytes.try_into().map_err(|_| InvalidRuntimeMasterKey)?; + Ok(Self { bytes }) + } + + pub fn creator_authority_key(&self) -> [u8; 32] { + blake3::derive_key(CREATOR_AUTHORITY_KEY_CONTEXT, &self.bytes) + } + + pub fn final_credential_key(&self) -> [u8; 32] { + blake3::derive_key(FINAL_CREDENTIAL_KEY_CONTEXT, &self.bytes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn derives_stable_distinct_keys_for_closed_runtime_domains() { + let encoded = URL_SAFE_NO_PAD.encode([7u8; 32]); + let first = RuntimeMasterKey::from_base64url(&encoded).unwrap(); + let second = RuntimeMasterKey::from_base64url(&encoded).unwrap(); + + assert_eq!( + first.creator_authority_key(), + second.creator_authority_key() + ); + assert_eq!(first.final_credential_key(), second.final_credential_key()); + assert_ne!(first.creator_authority_key(), first.final_credential_key()); + assert_ne!(first.creator_authority_key(), [7u8; 32]); + assert_ne!(first.final_credential_key(), [7u8; 32]); + } + + #[test] + fn rejects_invalid_or_wrong_length_values_without_exposing_input() { + for value in ["not-a-key***", &URL_SAFE_NO_PAD.encode([7u8; 31])] { + let error = RuntimeMasterKey::from_base64url(value).unwrap_err(); + let debug = format!("{error:?}"); + assert!(!debug.contains(value)); + } + } + + #[test] + fn debug_output_redacts_root_key() { + let encoded = URL_SAFE_NO_PAD.encode([9u8; 32]); + let key = RuntimeMasterKey::from_base64url(&encoded).unwrap(); + let debug = format!("{key:?}"); + + assert_eq!(debug, "RuntimeMasterKey(\"\")"); + assert!(!debug.contains(&encoded)); + } +} diff --git a/locks-service/src/infrastructure/verifiers/paykit_payment.rs b/locks-service/src/infrastructure/verifiers/paykit_payment.rs index 7cde36e..64ce80a 100644 --- a/locks-service/src/infrastructure/verifiers/paykit_payment.rs +++ b/locks-service/src/infrastructure/verifiers/paykit_payment.rs @@ -76,7 +76,7 @@ where .client .transaction_status(&request.creator, &request.bundle_id) .await - .map_err(|_| ApplicationError::VerificationPending)?; + .map_err(|_| ApplicationError::VerificationDependencyUnavailable)?; if !payment_status_satisfies(status, self.minimum_confirmations) { return Err(ApplicationError::VerificationPending); } @@ -230,12 +230,12 @@ mod tests { } #[tokio::test] - async fn status_client_errors_leave_task_pending() { + async fn status_client_errors_remain_distinct_from_healthy_pending() { let verifier = PaykitPaymentVerifier::new(FakeStatusClient::error(), 0); assert_eq!( verifier.verify(request()).await, - Err(ApplicationError::VerificationPending) + Err(ApplicationError::VerificationDependencyUnavailable) ); } @@ -257,7 +257,8 @@ mod tests { params: json!({ "recipient_pubky": "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy", "amount": "50000", - "asset": "BTC" + "asset": "BTC", + "payment_in": 24 }), }, proof: Proof { diff --git a/locks-service/tests/content_lock_deletion_executor.rs b/locks-service/tests/content_lock_deletion_executor.rs new file mode 100644 index 0000000..9ed8947 --- /dev/null +++ b/locks-service/tests/content_lock_deletion_executor.rs @@ -0,0 +1,17 @@ +use locks_service::application::use_cases::execute_content_lock_deletion_phase::{ + ContentLockDeletionPhaseExecutor, DeletionPhaseExecutionOutcome, +}; + +#[test] +fn deletion_phase_executor_exposes_closed_outcomes() { + let _ = std::mem::size_of::>(); + let outcomes = [ + DeletionPhaseExecutionOutcome::Progressed, + DeletionPhaseExecutionOutcome::Deferred, + DeletionPhaseExecutionOutcome::ClaimLost, + DeletionPhaseExecutionOutcome::TerminalFailed, + DeletionPhaseExecutionOutcome::TransientDependencyFailure, + DeletionPhaseExecutionOutcome::FatalFailure, + ]; + assert_eq!(outcomes.len(), 6); +} diff --git a/locks-service/tests/content_lock_deletions.rs b/locks-service/tests/content_lock_deletions.rs new file mode 100644 index 0000000..b66e7ee --- /dev/null +++ b/locks-service/tests/content_lock_deletions.rs @@ -0,0 +1,2186 @@ +use std::{collections::BTreeMap, str::FromStr, sync::Arc}; + +use locks_core::{ + ids::{BundleId, CreatorPubky, GuardedResourceHash, PubkyLockResource, TaskId}, + lock_policy::{ + AccessPolicy, CONTENT_LOCK_VERSION, ContentLock, GuardedResource, LockLogic, + LockServerConfig, VerifierType, + }, + verification::{Proof, SUBMITTED_PROOF_BUNDLE_VERSION, SubmittedProofBundle}, +}; +use locks_service::{ + application::{ + models::{ + AccessCredential, AccessCredentialLookupKey, AccessCredentialRecord, + AdvanceContentLockDeletionPhaseResult, ContentLockDeletionFailureCode, + ContentLockDeletionJob, ContentLockDeletionPhase, ContentLockDeletionState, + FinalAccessWindows, InitializeFinalAccessWindowsResult, PrepareForceDeletionResult, + VerificationTaskRecord, VerificationTaskStatus, + }, + ports::{ + AccessCredentialStore, Clock, ContentLockDeletionActionAcquireResult, + ContentLockDeletionActionClaim, ContentLockDeletionActionOwnership, + ContentLockDeletionRepository, VerificationTaskClaimer, VerificationTaskRepository, + }, + use_cases::no_paykit_deletion_drain::NoPaykitDeletionDrainUseCase, + }, + infrastructure::memory::{ + access_credentials::InMemoryAccessCredentialStore, + content_lock_deletion_action_ownership::InMemoryContentLockDeletionActionOwnership, + content_lock_deletions::InMemoryContentLockDeletionRepository, + verification_task_claims::InMemoryVerificationTaskClaimer, + verification_task_deletion_fence::InMemoryVerificationTaskDeletionFence, + verification_tasks::InMemoryVerificationTaskRepository, + }, +}; +use serde_json::json; +use time::macros::datetime; +use uuid::Uuid; + +struct PausingVerificationTaskRepository { + inner: Arc, + pause_next_get: std::sync::atomic::AtomicBool, + get_entered: tokio::sync::Notify, + release_get: tokio::sync::Notify, +} + +impl PausingVerificationTaskRepository { + fn new(inner: Arc) -> Self { + Self { + inner, + pause_next_get: std::sync::atomic::AtomicBool::new(true), + get_entered: tokio::sync::Notify::new(), + release_get: tokio::sync::Notify::new(), + } + } +} + +#[async_trait::async_trait] +impl VerificationTaskRepository for PausingVerificationTaskRepository { + async fn insert_verification_task( + &self, + task: VerificationTaskRecord, + ) -> Result<(), locks_service::application::errors::ApplicationError> { + self.inner.insert_verification_task(task).await + } + + async fn update_verification_task( + &self, + task: VerificationTaskRecord, + ) -> Result<(), locks_service::application::errors::ApplicationError> { + self.inner.update_verification_task(task).await + } + + async fn get_verification_task( + &self, + task_id: &TaskId, + ) -> Result, locks_service::application::errors::ApplicationError> + { + if self + .pause_next_get + .swap(false, std::sync::atomic::Ordering::SeqCst) + { + self.get_entered.notify_one(); + self.release_get.notified().await; + } + self.inner.get_verification_task(task_id).await + } + + async fn delete_verification_task( + &self, + task_id: &TaskId, + ) -> Result<(), locks_service::application::errors::ApplicationError> { + self.inner.delete_verification_task(task_id).await + } +} + +const CREATOR: &str = "pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy"; +const NOW: time::OffsetDateTime = datetime!(2026-08-12 05:00:00 UTC); +const LEASE_END: time::OffsetDateTime = datetime!(2026-08-12 05:05:00 UTC); + +#[derive(Debug)] +struct FixedClock(time::OffsetDateTime); + +impl Clock for FixedClock { + fn now(&self) -> time::OffsetDateTime { + self.0 + } +} + +#[derive(Debug)] +struct MutableClock(std::sync::Mutex); + +impl MutableClock { + fn new(now: time::OffsetDateTime) -> Self { + Self(std::sync::Mutex::new(now)) + } + + fn set(&self, now: time::OffsetDateTime) { + *self.0.lock().unwrap() = now; + } +} + +impl Clock for MutableClock { + fn now(&self) -> time::OffsetDateTime { + *self.0.lock().unwrap() + } +} + +#[tokio::test] +async fn in_memory_action_ownership_is_exclusive_and_reacquirable() { + let repository = Arc::new( + InMemoryContentLockDeletionRepository::with_verification_task_fence(Arc::new( + InMemoryVerificationTaskDeletionFence::with_clock(Arc::new(FixedClock(NOW))), + )), + ); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job).await.unwrap(); + let claimed = repository + .claim_next("worker", time::Duration::minutes(5)) + .await + .unwrap() + .unwrap(); + let ownership = InMemoryContentLockDeletionActionOwnership::new(repository); + let request = || ContentLockDeletionActionClaim { + job_id: claimed.job.job_id, + worker_id: "worker", + claim_token: claimed.claim_token, + expected_phase: claimed.job.phase, + force: false, + }; + + let ContentLockDeletionActionAcquireResult::Acquired(first) = + ownership.try_acquire(request()).await.unwrap() + else { + panic!("live claim must acquire") + }; + assert!(matches!( + ownership.try_acquire(request()).await.unwrap(), + ContentLockDeletionActionAcquireResult::Busy + )); + + first.release().await.unwrap(); + let ContentLockDeletionActionAcquireResult::Acquired(reacquired) = + ownership.try_acquire(request()).await.unwrap() + else { + panic!("released claim must reacquire") + }; + drop(reacquired); + assert!(matches!( + ownership.try_acquire(request()).await.unwrap(), + ContentLockDeletionActionAcquireResult::Acquired(_) + )); +} + +#[tokio::test] +async fn in_memory_action_ownership_rejects_expired_claim_at_half_open_boundary() { + let clock = Arc::new(MutableClock::new(NOW)); + let repository = Arc::new( + InMemoryContentLockDeletionRepository::with_verification_task_fence(Arc::new( + InMemoryVerificationTaskDeletionFence::with_clock(clock.clone()), + )), + ); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job).await.unwrap(); + let claimed = repository + .claim_next("worker", time::Duration::minutes(5)) + .await + .unwrap() + .unwrap(); + clock.set(NOW + time::Duration::minutes(5)); + let ownership = InMemoryContentLockDeletionActionOwnership::new(repository); + let result = ownership + .try_acquire(ContentLockDeletionActionClaim { + job_id: claimed.job.job_id, + worker_id: "worker", + claim_token: claimed.claim_token, + expected_phase: claimed.job.phase, + force: false, + }) + .await + .unwrap(); + assert!(matches!( + result, + ContentLockDeletionActionAcquireResult::ClaimLost + )); +} + +#[tokio::test] +async fn frozen_manifest_identity_is_immutable_and_creator_lock_unique() { + let repository = InMemoryContentLockDeletionRepository::with_verification_task_fence(Arc::new( + InMemoryVerificationTaskDeletionFence::with_clock(Arc::new(FixedClock(NOW))), + )); + let lock = content_lock(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock.clone(), NOW).unwrap(); + + repository.insert_job(job.clone()).await.unwrap(); + assert_eq!( + repository + .get_job(&job.creator, &job.lock_id) + .await + .unwrap(), + Some(job.clone()) + ); + + let duplicate = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + assert!(repository.insert_job(duplicate).await.is_err()); + + let mut duplicate_id = + ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + duplicate_id.job_id = job.job_id; + assert!(repository.insert_job(duplicate_id).await.is_err()); + + assert!(job.validate_frozen_identity().is_ok()); + let mut corrupted = job; + corrupted + .frozen_content_lock + .access_policy + .requested_credential_ttl_seconds += 1; + assert!(corrupted.validate_frozen_identity().is_err()); + + let mut malformed = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + malformed.state = ContentLockDeletionState::Running; + assert!(repository.insert_job(malformed).await.is_err()); +} + +#[tokio::test] +async fn due_claims_reclaim_with_fresh_tokens_and_fence_stale_writes() { + let clock = Arc::new(MutableClock::new(NOW)); + let repository = InMemoryContentLockDeletionRepository::with_verification_task_fence(Arc::new( + InMemoryVerificationTaskDeletionFence::with_clock(clock.clone()), + )); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + + let first = repository + .claim_next("worker-a", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + assert_eq!(first.job.state, ContentLockDeletionState::Running); + assert_eq!(first.job.attempt_count, 1); + assert!( + repository + .claim_next("worker-b", (LEASE_END) - (NOW)) + .await + .unwrap() + .is_none() + ); + + clock.set(LEASE_END); + let reclaimed = repository + .claim_next( + "worker-b", + (datetime!(2026-08-12 05:10:00 UTC)) - (datetime!(2026-08-12 05:05:01 UTC)), + ) + .await + .unwrap() + .unwrap(); + assert_ne!(first.claim_token, reclaimed.claim_token); + assert_eq!(reclaimed.job.attempt_count, 2); + + assert_eq!( + repository + .advance_phase( + job.job_id, + "worker-a", + first.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap(), + AdvanceContentLockDeletionPhaseResult::ClaimLost + ); + clock.set(datetime!(2026-08-12 05:06:00 UTC)); + let advanced = repository + .advance_phase( + job.job_id, + "worker-b", + reclaimed.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + assert_eq!(advanced.state, ContentLockDeletionState::Queued); + assert_eq!(advanced.phase, ContentLockDeletionPhase::StartPaymentDrain); + assert_eq!(advanced.attempt_count, 0); + + clock.set(datetime!(2026-08-12 05:06:01 UTC)); + let next_claim = repository + .claim_next( + "worker-c", + (datetime!(2026-08-12 05:11:00 UTC)) - (datetime!(2026-08-12 05:06:01 UTC)), + ) + .await + .unwrap() + .unwrap(); + assert!( + repository + .advance_phase( + job.job_id, + "worker-c", + next_claim.claim_token, + ContentLockDeletionPhase::DeleteContent, + ) + .await + .is_err() + ); + clock.set(datetime!(2026-08-12 05:07:00 UTC)); + let failed = repository + .finish( + job.job_id, + "worker-c", + next_claim.claim_token, + Some(ContentLockDeletionFailureCode::TombstoneMissing), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(failed.state, ContentLockDeletionState::Failed); + assert_eq!( + failed.failure_code, + Some(ContentLockDeletionFailureCode::TombstoneMissing) + ); +} + +#[tokio::test] +async fn healthy_defer_does_not_accumulate_transient_attempts_across_polls() { + let clock = Arc::new(MutableClock::new(NOW)); + let repository = InMemoryContentLockDeletionRepository::with_verification_task_fence(Arc::new( + InMemoryVerificationTaskDeletionFence::with_clock(clock.clone()), + )); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + + let mut due_at = NOW; + for poll in 1..=3 { + let claimed = repository + .claim_next("worker-a", (due_at + time::Duration::minutes(5)) - (due_at)) + .await + .unwrap() + .unwrap(); + assert_eq!(claimed.job.attempt_count, 1, "healthy poll {poll}"); + + due_at += time::Duration::minutes(1); + let deferred = repository + .defer( + job.job_id, + "worker-a", + claimed.claim_token, + time::Duration::minutes(1), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(deferred.state, ContentLockDeletionState::Queued); + assert_eq!(deferred.attempt_count, 0); + assert_eq!(deferred.next_attempt_at, Some(due_at)); + clock.set(due_at); + } +} + +#[tokio::test] +async fn defer_is_fenced_by_the_live_claim_token() { + let repository = InMemoryContentLockDeletionRepository::with_verification_task_fence(Arc::new( + InMemoryVerificationTaskDeletionFence::with_clock(Arc::new(FixedClock(NOW))), + )); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + let claimed = repository + .claim_next("worker-a", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + let next_poll = NOW + time::Duration::minutes(1); + + assert!( + repository + .defer(job.job_id, "worker-a", Uuid::new_v4(), (next_poll) - (NOW)) + .await + .unwrap() + .is_none() + ); + let still_running = repository + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .unwrap(); + assert_eq!(still_running.state, ContentLockDeletionState::Running); + assert_eq!(still_running.attempt_count, 1); + + let deferred = repository + .defer( + job.job_id, + "worker-a", + claimed.claim_token, + (next_poll) - (NOW), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(deferred.attempt_count, 0); +} + +#[test] +fn failure_codes_are_a_closed_stable_vocabulary() { + for (code, wire) in [ + ( + ContentLockDeletionFailureCode::TombstoneMissing, + "tombstone_missing", + ), + ( + ContentLockDeletionFailureCode::TombstoneReplaced, + "tombstone_replaced", + ), + ( + ContentLockDeletionFailureCode::ResourceReplaced, + "resource_replaced", + ), + ( + ContentLockDeletionFailureCode::RetryExhausted, + "retry_exhausted", + ), + ( + ContentLockDeletionFailureCode::StateCorrupt, + "state_corrupt", + ), + ] { + assert_eq!(code.as_str(), wire); + assert_eq!( + wire.parse::().unwrap(), + code + ); + } + assert!( + "backend: secret" + .parse::() + .is_err() + ); + assert!( + "final_credential_issuance_missed" + .parse::() + .is_err() + ); +} + +#[tokio::test] +async fn retry_due_time_and_force_receipts_are_durable_repository_facts() { + let clock = Arc::new(MutableClock::new(NOW)); + let repository = InMemoryContentLockDeletionRepository::with_verification_task_fence(Arc::new( + InMemoryVerificationTaskDeletionFence::with_clock(clock.clone()), + )); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + let claimed = repository + .claim_next("worker-a", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + let retry_at = datetime!(2026-08-12 05:06:00 UTC); + repository + .schedule_retry( + job.job_id, + "worker-a", + claimed.claim_token, + (retry_at) - (NOW), + ) + .await + .unwrap() + .unwrap(); + assert!( + repository + .claim_next("worker-b", (LEASE_END) - (NOW)) + .await + .unwrap() + .is_none() + ); + clock.set(retry_at); + assert!( + repository + .claim_next( + "worker-b", + (datetime!(2026-08-12 05:11:00 UTC)) - (retry_at) + ) + .await + .unwrap() + .is_some() + ); + + assert!(matches!( + repository + .prepare_force_deletion(&job.creator, &job.lock_id) + .await + .unwrap(), + PrepareForceDeletionResult::Active(_) + )); + assert!( + !repository + .has_force_receipt(&job.creator, &job.lock_id) + .await + .unwrap() + ); +} + +#[tokio::test] +async fn in_memory_active_force_completion_is_exactly_fenced_and_permanently_blocks_publication() { + let clock = Arc::new(MutableClock::new(NOW)); + let repository = InMemoryContentLockDeletionRepository::with_verification_task_fence(Arc::new( + InMemoryVerificationTaskDeletionFence::with_clock(clock.clone()), + )); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + let revoked = repository + .claim_next("worker-old", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + let forced_at = NOW + time::Duration::seconds(1); + clock.set(forced_at); + assert!(matches!( + repository + .prepare_force_deletion(&job.creator, &job.lock_id) + .await + .unwrap(), + PrepareForceDeletionResult::Active(_) + )); + assert!( + !repository + .complete_force_deletion(job.job_id, "worker-old", revoked.claim_token) + .await + .unwrap() + ); + + let expiring = repository + .claim_next( + "worker-expiring", + (forced_at + time::Duration::minutes(1)) - (forced_at), + ) + .await + .unwrap() + .unwrap(); + assert!( + !repository + .complete_force_deletion(job.job_id, "worker-expiring", Uuid::new_v4()) + .await + .unwrap() + ); + let reclaim_at = forced_at + time::Duration::minutes(2); + clock.set(reclaim_at); + assert!( + !repository + .complete_force_deletion(job.job_id, "worker-expiring", expiring.claim_token,) + .await + .unwrap() + ); + let live = repository + .claim_next( + "worker-live", + (reclaim_at + time::Duration::minutes(5)) - (reclaim_at), + ) + .await + .unwrap() + .unwrap(); + assert!( + !repository + .complete_force_deletion(job.job_id, "worker-expiring", expiring.claim_token,) + .await + .unwrap() + ); + assert!( + repository + .complete_force_deletion(job.job_id, "worker-live", live.claim_token) + .await + .unwrap() + ); + + assert!( + repository + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .is_none() + ); + assert!( + repository + .has_force_receipt(&job.creator, &job.lock_id) + .await + .unwrap() + ); + assert!( + !repository + .complete_force_deletion(job.job_id, "worker-live", live.claim_token) + .await + .unwrap() + ); + assert_eq!( + repository + .begin_publication(&job.creator, &job.lock_id, Uuid::new_v4()) + .await, + Err(locks_service::application::errors::ApplicationError::ContentLockDeletionInProgress) + ); +} + +#[tokio::test] +async fn in_memory_unforced_claim_cannot_create_a_force_receipt() { + let repository = InMemoryContentLockDeletionRepository::with_verification_task_fence(Arc::new( + InMemoryVerificationTaskDeletionFence::with_clock(Arc::new(FixedClock(NOW))), + )); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + repository.insert_job(job.clone()).await.unwrap(); + let claim = repository + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + + assert!( + !repository + .complete_force_deletion(job.job_id, "worker", claim.claim_token) + .await + .unwrap() + ); + assert!( + repository + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .is_some() + ); + assert!( + !repository + .has_force_receipt(&job.creator, &job.lock_id) + .await + .unwrap() + ); +} + +#[tokio::test] +async fn in_memory_deletion_enrolls_existing_ordinary_credentials_and_blocks_late_insertion() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let lock = content_lock(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), lock.clone(), NOW).unwrap(); + let task = verification_task(&job, VerificationTaskStatus::Completed); + verification_tasks + .insert_verification_task(task.clone()) + .await + .unwrap(); + + let ordinary = AccessCredential::new("ordinary-before-deletion"); + let ordinary_lookup = AccessCredentialLookupKey::derive(&ordinary); + let original_expiry = NOW + time::Duration::minutes(10); + access + .insert_access_credential( + &job.lock_id, + ordinary_lookup.clone(), + AccessCredentialRecord { + creator: job.creator.clone(), + bundle_id: task.submitted_proof_bundle.bundle_id.clone(), + expires_at: original_expiry, + }, + ) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + + let first = access + .prepare_deletion_read( + &ordinary_lookup, + "/priv/locks.app/content/post.json", + time::Duration::seconds(30), + ) + .await + .unwrap() + .unwrap(); + let replay = access + .prepare_deletion_read( + &ordinary_lookup, + "/priv/locks.app/content/post.json", + time::Duration::minutes(1), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(first.claim_token, None); + assert_eq!(replay, first); + assert_eq!( + access + .get_access_credential(&ordinary_lookup) + .await + .unwrap() + .unwrap() + .expires_at, + original_expiry + ); + assert!( + access + .prepare_deletion_read( + &ordinary_lookup, + "/priv/locks.app/content/not-in-frozen-manifest.json", + time::Duration::seconds(30), + ) + .await + .unwrap() + .is_none() + ); + + assert_eq!( + access + .insert_access_credential( + &job.lock_id, + AccessCredentialLookupKey::derive(&AccessCredential::new("late")), + AccessCredentialRecord { + creator: job.creator.clone(), + bundle_id: BundleId::from_str("000G40R40M30E209185GR38E1V").unwrap(), + expires_at: original_expiry, + }, + ) + .await, + Err(locks_service::application::errors::ApplicationError::ContentLockDeletionInProgress) + ); + + let drain_existing_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainExistingCredentials, + ) + .await; + assert!(matches!( + deletions + .advance_phase( + job.job_id, + "worker-final", + drain_existing_claim.claim_token, + ContentLockDeletionPhase::IssueFinalCredentials, + ) + .await, + Ok(AdvanceContentLockDeletionPhaseResult::ObligationsPending) + )); + + assert!(matches!( + deletions + .prepare_force_deletion(&job.creator, &job.lock_id) + .await + .unwrap(), + PrepareForceDeletionResult::Active(_) + )); + assert_eq!( + access + .get_access_credential(&ordinary_lookup) + .await + .unwrap() + .unwrap() + .expires_at, + original_expiry + ); +} + +#[tokio::test] +async fn in_memory_cutoff_is_captured_under_the_shared_fence_not_from_the_caller() { + let authoritative_cutoff = NOW + time::Duration::minutes(2); + let fence = Arc::new(InMemoryVerificationTaskDeletionFence::with_clock(Arc::new( + FixedClock(authoritative_cutoff), + ))); + let verification_tasks = Arc::new(InMemoryVerificationTaskRepository::with_deletion_fence( + Arc::clone(&fence), + )); + let access = Arc::new( + InMemoryAccessCredentialStore::with_verification_task_repository_and_deletion_fence( + verification_tasks.clone(), + Arc::clone(&fence), + ), + ); + let deletions = + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + Arc::clone(&access), + fence, + ); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let task = verification_task(&job, VerificationTaskStatus::Pending); + let credential = AccessCredential::new("expired-while-waiting-for-cutoff-fence"); + let lookup = AccessCredentialLookupKey::derive(&credential); + verification_tasks + .insert_verification_task(task.clone()) + .await + .unwrap(); + access + .insert_access_credential( + &job.lock_id, + lookup.clone(), + AccessCredentialRecord { + creator: job.creator.clone(), + bundle_id: task.submitted_proof_bundle.bundle_id, + expires_at: NOW + time::Duration::minutes(1), + }, + ) + .await + .unwrap(); + + deletions.insert_job(job.clone()).await.unwrap(); + + assert_eq!( + deletions + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .unwrap() + .deletion_started_at, + authoritative_cutoff + ); + assert!(!access.deletion_credential_enrolled(&lookup).await.unwrap()); +} + +#[tokio::test] +async fn failed_access_registration_leaves_no_job_or_task_ownership() { + let lock = content_lock(); + let first_job = ContentLockDeletionJob::new(Uuid::new_v4(), lock.clone(), NOW).unwrap(); + let first_fence = Arc::new(InMemoryVerificationTaskDeletionFence::with_clock(Arc::new( + FixedClock(NOW), + ))); + let first_tasks = Arc::new(InMemoryVerificationTaskRepository::with_deletion_fence( + Arc::clone(&first_fence), + )); + let access = Arc::new( + InMemoryAccessCredentialStore::with_verification_task_repository_and_deletion_fence( + first_tasks, + Arc::clone(&first_fence), + ), + ); + let first_deletions = + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + Arc::clone(&access), + first_fence, + ); + tokio::time::timeout( + std::time::Duration::from_secs(1), + first_deletions.insert_job(first_job), + ) + .await + .expect("first deletion admission must not deadlock") + .unwrap(); + + let second_fence = Arc::new(InMemoryVerificationTaskDeletionFence::with_clock(Arc::new( + FixedClock(NOW), + ))); + let second_tasks = Arc::new(InMemoryVerificationTaskRepository::with_deletion_fence( + Arc::clone(&second_fence), + )); + let second_job = ContentLockDeletionJob::new(Uuid::new_v4(), lock, NOW).unwrap(); + let task = verification_task(&second_job, VerificationTaskStatus::Pending); + second_tasks + .insert_verification_task(task.clone()) + .await + .unwrap(); + let claimer = + InMemoryVerificationTaskClaimer::with_deletion_fence(vec![task], Arc::clone(&second_fence)); + let second_deletions = + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + access, + second_fence, + ); + + assert_eq!( + tokio::time::timeout( + std::time::Duration::from_secs(1), + second_deletions.insert_job(second_job.clone()), + ) + .await + .expect("failed deletion admission must not deadlock"), + Err(locks_service::application::errors::ApplicationError::ContentLockDeletionInProgress) + ); + assert!( + second_deletions + .get_job(&second_job.creator, &second_job.lock_id) + .await + .unwrap() + .is_none() + ); + assert!( + tokio::time::timeout( + std::time::Duration::from_secs(1), + claimer.claim_next_verification_task("worker", (LEASE_END) - (NOW)), + ) + .await + .expect("failed deletion admission must release task ownership locks") + .unwrap() + .is_some() + ); +} + +#[tokio::test] +async fn in_memory_final_credential_is_exactly_replayable_and_reads_are_lease_fenced() { + let clock = Arc::new(MutableClock::new(NOW)); + let (verification_tasks, access, deletions) = in_memory_access_stack_with_clock(clock.clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let completed = verification_task(&job, VerificationTaskStatus::Pending) + .transition_to(VerificationTaskStatus::InProgress, NOW, None) + .unwrap() + .transition_to(VerificationTaskStatus::Completed, NOW, None) + .unwrap(); + let bundle_id = completed.submitted_proof_bundle.bundle_id.clone(); + verification_tasks + .insert_verification_task(completed) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let claimed = advance_to_final_issuance(&deletions, &access, job.job_id).await; + let issuance_deadline = NOW + time::Duration::minutes(15); + let read_deadline = NOW + time::Duration::minutes(30); + let initialized = access + .initialize_final_access_windows( + job.job_id, + "worker-final", + claimed.claim_token, + time::Duration::minutes(15), + time::Duration::minutes(15), + ) + .await + .unwrap(); + assert_eq!( + initialized, + InitializeFinalAccessWindowsResult::Initialized(FinalAccessWindows { + issuance_started_at: NOW, + credential_issuance_deadline: issuance_deadline, + read_deadline, + }) + ); + assert_eq!( + access + .initialize_final_access_windows( + job.job_id, + "worker-final", + claimed.claim_token, + time::Duration::minutes(20), + time::Duration::minutes(20), + ) + .await + .unwrap(), + initialized + ); + + let candidate = AccessCredential::new("final-secret-bearer"); + let first = access + .issue_or_replay_final_credential(&job.creator, &bundle_id, NOW, candidate.clone()) + .await + .unwrap() + .unwrap(); + let replay = access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + NOW + time::Duration::minutes(1), + AccessCredential::new("different-candidate-must-not-win"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(first, replay); + assert_eq!(first.credential, candidate); + assert_eq!(first.expires_at, read_deadline); + assert!(!format!("{access:?}").contains("final-secret-bearer")); + let boundary_replay = access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + issuance_deadline, + AccessCredential::new("boundary-candidate-must-not-win"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(boundary_replay, first); + + let advanced = deletions + .advance_phase( + job.job_id, + "worker-final", + claimed.claim_token, + ContentLockDeletionPhase::DrainFinalReads, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + assert_eq!(advanced.phase, ContentLockDeletionPhase::DrainFinalReads); + let phase_replay = access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + issuance_deadline, + AccessCredential::new("post-advance-candidate-must-not-win"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(phase_replay, first); + + let lookup = AccessCredentialLookupKey::derive(&first.credential); + let path = "/priv/locks.app/content/post.json"; + let first_claim = access + .prepare_deletion_read(&lookup, path, time::Duration::minutes(1)) + .await + .unwrap() + .unwrap() + .claim_token + .unwrap(); + clock.set(NOW + time::Duration::seconds(30)); + let equality_reclaim = access + .prepare_deletion_read(&lookup, path, time::Duration::seconds(90)) + .await + .unwrap() + .unwrap() + .claim_token + .unwrap(); + assert_ne!(equality_reclaim, first_claim); + assert!( + !access + .release_deletion_read(&lookup, path, Uuid::new_v4(), NOW) + .await + .unwrap() + ); + assert!( + !access + .release_deletion_read( + &lookup, + path, + first_claim, + NOW + time::Duration::seconds(30), + ) + .await + .unwrap() + ); + assert!( + access + .release_deletion_read( + &lookup, + path, + equality_reclaim, + NOW + time::Duration::seconds(30), + ) + .await + .unwrap() + ); + let stale_claim = access + .prepare_deletion_read( + &lookup, + path, + time::Duration::minutes(59) + time::Duration::seconds(30), + ) + .await + .unwrap() + .unwrap() + .claim_token + .unwrap(); + let reclaim_time = NOW + time::Duration::seconds(60); + clock.set(reclaim_time); + assert!( + !access + .consume_deletion_read(&lookup, path, stale_claim) + .await + .unwrap() + ); + let recovered_claim = access + .prepare_deletion_read( + &lookup, + path, + (NOW + time::Duration::hours(1)) - reclaim_time, + ) + .await + .unwrap() + .unwrap() + .claim_token + .unwrap(); + assert!( + !access + .consume_deletion_read(&lookup, path, stale_claim) + .await + .unwrap() + ); + assert!( + access + .consume_deletion_read(&lookup, path, recovered_claim) + .await + .unwrap() + ); + clock.set(NOW + time::Duration::minutes(3)); + assert!( + access + .prepare_deletion_read(&lookup, path, time::Duration::minutes(1)) + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn mutable_task_completion_cannot_resolve_the_immutable_deletion_snapshot() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let pending = verification_task(&job, VerificationTaskStatus::Pending); + let bundle_id = pending.submitted_proof_bundle.bundle_id.clone(); + let task_id = pending.task_id; + verification_tasks + .insert_verification_task(pending.clone()) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + + let drain_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainPayments, + ) + .await; + let completed = pending + .transition_to(VerificationTaskStatus::InProgress, NOW, None) + .unwrap() + .transition_to(VerificationTaskStatus::Completed, NOW, None) + .unwrap(); + verification_tasks + .update_verification_task(completed) + .await + .unwrap(); + + assert!( + !access + .final_credential_available(&job.creator, &bundle_id, NOW) + .await + .unwrap() + ); + assert!( + access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + NOW, + AccessCredential::new("mutable-completion-must-not-win"), + ) + .await + .unwrap() + .is_none() + ); + assert!( + access + .resolve_deletion_payment( + job.job_id, + "worker-final", + drain_claim.claim_token, + NOW, + &task_id, + VerificationTaskStatus::Completed, + ) + .await + .unwrap() + ); + assert!( + access + .complete_deletion_payment_aggregate( + job.job_id, + "worker-final", + drain_claim.claim_token, + NOW, + ) + .await + .unwrap() + ); + + deletions + .advance_phase( + job.job_id, + "worker-final", + drain_claim.claim_token, + ContentLockDeletionPhase::DrainExistingCredentials, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let existing_drain_claim = deletions + .claim_next("worker-final", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + deletions + .advance_phase( + job.job_id, + "worker-final", + existing_drain_claim.claim_token, + ContentLockDeletionPhase::IssueFinalCredentials, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let final_claim = deletions + .claim_next("worker-final", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + access + .initialize_final_access_windows( + job.job_id, + "worker-final", + final_claim.claim_token, + time::Duration::minutes(15), + time::Duration::minutes(15), + ) + .await + .unwrap(); + assert!( + access + .final_credential_available(&job.creator, &bundle_id, NOW) + .await + .unwrap() + ); +} + +#[tokio::test] +async fn in_memory_payment_drain_waits_for_pending_non_paykit_snapshot() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let mut pending = verification_task(&job, VerificationTaskStatus::Pending); + pending.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::DevStatic; + verification_tasks + .insert_verification_task(pending) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let drain_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainPayments, + ) + .await; + + assert!(matches!( + tokio::time::timeout( + std::time::Duration::from_secs(1), + deletions.advance_phase( + job.job_id, + "worker-final", + drain_claim.claim_token, + ContentLockDeletionPhase::DrainExistingCredentials, + ), + ) + .await + .expect("pending non-Paykit guard must not deadlock"), + Err( + locks_service::application::errors::ApplicationError::InvalidContentLockDeletionState { .. } + ) + )); +} + +#[tokio::test] +async fn no_paykit_drain_expires_pending_task_and_advances_without_an_aggregate() { + let (verification_tasks, _access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let mut pending = verification_task(&job, VerificationTaskStatus::Pending); + pending.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::DevStatic; + let task_id = pending.task_id; + verification_tasks + .insert_verification_task(pending) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + + let withdraw = deletions + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + deletions + .advance_phase( + job.job_id, + "worker", + withdraw.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + + let use_case = NoPaykitDeletionDrainUseCase::new(&deletions, &FixedClock(NOW)); + let start = deletions + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + assert!(use_case.execute_claimed(start, "worker").await.unwrap()); + let drain = deletions + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + assert!(use_case.execute_claimed(drain, "worker").await.unwrap()); + + assert_eq!( + verification_tasks + .get_verification_task(&task_id) + .await + .unwrap() + .unwrap() + .status, + VerificationTaskStatus::Expired + ); + assert_eq!( + deletions + .get_job(&job.creator, &job.lock_id) + .await + .unwrap() + .unwrap() + .phase, + ContentLockDeletionPhase::DrainExistingCredentials + ); +} + +#[tokio::test] +async fn no_paykit_drain_preserves_terminal_tasks() { + for status in [ + VerificationTaskStatus::Completed, + VerificationTaskStatus::Failed, + VerificationTaskStatus::Expired, + ] { + let (verification_tasks, _access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let mut task = verification_task(&job, status); + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::DevStatic; + task.started_at = (status != VerificationTaskStatus::Expired).then_some(NOW); + task.completed_at = Some(NOW); + task.failure_message = + (status == VerificationTaskStatus::Failed).then(|| "closed failure".to_owned()); + let task_id = task.task_id; + verification_tasks + .insert_verification_task(task) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let withdraw = deletions + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + deletions + .advance_phase( + job.job_id, + "worker", + withdraw.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let claim = deletions + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + NoPaykitDeletionDrainUseCase::new(&deletions, &FixedClock(NOW)) + .execute_claimed(claim, "worker") + .await + .unwrap(); + assert_eq!( + verification_tasks + .get_verification_task(&task_id) + .await + .unwrap() + .unwrap() + .status, + status + ); + } +} + +#[tokio::test] +async fn no_paykit_drain_fences_stale_claim() { + let clock = Arc::new(MutableClock::new(NOW)); + let (verification_tasks, _access, deletions) = in_memory_access_stack_with_clock(clock.clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let mut task = verification_task(&job, VerificationTaskStatus::Pending); + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::DevStatic; + let task_id = task.task_id; + verification_tasks + .insert_verification_task(task) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let withdraw = deletions + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + deletions + .advance_phase( + job.job_id, + "worker", + withdraw.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let claim = deletions + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + clock.set(LEASE_END + time::Duration::seconds(1)); + assert!( + !NoPaykitDeletionDrainUseCase::new(&deletions, &FixedClock(NOW)) + .execute_claimed(claim, "worker") + .await + .unwrap() + ); + assert_eq!( + verification_tasks + .get_verification_task(&task_id) + .await + .unwrap() + .unwrap() + .status, + VerificationTaskStatus::Pending + ); +} + +#[tokio::test] +async fn in_memory_payment_resolution_rejects_claim_at_expiry_equality() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let task = verification_task(&job, VerificationTaskStatus::Pending); + let task_id = task.task_id; + verification_tasks + .insert_verification_task(task) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainPayments, + ) + .await; + + assert!( + !access + .resolve_deletion_payment( + job.job_id, + "worker-final", + claim.claim_token, + LEASE_END, + &task_id, + VerificationTaskStatus::Completed, + ) + .await + .unwrap() + ); +} + +#[tokio::test] +async fn in_memory_payment_aggregate_completion_rejects_claim_at_expiry_equality() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + verification_tasks + .insert_verification_task(verification_task(&job, VerificationTaskStatus::Completed)) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainPayments, + ) + .await; + + assert!( + !access + .complete_deletion_payment_aggregate( + job.job_id, + "worker-final", + claim.claim_token, + LEASE_END, + ) + .await + .unwrap() + ); +} + +#[tokio::test] +async fn in_memory_non_paykit_expiry_rejects_claim_at_expiry_equality() { + let clock = Arc::new(MutableClock::new(NOW)); + let (verification_tasks, _access, deletions) = in_memory_access_stack_with_clock(clock.clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let mut task = verification_task(&job, VerificationTaskStatus::Pending); + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::DevStatic; + let task_id = task.task_id; + verification_tasks + .insert_verification_task(task) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let withdraw = deletions + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + deletions + .advance_phase( + job.job_id, + "worker", + withdraw.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let claim = deletions + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + + clock.set(LEASE_END); + assert!( + !deletions + .expire_unresolved_non_paykit_tasks(job.job_id, "worker", claim.claim_token) + .await + .unwrap() + ); + assert_eq!( + verification_tasks + .get_verification_task(&task_id) + .await + .unwrap() + .unwrap() + .status, + VerificationTaskStatus::Pending + ); +} + +#[tokio::test] +async fn no_paykit_task_expiry_serializes_with_force_claim_revocation() { + let fence = Arc::new(InMemoryVerificationTaskDeletionFence::with_clock(Arc::new( + FixedClock(NOW), + ))); + let verification_tasks = Arc::new(InMemoryVerificationTaskRepository::with_deletion_fence( + Arc::clone(&fence), + )); + let pausing_tasks = Arc::new(PausingVerificationTaskRepository::new( + verification_tasks.clone(), + )); + let access = Arc::new( + InMemoryAccessCredentialStore::with_verification_task_repository_and_deletion_fence( + pausing_tasks.clone(), + Arc::clone(&fence), + ), + ); + let deletions = Arc::new( + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + access, fence, + ), + ); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let mut task = verification_task(&job, VerificationTaskStatus::Pending); + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::DevStatic; + let task_id = task.task_id; + verification_tasks + .insert_verification_task(task) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let withdraw = deletions + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + deletions + .advance_phase( + job.job_id, + "worker", + withdraw.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let claim = deletions + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + + let expiry_repository = deletions.clone(); + let job_id = job.job_id; + let expiry = tokio::spawn(async move { + expiry_repository + .expire_unresolved_non_paykit_tasks(job_id, "worker", claim.claim_token) + .await + }); + pausing_tasks.get_entered.notified().await; + + let force_repository = deletions.clone(); + let creator = job.creator.clone(); + let lock_id = job.lock_id.clone(); + let mut force = tokio::spawn(async move { + force_repository + .prepare_force_deletion(&creator, &lock_id) + .await + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), &mut force) + .await + .is_err(), + "force must wait until task and snapshot terminalization commits" + ); + + pausing_tasks.release_get.notify_one(); + assert!(expiry.await.unwrap().unwrap()); + assert!(matches!( + force.await.unwrap().unwrap(), + PrepareForceDeletionResult::Active(_) + )); + assert_eq!( + verification_tasks + .get_verification_task(&task_id) + .await + .unwrap() + .unwrap() + .status, + VerificationTaskStatus::Expired + ); +} + +#[tokio::test] +async fn no_paykit_drain_rejects_paykit_snapshot() { + let (verification_tasks, _access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + verification_tasks + .insert_verification_task(verification_task(&job, VerificationTaskStatus::Pending)) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let withdraw = deletions + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + deletions + .advance_phase( + job.job_id, + "worker", + withdraw.claim_token, + ContentLockDeletionPhase::StartPaymentDrain, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let claim = deletions + .claim_next("worker", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + assert!(matches!( + NoPaykitDeletionDrainUseCase::new(&deletions, &FixedClock(NOW)) + .execute_claimed(claim, "worker") + .await, + Err( + locks_service::application::errors::ApplicationError::InvalidContentLockDeletionState { .. } + ) + )); +} + +#[tokio::test] +async fn in_memory_payment_drain_waits_for_pending_paykit_snapshot() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + verification_tasks + .insert_verification_task(verification_task(&job, VerificationTaskStatus::Pending)) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let drain_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainPayments, + ) + .await; + + assert!(matches!( + tokio::time::timeout( + std::time::Duration::from_secs(1), + deletions.advance_phase( + job.job_id, + "worker-final", + drain_claim.claim_token, + ContentLockDeletionPhase::DrainExistingCredentials, + ), + ) + .await + .expect("pending Paykit guard must not deadlock"), + Err( + locks_service::application::errors::ApplicationError::InvalidContentLockDeletionState { .. } + ) + )); +} + +#[tokio::test] +async fn in_memory_payment_drain_waits_for_completed_aggregate() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + verification_tasks + .insert_verification_task(verification_task(&job, VerificationTaskStatus::Completed)) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let drain_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainPayments, + ) + .await; + + assert!( + !access + .complete_deletion_payment_aggregate( + job.job_id, + "different-worker", + drain_claim.claim_token, + NOW, + ) + .await + .unwrap() + ); + assert!( + !access + .complete_deletion_payment_aggregate(job.job_id, "worker-final", Uuid::new_v4(), NOW,) + .await + .unwrap() + ); + assert!(matches!( + tokio::time::timeout( + std::time::Duration::from_secs(1), + deletions.advance_phase( + job.job_id, + "worker-final", + drain_claim.claim_token, + ContentLockDeletionPhase::DrainExistingCredentials, + ), + ) + .await + .expect("payment aggregate guard must not deadlock"), + Err( + locks_service::application::errors::ApplicationError::InvalidContentLockDeletionState { .. } + ) + )); +} + +#[tokio::test] +async fn in_memory_phase_and_success_finish_guards_preserve_access_obligations() { + let clock = Arc::new(MutableClock::new(NOW)); + let (verification_tasks, access, deletions) = in_memory_access_stack_with_clock(clock.clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let completed = verification_task(&job, VerificationTaskStatus::Pending) + .transition_to(VerificationTaskStatus::InProgress, NOW, None) + .unwrap() + .transition_to(VerificationTaskStatus::Completed, NOW, None) + .unwrap(); + let bundle_id = completed.submitted_proof_bundle.bundle_id.clone(); + verification_tasks + .insert_verification_task(completed) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + + let issue_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::IssueFinalCredentials, + ) + .await; + access + .initialize_final_access_windows( + job.job_id, + "worker-final", + issue_claim.claim_token, + time::Duration::minutes(15), + time::Duration::minutes(15), + ) + .await + .unwrap(); + let deadline = NOW + time::Duration::minutes(15); + clock.set(deadline); + let deadline_claim = deletions + .claim_next( + "worker-final", + (deadline + time::Duration::minutes(5)) - (deadline), + ) + .await + .unwrap() + .unwrap(); + assert!(matches!( + deletions + .advance_phase( + job.job_id, + "worker-final", + deadline_claim.claim_token, + ContentLockDeletionPhase::DrainFinalReads, + ) + .await, + Ok(AdvanceContentLockDeletionPhaseResult::TerminalFailure( + ContentLockDeletionFailureCode::StateCorrupt + )) + )); + assert!( + access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + deadline, + AccessCredential::new("fresh-at-deadline-must-not-issue"), + ) + .await + .unwrap() + .is_none() + ); + assert!(matches!( + deletions + .finish(job.job_id, "worker-final", deadline_claim.claim_token, None,) + .await, + Err( + locks_service::application::errors::ApplicationError::InvalidContentLockDeletionState { .. } + ) + )); +} + +#[tokio::test] +async fn in_memory_final_credential_eligibility_does_not_change_when_cutoff_credential_is_deleted() +{ + let clock = Arc::new(MutableClock::new(NOW)); + let (verification_tasks, access, deletions) = in_memory_access_stack_with_clock(clock.clone()); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let completed = verification_task(&job, VerificationTaskStatus::Pending) + .transition_to(VerificationTaskStatus::InProgress, NOW, None) + .unwrap() + .transition_to(VerificationTaskStatus::Completed, NOW, None) + .unwrap(); + let bundle_id = completed.submitted_proof_bundle.bundle_id.clone(); + verification_tasks + .insert_verification_task(completed) + .await + .unwrap(); + + let ordinary = AccessCredential::new("active-at-cutoff"); + let ordinary_lookup = AccessCredentialLookupKey::derive(&ordinary); + let original_expiry = NOW + time::Duration::minutes(10); + access + .insert_access_credential( + &job.lock_id, + ordinary_lookup.clone(), + AccessCredentialRecord { + creator: job.creator.clone(), + bundle_id: bundle_id.clone(), + expires_at: original_expiry, + }, + ) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let drain_existing_claim = advance_to_phase( + &deletions, + &access, + job.job_id, + ContentLockDeletionPhase::DrainExistingCredentials, + ) + .await; + assert_eq!( + deletions + .advance_phase( + job.job_id, + "worker-final", + drain_existing_claim.claim_token, + ContentLockDeletionPhase::IssueFinalCredentials, + ) + .await + .unwrap(), + AdvanceContentLockDeletionPhaseResult::ObligationsPending + ); + + let after_expiry = original_expiry; + access + .delete_access_credential(&ordinary_lookup) + .await + .unwrap(); + clock.set(after_expiry); + let after_expiry_claim = deletions + .claim_next( + "worker-final", + (after_expiry + time::Duration::minutes(5)) - (after_expiry), + ) + .await + .unwrap() + .unwrap(); + deletions + .advance_phase( + job.job_id, + "worker-final", + after_expiry_claim.claim_token, + ContentLockDeletionPhase::IssueFinalCredentials, + ) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + let claimed = deletions + .claim_next( + "worker-final", + (after_expiry + time::Duration::minutes(5)) - (after_expiry), + ) + .await + .unwrap() + .unwrap(); + assert!(matches!( + access + .initialize_final_access_windows( + job.job_id, + "worker-final", + claimed.claim_token, + time::Duration::minutes(15), + time::Duration::minutes(15), + ) + .await + .unwrap(), + InitializeFinalAccessWindowsResult::Initialized(_) + )); + + access + .delete_access_credential(&ordinary_lookup) + .await + .unwrap(); + + assert!( + !access + .final_credential_available(&job.creator, &bundle_id, after_expiry) + .await + .unwrap() + ); + assert!( + access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + after_expiry, + AccessCredential::new("must-remain-ineligible"), + ) + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn in_memory_final_credential_rejects_completed_non_paykit_snapshot() { + let (verification_tasks, access, deletions) = in_memory_access_stack(); + let job = ContentLockDeletionJob::new(Uuid::new_v4(), content_lock(), NOW).unwrap(); + let mut task = verification_task(&job, VerificationTaskStatus::Completed); + task.submitted_proof_bundle.proofs[0].verifier_type = VerifierType::DevStatic; + let bundle_id = task.submitted_proof_bundle.bundle_id.clone(); + verification_tasks + .insert_verification_task(task.clone()) + .await + .unwrap(); + deletions.insert_job(job.clone()).await.unwrap(); + let claimed = advance_to_final_issuance(&deletions, &access, job.job_id).await; + assert!(matches!( + access + .initialize_final_access_windows( + job.job_id, + "worker-final", + claimed.claim_token, + time::Duration::minutes(15), + time::Duration::minutes(15), + ) + .await + .unwrap(), + InitializeFinalAccessWindowsResult::Initialized(_) + )); + + assert!( + !access + .final_credential_available(&job.creator, &bundle_id, NOW) + .await + .unwrap() + ); + assert!( + access + .issue_or_replay_final_credential( + &job.creator, + &bundle_id, + NOW, + AccessCredential::new("must-not-be-issued"), + ) + .await + .unwrap() + .is_none() + ); +} + +fn in_memory_access_stack() -> ( + Arc, + Arc, + InMemoryContentLockDeletionRepository, +) { + in_memory_access_stack_with_clock(Arc::new(FixedClock(NOW))) +} + +fn in_memory_access_stack_with_clock( + clock: Arc, +) -> ( + Arc, + Arc, + InMemoryContentLockDeletionRepository, +) { + let fence = Arc::new(InMemoryVerificationTaskDeletionFence::with_clock(clock)); + let verification_tasks = Arc::new(InMemoryVerificationTaskRepository::with_deletion_fence( + Arc::clone(&fence), + )); + let access = Arc::new( + InMemoryAccessCredentialStore::with_verification_task_repository_and_deletion_fence( + verification_tasks.clone(), + Arc::clone(&fence), + ), + ); + let deletions = + InMemoryContentLockDeletionRepository::with_access_credentials_and_verification_task_fence( + Arc::clone(&access), + fence, + ); + (verification_tasks, access, deletions) +} + +async fn advance_to_phase( + deletions: &InMemoryContentLockDeletionRepository, + access: &InMemoryAccessCredentialStore, + job_id: Uuid, + target: ContentLockDeletionPhase, +) -> locks_service::application::models::ClaimedContentLockDeletionJob { + for next_phase in [ + ContentLockDeletionPhase::StartPaymentDrain, + ContentLockDeletionPhase::DrainPayments, + ContentLockDeletionPhase::DrainExistingCredentials, + ContentLockDeletionPhase::IssueFinalCredentials, + ] { + let claimed = deletions + .claim_next("worker-final", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + if next_phase == ContentLockDeletionPhase::DrainExistingCredentials { + assert!( + access + .complete_deletion_payment_aggregate( + job_id, + "worker-final", + claimed.claim_token, + NOW, + ) + .await + .unwrap() + ); + } + deletions + .advance_phase(job_id, "worker-final", claimed.claim_token, next_phase) + .await + .unwrap() + .advanced() + .expect("live claim should advance phase"); + if next_phase == target { + return deletions + .claim_next("worker-final", (LEASE_END) - (NOW)) + .await + .unwrap() + .unwrap(); + } + } + panic!("unsupported test target phase"); +} + +async fn advance_to_final_issuance( + deletions: &InMemoryContentLockDeletionRepository, + access: &InMemoryAccessCredentialStore, + job_id: Uuid, +) -> locks_service::application::models::ClaimedContentLockDeletionJob { + advance_to_phase( + deletions, + access, + job_id, + ContentLockDeletionPhase::IssueFinalCredentials, + ) + .await +} + +fn verification_task( + job: &ContentLockDeletionJob, + status: VerificationTaskStatus, +) -> VerificationTaskRecord { + VerificationTaskRecord { + task_id: TaskId::from_str("018fc6ec-2f3d-4f7e-8b7d-6f5c4b3a2d10").unwrap(), + creator: job.creator.clone(), + submitted_proof_bundle: SubmittedProofBundle { + version: SUBMITTED_PROOF_BUNDLE_VERSION, + bundle_id: BundleId::from_str("000G40R40M30E209185GR38E1W").unwrap(), + pubky_lock_resource: PubkyLockResource::from_str(&format!( + "{}/pub/locks.app/{}.json", + job.creator, job.lock_id + )) + .unwrap(), + reader_public_key: None, + proofs: vec![Proof { + criterion_id: "criterion-1".to_owned(), + verifier_type: VerifierType::PaykitPayment, + payload: json!({}), + }], + }, + status, + submitted_at: NOW, + started_at: None, + completed_at: None, + failure_message: None, + } +} + +fn content_lock() -> ContentLock { + ContentLock { + version: CONTENT_LOCK_VERSION, + creator: CreatorPubky::from_str(CREATOR).unwrap(), + primary_resource: Some( + GuardedResource::new( + "/priv/locks.app/content/post.json".to_owned(), + GuardedResourceHash::from_bytes([7; 32]), + "application/json".to_owned(), + 42, + ) + .unwrap(), + ), + secondary_resources: BTreeMap::new(), + criteria: vec![], + lock_logic: LockLogic::All { criteria: vec![] }, + access_policy: AccessPolicy { + requested_credential_ttl_seconds: 900, + }, + lock_server: LockServerConfig { override_: None }, + created_at: datetime!(2026-08-12 04:00:00 UTC), + } +} diff --git a/locks-service/tests/content_lock_tombstones.rs b/locks-service/tests/content_lock_tombstones.rs new file mode 100644 index 0000000..27a0940 --- /dev/null +++ b/locks-service/tests/content_lock_tombstones.rs @@ -0,0 +1,417 @@ +use std::str::FromStr; +use std::sync::Mutex; + +use async_trait::async_trait; +use locks_core::content_lock_deletion::ContentLockDeletionTombstone; +use locks_core::ids::{ContentLockPath, CreatorPubky, LockId}; +use locks_service::application::errors::ApplicationError; +use locks_service::application::ports::{ + ContentLockRepository, ContentLockTombstoneRepository, TombstoneReadback, +}; +use locks_service::infrastructure::memory::{ + content_lock_tombstones::InMemoryContentLockTombstoneRepository, + content_locks::InMemoryContentLockRepository, + public_content_locks::InMemoryPublicContentLockStore, +}; +use locks_service::infrastructure::pubky::{ + PubkyBytesResource, PubkyContentLockTombstoneRepository, PubkyHomeserverStorageClient, +}; +use time::macros::datetime; + +const LOCK_ID: &str = "000G40R40M30E209185GR38E1W8124GK2GAHC5RR34D1P70X3RFG"; + +#[tokio::test] +async fn memory_content_lock_and_tombstone_adapters_share_one_canonical_public_path() { + let store = InMemoryPublicContentLockStore::new(); + let content_locks = InMemoryContentLockRepository::with_public_store(store.clone()); + let tombstones = InMemoryContentLockTombstoneRepository::with_public_store(store); + let creator = creator(); + let path = path(); + let original: locks_core::lock_policy::ContentLock = + serde_json::from_value(serde_json::json!({ + "version": 1, + "creator": creator, + "primary_resource": null, + "secondary_resources": {}, + "criteria": [], + "lock_logic": { "type": "all", "criteria": [] }, + "access_policy": { "requested_credential_ttl_seconds": 900 }, + "lock_server": { "override": null }, + "created_at": "2026-08-12T04:00:00Z" + })) + .unwrap(); + content_locks + .upsert_content_lock(creator.clone(), path.clone(), original.clone()) + .await + .unwrap(); + + let tombstone = tombstone_at_five(); + assert_eq!( + tombstones + .withdraw_content_lock(creator.clone(), path.clone(), &original, &tombstone) + .await + .unwrap(), + TombstoneReadback::Exact + ); + assert!( + content_locks + .get_content_lock(&creator, &path) + .await + .is_err() + ); + + assert_eq!( + tombstones + .read_tombstone(&creator, &path, &tombstone) + .await + .unwrap(), + TombstoneReadback::Exact + ); + assert!( + content_locks + .get_content_lock(&creator, &path) + .await + .is_err() + ); +} + +#[tokio::test] +async fn memory_withdrawal_is_exact_and_replacement_is_classified_without_parsing() { + let store = InMemoryPublicContentLockStore::new(); + let content_locks = InMemoryContentLockRepository::with_public_store(store.clone()); + let repository = InMemoryContentLockTombstoneRepository::with_public_store(store); + let creator = creator(); + let path = path(); + let original = original_content_lock(); + let expected = tombstone_at_five(); + + assert_eq!( + repository + .read_tombstone(&creator, &path, &expected) + .await + .unwrap(), + TombstoneReadback::Missing + ); + assert_eq!( + repository + .withdraw_content_lock(creator.clone(), path.clone(), &original, &expected) + .await + .unwrap(), + TombstoneReadback::Missing + ); + + content_locks + .upsert_content_lock(creator.clone(), path.clone(), original.clone()) + .await + .unwrap(); + assert_eq!( + repository + .withdraw_content_lock(creator.clone(), path.clone(), &original, &expected) + .await + .unwrap(), + TombstoneReadback::Exact + ); + + let mut replacement = original.clone(); + replacement.created_at = datetime!(2026-08-12 05:00:01 UTC); + content_locks + .upsert_content_lock(creator.clone(), path.clone(), replacement.clone()) + .await + .unwrap(); + + assert_eq!( + repository + .read_tombstone(&creator, &path, &expected) + .await + .unwrap(), + TombstoneReadback::Replaced + ); + assert_eq!( + content_locks + .get_content_lock(&creator, &path) + .await + .unwrap(), + Some(replacement) + ); +} + +#[tokio::test] +async fn memory_force_delete_removes_tombstone_and_missing_retry_succeeds() { + let store = InMemoryPublicContentLockStore::new(); + let content_locks = InMemoryContentLockRepository::with_public_store(store.clone()); + let repository = InMemoryContentLockTombstoneRepository::with_public_store(store); + let creator = creator(); + let path = path(); + let original = original_content_lock(); + let expected = tombstone_at_five(); + content_locks + .upsert_content_lock(creator.clone(), path.clone(), original.clone()) + .await + .unwrap(); + repository + .withdraw_content_lock(creator.clone(), path.clone(), &original, &expected) + .await + .unwrap(); + + repository + .force_delete_content_lock_and_verify_absent(&creator, &path) + .await + .unwrap(); + repository + .force_delete_content_lock_and_verify_absent(&creator, &path) + .await + .unwrap(); + assert_eq!( + repository + .read_tombstone(&creator, &path, &expected) + .await + .unwrap(), + TombstoneReadback::Missing + ); +} + +#[tokio::test] +async fn pubky_withdrawal_writes_canonical_bytes_to_exact_path_and_reads_them_back() { + let repository = PubkyContentLockTombstoneRepository::new(FakePubkyStorage::default()); + let creator = creator(); + let path = path(); + let original = original_content_lock(); + let expected = tombstone_at_five(); + repository + .client() + .replace_bytes(original.canonical_json_bytes().unwrap()); + + assert_eq!( + repository + .withdraw_content_lock(creator.clone(), path.clone(), &original, &expected) + .await + .unwrap(), + TombstoneReadback::Exact + ); + assert_eq!( + repository.client().written_bytes(), + serde_json::to_vec(&expected).unwrap() + ); + assert_eq!( + repository.client().operations(), + vec![ + format!("get_bytes {creator} {path}"), + format!("put_bytes {creator} {path} application/json"), + format!("get_bytes {creator} {path}"), + ] + ); + + assert_eq!( + repository + .withdraw_content_lock(creator.clone(), path.clone(), &original, &expected) + .await + .unwrap(), + TombstoneReadback::Exact + ); + assert_eq!( + repository + .client() + .operations() + .iter() + .filter(|operation| operation.starts_with("put_bytes ")) + .count(), + 1 + ); +} + +#[tokio::test] +async fn pubky_readback_classifies_missing_and_non_tombstone_replacement_as_raw_bytes() { + let repository = PubkyContentLockTombstoneRepository::new(FakePubkyStorage::default()); + let creator = creator(); + let path = path(); + let expected = tombstone_at_five(); + + assert_eq!( + repository + .read_tombstone(&creator, &path, &expected) + .await + .unwrap(), + TombstoneReadback::Missing + ); + + repository + .client() + .replace_bytes(br#"{"version":1,"creator":"not-a-tombstone"}"#.to_vec()); + assert_eq!( + repository + .read_tombstone(&creator, &path, &expected) + .await + .unwrap(), + TombstoneReadback::Replaced + ); + + assert_eq!( + repository + .withdraw_content_lock( + creator.clone(), + path.clone(), + &original_content_lock(), + &expected, + ) + .await + .unwrap(), + TombstoneReadback::Replaced + ); + assert!( + repository + .client() + .operations() + .iter() + .all(|operation| !operation.starts_with("put_bytes ")) + ); +} + +#[tokio::test] +async fn pubky_force_delete_removes_original_bytes_tombstone_and_replacement_with_missing_retries() +{ + let repository = PubkyContentLockTombstoneRepository::new(FakePubkyStorage::default()); + let creator = creator(); + let path = path(); + + for bytes in [ + br#"{"version":1,"creator":"original-content-lock"}"#.to_vec(), + serde_json::to_vec(&tombstone_at_five()).unwrap(), + br#"replacement bytes that are not json"#.to_vec(), + ] { + repository.client().replace_bytes(bytes); + repository + .force_delete_content_lock_and_verify_absent(&creator, &path) + .await + .unwrap(); + } + + repository + .force_delete_content_lock_and_verify_absent(&creator, &path) + .await + .unwrap(); + let expected_last = format!("get_bytes {creator} {path}"); + assert_eq!( + repository.client().operations().last(), + Some(&expected_last) + ); +} + +#[derive(Debug, Default)] +struct FakePubkyStorage { + bytes: Mutex>>, + written_bytes: Mutex>>, + operations: Mutex>, +} + +impl FakePubkyStorage { + fn replace_bytes(&self, bytes: Vec) { + *self.bytes.lock().unwrap() = Some(bytes); + } + + fn written_bytes(&self) -> Vec { + self.written_bytes.lock().unwrap().clone().unwrap() + } + + fn operations(&self) -> Vec { + self.operations.lock().unwrap().clone() + } +} + +#[async_trait] +impl PubkyHomeserverStorageClient for FakePubkyStorage { + async fn put_json_value_as_creator( + &self, + _creator: &CreatorPubky, + _path: &str, + _body: serde_json::Value, + ) -> Result<(), ApplicationError> { + unreachable!("tombstones must use raw canonical bytes") + } + + async fn get_json_value_as_creator( + &self, + _creator: &CreatorPubky, + _path: &str, + ) -> Result, ApplicationError> { + unreachable!("tombstones must not be parsed as JSON or ContentLock") + } + + async fn put_bytes_as_creator( + &self, + creator: &CreatorPubky, + path: &str, + bytes: Vec, + content_type: &str, + ) -> Result<(), ApplicationError> { + self.operations + .lock() + .unwrap() + .push(format!("put_bytes {creator} {path} {content_type}")); + *self.written_bytes.lock().unwrap() = Some(bytes.clone()); + *self.bytes.lock().unwrap() = Some(bytes); + Ok(()) + } + + async fn get_bytes_as_creator( + &self, + creator: &CreatorPubky, + path: &str, + ) -> Result, ApplicationError> { + self.operations + .lock() + .unwrap() + .push(format!("get_bytes {creator} {path}")); + Ok(self + .bytes + .lock() + .unwrap() + .clone() + .map(|bytes| PubkyBytesResource { + bytes, + content_type: Some("application/json".to_owned()), + })) + } + + async fn delete_as_creator( + &self, + creator: &CreatorPubky, + path: &str, + ) -> Result<(), ApplicationError> { + self.operations + .lock() + .unwrap() + .push(format!("delete {creator} {path}")); + *self.bytes.lock().unwrap() = None; + Ok(()) + } +} + +fn creator() -> CreatorPubky { + CreatorPubky::from_str("pubkytkrq8zmwb8a3m9k15csu3q17qmfgqnp9dskbrg9uq1rydpyxp7qy").unwrap() +} + +fn path() -> ContentLockPath { + ContentLockPath::from_lock_id(LockId::from_str(LOCK_ID).unwrap()) +} + +fn tombstone_at_five() -> ContentLockDeletionTombstone { + ContentLockDeletionTombstone::new( + LockId::from_str(LOCK_ID).unwrap(), + datetime!(2026-08-12 05:00:00 UTC), + ) +} + +fn original_content_lock() -> locks_core::lock_policy::ContentLock { + serde_json::from_value(serde_json::json!({ + "version": 1, + "creator": creator(), + "primary_resource": null, + "secondary_resources": {}, + "criteria": [], + "lock_logic": { "type": "all", "criteria": [] }, + "access_policy": { "requested_credential_ttl_seconds": 900 }, + "lock_server": { "override": null }, + "created_at": "2026-08-12T04:00:00Z" + })) + .unwrap() +} diff --git a/scripts/test-compose-bootstrap.sh b/scripts/test-compose-bootstrap.sh index e7e192a..2e303d8 100755 --- a/scripts/test-compose-bootstrap.sh +++ b/scripts/test-compose-bootstrap.sh @@ -3,7 +3,7 @@ set -eu repo_root="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" entrypoint="$repo_root/docker/locks-server-compose-entrypoint.sh" -key_name="PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" +key_name="PUBKY_LOCK_RUNTIME_MASTER_KEY" env -u "$key_name" docker compose -f "$repo_root/docker-compose.yml" config --quiet @@ -22,7 +22,7 @@ EOF cat > "$bin_dir/locks-server" <<'EOF' #!/bin/sh set -eu -printf '%s' "$PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY" > "$LOCKS_TEST_KEY_CAPTURE" +printf '%s' "$PUBKY_LOCK_RUNTIME_MASTER_KEY" > "$LOCKS_TEST_KEY_CAPTURE" EOF chmod +x "$bin_dir/locks-server" @@ -44,7 +44,7 @@ file_mode() { } run_entrypoint -key_file="$service_home/creator-authority-encryption-key" +key_file="$service_home/runtime-master-key" test -f "$key_file" test "$(wc -c < "$key_file" | tr -d ' ')" -eq 43 grep -Eq '^[A-Za-z0-9_-]{43}$' "$key_file" @@ -56,13 +56,71 @@ run_entrypoint test "$(cat "$capture")" = "$first_key" override='AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' -PUBKY_LOCK_CREATOR_AUTH_ENCRYPTION_KEY="$override" \ +if PUBKY_LOCK_RUNTIME_MASTER_KEY="$override" \ + PATH="$bin_dir:$PATH" \ + LOCKS_SERVICE_HOME="$service_home" \ + LOCKS_COMPOSE_CONFIG="$tmp/config.compose.toml" \ + LOCKS_TEST_KEY_CAPTURE="$capture" \ + sh "$entrypoint" >"$tmp/mismatched-key.stdout" 2>"$tmp/mismatched-key.stderr"; then + echo "entrypoint replaced an existing runtime master key" >&2 + exit 1 +fi +grep -q "does not match the persisted runtime master key" "$tmp/mismatched-key.stderr" +test "$(cat "$key_file")" = "$first_key" + +# Explicitly discarding local encrypted state includes discarding its key. A +# valid override may establish the key only once that reset has happened. +rm "$key_file" +PUBKY_LOCK_RUNTIME_MASTER_KEY="$override" \ PATH="$bin_dir:$PATH" \ LOCKS_SERVICE_HOME="$service_home" \ LOCKS_COMPOSE_CONFIG="$tmp/config.compose.toml" \ LOCKS_TEST_KEY_CAPTURE="$capture" \ sh "$entrypoint" test "$(cat "$capture")" = "$override" -test "$(cat "$key_file")" = "$first_key" +test "$(cat "$key_file")" = "$override" +test "$(file_mode "$key_file")" = 600 + +run_entrypoint +test "$(cat "$capture")" = "$override" + +invalid_override='AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' +if PUBKY_LOCK_RUNTIME_MASTER_KEY="$invalid_override" \ + PATH="$bin_dir:$PATH" \ + LOCKS_SERVICE_HOME="$service_home" \ + LOCKS_COMPOSE_CONFIG="$tmp/config.compose.toml" \ + LOCKS_TEST_KEY_CAPTURE="$capture" \ + sh "$entrypoint" >"$tmp/invalid-key.stdout" 2>"$tmp/invalid-key.stderr"; then + echo "entrypoint accepted a padded-or-wrong-length runtime master key" >&2 + exit 1 +fi +grep -q "must be an unpadded base64url-encoded 32-byte key" "$tmp/invalid-key.stderr" +test "$(cat "$key_file")" = "$override" + +# The final base64url character for 32 bytes carries only two data bits. `B` +# has non-zero trailing bits and must not be accepted as an alias for `A`. +noncanonical_override='AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB' +if PUBKY_LOCK_RUNTIME_MASTER_KEY="$noncanonical_override" \ + PATH="$bin_dir:$PATH" \ + LOCKS_SERVICE_HOME="$service_home" \ + LOCKS_COMPOSE_CONFIG="$tmp/config.compose.toml" \ + LOCKS_TEST_KEY_CAPTURE="$capture" \ + sh "$entrypoint" >"$tmp/noncanonical-key.stdout" 2>"$tmp/noncanonical-key.stderr"; then + echo "entrypoint accepted a noncanonical runtime master key" >&2 + exit 1 +fi +grep -q "must be an unpadded base64url-encoded 32-byte key" \ + "$tmp/noncanonical-key.stderr" +test "$(cat "$key_file")" = "$override" + +retired_key_file="$service_home/creator-authority-encryption-key" +: > "$retired_key_file" +if run_entrypoint >"$tmp/retired-key.stdout" 2>"$tmp/retired-key.stderr"; then + echo "entrypoint accepted retired creator-authority key" >&2 + exit 1 +fi +grep -q "retired creator-authority key detected" "$tmp/retired-key.stderr" +grep -q "discard and reacquire creator authority rows or recreate the local database" \ + "$tmp/retired-key.stderr" printf 'compose bootstrap regression passed\n'