feat: support partial (multi-request) scene uploads

.env.default

@@ -81,3 +81,7 @@ SHARED_SECRET_MAX_ATTEMPTS_PER_MINUTE=3
 ######################################
 # How long to keep UNDEPLOYED scenes before permanent deletion (ms). Default: 7 days
 SCENE_EVICTION_TTL_MS=604800000
+# How long a partial (multi-request) deployment may stay pending before it is reclaimed (ms). Default 24h.
+PENDING_DEPLOYMENT_TTL=86400000
+# Max concurrent non-expired pending (partial) uploads a single deployer may have in flight. Default 10.
+MAX_PENDING_DEPLOYMENTS_PER_DEPLOYER=10

docs/database-schema.md

@@ -11,7 +11,20 @@ The database contains the following main tables:
 3. **`world_permissions`** - Stores deployment and streaming permission grants
 4. **`world_permission_parcels`** - Stores parcel-level permission restrictions (normalized)
 5. **`blocked`** - Stores blocked wallet addresses
-6. **`migrations`** - Tracks executed database migrations (internal, managed automatically)
+6. **`pending_scenes`** - Stores partial (multi-request) deployments still being uploaded (not yet live)
+7. **`migrations`** - Tracks executed database migrations (internal, managed automatically)
+
+### Table: `pending_scenes`
+
+Staging area for partial deployments: a scene's content can be uploaded across several `POST /entities`
+requests (with `partial=true`), and the entity only becomes a live `world_scenes` row once every
+referenced file is present. Intentionally has **no** foreign key to `worlds` — a half-uploaded world must
+not create a `worlds` row (which would leak into listings and world-validity checks) before it goes live.
+The authoritative entity bytes live in content storage under the entity id; the `entity` JSONB here is a
+copy used by garbage collection. Columns: `entity_id` (PK), `world_name`, `parcels` (TEXT[]), `entity`
+(JSONB), `deployer`, `created_at`/`updated_at` (TIMESTAMPTZ). At most one non-expired pending scene may
+exist per world + overlapping parcels; rows expire after `PENDING_DEPLOYMENT_TTL` (default 24h) and are
+removed by the daily eviction job. `created_at` anchors the deployment-TTL check for staged uploads.
 
 ---
 

docs/openapi.yaml

@@ -306,8 +306,19 @@ paths:
         - Content Server
       summary: /entities
       description: |
-        Deploy a new scene to a world. The deployment must include entity metadata and all content files.
-        Requires authentication via auth chain.
+        Deploy a new scene to a world. Requires authentication via auth chain.
+
+        A standard deployment includes the entity metadata file and every content file it references
+        in a single request, and returns `200`.
+
+        **Partial (multi-request) deployments.** For scenes too large for one request, send `partial`
+        set to `"true"` and split the content files across several requests. The first partial request
+        must include the entity metadata file; later requests may omit it (it is read back from
+        storage). Each request stores the files it carries; the scene only goes live once every
+        referenced file is present. A request that still leaves files missing returns `202` with the
+        list of outstanding hashes; the request that completes the set runs full validation, deploys,
+        and returns `200`. At most one in-flight partial deployment is kept per set of parcels (a newer
+        one replaces it), and staged content expires if the upload is not completed in time.
       operationId: worldsContentServer_deployEntity
       requestBody:
         required: true
@@ -322,13 +333,19 @@ paths:
                   type: string
                   description: The entity ID (IPFS hash)
                   example: bafkreiabi74jam7uh5gmm6x4hox664j53bmt7dy4alrltkhyj3ozcxs2iu
+                partial:
+                  type: string
+                  enum: ['true']
+                  description: >
+                    Set to "true" to mark this request as a staging request of a partial
+                    (multi-request) deployment. Omit for a standard single-request deployment.
               additionalProperties:
                 type: string
                 format: binary
                 description: Content files referenced by the entity
       responses:
         '200':
-          description: Entity deployed successfully
+          description: Entity deployed successfully (all content present)
           content:
             application/json:
               schema:
@@ -340,6 +357,20 @@ paths:
                   message:
                     type: string
                     description: Deployment confirmation message
+        '202':
+          description: >
+            Partial deployment accepted but not yet complete: some referenced content files are still
+            missing. Send them in further requests to complete the deployment.
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  missing:
+                    type: array
+                    items:
+                      type: string
+                    description: Content hashes still missing before the scene can go live
         '400':
           description: Invalid request or deployment failed validation
           content:

src/adapters/eviction-job.ts

@@ -5,9 +5,9 @@ const DEFAULT_TTL_MS = 7 * 24 * 60 * 60 * 1000 // 7 days
 const ONE_DAY_MS = 24 * 60 * 60 * 1000
 
 export async function createEvictionJob(
-  components: Pick<AppComponents, 'config' | 'logs' | 'worlds'>
+  components: Pick<AppComponents, 'config' | 'logs' | 'worlds' | 'pendingScenesManager'>
 ): Promise<IJobComponent> {
-  const { config, logs, worlds } = components
+  const { config, logs, worlds, pendingScenesManager } = components
   const logger = logs.getLogger('eviction-job')
   const evictionTtlMs = (await config.getNumber('SCENE_EVICTION_TTL_MS')) ?? DEFAULT_TTL_MS
 
@@ -16,7 +16,9 @@ export async function createEvictionJob(
     async () => {
       logger.info('Running eviction job...')
       const evicted = await worlds.evictUndeployedWorlds(evictionTtlMs)
-      logger.info(`Eviction completed. Deleted ${evicted} scene(s).`)
+      // The pending-scenes manager owns the PENDING_DEPLOYMENT_TTL, so expiry uses its configured value.
+      const expiredPending = await pendingScenesManager.deleteExpired()
+      logger.info(`Eviction completed. Deleted ${evicted} scene(s) and ${expiredPending} expired pending upload(s).`)
     },
     ONE_DAY_MS,
     { repeat: true, onError: (err) => logger.error(`Eviction job failed: ${err}`) }

src/adapters/pending-scenes-manager.ts

@@ -0,0 +1,166 @@
+import SQL from 'sql-template-strings'
+import { Entity } from '@dcl/schemas'
+import { InvalidRequestError } from '@dcl/http-commons'
+import { AppComponents, IPendingScenesManager, PendingScene, UpsertPendingScene } from '../types'
+
+const DEFAULT_PENDING_DEPLOYMENT_TTL_MS = 24 * 60 * 60 * 1000 // 24 hours
+
+type PendingSceneRow = {
+  entity_id: string
+  world_name: string
+  parcels: string[]
+  entity: Entity
+  deployer: string
+  created_at: Date
+  updated_at: Date
+}
+
+function toPendingScene(row: PendingSceneRow): PendingScene {
+  return {
+    entityId: row.entity_id,
+    worldName: row.world_name,
+    parcels: row.parcels,
+    entity: row.entity,
+    deployer: row.deployer,
+    createdAt: row.created_at,
+    updatedAt: row.updated_at
+  }
+}
+
+export async function createPendingScenesManager(
+  components: Pick<AppComponents, 'config' | 'database' | 'logs'>
+): Promise<IPendingScenesManager> {
+  const { config, database, logs } = components
+  const logger = logs.getLogger('pending-scenes-manager')
+  const ttlMs = (await config.getNumber('PENDING_DEPLOYMENT_TTL')) ?? DEFAULT_PENDING_DEPLOYMENT_TTL_MS
+
+  async function getByEntityId(entityId: string): Promise<PendingScene | undefined> {
+    const cutoff = new Date(Date.now() - ttlMs)
+    const result = await database.query<PendingSceneRow>(
+      SQL`SELECT entity_id, world_name, parcels, entity, deployer, created_at, updated_at
+          FROM pending_scenes
+          WHERE entity_id = ${entityId} AND created_at >= ${cutoff}
+          LIMIT 1`
+    )
+    return result.rows.length > 0 ? toPendingScene(result.rows[0]) : undefined
+  }
+
+  async function upsert(
+    input: UpsertPendingScene,
+    limit: { maxPendingPerDeployer: number; isNewUpload: boolean }
+  ): Promise<PendingScene> {
+    const worldName = input.worldName.toLowerCase()
+    const deployer = input.deployer.toLowerCase()
+    const expiryCutoff = new Date(Date.now() - ttlMs)
+
+    return await database.withAsyncContextTransaction(async () => {
+      // Per-deployer lock FIRST (acquired before the per-world lock below, a consistent order that can't
+      // deadlock). It serializes a deployer's concurrent staging requests — even to different worlds —
+      // so the concurrent-pending cap can't be raced: without it, several new uploads could each read a
+      // count below the cap and then all insert.
+      await database.query(SQL`SELECT pg_advisory_xact_lock(hashtextextended(${'pending_deployer:' + deployer}, 0))`)
+      if (limit.isNewUpload) {
+        const active = await countActiveByDeployer(deployer)
+        if (active >= limit.maxPendingPerDeployer) {
+          throw new InvalidRequestError(
+            `Too many partial uploads in progress for this account (max ${limit.maxPendingPerDeployer}). Finalize or abandon an existing upload before starting another.`
+          )
+        }
+      }
+
+      // Serialize the "replace overlapping + insert" critical section per world (also across
+      // processes) so two concurrent uploads for the same world+parcels can't both insert.
+      await database.query(SQL`SELECT pg_advisory_xact_lock(hashtextextended(${'pending_scenes:' + worldName}, 0))`)
+
+      // The single pending slot per parcel set goes to the NEWEST scene (Decentraland deployment
+      // ordering: greater entity.timestamp, tie broken by greater entity id). Reject rather than
+      // replace when a strictly-newer overlapping upload is already in flight, so a stale/older upload
+      // can't evict a newer competitor's staged content (and two clients can't ping-pong evicting each
+      // other). A resume (same entity id) is excluded and never conflicts with itself.
+      const newer = await database.query(SQL`
+        SELECT 1 FROM pending_scenes
+        WHERE world_name = ${worldName}
+          AND parcels && ${input.parcels}::text[]
+          AND entity_id != ${input.entityId}
+          AND created_at >= ${expiryCutoff}
+          AND ( (entity->>'timestamp')::bigint > ${input.entity.timestamp}
+                OR ((entity->>'timestamp')::bigint = ${input.entity.timestamp} AND entity_id > ${input.entityId}) )
+        LIMIT 1
+      `)
+      if (newer.rowCount > 0) {
+        throw new InvalidRequestError('A newer partial upload is already in progress for one or more of these parcels.')
+      }
+
+      // Purge expired rows and replace any non-expired pending scene of this world whose parcels
+      // overlap the new one (a different entity id). Having rejected the newer-conflict above, every
+      // remaining overlapping row is strictly older, so replacing it is the intended "newest wins".
+      await database.query(SQL`
+        DELETE FROM pending_scenes
+        WHERE created_at < ${expiryCutoff}
+           OR (world_name = ${worldName} AND parcels && ${input.parcels}::text[] AND entity_id != ${input.entityId})
+      `)
+
+      const result = await database.query<PendingSceneRow>(SQL`
+        INSERT INTO pending_scenes (entity_id, world_name, parcels, entity, deployer, created_at, updated_at)
+        VALUES (${input.entityId}, ${worldName}, ${input.parcels}::text[], ${input.entity}::jsonb, ${input.deployer.toLowerCase()}, now(), now())
+        ON CONFLICT (entity_id) DO UPDATE SET updated_at = now()
+        RETURNING entity_id, world_name, parcels, entity, deployer, created_at, updated_at
+      `)
+      return toPendingScene(result.rows[0])
+    })
+  }
+
+  async function deleteByEntityId(entityId: string): Promise<void> {
+    await database.query(SQL`DELETE FROM pending_scenes WHERE entity_id = ${entityId}`)
+  }
+
+  async function countActiveByDeployer(deployer: string): Promise<number> {
+    const cutoff = new Date(Date.now() - ttlMs)
+    const result = await database.query<{ count: string }>(
+      SQL`SELECT COUNT(*) as count FROM pending_scenes
+          WHERE deployer = ${deployer.toLowerCase()} AND created_at >= ${cutoff}`
+    )
+    return parseInt(result.rows[0].count, 10)
+  }
+
+  async function deleteExpired(): Promise<number> {
+    const cutoff = new Date(Date.now() - ttlMs)
+    const result = await database.query(SQL`DELETE FROM pending_scenes WHERE created_at < ${cutoff}`)
+    const removed = result.rowCount ?? 0
+    if (removed > 0) {
+      logger.info(`Removed ${removed} expired pending scene(s)`)
+    }
+    return removed
+  }
+
+  async function getActivePendingKeys(): Promise<Set<string>> {
+    const cutoff = new Date(Date.now() - ttlMs)
+    // Project only the content hashes out of the entity JSONB instead of shipping every pending
+    // scene's full manifest: GC calls this once per delete batch, so on a large sweep the payload
+    // size matters more than the (tiny) row count. The jsonb_typeof guard keeps a row whose `content`
+    // is absent, null, or a non-array from erroring `jsonb_array_elements` ('cannot extract elements
+    // from a scalar') — one such row would otherwise fail the whole query and wedge GC server-wide.
+    const result = await database.query<{ entity_id: string; hashes: string[] | null }>(
+      SQL`SELECT entity_id,
+                 ARRAY(
+                   SELECT jsonb_array_elements(entity->'content')->>'hash'
+                   WHERE jsonb_typeof(entity->'content') = 'array'
+                 ) AS hashes
+          FROM pending_scenes
+          WHERE created_at >= ${cutoff}`
+    )
+    const keys = new Set<string>()
+    for (const row of result.rows) {
+      // The staged entity JSON, its auth-chain blob, and every content file it references are all
+      // referenced by the in-flight upload even though no world_scenes row exists yet.
+      keys.add(row.entity_id)
+      keys.add(`${row.entity_id}.auth`)
+      for (const hash of row.hashes ?? []) {
+        keys.add(hash)
+      }
+    }
+    return keys
+  }
+
+  return { getByEntityId, upsert, deleteByEntityId, deleteExpired, getActivePendingKeys, countActiveByDeployer }
+}

src/adapters/worlds-manager.ts

@@ -29,6 +29,7 @@ import {
 } from '../types'
 import { streamToBuffer } from '@dcl/catalyst-storage'
 import { Entity, EthAddress } from '@dcl/schemas'
+import { InvalidRequestError } from '@dcl/http-commons'
 import SQL from 'sql-template-strings'
 import { buildWorldRuntimeMetadata, shouldShowInPlaces } from '../logic/world-runtime-metadata-utils'
 import { AccessSetting, defaultAccess } from '../logic/access'
@@ -228,6 +229,35 @@ export async function createWorldsManagerComponent({
     const thumbnailHash = thumbnailContent?.hash || null
 
     await database.withAsyncContextTransaction(async () => {
+      // Serialize concurrent deploys to the same world so the "reject if a newer scene already holds
+      // these parcels" check below is atomic with the soft-delete + insert. Both the vanilla and the
+      // partial-finalize paths reach deployScene, so this is the single place deployment ordering is
+      // enforced for world scenes — without the lock, a newer deploy could commit between another
+      // deploy's check and its write, and the older one would silently undeploy it.
+      await database.query(
+        SQL`SELECT pg_advisory_xact_lock(hashtextextended(${'world_scene_deploy:' + worldName.toLowerCase()}, 0))`
+      )
+
+      // Reject if a strictly-newer scene (Decentraland ordering: greater entity.timestamp, tie broken
+      // by greater entity id) already occupies any of these parcels. This makes an older deploy — most
+      // importantly a stale partial finalize — unable to overwrite a newer one, atomically under the
+      // lock above rather than via a racy pre-write check in the caller.
+      const newer = await database.query(SQL`
+        SELECT 1 FROM world_scenes
+        WHERE world_name = ${worldName.toLowerCase()}
+          AND status = 'DEPLOYED'
+          AND parcels && ${parcels}::text[]
+          AND entity_id != ${scene.id}
+          AND ( (entity->>'timestamp')::bigint > ${scene.timestamp}
+                OR ((entity->>'timestamp')::bigint = ${scene.timestamp} AND entity_id > ${scene.id}) )
+        LIMIT 1
+      `)
+      if (newer.rowCount > 0) {
+        throw new InvalidRequestError(
+          `Deployment failed: a newer scene is already deployed on one or more of these parcels.`
+        )
+      }
+
       // Ensure world record exists, update if it does
       // On first deployment (INSERT), set settings from scene metadata
       // On subsequent deployments (UPDATE), preserve existing settings

src/components.ts

@@ -23,6 +23,8 @@ import { createWorldsIndexerComponent } from './adapters/worlds-indexer'
 
 import { createValidator } from './logic/validations'
 import { createEntityDeployer } from './adapters/entity-deployer'
+import { createPendingScenesManager } from './adapters/pending-scenes-manager'
+import { createPartialDeploymentsComponent } from './logic/partial-deployments'
 import { createMigrationExecutor } from './adapters/migration-executor'
 import { createNameDenyListChecker } from './adapters/name-deny-list-checker'
 import { createDatabaseComponent } from './adapters/database-component'
@@ -217,6 +219,20 @@ export async function initComponents(): Promise<AppComponents> {
     worldsManager
   })
 
+  const pendingScenesManager = await createPendingScenesManager({ config, database, logs })
+
+  const partialDeployments = await createPartialDeploymentsComponent({
+    config,
+    coordinates,
+    entityDeployer,
+    limitsManager,
+    logs,
+    pendingScenesManager,
+    storage,
+    validator,
+    worldsManager
+  })
+
   const migrationExecutor = createMigrationExecutor({ logs, database: database, nameOwnership, storage, worldsManager })
 
   const notificationService = await createNotificationsClientComponent({ config, fetch, logs })
@@ -244,7 +260,7 @@ export async function initComponents(): Promise<AppComponents> {
 
   const worlds = createWorldsComponent({ worldsManager, snsClient })
 
-  const evictionJob = await createEvictionJob({ config, logs, worlds })
+  const evictionJob = await createEvictionJob({ config, logs, worlds, pendingScenesManager })
 
   const denyList = await createDenyListComponent({ config, fetch, logs })
   const bans = await createBansComponent({ config, fetch, logs })
@@ -302,6 +318,8 @@ export async function initComponents(): Promise<AppComponents> {
     nats,
     notificationService,
     participantKicker,
+    partialDeployments,
+    pendingScenesManager,
     peersRegistry,
     permissions,
     permissionsManager,