Separate blocks from items

pydatalab/src/pydatalab/export.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 
 from pydatalab import __version__
+from pydatalab.blocks.store import load_blocks_obj
 from pydatalab.config import CONFIG
 from pydatalab.logger import LOGGER
 from pydatalab.models import ITEM_MODELS
@@ -377,6 +378,8 @@ def create_eln_file(
         try:
             item_data = list(cursor)[0]
 
+            if item_data.get("blocks_obj"):
+                item_data["blocks_obj"] = load_blocks_obj(item_data)
             ItemModel = ITEM_MODELS[item_data["type"]]
             item_data = ItemModel(**item_data).model_dump()
 
@@ -419,6 +422,8 @@ def create_eln_file(
     _all_items = []
 
     for ind, item in enumerate(all_items):
+        if item.get("blocks_obj"):
+            item["blocks_obj"] = load_blocks_obj(item)
         ItemModel = ITEM_MODELS[item["type"]]
         _all_items.append(ItemModel(**item).model_dump())
 

pydatalab/src/pydatalab/models/__init__.py

@@ -1,5 +1,6 @@
 import functools
 
+from pydatalab.models.blocks import Block
 from pydatalab.models.cells import Cell
 from pydatalab.models.collections import Collection
 from pydatalab.models.equipment import Equipment
@@ -8,7 +9,7 @@
 from pydatalab.models.people import Person
 from pydatalab.models.samples import Sample
 from pydatalab.models.starting_materials import StartingMaterial
-from pydatalab.models.versions import ItemVersion
+from pydatalab.models.versions import BlockVersion, ItemVersion
 
 
 @functools.lru_cache(maxsize=1)
@@ -31,6 +32,8 @@ def generate_schemas() -> dict[str, dict]:
 ITEM_SCHEMAS = generate_schemas()
 
 __all__ = (
+    "Block",
+    "BlockVersion",
     "File",
     "Sample",
     "StartingMaterial",

pydatalab/src/pydatalab/models/blocks.py

@@ -1,5 +1,9 @@
+from typing import Any, Literal
+
 from pydantic import ConfigDict, Field
 
+from pydatalab.models.entries import Entry
+from pydatalab.models.traits import HasOwner, HasRevisionControl
 from pydatalab.models.utils import BaseModel, PyObjectId
 
 
@@ -72,3 +76,42 @@ class DataBlockResponse(BaseModel):
     )
     """Any structured metadata associated with the block, for example,
     experimental acquisition parameters."""
+
+
+# Here to avoid circular import
+class HasBlocks(BaseModel):
+    """Trait mixin for models that can have data blocks attached to them."""
+
+    blocks_obj: dict[str, DataBlockResponse] = Field({})
+    """A mapping from block ID to block data."""
+
+    display_order: list[str] = Field([])
+    """The order in which to display block data in the UI."""
+
+
+class Block(Entry, HasOwner, HasRevisionControl):
+    """A model for a data block stored as its own document in the `blocks` collection.
+
+    This is the persistence envelope around a block's payload (the output of
+    `DataBlock.to_db()`, stored verbatim under `data`); the payload itself is
+    described by `DataBlockResponse` and its per-block-type subclasses.
+    """
+
+    type: Literal["blocks"] = "blocks"
+
+    block_id: str
+    """The runtime-generated shorthand ID for the block, used as the key in the
+    parent item's `blocks_obj`/`display_order` and in the DOM."""
+
+    blocktype: str
+    """A short string key specifying the type (technique) of the block."""
+
+    data: dict[str, Any] = Field(default_factory=dict)
+    """The block payload, exactly as produced by `DataBlock.to_db()`."""
+
+    version: int = 0
+    """The latest committed version number of this block in `block_versions`.
+
+    A newly created block starts at 0, i.e., with no committed version; the first
+    version is cut when an item version snapshot is next saved. Block creation
+    does not create a versioned entry. Only when item is saved."""

pydatalab/src/pydatalab/models/collections.py

@@ -4,8 +4,9 @@
     model_validator,
 )
 
+from pydatalab.models.blocks import HasBlocks
 from pydatalab.models.entries import Entry
-from pydatalab.models.traits import HasBlocks, HasOwner
+from pydatalab.models.traits import HasOwner
 from pydatalab.models.utils import HumanReadableIdentifier
 
 

pydatalab/src/pydatalab/models/items.py

@@ -2,10 +2,10 @@
 
 from pydantic import field_validator
 
+from pydatalab.models.blocks import HasBlocks
 from pydatalab.models.entries import Entry
 from pydatalab.models.files import HasFiles
 from pydatalab.models.traits import (
-    HasBlocks,
     HasOwner,
     HasRevisionControl,
     IsCollectable,

pydatalab/src/pydatalab/models/traits.py

@@ -2,7 +2,6 @@
 
 from pydantic import AliasChoices, ConfigDict, Field, field_validator, model_validator
 
-from pydatalab.models.blocks import DataBlockResponse
 from pydatalab.models.people import Group, Person
 from pydatalab.models.utils import BaseModel, Constituent, InlineSubstance, PyObjectId
 
@@ -12,7 +11,6 @@
 __all__ = (
     "HasOwner",
     "HasRevisionControl",
-    "HasBlocks",
     "IsCollectable",
     "HasSynthesisInfo",
     "HasSubstanceInfo",
@@ -48,16 +46,6 @@ class HasRevisionControl(BaseModel):
     """The version number used by the version control system for tracking snapshots."""
 
 
-class HasBlocks(BaseModel):
-    """Trait mixin for models that can have data blocks attached to them."""
-
-    blocks_obj: dict[str, DataBlockResponse] = Field({})
-    """A mapping from block ID to block data."""
-
-    display_order: list[str] = Field([])
-    """The order in which to display block data in the UI."""
-
-
 class CollectionReference(BaseModel):
     """A reference to a collection, used for inlining collection info within other models."""
 

pydatalab/src/pydatalab/models/versions.py

@@ -69,6 +69,54 @@ def validate_restored_from_version(self):
         return self
 
 
+class BlockVersion(BaseModel):
+    """A snapshot of a data block's payload at a specific point in time.
+
+    This model represents a version entry in the `block_versions` collection.
+    Entries are created whenever an item version snapshot is cut and the
+    block's payload has changed since its last committed version (or by a
+    version restore).
+    """
+
+    block_immutable_id: PyObjectId
+    """The immutable ID (`_id`) of the `blocks` document this version belongs to"""
+
+    block_id: str
+    """The shorthand block ID, denormalised for lookups by block ID"""
+
+    version: int = Field(ge=1)
+    """Sequential version number (1-indexed), scoped to the block"""
+
+    timestamp: datetime
+    """When this version was created (ISO format with timezone)"""
+
+    action: VersionAction
+    """The action that triggered this version, matching the item snapshot that cut it"""
+
+    user_id: PyObjectId | None = None
+    """User's ObjectId for efficient querying and indexing"""
+
+    datalab_version: str
+    """Version of datalab-server that created this snapshot"""
+
+    data: dict
+    """Complete snapshot of the block payload at this version"""
+
+    restored_from_version: PyObjectId | None = None
+    """ObjectId of the `block_versions` entry that was restored from (only present if action='restored')"""
+
+    @model_validator(mode="after")
+    def validate_restored_from_version(self):
+        """Ensure restored_from_version is only present when action='restored'."""
+        if self.action == VersionAction.RESTORED and self.restored_from_version is None:
+            raise ValueError("restored_from_version must be provided when action='restored'")
+        if self.action != VersionAction.RESTORED and self.restored_from_version is not None:
+            raise ValueError(
+                f"restored_from_version should only be present when action='restored', got action='{self.action}'"
+            )
+        return self
+
+
 class VersionCounter(BaseModel):
     """Atomic counter for tracking version numbers per item.
 

pydatalab/src/pydatalab/mongo.py

@@ -435,6 +435,22 @@ def create_group_fts():
         "refcode", unique=True, name="unique refcode counter", background=background
     )
 
+    # Block storage indexes. `block_id` is only effectively unique (random,
+    # historically unique within a single item), so is not a unique index.
+    ret += db.blocks.create_index("block_id", name="block ID", background=background)
+    ret += db.blocks.create_index("blocktype", name="block type", background=background)
+    ret += db.blocks.create_index("creator_ids", name="block creators", background=background)
+    ret += db.blocks.create_index("group_ids", name="block groups", background=background)
+    ret += db.block_versions.create_index(
+        [("block_immutable_id", pymongo.ASCENDING), ("version", pymongo.DESCENDING)],
+        unique=True,
+        name="block immutable ID and version",
+        background=background,
+    )
+    ret += db.block_versions.create_index(
+        "block_id", name="block version block ID", background=background
+    )
+
     return ret