diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7bc4c2488..bbd34c4f3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,8 +29,6 @@ updates: - minor - patch ignore: - - dependency-name: "pydantic" - versions: [ ">=2" ] - dependency-name: "bokeh" versions: [ ">=3" ] - dependency-name: "langchain" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb8d6afca..5364b0654 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -286,6 +286,9 @@ jobs: cp webapp/.env.test_e2e .env echo "PYDATALAB_TESTING=true" >> .env echo "PYDATALAB_TESTING=true" >> pydatalab/.env + # Enable the tags feature for the e2e backend (frontend flag is in .env.test_e2e). + echo "PYDATALAB_ENABLE_TAGS=true" >> .env + echo "PYDATALAB_ENABLE_TAGS=true" >> pydatalab/.env - name: Build Docker images uses: docker/bake-action@v7 @@ -323,6 +326,10 @@ jobs: exit 1 fi + - name: Create the test admin user + # The first admin cannot be created over the API, so create one directly. + run: docker compose exec -T api uv run invoke admin.seed-e2e-admin + - name: Run end-to-end tests uses: cypress-io/github-action@v7 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aeb026a94..f359d5dc3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,7 +67,7 @@ repos: hooks: - id: mypy additional_dependencies: - ["types-python-dateutil", "types-requests", "types-paramiko", "pydantic~=1.10"] + ["types-python-dateutil", "types-requests", "types-paramiko", "pydantic", "pydantic-settings"] args: ["--config-file", "pydatalab/pyproject.toml"] - repo: local diff --git a/pydatalab/pyproject.toml b/pydatalab/pyproject.toml index e8ac75338..ce8c670c8 100644 --- a/pydatalab/pyproject.toml +++ b/pydatalab/pyproject.toml @@ -20,7 +20,9 @@ dependencies = [ "bokeh ~= 2.4, < 3.0", "matplotlib ~= 3.8", "periodictable ~= 2.1", - "pydantic[email, dotenv] < 2.0", + "pydantic[email] >= 2.11, < 3.0", + "pydantic-settings ~= 2.10", + "python-dotenv ~= 1.1", "pint ~= 0.24", "pandas[excel] ~= 2.2", "pymongo ~= 4.7", diff --git a/pydatalab/schemas/cell.json b/pydatalab/schemas/cell.json index 17104a678..55c7211da 100644 --- a/pydatalab/schemas/cell.json +++ b/pydatalab/schemas/cell.json @@ -1,958 +1,1581 @@ { - "title": "Cell", - "description": "A model for representing electrochemical cells.\n\nA cell is an electrochemical device assembled from other items, recording its\ncomponents and the format it was built in.", - "type": "object", - "properties": { - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "cells", - "const": "cells", - "pattern": "^cells$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Name", + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" + ], + "title": "AccountStatus", "type": "string" }, - "status": { - "default": "active", - "allOf": [ - { - "$ref": "#/definitions/CellStatus" + "CellComponent": { + "properties": { + "item": { + "anyOf": [ + { + "$ref": "#/$defs/EntryReference" + }, + { + "$ref": "#/$defs/InlineSubstance" + } + ], + "description": "A reference to item (sample or starting material) entry for the constituent substance.", + "title": "Item" + }, + "quantity": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the constituent material used to create the sample.", + "title": "Quantity" + }, + "unit": { + "default": "g", + "description": "The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).", + "title": "Unit", + "type": "string" } - ] - }, - "cell_format": { - "$ref": "#/definitions/CellFormat" - }, - "cell_format_description": { - "title": "Cell Format Description", - "type": "string" - }, - "cell_preparation_description": { - "title": "Cell Preparation Description", - "type": "string" - }, - "characteristic_mass": { - "title": "Characteristic Mass", - "type": "number" - }, - "characteristic_chemical_formula": { - "title": "Characteristic Chemical Formula", - "type": "string" - }, - "characteristic_molar_mass": { - "title": "Characteristic Molar Mass", - "type": "number" - }, - "positive_electrode": { - "title": "Positive Electrode", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/CellComponent" - } - }, - "negative_electrode": { - "title": "Negative Electrode", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/CellComponent" - } - }, - "electrolyte": { - "title": "Electrolyte", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/CellComponent" - } + }, + "required": [ + "item" + ], + "title": "CellComponent", + "type": "object" }, - "active_ion_charge": { - "title": "Active Ion Charge", - "default": 1, - "type": "number" - } - }, - "required": [ - "item_id" - ], - "definitions": { - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "CellFormat": { "enum": [ - "parent", - "child", - "sibling", - "is_part_of", + "coin", + "pouch", + "in situ (XRD)", + "in situ (NMR)", + "in situ (SQUID)", + "in situ (optical)", + "swagelok", + "cylindrical", "other" ], + "title": "CellFormat", "type": "string" }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "CellStatus": { + "description": "An enumeration of the status of cells", "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" + "active", + "planned", + "disposed", + "cycled", + "shorted", + "dismantled", + "other" ], + "title": "CellStatus", "type": "string" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "description": { - "title": "Description", - "type": "string" - }, - "relation": { - "$ref": "#/definitions/RelationshipType" - }, - "type": { - "$ref": "#/definitions/KnownType" - }, "immutable_id": { + "description": "The immutable ID of the collection.", + "format": "objectid", "title": "Immutable Id", "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" } }, "required": [ - "type" - ] - }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" + "immutable_id" ], - "type": "string" + "title": "CollectionReference", + "type": "object" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "DataBlockResponse": { + "additionalProperties": true, + "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" - }, - "identifier": { - "title": "Identifier", + "blocktype": { + "description": "The type of the block.", + "title": "Blocktype", "type": "string" }, - "name": { - "title": "Name", + "block_id": { + "description": "A shorthand random ID for the block.", + "title": "Block Id", "type": "string" }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" - }, - "display_name": { - "title": "Display Name", - "type": "string" - } - }, - "required": [ - "identity_type", - "identifier", - "name" - ] - }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" - ], - "type": "string" - }, - "Group": { - "title": "Group", - "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "default": "groups", - "const": "groups", - "type": "string" + "item_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" + "collection_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } + "freeform_comment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" }, - "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "file_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" }, - "members": { - "title": "Members", - "type": "array", - "items": { - "type": "object" - } + "file_ids": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "errors": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" }, - "description": { - "title": "Description", - "type": "string" + "warnings": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "anyOf": [ - { + "b64_encoded_image": { + "anyOf": [ + { + "additionalProperties": { "type": "string" }, - { - "type": "object" - } - ] - } + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "default": null, + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" + }, + "bokeh_plot_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "default": null, + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" + }, + "computed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" + }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" } - } - }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", - "enum": [ - "active", - "unverified", - "deactivated" + }, + "required": [ + "blocktype", + "block_id" ], - "type": "string" + "title": "DataBlockResponse", + "type": "object" }, - "Person": { - "title": "Person", - "description": "A model that describes an individual and their digital identities.", - "type": "object", + "EntryReference": { + "additionalProperties": true, + "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", "properties": { "type": { "title": "Type", - "default": "people", - "const": "people", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "identities": { - "title": "Identities", - "type": "array", - "items": { - "$ref": "#/definitions/Identity" - } - }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" - }, - "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" - }, - "gravatar_hash": { - "title": "Gravatar Hash", "type": "string" }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" }, - "role": { - "default": "user", - "allOf": [ + "immutable_id": { + "anyOf": [ { - "$ref": "#/definitions/UserRole" + "format": "objectid", + "type": "string" + }, + { + "type": "null" } - ] + ], + "default": null, + "title": "Immutable Id" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Item Id" }, - "account_status": { - "default": "unverified", - "allOf": [ + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] - } - } - }, - "FileChecksums": { - "title": "FileChecksums", - "description": "Content checksums for a file.", - "type": "object", - "properties": { - "md5": { - "title": "Md5", - "type": "string" + ], + "default": null, + "title": "Refcode" }, - "sha256": { - "title": "Sha256", - "type": "string" + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" } }, "required": [ - "md5", - "sha256" - ] + "type" + ], + "title": "EntryReference", + "type": "object" }, "File": { - "title": "File", "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", "properties": { "revision": { - "title": "Revision", "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", "type": "integer" }, "revisions": { - "title": "Revisions", - "type": "object" + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" }, "version": { - "title": "Version", "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", "type": "integer" }, "creator_ids": { - "title": "Creator Ids", "default": [], - "type": "array", + "description": "The database IDs of the user(s) who created the item.", "items": { + "format": "objectid", "type": "string" - } + }, + "title": "Creator Ids", + "type": "array" }, "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" }, "group_ids": { - "title": "Group Ids", "default": [], - "type": "array", + "description": "The database IDs of the group(s) that have read-access to this item.", "items": { + "format": "objectid", "type": "string" - } + }, + "title": "Group Ids", + "type": "array" }, "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" }, "type": { - "title": "Type", - "default": "files", "const": "files", - "pattern": "^files$", + "default": "files", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "size": { - "title": "Size", - "type": "integer" + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" }, "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" }, "item_ids": { - "title": "Item Ids", - "type": "array", + "description": "A list of item IDs associated with this file.", "items": { "type": "string" - } + }, + "title": "Item Ids", + "type": "array" }, "blocks": { - "title": "Blocks", - "type": "array", + "description": "A list of block IDs associated with this file.", "items": { "type": "string" - } + }, + "title": "Blocks", + "type": "array" }, "name": { + "description": "The filename on disk.", "title": "Name", "type": "string" }, "extension": { + "description": "The file extension that the file was uploaded with.", "title": "Extension", "type": "string" }, "original_name": { - "title": "Original Name", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" }, "location": { - "title": "Location", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" }, "url_path": { - "title": "Url Path", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" }, "source": { - "title": "Source", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" }, "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp for the original file upload.", + "title": "Time Added" }, "metadata": { - "title": "Metadata", - "type": "object" + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" }, "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, "title": "Representation" }, "source_server_name": { - "title": "Source Server Name", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" }, "source_path": { - "title": "Source Path", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" }, "is_live": { + "description": "Whether or not the file should be watched for future updates.", "title": "Is Live", "type": "boolean" }, "checksums": { - "$ref": "#/definitions/FileChecksums" + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ - "item_ids", - "blocks", "name", "extension", - "time_added", "is_live" - ] + ], + "title": "File", + "type": "object" }, - "DataBlockResponse": { - "title": "DataBlockResponse", - "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", + "FileChecksums": { + "description": "Content checksums for a file.", "properties": { - "blocktype": { - "title": "Blocktype", + "md5": { + "description": "The MD5 hex digest of the file contents.", + "title": "Md5", "type": "string" }, - "block_id": { - "title": "Block Id", + "sha256": { + "description": "The SHA-256 hex digest of the file contents.", + "title": "Sha256", "type": "string" - }, - "item_id": { - "title": "Item Id", + } + }, + "required": [ + "md5", + "sha256" + ], + "title": "FileChecksums", + "type": "object" + }, + "Group": { + "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", + "properties": { + "type": { + "const": "groups", + "default": "groups", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, - "collection_id": { - "title": "Collection Id", + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "group_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" + }, + "members": { + "description": "A list of people that belong to this group; stored on the user objects.", + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Members", + "type": "array" + }, + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" + }, + "managers": { + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", + "items": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + } + ] + }, + "title": "Managers", + "type": "array" + } + }, + "title": "Group", + "type": "object" + }, + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", + "properties": { + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." + }, + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", "type": "string" }, - "title": { - "title": "Title", + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", "type": "string" }, - "freeform_comment": { - "title": "Freeform Comment", + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" + }, + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" + } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "InlineSubstance": { + "properties": { + "name": { + "title": "Name", "type": "string" }, - "file_id": { - "title": "File Id", + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "name" + ], + "title": "InlineSubstance", + "type": "object" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" + ], + "title": "KnownType", + "type": "string" + }, + "Person": { + "description": "A model that describes an individual and their digital identities.", + "properties": { + "type": { + "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, - "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" }, - "errors": { - "title": "Errors", - "type": "array", + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", "items": { - "type": "string" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "warnings": { - "title": "Warnings", - "type": "array", + "identities": { + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", "items": { - "type": "string" - } + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" }, - "b64_encoded_image": { - "title": "B64 Encoded Image", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" }, - "bokeh_plot_data": { - "title": "Bokeh Plot Data", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object" + "contact_email": { + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" }, - "computed": { - "title": "Computed", - "datalab_exclude_from_load": true, - "type": "object" + "gravatar_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" }, - "metadata": { - "title": "Metadata", - "datalab_exclude_from_load": true, + "managers": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" + }, + "role": { + "$ref": "#/$defs/UserRole", + "default": "user", + "description": "The role assigned to this person." + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" + }, + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." + } + }, + "title": "Person", + "type": "object" + }, + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" + }, + "TypedRelationship": { + "additionalProperties": true, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" + }, + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." + }, + "type": { + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" + }, + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" + } + }, + "required": [ + "type" + ], + "title": "TypedRelationship", + "type": "object" + }, + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing electrochemical cells.\n\nA cell is an electrochemical device assembled from other items, recording its\ncomponents and the format it was built in.", + "properties": { + "tags": { + "description": "Tags applied to this entry: references to `tags` entries (by\n`immutable_id`).", + "items": { + "$ref": "#/$defs/EntryReference" + }, + "title": "Tags", + "type": "array" + }, + "files": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this item.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, "type": "object" + }, + { + "type": "null" } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" }, - "required": [ - "blocktype", - "block_id" - ] + "title": "Creator Ids", + "type": "array" }, - "Collection": { - "title": "Collection", - "description": "A model for representing a group of related items, for example all the samples\nbelonging to one batch or project.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", + "creators": { + "anyOf": [ + { "items": { - "type": "string" - } + "$ref": "#/$defs/Person" + }, + "type": "array" }, - "groups": { - "title": "Groups", - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { "items": { - "$ref": "#/definitions/Group" - } + "$ref": "#/$defs/Group" + }, + "type": "array" }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "cells", + "default": "cells", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", "type": "string" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "title": { - "title": "Title", - "type": "string" - }, - "description": { - "title": "Description", + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", "type": "string" }, - "num_items": { - "title": "Num Items", - "type": "integer" + { + "type": "null" } - } - }, - "CellStatus": { - "title": "CellStatus", - "description": "An enumeration of the status of cells", - "enum": [ - "active", - "planned", - "disposed", - "cycled", - "shorted", - "dismantled", - "other" ], - "type": "string" + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" }, - "CellFormat": { - "title": "CellFormat", - "description": "An enumeration.", - "enum": [ - "coin", - "pouch", - "in situ (XRD)", - "in situ (NMR)", - "in situ (SQUID)", - "in situ (optical)", - "swagelok", - "cylindrical", - "other" - ], + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", "type": "string" }, - "EntryReference": { - "title": "EntryReference", - "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", - "type": "object", - "properties": { - "type": { - "title": "Type", + "description": { + "anyOf": [ + { "type": "string" }, - "name": { - "title": "Name", + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "immutable_id": { - "title": "Immutable Id", + { + "type": "null" + } + ], + "default": null, + "description": "A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).", + "title": "Date" + }, + "name": { + "anyOf": [ + { "type": "string" }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + { + "type": "null" + } + ], + "default": null, + "description": "An optional human-readable/usable name for the entry.", + "title": "Name" + }, + "status": { + "$ref": "#/$defs/CellStatus", + "default": "active", + "description": "The status of the cells, indicating its current state." + }, + "cell_format": { + "anyOf": [ + { + "$ref": "#/$defs/CellFormat" }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + { + "type": "null" + } + ], + "default": null, + "description": "The form factor of the cell, e.g., coin, pouch, in situ or otherwise." + }, + "cell_format_description": { + "anyOf": [ + { "type": "string" }, - "chemform": { - "title": "Chemform", - "type": "string" + { + "type": "null" } - }, - "required": [ - "type" - ] + ], + "default": null, + "description": "Additional human-readable description of the cell form factor, e.g., 18650, AMPIX, CAMPIX", + "title": "Cell Format Description" }, - "InlineSubstance": { - "title": "InlineSubstance", - "type": "object", - "properties": { - "name": { - "title": "Name", + "cell_preparation_description": { + "anyOf": [ + { "type": "string" }, - "chemform": { - "title": "Chemform", - "type": "string" + { + "type": "null" } - }, - "required": [ - "name" - ] + ], + "default": null, + "description": "Description of how the cell was prepared.", + "title": "Cell Preparation Description" }, - "CellComponent": { - "title": "CellComponent", - "description": "A constituent of a sample.", - "type": "object", - "properties": { - "item": { - "title": "Item", - "anyOf": [ - { - "$ref": "#/definitions/EntryReference" - }, - { - "$ref": "#/definitions/InlineSubstance" - } - ] - }, - "quantity": { - "title": "Quantity", - "minimum": 0, + "characteristic_mass": { + "anyOf": [ + { "type": "number" }, - "unit": { - "title": "Unit", - "default": "g", + { + "type": "null" + } + ], + "default": null, + "description": "The characteristic mass of the cell in milligrams. Can be used to normalize capacities.", + "title": "Characteristic Mass" + }, + "characteristic_chemical_formula": { + "anyOf": [ + { "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chemical formula of the active material. Can be used to calculated molar mass in g/mol for normalizing capacities.", + "title": "Characteristic Chemical Formula" + }, + "characteristic_molar_mass": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" } + ], + "default": null, + "description": "The molar mass of the active material, in g/mol. Will be inferred from the chemical formula, or can be supplied if it cannot be supplied", + "title": "Characteristic Molar Mass" + }, + "positive_electrode": { + "description": "The components of the positive electrode (cathode) of the cell.", + "items": { + "$ref": "#/$defs/CellComponent" }, - "required": [ - "item", - "quantity" - ] + "title": "Positive Electrode", + "type": "array" + }, + "negative_electrode": { + "description": "The components of the negative electrode (anode) of the cell.", + "items": { + "$ref": "#/$defs/CellComponent" + }, + "title": "Negative Electrode", + "type": "array" + }, + "electrolyte": { + "description": "The components of the electrolyte of the cell.", + "items": { + "$ref": "#/$defs/CellComponent" + }, + "title": "Electrolyte", + "type": "array" + }, + "active_ion": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The active ion species.", + "title": "Active Ion" + }, + "active_ion_charge": { + "default": 1, + "description": "The charge of the active ion species.", + "title": "Active Ion Charge", + "type": "number" } - } + }, + "required": [ + "item_id" + ], + "title": "Cell", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/equipment.json b/pydatalab/schemas/equipment.json index 33d5fca20..a641f79f1 100644 --- a/pydatalab/schemas/equipment.json +++ b/pydatalab/schemas/equipment.json @@ -1,816 +1,1431 @@ { - "title": "Equipment", - "description": "A model for representing a piece of equipment.\n\nEquipment represents an instrument or apparatus in the lab, which can be linked to\nthe items measured on it.", - "type": "object", - "properties": { - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "equipment", - "const": "equipment", - "pattern": "^equipment$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Name", + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" + ], + "title": "AccountStatus", "type": "string" }, - "status": { - "default": "working", - "allOf": [ - { - "$ref": "#/definitions/EquipmentStatus" + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", + "properties": { + "immutable_id": { + "description": "The immutable ID of the collection.", + "format": "objectid", + "title": "Immutable Id", + "type": "string" } - ] - }, - "serial_numbers": { - "title": "Serial Numbers", - "type": "string" - }, - "manufacturer": { - "title": "Manufacturer", - "type": "string" + }, + "required": [ + "immutable_id" + ], + "title": "CollectionReference", + "type": "object" }, - "location": { - "title": "Location", - "type": "string" + "DataBlockResponse": { + "additionalProperties": true, + "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", + "properties": { + "blocktype": { + "description": "The type of the block.", + "title": "Blocktype", + "type": "string" + }, + "block_id": { + "description": "A shorthand random ID for the block.", + "title": "Block Id", + "type": "string" + }, + "item_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" + }, + "collection_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" + }, + "freeform_comment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" + }, + "file_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" + }, + "file_ids": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" + }, + "errors": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" + }, + "warnings": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" + }, + "b64_encoded_image": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "default": null, + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" + }, + "bokeh_plot_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "default": null, + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" + }, + "computed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" + }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" + } + }, + "required": [ + "blocktype", + "block_id" + ], + "title": "DataBlockResponse", + "type": "object" }, - "contact": { - "title": "Contact", - "type": "string" - } - }, - "required": [ - "item_id" - ], - "definitions": { - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" + "EntryReference": { + "additionalProperties": true, + "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", + "properties": { + "type": { + "title": "Type", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Immutable Id" + }, + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Item Id" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Refcode" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "type" ], - "type": "string" + "title": "EntryReference", + "type": "object" }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "EquipmentStatus": { + "description": "An enumeration of the status of equipments", "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" + "working", + "broken", + "being_fixed", + "defunct", + "not_being_fixed", + "other" ], + "title": "EquipmentStatus", "type": "string" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", "properties": { - "description": { - "title": "Description", - "type": "string" + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" }, - "relation": { - "$ref": "#/definitions/RelationshipType" + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" }, "type": { - "$ref": "#/definitions/KnownType" + "const": "files", + "default": "files", + "title": "Type", + "type": "string" }, "immutable_id": { - "title": "Immutable Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" + }, + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" + }, + "item_ids": { + "description": "A list of item IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Item Ids", + "type": "array" + }, + "blocks": { + "description": "A list of block IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Blocks", + "type": "array" + }, + "name": { + "description": "The filename on disk.", + "title": "Name", "type": "string" }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", "type": "string" + }, + "original_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" + }, + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" + }, + "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp for the original file upload.", + "title": "Time Added" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" + }, + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" + }, + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" + }, + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", + "type": "boolean" + }, + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ - "type" - ] - }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" + "name", + "extension", + "is_live" ], - "type": "string" + "title": "File", + "type": "object" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "FileChecksums": { + "description": "Content checksums for a file.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" - }, - "identifier": { - "title": "Identifier", - "type": "string" - }, - "name": { - "title": "Name", + "md5": { + "description": "The MD5 hex digest of the file contents.", + "title": "Md5", "type": "string" }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" - }, - "display_name": { - "title": "Display Name", + "sha256": { + "description": "The SHA-256 hex digest of the file contents.", + "title": "Sha256", "type": "string" } }, "required": [ - "identity_type", - "identifier", - "name" - ] - }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" + "md5", + "sha256" ], - "type": "string" + "title": "FileChecksums", + "type": "object" }, "Group": { - "title": "Group", "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", "properties": { "type": { - "title": "Type", - "default": "groups", "const": "groups", + "default": "groups", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, "members": { - "title": "Members", - "type": "array", + "description": "A list of people that belong to this group; stored on the user objects.", "items": { + "additionalProperties": true, "type": "object" - } + }, + "title": "Members", + "type": "array" }, "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" }, "description": { - "title": "Description", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" }, "managers": { - "title": "Managers", - "type": "array", + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", "items": { "anyOf": [ { + "format": "objectid", "type": "string" }, { + "additionalProperties": true, "type": "object" } ] - } + }, + "title": "Managers", + "type": "array" + } + }, + "title": "Group", + "type": "object" + }, + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", + "properties": { + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." + }, + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", + "type": "string" + }, + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", + "type": "string" + }, + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" + }, + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" } - } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", "enum": [ - "active", - "unverified", - "deactivated" + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" ], + "title": "KnownType", "type": "string" }, "Person": { - "title": "Person", "description": "A model that describes an individual and their digital identities.", - "type": "object", "properties": { "type": { - "title": "Type", - "default": "people", "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "identities": { - "title": "Identities", - "type": "array", + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", "items": { - "$ref": "#/definitions/Identity" - } + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" }, "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" }, "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" }, "gravatar_hash": { - "title": "Gravatar Hash", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" }, "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" }, "role": { + "$ref": "#/$defs/UserRole", "default": "user", - "allOf": [ - { - "$ref": "#/definitions/UserRole" - } - ] + "description": "The role assigned to this person." }, "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "account_status": { - "default": "unverified", - "allOf": [ + "anyOf": [ { - "$ref": "#/definitions/AccountStatus" + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" } - ] - } - } - }, - "FileChecksums": { - "title": "FileChecksums", - "description": "Content checksums for a file.", - "type": "object", - "properties": { - "md5": { - "title": "Md5", - "type": "string" + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" }, - "sha256": { - "title": "Sha256", - "type": "string" + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." } }, - "required": [ - "md5", - "sha256" - ] + "title": "Person", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", - "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "size": { - "title": "Size", - "type": "integer" - }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" - }, - "item_ids": { - "title": "Item Ids", - "type": "array", - "items": { - "type": "string" - } - }, - "blocks": { - "title": "Blocks", - "type": "array", - "items": { - "type": "string" - } - }, - "name": { - "title": "Name", - "type": "string" - }, - "extension": { - "title": "Extension", - "type": "string" - }, - "original_name": { - "title": "Original Name", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" - }, - "url_path": { - "title": "Url Path", - "type": "string" - }, - "source": { - "title": "Source", - "type": "string" - }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" - }, - "metadata": { - "title": "Metadata", - "type": "object" - }, - "representation": { - "title": "Representation" - }, - "source_server_name": { - "title": "Source Server Name", - "type": "string" - }, - "source_path": { - "title": "Source Path", - "type": "string" - }, - "is_live": { - "title": "Is Live", - "type": "boolean" - }, - "checksums": { - "$ref": "#/definitions/FileChecksums" - } - }, - "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" }, - "DataBlockResponse": { - "title": "DataBlockResponse", - "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", + "TypedRelationship": { + "additionalProperties": true, "properties": { - "blocktype": { - "title": "Blocktype", - "type": "string" - }, - "block_id": { - "title": "Block Id", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "type": "string" - }, - "collection_id": { - "title": "Collection Id", - "type": "string" - }, - "title": { - "title": "Title", - "type": "string" - }, - "freeform_comment": { - "title": "Freeform Comment", - "type": "string" - }, - "file_id": { - "title": "File Id", - "type": "string" - }, - "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } - }, - "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" }, - "warnings": { - "title": "Warnings", - "type": "array", - "items": { - "type": "string" - } + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." }, - "b64_encoded_image": { - "title": "B64 Encoded Image", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } + "type": { + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." }, - "bokeh_plot_data": { - "title": "Bokeh Plot Data", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object" + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" }, - "computed": { - "title": "Computed", - "datalab_exclude_from_load": true, - "type": "object" + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" }, - "metadata": { - "title": "Metadata", - "datalab_exclude_from_load": true, - "type": "object" + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" } }, "required": [ - "blocktype", - "block_id" - ] + "type" + ], + "title": "TypedRelationship", + "type": "object" }, - "Collection": { - "title": "Collection", - "description": "A model for representing a group of related items, for example all the samples\nbelonging to one batch or project.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing a piece of equipment.\n\nEquipment represents an instrument or apparatus in the lab, which can be linked to\nthe items measured on it.", + "properties": { + "tags": { + "description": "Tags applied to this entry: references to `tags` entries (by\n`immutable_id`).", + "items": { + "$ref": "#/$defs/EntryReference" + }, + "title": "Tags", + "type": "array" + }, + "files": { + "anyOf": [ + { "items": { - "type": "string" - } + "$ref": "#/$defs/File" + }, + "type": "array" }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this item.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" }, - "creators": { - "title": "Creators", - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { "items": { - "$ref": "#/definitions/Person" - } + "$ref": "#/$defs/Person" + }, + "type": "array" }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { "items": { - "type": "string" - } + "$ref": "#/$defs/Group" + }, + "type": "array" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "equipment", + "default": "equipment", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { + "type": "string" }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", + "type": "string" }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + { + "type": "null" + } + ], + "default": null, + "description": "A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).", + "title": "Date" + }, + "name": { + "anyOf": [ + { "type": "string" }, - "title": { - "title": "Title", + { + "type": "null" + } + ], + "default": null, + "description": "An optional human-readable/usable name for the entry.", + "title": "Name" + }, + "status": { + "$ref": "#/$defs/EquipmentStatus", + "default": "working", + "description": "The status of the equipment, indicating its current state." + }, + "serial_numbers": { + "anyOf": [ + { "type": "string" }, - "description": { - "title": "Description", + { + "type": "null" + } + ], + "default": null, + "description": "A string describing one or more serial numbers for the instrument.", + "title": "Serial Numbers" + }, + "manufacturer": { + "anyOf": [ + { "type": "string" }, - "num_items": { - "title": "Num Items", - "type": "integer" + { + "type": "null" } - } + ], + "default": null, + "description": "The manufacturer of this piece of equipment", + "title": "Manufacturer" }, - "EquipmentStatus": { - "title": "EquipmentStatus", - "description": "An enumeration of the status of equipments", - "enum": [ - "working", - "broken", - "being_fixed", - "defunct", - "not_being_fixed", - "other" + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "type": "string" + "default": null, + "description": "Place where the equipment is located", + "title": "Location" + }, + "contact": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Contact information for equipment (e.g., email address or phone number).", + "title": "Contact" } - } + }, + "required": [ + "item_id" + ], + "title": "Equipment", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/plugin_config.json b/pydatalab/schemas/plugin_config.json index ae64bcb3c..71db64db6 100644 --- a/pydatalab/schemas/plugin_config.json +++ b/pydatalab/schemas/plugin_config.json @@ -1,96 +1,134 @@ { - "title": "PluginConfigModel", - "description": "The schema for the top-level plugins.toml file.", - "type": "object", - "properties": { - "dependencies": { - "title": "Dependencies", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "tool": { - "title": "Tool", - "default": { + "$defs": { + "ToolSection": { + "additionalProperties": false, + "properties": { "uv": { - "sources": {} + "$ref": "#/$defs/UvSection", + "default": { + "sources": {} + } } }, - "allOf": [ - { - "$ref": "#/definitions/ToolSection" + "title": "ToolSection", + "type": "object" + }, + "UvSection": { + "additionalProperties": false, + "properties": { + "sources": { + "additionalProperties": { + "$ref": "#/$defs/UvSource" + }, + "default": {}, + "title": "Sources", + "type": "object" } - ] - } - }, - "additionalProperties": false, - "definitions": { + }, + "title": "UvSection", + "type": "object" + }, "UvSource": { - "title": "UvSource", + "additionalProperties": false, "description": "A single entry under `[tool.uv.sources]` in plugins.toml.", - "type": "object", "properties": { "git": { - "title": "Git", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Git" }, "rev": { - "title": "Rev", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Rev" }, "branch": { - "title": "Branch", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Branch" }, "tag": { - "title": "Tag", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tag" }, "path": { - "title": "Path", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" }, "editable": { - "title": "Editable", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Editable" } }, - "additionalProperties": false - }, - "UvSection": { - "title": "UvSection", - "type": "object", - "properties": { - "sources": { - "title": "Sources", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/UvSource" - } - } + "title": "UvSource", + "type": "object" + } + }, + "additionalProperties": false, + "description": "The schema for the top-level plugins.toml file.", + "properties": { + "dependencies": { + "default": [], + "items": { + "type": "string" }, - "additionalProperties": false + "title": "Dependencies", + "type": "array" }, - "ToolSection": { - "title": "ToolSection", - "type": "object", - "properties": { + "tool": { + "$ref": "#/$defs/ToolSection", + "default": { "uv": { - "title": "Uv", - "default": { - "sources": {} - }, - "allOf": [ - { - "$ref": "#/definitions/UvSection" - } - ] + "sources": {} } - }, - "additionalProperties": false + } } - } + }, + "title": "PluginConfigModel", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/sample.json b/pydatalab/schemas/sample.json index 66fe4e4d9..a7de22c0d 100644 --- a/pydatalab/schemas/sample.json +++ b/pydatalab/schemas/sample.json @@ -1,944 +1,1570 @@ { - "title": "Sample", - "description": "A model for representing an experimental sample.\n\nA physical thing in the lab that can be created, characterised\nand connected to other items.", - "type": "object", - "properties": { - "chemform": { - "title": "Chemform", - "example": [ - "Na3P", - "Na3P", - "LiNiO2@C", - "Na3+xP", - "LiNi1/3Co0.1Mn0.1O2" - ], - "type": "string" - }, - "smiles": { - "title": "Smiles", - "aliases": [ - "SMILES", - "smiles_representation" - ], - "type": "string" - }, - "inchi": { - "title": "Inchi", - "type": "string" - }, - "inchi_key": { - "title": "Inchi Key", - "type": "string" - }, - "GHS_codes": { - "title": "Ghs H-Codes", - "examples": [ - "H224", - "H303, H316, H319" + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" ], + "title": "AccountStatus", "type": "string" }, - "molar_mass": { - "title": "Molecular Weight", - "type": "number" - }, - "CAS": { - "title": "Substance Cas", - "type": "string" - }, - "synthesis_constituents": { - "title": "Synthesis Constituents", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Constituent" - } - }, - "synthesis_description": { - "title": "Synthesis Description", - "type": "string" - }, - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "samples", - "const": "samples", - "pattern": "^samples$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Name", - "type": "string" - }, - "status": { - "default": "active", - "allOf": [ - { - "$ref": "#/definitions/ItemStatus" - } - ] - } - }, - "required": [ - "item_id" - ], - "definitions": { - "EntryReference": { - "title": "EntryReference", - "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", - "type": "object", + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "type": { - "title": "Type", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, "immutable_id": { + "description": "The immutable ID of the collection.", + "format": "objectid", "title": "Immutable Id", "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "chemform": { - "title": "Chemform", - "type": "string" - } - }, - "required": [ - "type" - ] - }, - "InlineSubstance": { - "title": "InlineSubstance", - "type": "object", - "properties": { - "name": { - "title": "Name", - "type": "string" - }, - "chemform": { - "title": "Chemform", - "type": "string" } }, "required": [ - "name" - ] + "immutable_id" + ], + "title": "CollectionReference", + "type": "object" }, "Constituent": { - "title": "Constituent", "description": "A constituent of a sample.", - "type": "object", "properties": { "item": { - "title": "Item", "anyOf": [ { - "$ref": "#/definitions/EntryReference" + "$ref": "#/$defs/EntryReference" }, { - "$ref": "#/definitions/InlineSubstance" + "$ref": "#/$defs/InlineSubstance" } - ] + ], + "description": "A reference to item (sample or starting material) entry for the constituent substance.", + "title": "Item" }, "quantity": { - "title": "Quantity", - "minimum": 0, - "type": "number" + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the constituent material used to create the sample.", + "title": "Quantity" }, "unit": { - "title": "Unit", "default": "g", + "description": "The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).", + "title": "Unit", "type": "string" } }, "required": [ - "item", - "quantity" - ] - }, - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" + "item" ], - "type": "string" - }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", - "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" - ], - "type": "string" + "title": "Constituent", + "type": "object" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "DataBlockResponse": { + "additionalProperties": true, + "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", "properties": { - "description": { - "title": "Description", + "blocktype": { + "description": "The type of the block.", + "title": "Blocktype", "type": "string" }, - "relation": { - "$ref": "#/definitions/RelationshipType" - }, - "type": { - "$ref": "#/definitions/KnownType" - }, - "immutable_id": { - "title": "Immutable Id", + "block_id": { + "description": "A shorthand random ID for the block.", + "title": "Block Id", "type": "string" }, "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - } - }, - "required": [ - "type" - ] + "collection_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" + }, + "freeform_comment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" + }, + "file_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" + }, + "file_ids": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" + }, + "errors": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" + }, + "warnings": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" + }, + "b64_encoded_image": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "default": null, + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" + }, + "bokeh_plot_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "default": null, + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" + }, + "computed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" + }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" + } + }, + "required": [ + "blocktype", + "block_id" + ], + "title": "DataBlockResponse", + "type": "object" }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" + "EntryReference": { + "additionalProperties": true, + "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", + "properties": { + "type": { + "title": "Type", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Immutable Id" + }, + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Item Id" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Refcode" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "type" ], - "type": "string" + "title": "EntryReference", + "type": "object" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "files", + "default": "files", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" + }, + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" + }, + "item_ids": { + "description": "A list of item IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Item Ids", + "type": "array" + }, + "blocks": { + "description": "A list of block IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Blocks", + "type": "array" + }, + "name": { + "description": "The filename on disk.", + "title": "Name", + "type": "string" + }, + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", + "type": "string" + }, + "original_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" }, - "identifier": { - "title": "Identifier", - "type": "string" + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" }, - "name": { - "title": "Name", - "type": "string" + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" }, - "verified": { - "title": "Verified", - "default": false, + "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp for the original file upload.", + "title": "Time Added" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" + }, + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" + }, + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" + }, + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", "type": "boolean" }, - "display_name": { - "title": "Display Name", - "type": "string" + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ - "identity_type", - "identifier", - "name" - ] + "name", + "extension", + "is_live" + ], + "title": "File", + "type": "object" }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" + "FileChecksums": { + "description": "Content checksums for a file.", + "properties": { + "md5": { + "description": "The MD5 hex digest of the file contents.", + "title": "Md5", + "type": "string" + }, + "sha256": { + "description": "The SHA-256 hex digest of the file contents.", + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "md5", + "sha256" ], - "type": "string" + "title": "FileChecksums", + "type": "object" }, "Group": { - "title": "Group", "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", "properties": { "type": { - "title": "Type", - "default": "groups", "const": "groups", + "default": "groups", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, "members": { - "title": "Members", - "type": "array", + "description": "A list of people that belong to this group; stored on the user objects.", "items": { + "additionalProperties": true, "type": "object" - } + }, + "title": "Members", + "type": "array" }, "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" }, "description": { - "title": "Description", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" }, "managers": { - "title": "Managers", - "type": "array", + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", "items": { "anyOf": [ { + "format": "objectid", "type": "string" }, { + "additionalProperties": true, "type": "object" } ] - } + }, + "title": "Managers", + "type": "array" } - } + }, + "title": "Group", + "type": "object" }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", + "properties": { + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." + }, + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", + "type": "string" + }, + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", + "type": "string" + }, + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" + }, + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" + } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "InlineSubstance": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "name" + ], + "title": "InlineSubstance", + "type": "object" + }, + "ItemStatus": { + "description": "An enumeration of the status of items", "enum": [ "active", - "unverified", - "deactivated" + "planned", + "disposed", + "completed", + "failed", + "other" + ], + "title": "ItemStatus", + "type": "string" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" ], + "title": "KnownType", "type": "string" }, "Person": { - "title": "Person", "description": "A model that describes an individual and their digital identities.", - "type": "object", "properties": { "type": { - "title": "Type", - "default": "people", "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "identities": { - "title": "Identities", - "type": "array", + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", "items": { - "$ref": "#/definitions/Identity" - } + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" }, "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" }, "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" }, "gravatar_hash": { - "title": "Gravatar Hash", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" }, "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" }, "role": { + "$ref": "#/$defs/UserRole", "default": "user", - "allOf": [ - { - "$ref": "#/definitions/UserRole" - } - ] + "description": "The role assigned to this person." }, "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "account_status": { - "default": "unverified", - "allOf": [ + "anyOf": [ { - "$ref": "#/definitions/AccountStatus" + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" } - ] - } - } - }, - "FileChecksums": { - "title": "FileChecksums", - "description": "Content checksums for a file.", - "type": "object", - "properties": { - "md5": { - "title": "Md5", - "type": "string" + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" }, - "sha256": { - "title": "Sha256", - "type": "string" + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." } }, - "required": [ - "md5", - "sha256" - ] + "title": "Person", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" + }, + "TypedRelationship": { + "additionalProperties": true, "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." }, "type": { - "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", - "type": "string" + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." }, "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "size": { - "title": "Size", - "type": "integer" - }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" - }, - "item_ids": { - "title": "Item Ids", - "type": "array", - "items": { - "type": "string" - } - }, - "blocks": { - "title": "Blocks", - "type": "array", - "items": { - "type": "string" - } - }, - "name": { - "title": "Name", - "type": "string" - }, - "extension": { - "title": "Extension", - "type": "string" - }, - "original_name": { - "title": "Original Name", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" - }, - "url_path": { - "title": "Url Path", - "type": "string" - }, - "source": { - "title": "Source", - "type": "string" - }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" - }, - "metadata": { - "title": "Metadata", - "type": "object" - }, - "representation": { - "title": "Representation" - }, - "source_server_name": { - "title": "Source Server Name", - "type": "string" - }, - "source_path": { - "title": "Source Path", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" }, - "is_live": { - "title": "Is Live", - "type": "boolean" + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" }, - "checksums": { - "$ref": "#/definitions/FileChecksums" + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" } }, "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + "type" + ], + "title": "TypedRelationship", + "type": "object" }, - "DataBlockResponse": { - "title": "DataBlockResponse", - "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", - "properties": { - "blocktype": { - "title": "Blocktype", + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing an experimental sample.\n\nA physical thing in the lab that can be created, characterised\nand connected to other items.", + "properties": { + "chemform": { + "anyOf": [ + { "type": "string" }, - "block_id": { - "title": "Block Id", + { + "type": "null" + } + ], + "default": null, + "description": "A string representation of the chemical formula or composition associated with this sample.\n\nThe representation is relatively free-form; clients are expected parse and interpret HTML markup for subscripts\nand accept unicode characters for greek letters.", + "examples": [ + "Na3P", + "Na3P", + "LiNiO2@C", + "Na3+xP", + "LiNi1/3Co0.1Mn0.1O2" + ], + "title": "Chemform" + }, + "smiles": { + "anyOf": [ + { "type": "string" }, - "item_id": { - "title": "Item Id", + { + "type": "null" + } + ], + "default": null, + "description": "A SMILES string representation of the chemical structure associated with this sample.", + "title": "Smiles" + }, + "inchi": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An International Chemical Identifier (InChI) string representation of chemicals/molecules associated with this sample.", + "title": "Inchi" + }, + "inchi_key": { + "anyOf": [ + { "type": "string" }, - "collection_id": { - "title": "Collection Id", + { + "type": "null" + } + ], + "default": null, + "description": "A unique key derived from the InChI.", + "title": "Inchi Key" + }, + "GHS_codes": { + "anyOf": [ + { "type": "string" }, - "title": { - "title": "Title", - "type": "string" + { + "type": "null" + } + ], + "default": null, + "description": "A string describing any GHS hazard codes associated with this item. See https://pubchem.ncbi.nlm.nih.gov/ghs/ for code definitions.", + "examples": [ + "H224", + "H303, H316, H319" + ], + "title": "Ghs Codes" + }, + "molar_mass": { + "anyOf": [ + { + "type": "number" }, - "freeform_comment": { - "title": "Freeform Comment", + { + "type": "null" + } + ], + "default": null, + "description": "Mass per formula unit, in g/mol.", + "title": "Molar Mass" + }, + "CAS": { + "anyOf": [ + { "type": "string" }, - "file_id": { - "title": "File Id", + { + "type": "null" + } + ], + "default": null, + "description": "The CAS Registry Number for the substance described by this entry.", + "title": "Cas" + }, + "synthesis_constituents": { + "default": [], + "description": "A list of references to constituent materials giving the amount and relevant inlined details of consituent items.", + "items": { + "$ref": "#/$defs/Constituent" + }, + "title": "Synthesis Constituents", + "type": "array" + }, + "synthesis_description": { + "anyOf": [ + { "type": "string" }, - "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } - }, - "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } - }, - "warnings": { - "title": "Warnings", - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "Free-text details of the procedure applied to synthesise the sample", + "title": "Synthesis Description" + }, + "tags": { + "description": "Tags applied to this entry: references to `tags` entries (by\n`immutable_id`).", + "items": { + "$ref": "#/$defs/EntryReference" + }, + "title": "Tags", + "type": "array" + }, + "files": { + "anyOf": [ + { "items": { - "type": "string" - } - }, - "b64_encoded_image": { - "title": "B64 Encoded Image", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "bokeh_plot_data": { - "title": "Bokeh Plot Data", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object" + "$ref": "#/$defs/File" + }, + "type": "array" }, - "computed": { - "title": "Computed", - "datalab_exclude_from_load": true, + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this item.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, "type": "object" }, - "metadata": { - "title": "Metadata", - "datalab_exclude_from_load": true, - "type": "object" + { + "type": "null" } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" }, - "required": [ - "blocktype", - "block_id" - ] + "title": "Creator Ids", + "type": "array" }, - "Collection": { - "title": "Collection", - "description": "A model for representing a group of related items, for example all the samples\nbelonging to one batch or project.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", + "creators": { + "anyOf": [ + { "items": { - "type": "string" - } + "$ref": "#/$defs/Person" + }, + "type": "array" }, - "groups": { - "title": "Groups", - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { "items": { - "$ref": "#/definitions/Group" - } + "$ref": "#/$defs/Group" + }, + "type": "array" }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "samples", + "default": "samples", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", "type": "string" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", "type": "string" }, - "title": { - "title": "Title", + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { "type": "string" }, - "description": { - "title": "Description", + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "num_items": { - "title": "Num Items", - "type": "integer" + { + "type": "null" } - } + ], + "default": null, + "description": "A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).", + "title": "Date" }, - "ItemStatus": { - "title": "ItemStatus", - "description": "An enumeration of the status of items", - "enum": [ - "active", - "planned", - "disposed", - "completed", - "failed", - "other" + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "type": "string" + "default": null, + "description": "An optional human-readable/usable name for the entry.", + "title": "Name" + }, + "status": { + "$ref": "#/$defs/ItemStatus", + "default": "active", + "description": "The status of the sample, indicating its current state." } - } + }, + "required": [ + "item_id" + ], + "title": "Sample", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/startingmaterial.json b/pydatalab/schemas/startingmaterial.json index 6e5306c02..07e4bfccf 100644 --- a/pydatalab/schemas/startingmaterial.json +++ b/pydatalab/schemas/startingmaterial.json @@ -1,982 +1,1689 @@ { - "title": "StartingMaterial", - "description": "A model for representing a starting material, i.e., a chemical or precursor held\nin the lab's inventory, from which samples are made.\n\nThe model mixes container-level and substance-level information and can be used to\nrepresent either depending on preference.", - "type": "object", - "properties": { - "chemform": { - "title": "Chemform", - "example": [ - "Na3P", - "Na3P", - "LiNiO2@C", - "Na3+xP", - "LiNi1/3Co0.1Mn0.1O2" - ], - "type": "string" - }, - "smiles": { - "title": "Smiles", - "aliases": [ - "SMILES", - "smiles_representation" - ], - "type": "string" - }, - "inchi": { - "title": "Inchi", - "type": "string" - }, - "inchi_key": { - "title": "Inchi Key", - "type": "string" - }, - "GHS_codes": { - "title": "Ghs H-Codes", - "examples": [ - "H224", - "H303, H316, H319" + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" ], + "title": "AccountStatus", "type": "string" }, - "molar_mass": { - "title": "Molecular Weight", - "type": "number" - }, - "CAS": { - "title": "Substance Cas", - "type": "string" - }, - "synthesis_constituents": { - "title": "Synthesis Constituents", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Constituent" - } - }, - "synthesis_description": { - "title": "Synthesis Description", - "type": "string" - }, - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "starting_materials", - "const": "starting_materials", - "pattern": "^starting_materials$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date Acquired", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Container Name", - "type": "string" - }, - "status": { - "default": "available", - "allOf": [ - { - "$ref": "#/definitions/StartingMaterialsStatus" - } - ] - }, - "barcode": { - "title": "Barcode", - "type": "string" - }, - "date_opened": { - "title": "Date Opened", - "type": "string", - "format": "date-time" - }, - "chemical_purity": { - "title": "Chemical Purity", - "type": "string" - }, - "full_percent": { - "title": "Full %", - "type": "string" - }, - "size": { - "title": "Container Size", - "type": "string" - }, - "size_unit": { - "title": "Unit", - "type": "string" - }, - "supplier": { - "title": "Supplier", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" - }, - "comment": { - "title": "Comments", - "type": "string" - } - }, - "required": [ - "item_id" - ], - "definitions": { - "EntryReference": { - "title": "EntryReference", - "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", - "type": "object", + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "type": { - "title": "Type", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, "immutable_id": { + "description": "The immutable ID of the collection.", + "format": "objectid", "title": "Immutable Id", "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "chemform": { - "title": "Chemform", - "type": "string" - } - }, - "required": [ - "type" - ] - }, - "InlineSubstance": { - "title": "InlineSubstance", - "type": "object", - "properties": { - "name": { - "title": "Name", - "type": "string" - }, - "chemform": { - "title": "Chemform", - "type": "string" } }, "required": [ - "name" - ] + "immutable_id" + ], + "title": "CollectionReference", + "type": "object" }, "Constituent": { - "title": "Constituent", "description": "A constituent of a sample.", - "type": "object", "properties": { "item": { - "title": "Item", "anyOf": [ { - "$ref": "#/definitions/EntryReference" + "$ref": "#/$defs/EntryReference" }, { - "$ref": "#/definitions/InlineSubstance" + "$ref": "#/$defs/InlineSubstance" } - ] + ], + "description": "A reference to item (sample or starting material) entry for the constituent substance.", + "title": "Item" }, "quantity": { - "title": "Quantity", - "minimum": 0, - "type": "number" + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the constituent material used to create the sample.", + "title": "Quantity" }, "unit": { - "title": "Unit", "default": "g", + "description": "The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).", + "title": "Unit", "type": "string" } }, "required": [ - "item", - "quantity" - ] - }, - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" - ], - "type": "string" - }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", - "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" + "item" ], - "type": "string" + "title": "Constituent", + "type": "object" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "DataBlockResponse": { + "additionalProperties": true, + "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", "properties": { - "description": { - "title": "Description", - "type": "string" + "blocktype": { + "description": "The type of the block.", + "title": "Blocktype", + "type": "string" }, - "relation": { - "$ref": "#/definitions/RelationshipType" + "block_id": { + "description": "A shorthand random ID for the block.", + "title": "Block Id", + "type": "string" + }, + "item_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" + }, + "collection_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" + }, + "freeform_comment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" + }, + "file_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" + }, + "file_ids": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" + }, + "errors": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" + }, + "warnings": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" }, + "b64_encoded_image": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "default": null, + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" + }, + "bokeh_plot_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "default": null, + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" + }, + "computed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" + }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" + } + }, + "required": [ + "blocktype", + "block_id" + ], + "title": "DataBlockResponse", + "type": "object" + }, + "EntryReference": { + "additionalProperties": true, + "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", + "properties": { "type": { - "$ref": "#/definitions/KnownType" + "title": "Type", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" }, "immutable_id": { - "title": "Immutable Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Immutable Id" }, "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Item Id" }, "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Refcode" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" } }, "required": [ "type" - ] + ], + "title": "EntryReference", + "type": "object" }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", + "properties": { + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "files", + "default": "files", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" + }, + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" + }, + "item_ids": { + "description": "A list of item IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Item Ids", + "type": "array" + }, + "blocks": { + "description": "A list of block IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Blocks", + "type": "array" + }, + "name": { + "description": "The filename on disk.", + "title": "Name", + "type": "string" + }, + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", + "type": "string" + }, + "original_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" + }, + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" + }, + "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp for the original file upload.", + "title": "Time Added" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" + }, + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" + }, + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" + }, + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", + "type": "boolean" + }, + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Content checksums (MD5 and SHA-256) of the file." + } + }, + "required": [ + "name", + "extension", + "is_live" ], - "type": "string" + "title": "File", + "type": "object" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "FileChecksums": { + "description": "Content checksums for a file.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" - }, - "identifier": { - "title": "Identifier", - "type": "string" - }, - "name": { - "title": "Name", + "md5": { + "description": "The MD5 hex digest of the file contents.", + "title": "Md5", "type": "string" }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" - }, - "display_name": { - "title": "Display Name", + "sha256": { + "description": "The SHA-256 hex digest of the file contents.", + "title": "Sha256", "type": "string" } }, "required": [ - "identity_type", - "identifier", - "name" - ] - }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" + "md5", + "sha256" ], - "type": "string" + "title": "FileChecksums", + "type": "object" }, "Group": { - "title": "Group", "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", "properties": { "type": { - "title": "Type", - "default": "groups", "const": "groups", + "default": "groups", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, "members": { - "title": "Members", - "type": "array", + "description": "A list of people that belong to this group; stored on the user objects.", "items": { + "additionalProperties": true, "type": "object" - } + }, + "title": "Members", + "type": "array" }, "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" }, "description": { - "title": "Description", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" }, "managers": { - "title": "Managers", - "type": "array", + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", "items": { "anyOf": [ { + "format": "objectid", "type": "string" }, { + "additionalProperties": true, "type": "object" } ] - } + }, + "title": "Managers", + "type": "array" } - } + }, + "title": "Group", + "type": "object" }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", + "properties": { + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." + }, + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", + "type": "string" + }, + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", + "type": "string" + }, + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" + }, + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" + } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", "enum": [ - "active", - "unverified", - "deactivated" + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "InlineSubstance": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "name" + ], + "title": "InlineSubstance", + "type": "object" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" ], + "title": "KnownType", "type": "string" }, "Person": { - "title": "Person", "description": "A model that describes an individual and their digital identities.", - "type": "object", "properties": { "type": { - "title": "Type", - "default": "people", "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "identities": { - "title": "Identities", - "type": "array", + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", "items": { - "$ref": "#/definitions/Identity" - } + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" }, "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" }, "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" + }, + "gravatar_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" + }, + "managers": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" + }, + "role": { + "$ref": "#/$defs/UserRole", + "default": "user", + "description": "The role assigned to this person." + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" + }, + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." + } + }, + "title": "Person", + "type": "object" + }, + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" + }, + "StartingMaterialsStatus": { + "description": "An enumeration of the status of starting materials", + "enum": [ + "ordered", + "disposed", + "planned", + "available", + "unavailable", + "exhausted", + "other" + ], + "title": "StartingMaterialsStatus", + "type": "string" + }, + "TypedRelationship": { + "additionalProperties": true, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" }, - "gravatar_hash": { - "title": "Gravatar Hash", - "type": "string" + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } + "type": { + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." }, - "role": { - "default": "user", - "allOf": [ + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, { - "$ref": "#/definitions/UserRole" + "type": "null" } - ] + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "account_status": { - "default": "unverified", - "allOf": [ + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] - } - } - }, - "FileChecksums": { - "title": "FileChecksums", - "description": "Content checksums for a file.", - "type": "object", - "properties": { - "md5": { - "title": "Md5", - "type": "string" + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" }, - "sha256": { - "title": "Sha256", - "type": "string" + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" } }, "required": [ - "md5", - "sha256" - ] + "type" + ], + "title": "TypedRelationship", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", - "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing a starting material, i.e., a chemical or precursor held\nin the lab's inventory, from which samples are made.\n\nThe model mixes container-level and substance-level information and can be used to\nrepresent either depending on preference.", + "properties": { + "chemform": { + "anyOf": [ + { "type": "string" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + { + "type": "null" + } + ], + "default": null, + "description": "A string representation of the chemical formula or composition associated with this sample.\n\nThe representation is relatively free-form; clients are expected parse and interpret HTML markup for subscripts\nand accept unicode characters for greek letters.", + "examples": [ + "Na3P", + "Na3P", + "LiNiO2@C", + "Na3+xP", + "LiNi1/3Co0.1Mn0.1O2" + ], + "title": "Chemform" + }, + "smiles": { + "anyOf": [ + { "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "size": { - "title": "Size", - "type": "integer" - }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" - }, - "item_ids": { - "title": "Item Ids", - "type": "array", - "items": { - "type": "string" - } - }, - "blocks": { - "title": "Blocks", - "type": "array", - "items": { - "type": "string" - } - }, - "name": { - "title": "Name", + { + "type": "null" + } + ], + "default": null, + "description": "A SMILES string representation of the chemical structure associated with this sample.", + "title": "Smiles" + }, + "inchi": { + "anyOf": [ + { "type": "string" }, - "extension": { - "title": "Extension", + { + "type": "null" + } + ], + "default": null, + "description": "An International Chemical Identifier (InChI) string representation of chemicals/molecules associated with this sample.", + "title": "Inchi" + }, + "inchi_key": { + "anyOf": [ + { "type": "string" }, - "original_name": { - "title": "Original Name", + { + "type": "null" + } + ], + "default": null, + "description": "A unique key derived from the InChI.", + "title": "Inchi Key" + }, + "GHS_codes": { + "anyOf": [ + { "type": "string" }, - "location": { - "title": "Location", - "type": "string" + { + "type": "null" + } + ], + "default": null, + "description": "A string describing any GHS hazard codes associated with this item. See https://pubchem.ncbi.nlm.nih.gov/ghs/ for code definitions.", + "examples": [ + "H224", + "H303, H316, H319" + ], + "title": "Ghs Codes" + }, + "molar_mass": { + "anyOf": [ + { + "type": "number" }, - "url_path": { - "title": "Url Path", + { + "type": "null" + } + ], + "default": null, + "description": "Mass per formula unit, in g/mol.", + "title": "Molar Mass" + }, + "CAS": { + "anyOf": [ + { "type": "string" }, - "source": { - "title": "Source", + { + "type": "null" + } + ], + "default": null, + "description": "The CAS Registry Number for the substance described by this entry.", + "title": "Cas" + }, + "synthesis_constituents": { + "default": [], + "description": "A list of references to constituent materials giving the amount and relevant inlined details of consituent items.", + "items": { + "$ref": "#/$defs/Constituent" + }, + "title": "Synthesis Constituents", + "type": "array" + }, + "synthesis_description": { + "anyOf": [ + { "type": "string" }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" + { + "type": "null" + } + ], + "default": null, + "description": "Free-text details of the procedure applied to synthesise the sample", + "title": "Synthesis Description" + }, + "tags": { + "description": "Tags applied to this entry: references to `tags` entries (by\n`immutable_id`).", + "items": { + "$ref": "#/$defs/EntryReference" + }, + "title": "Tags", + "type": "array" + }, + "files": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" }, - "metadata": { - "title": "Metadata", + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this item.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, "type": "object" }, - "representation": { - "title": "Representation" + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" }, - "source_server_name": { - "title": "Source Server Name", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "starting_materials", + "default": "starting_materials", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", "type": "string" }, - "source_path": { - "title": "Source Path", + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "is_live": { - "title": "Is Live", - "type": "boolean" - }, - "checksums": { - "$ref": "#/definitions/FileChecksums" + { + "type": "null" } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" }, - "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + "title": "Relationships", + "type": "array" }, - "DataBlockResponse": { - "title": "DataBlockResponse", - "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", - "properties": { - "blocktype": { - "title": "Blocktype", + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", "type": "string" }, - "block_id": { - "title": "Block Id", + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { "type": "string" }, - "item_id": { - "title": "Item Id", + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "collection_id": { - "title": "Collection Id", + { + "type": "null" + } + ], + "default": null, + "description": "The date the item was acquired", + "title": "Date" + }, + "name": { + "anyOf": [ + { "type": "string" }, - "title": { - "title": "Title", + { + "type": "null" + } + ], + "default": null, + "description": "The name of the substance in the container.", + "title": "Name" + }, + "status": { + "$ref": "#/$defs/StartingMaterialsStatus", + "default": "available", + "description": "The status of the starting materials, indicating its current state." + }, + "barcode": { + "anyOf": [ + { "type": "string" }, - "freeform_comment": { - "title": "Freeform Comment", + { + "type": "null" + } + ], + "default": null, + "description": "A unique barcode provided by an external source, e.g., cheminventory.", + "title": "Barcode" + }, + "date_opened": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "file_id": { - "title": "File Id", + { + "type": "null" + } + ], + "default": null, + "description": "The date the item was opened", + "title": "Date Opened" + }, + "chemical_purity": { + "anyOf": [ + { "type": "string" }, - "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } - }, - "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } - }, - "warnings": { - "title": "Warnings", - "type": "array", - "items": { - "type": "string" - } - }, - "b64_encoded_image": { - "title": "B64 Encoded Image", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "bokeh_plot_data": { - "title": "Bokeh Plot Data", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object" - }, - "computed": { - "title": "Computed", - "datalab_exclude_from_load": true, - "type": "object" - }, - "metadata": { - "title": "Metadata", - "datalab_exclude_from_load": true, - "type": "object" + { + "type": "null" } - }, - "required": [ - "blocktype", - "block_id" - ] + ], + "default": null, + "description": "The chemical purity of this container with regards to the defined substance.", + "title": "Chemical Purity" }, - "Collection": { - "title": "Collection", - "description": "A model for representing a group of related items, for example all the samples\nbelonging to one batch or project.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", + "full_percent": { + "anyOf": [ + { "type": "string" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the defined substance remaining in the container, expressed as a percentage.", + "title": "Full Percent" + }, + "size": { + "anyOf": [ + { "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + { + "type": "null" + } + ], + "default": null, + "description": "The total size of the container, in units of `size_unit`.", + "title": "Size" + }, + "size_unit": { + "anyOf": [ + { "type": "string" }, - "title": { - "title": "Title", + { + "type": "null" + } + ], + "default": null, + "description": "Units for the 'size' field.", + "title": "Size Unit" + }, + "supplier": { + "anyOf": [ + { "type": "string" }, - "description": { - "title": "Description", + { + "type": "null" + } + ], + "default": null, + "description": "Supplier or manufacturer of the chemical.", + "title": "Supplier" + }, + "location": { + "anyOf": [ + { "type": "string" }, - "num_items": { - "title": "Num Items", - "type": "integer" + { + "type": "null" } - } + ], + "default": null, + "description": "The place where the container is located.", + "title": "Location" }, - "StartingMaterialsStatus": { - "title": "StartingMaterialsStatus", - "description": "An enumeration of the status of starting materials", - "enum": [ - "ordered", - "disposed", - "planned", - "available", - "unavailable", - "exhausted", - "other" + "comment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "type": "string" + "default": null, + "description": "Any additional comments or notes about the container.", + "title": "Comment" } - } + }, + "required": [ + "item_id" + ], + "title": "StartingMaterial", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/src/pydatalab/apps/__init__.py b/pydatalab/src/pydatalab/apps/__init__.py index 9f49a0950..f48a6427e 100644 --- a/pydatalab/src/pydatalab/apps/__init__.py +++ b/pydatalab/src/pydatalab/apps/__init__.py @@ -3,8 +3,11 @@ """ +import warnings from typing import TYPE_CHECKING +from pydantic.warnings import PydanticDeprecatedSince20 + if TYPE_CHECKING: # This import is required to prevent circular imports for application-specific blocks from pydatalab.blocks.base import DataBlock # noqa @@ -116,7 +119,13 @@ def load_block_plugins(): block_plugins: dict[str, type[DataBlock]] = {} for entry_point in entry_points(group="pydatalab.apps.plugins"): - block = entry_point.load() + # Plugins may not yet be migrated to pydantic v2, so tolerate the v2 + # deprecation warnings their models emit at import time (e.g. extra `Field` + # kwargs like `datalab_exclude_*`) rather than letting them error out plugin + # loading. datalab's own deprecations are still surfaced as errors. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", PydanticDeprecatedSince20) + block = entry_point.load() if not issubclass(block, DataBlock): raise ValueError(f"Plugin {block} must be a subclass of DataBlock") diff --git a/pydatalab/src/pydatalab/apps/chat/blocks.py b/pydatalab/src/pydatalab/apps/chat/blocks.py index c2bce7df3..61e73ae10 100644 --- a/pydatalab/src/pydatalab/apps/chat/blocks.py +++ b/pydatalab/src/pydatalab/apps/chat/blocks.py @@ -2,7 +2,7 @@ import warnings from langchain_core.messages import AIMessage, HumanMessage, SystemMessage -from pydantic import Field, validator +from pydantic import Field, field_validator from pydatalab.blocks.base import DataBlock from pydatalab.models import ITEM_MODELS @@ -16,15 +16,18 @@ class ChatBlockResponse(DataBlockResponse): messages: list[dict] = Field(default_factory=list) - prompt: str | None + prompt: str | None = None model: str available_models: dict[str, ModelCard] | None = Field( - datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, + validate_default=True, + json_schema_extra={"datalab_exclude_from_db": True, "datalab_exclude_from_load": True}, ) - token_count: int | None + token_count: int | None = None temperature: float - @validator("available_models", pre=True, always=True) + @field_validator("available_models", mode="before") + @classmethod def set_available_models(cls, _): return AVAILABLE_MODELS @@ -204,7 +207,7 @@ def _prepare_item_json_for_chat(self, item_id: str, item_data: dict | None = Non item_model.blocks_obj = { k: block for k, block in item_model.blocks_obj.items() if block.blocktype != "chat" } - item_data = item_model.dict(exclude_none=True, exclude_unset=True) + item_data = item_model.model_dump(exclude_none=True, exclude_unset=True) item_data["type"] = item_model.type # strip irrelevant or large fields diff --git a/pydatalab/src/pydatalab/apps/chat/models.py b/pydatalab/src/pydatalab/apps/chat/models.py index 03efc7246..756ca1970 100644 --- a/pydatalab/src/pydatalab/apps/chat/models.py +++ b/pydatalab/src/pydatalab/apps/chat/models.py @@ -9,7 +9,7 @@ class ModelCard(BaseModel): context_window: int input_cost_usd_per_MTok: float output_cost_usd_per_MTok: float - chat_client: type[BaseChatModel] | None = Field(exclude=True) + chat_client: type[BaseChatModel] | None = Field(None, exclude=True) __all__ = ("AVAILABLE_MODELS", "ModelCard") diff --git a/pydatalab/src/pydatalab/apps/nmr/blocks.py b/pydatalab/src/pydatalab/apps/nmr/blocks.py index 9ba68ed9d..de8680bf4 100644 --- a/pydatalab/src/pydatalab/apps/nmr/blocks.py +++ b/pydatalab/src/pydatalab/apps/nmr/blocks.py @@ -130,7 +130,7 @@ def read_bruker_nmr_data( metadata["pulse_program_name"] = a_dic["acqus"]["PULPROG"] metadata["title"] = topspin_title - self.data["metadata"] = NMRMetadata(**metadata).dict() + self.data["metadata"] = NMRMetadata(**metadata).model_dump() return serialized_df, metadata @@ -209,7 +209,7 @@ def read_jcamp_nmr_data( pass serialized_df = df.to_dict() if (df is not None) else None - self.data["metadata"] = NMRMetadata(**metadata).dict() + self.data["metadata"] = NMRMetadata(**metadata).model_dump() return serialized_df, metadata @@ -293,7 +293,7 @@ def read_jeol_nmr_data( metadata["nscans"] = nscans serialized_df = df.to_dict() if (df is not None) else None - self.data["metadata"] = NMRMetadata(**metadata).dict() + self.data["metadata"] = NMRMetadata(**metadata).model_dump() return serialized_df, metadata diff --git a/pydatalab/src/pydatalab/apps/nmr/models.py b/pydatalab/src/pydatalab/apps/nmr/models.py index 269684c26..73a981b4a 100644 --- a/pydatalab/src/pydatalab/apps/nmr/models.py +++ b/pydatalab/src/pydatalab/apps/nmr/models.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from pydatalab.models.blocks import DataBlockResponse @@ -46,8 +46,7 @@ class NMRMetadata(BaseModel): title: str | None = None """Title or description of the NMR experiment.""" - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") class NMRModel(DataBlockResponse): diff --git a/pydatalab/src/pydatalab/apps/xrd/blocks.py b/pydatalab/src/pydatalab/apps/xrd/blocks.py index cf7f327a3..6393464f4 100644 --- a/pydatalab/src/pydatalab/apps/xrd/blocks.py +++ b/pydatalab/src/pydatalab/apps/xrd/blocks.py @@ -264,7 +264,7 @@ def generate_xrd_plot(self, filenames: list[str | Path] | None = None) -> None: pattern_df.attrs["wavelength"] = ( f"{self.data.get('wavelength', self.defaults['wavelength'])} Å" ) - peak_information[str(f["immutable_id"])] = PeakInformation(**peak_data).dict() + peak_information[str(f["immutable_id"])] = PeakInformation(**peak_data).model_dump() if len(all_files) > 1: pattern_df["normalized intensity (staggered)"] += ind pattern_dfs.append(pattern_df) diff --git a/pydatalab/src/pydatalab/backups.py b/pydatalab/src/pydatalab/backups.py index bea81e84d..f058a62a6 100644 --- a/pydatalab/src/pydatalab/backups.py +++ b/pydatalab/src/pydatalab/backups.py @@ -92,7 +92,7 @@ def take_snapshot(snapshot_path: Path, encrypt: bool = False) -> None: LOGGER.debug("Dumping server config.") with tempfile.TemporaryDirectory() as temp_dir: with open(tmp_config := Path(temp_dir) / "config.json", "w") as f: - data = CONFIG.json(indent=2, exclude_unset=True) + data = CONFIG.model_dump_json(indent=2, exclude_unset=True) f.write(data) tar.add( diff --git a/pydatalab/src/pydatalab/blocks/base.py b/pydatalab/src/pydatalab/blocks/base.py index 3d88e1961..b149f7ad2 100644 --- a/pydatalab/src/pydatalab/blocks/base.py +++ b/pydatalab/src/pydatalab/blocks/base.py @@ -195,10 +195,10 @@ def to_db(self) -> dict: ) exclude_fields: set[str] = { f - for (f, s) in self.block_db_model.schema()["properties"].items() + for (f, s) in self.block_db_model.model_json_schema()["properties"].items() if s.get("datalab_exclude_from_db") } - return self.block_db_model(**self.data).dict( + return self.block_db_model(**self.data).model_dump( exclude=exclude_fields, exclude_unset=True, exclude_none=True, @@ -254,7 +254,7 @@ def to_web(self) -> dict[str, Any]: else: self.data.pop("warnings", None) - return self.block_db_model(**self.data).dict(exclude_unset=True, exclude_none=True) + return self.block_db_model(**self.data).model_dump(exclude_unset=True, exclude_none=True) def process_events(self, events: list[dict] | dict): """Handle any supported events passed to the block.""" @@ -350,11 +350,11 @@ def update_from_web(self, data: dict, stored_data: dict | None = None): ) exclude_fields: set[str] = { f - for (f, s) in self.block_db_model.schema()["properties"].items() + for (f, s) in self.block_db_model.model_json_schema()["properties"].items() if s.get("datalab_exclude_from_load") } [data.pop(f, None) for f in exclude_fields] - self.data.update(self.block_db_model(**data).dict()) + self.data.update(self.block_db_model(**data).model_dump(exclude_unset=True)) # Fields stripped above (e.g., `metadata`, `computed`) are server-authoritative: # writable only via block events or block code, never from the web payload. # Without restoring them from the stored state here, the model defaults diff --git a/pydatalab/src/pydatalab/config.py b/pydatalab/src/pydatalab/config.py index 74c6b8c20..1b58a1c29 100644 --- a/pydatalab/src/pydatalab/config.py +++ b/pydatalab/src/pydatalab/config.py @@ -9,12 +9,13 @@ from pydantic import ( AnyUrl, BaseModel, - BaseSettings, + ConfigDict, Field, ValidationError, - root_validator, - validator, + field_validator, + model_validator, ) +from pydantic_settings import BaseSettings, SettingsConfigDict from pydatalab.models import Person from pydatalab.models.utils import RandomAlphabeticalRefcodeFactory, RefCodeFactory @@ -24,7 +25,7 @@ config_logger = logging.getLogger("pydatalab.config") -def config_file_settings(settings: BaseSettings) -> dict[str, Any]: +def config_file_settings(settings_cls: type[BaseSettings] | None = None) -> dict[str, Any]: """Returns a dictionary of server settings loaded from the default or specified JSON config file location (via the env var `PYDATALAB_CONFIG_FILE`). @@ -34,7 +35,7 @@ def config_file_settings(settings: BaseSettings) -> dict[str, Any]: res = {} if config_file.is_file(): config_logger.debug("Loading from config file at %s", config_file) - config_file_content = config_file.read_text(encoding=settings.__config__.env_file_encoding) + config_file_content = config_file.read_text(encoding="utf-8") try: res = json.loads(config_file_content) @@ -51,20 +52,20 @@ def config_file_settings(settings: BaseSettings) -> dict[str, Any]: class DeploymentMetadata(BaseModel): """A model for specifying metadata about a datalab deployment.""" - maintainer: Person | None + maintainer: Person | None = None issue_tracker: AnyUrl | None = Field("https://github.com/datalab-org/datalab/issues") - homepage: AnyUrl | None + homepage: AnyUrl | None = None source_repository: AnyUrl | None = Field("https://github.com/datalab-org/datalab") - @validator("maintainer") + @field_validator("maintainer") + @classmethod def strip_fields_from_person(cls, v): if not v.contact_email: raise ValueError("Must provide contact email for maintainer.") return Person(contact_email=v.contact_email, display_name=v.display_name) - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class BackupStrategy(BaseModel): @@ -75,7 +76,8 @@ class BackupStrategy(BaseModel): description="Whether this backup strategy is active; i.e., whether it is actually used. All strategies will be disabled in testing scenarios.", ) hostname: str | None = Field( - description="The hostname of the SSH-accessible server on which to store the backup (`None` indicates local backups)." + None, + description="The hostname of the SSH-accessible server on which to store the backup (`None` indicates local backups).", ) location: Path = Field( description="The location under which to store the backups on the host. Each backup will be date-stamped and stored in a subdirectory of this location." @@ -87,7 +89,7 @@ class BackupStrategy(BaseModel): frequency: str | None = Field( None, description="The frequency of the backup, described in the crontab syntax.", - pattern=r"^(?:\*|\d+(?:-\d+)?)(?:\/\d+)?(?:,\d+(?:-\d+)?(?:\/\d+)?)*$", + examples=["5 4 * * *", "5 2 1 1,4,7,10 *"], ) notification_email_address: str | None = Field( None, description="An email address to send backup notifications to." @@ -143,7 +145,7 @@ class ServerConfig(BaseSettings): False, description="Whether to run the server in testing mode, i.e., without user auth." ) - SECRET_KEY: str = Field( + SECRET_KEY: str | None = Field( None, description="The secret key to use for Flask. This value should be changed and/or loaded from an environment variable for production deployments.", ) @@ -251,6 +253,11 @@ class ServerConfig(BaseSettings): description="Maximum number of items that can be created in a single batch operation.", ) + ENABLE_TAGS: bool = Field( + False, + description="Whether to enable the (experimental) item tags feature and its `/tags` API routes.", + ) + ASYNC_BLOCK_TYPES: list[str] = Field( [], description="A list of block type slugs (e.g. ['cycle', 'xrd']) that should be processed asynchronously via the task queue. Defaults to no blocks.", @@ -280,18 +287,26 @@ class ServerConfig(BaseSettings): description="The desired backup configuration.", ) - @root_validator - def validate_cache_ages(cls, values): - if values.get("REMOTE_CACHE_MIN_AGE") > values.get("REMOTE_CACHE_MAX_AGE"): + @model_validator(mode="after") + def validate_cache_ages(self): + """Check the cache ages after parsing, so that the defaults are covered even + when only one of the two values is set explicitly. + """ + min_age = self.REMOTE_CACHE_MIN_AGE + max_age = self.REMOTE_CACHE_MAX_AGE + + if min_age > max_age: raise RuntimeError( - f"The maximum cache age must be greater than the minimum cache age: min {values.get('REMOTE_CACHE_MIN_AGE')=}, max {values.get('REMOTE_CACHE_MAX_AGE')=}" + f"The maximum cache age must be greater than the minimum cache age: min {min_age=}, max {max_age=}" ) - return values + return self - @validator("SECRET_KEY", pre=True, always=True) - def validate_secret_key(cls, v, values): + @field_validator("SECRET_KEY", mode="before") + @classmethod + def validate_secret_key(cls, v, info): if v is None: - if values.get("TESTING"): + data = info.data if hasattr(info, "data") else {} + if data.get("TESTING"): config_logger.error( "`CONFIG.TESTING` is enabled - generating a deterministic secret key for testing purposes. This MUST be updated for production deployments." ) @@ -301,7 +316,8 @@ def validate_secret_key(cls, v, values): return v - @validator("ROOT_PATH") + @field_validator("ROOT_PATH", mode="before") + @classmethod def validate_root_path(cls, v): if not v.startswith("/"): v = "/" + v @@ -311,20 +327,22 @@ def validate_root_path(cls, v): return v - @validator("IDENTIFIER_PREFIX", pre=True, always=True) - def validate_identifier_prefix(cls, v, values): + @field_validator("IDENTIFIER_PREFIX", mode="before") + @classmethod + def validate_identifier_prefix(cls, v, info): """Make sure that the identifier prefix is set and is valid, raising clear error messages if not. If in testing mode, then set the prefix to 'test' too. The app startup will test for this value and should also warn aggressively that this is unset. """ - if values.get("TESTING") or v is None: + data = info.data if hasattr(info, "data") else {} + if data.get("TESTING") or v is None: return "test" if len(v) > 12: raise RuntimeError( - "Identifier prefix must be less than 12 characters long, received {v=}" + f"Identifier prefix must be less than 12 characters long, received {v=}" ) # test a trial refcode @@ -336,18 +354,23 @@ def validate_identifier_prefix(cls, v, values): raise RuntimeError( f"Invalid identifier prefix: {v}. Validation with refcode `AAAAAA` returned error: {exc}" ) - return v - @root_validator - def deactivate_backup_strategies_during_testing(cls, values): - if values.get("TESTING"): - for name in values.get("BACKUP_STRATEGIES", {}): - values["BACKUP_STRATEGIES"][name].active = False + @model_validator(mode="after") + def deactivate_backup_strategies_during_testing(self): + """Disable every backup strategy when running in testing mode. - return values + This runs after parsing so that the default strategies are covered (defaults + never appear in the raw input) and so that strategies loaded from a config + file are `BackupStrategy` instances rather than plain dicts. + """ + if self.TESTING and self.BACKUP_STRATEGIES: + for strategy in self.BACKUP_STRATEGIES.values(): + strategy.active = False + return self - @validator("LOG_FILE") + @field_validator("LOG_FILE", mode="before") + @classmethod def make_missing_log_directory(cls, v): """Make sure that the log directory exists and is writable.""" if v is None: @@ -360,25 +383,40 @@ def make_missing_log_directory(cls, v): raise RuntimeError(f"Unable to create log file at {v}") from exc return v - class Config: - env_prefix = "pydatalab_" - extra = "allow" - env_file = ".env" - env_file_encoding = "utf-8" - validate_assignment = True - - @classmethod - def customise_sources( - cls, + def update(self, values: dict): + """Update the configuration with new values, following Pydantic v1 behavior.""" + for key, value in values.items(): + key_upper = key.upper() + if hasattr(self, key_upper): + setattr(self, key_upper, value) + else: + setattr(self, key_upper, value) + + model_config = SettingsConfigDict( + env_prefix="PYDATALAB_", + extra="allow", + env_file=".env", + env_file_encoding="utf-8", + validate_assignment=True, + case_sensitive=False, + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings, + env_settings, + dotenv_settings, + file_secret_settings, + ): + return ( init_settings, env_settings, + dotenv_settings, + config_file_settings, file_secret_settings, - ): - return (init_settings, env_settings, config_file_settings, file_secret_settings) - - def update(self, mapping): - for key in mapping: - setattr(self, key.upper(), mapping[key]) + ) CONFIG: ServerConfig = ServerConfig() diff --git a/pydatalab/src/pydatalab/errors.py b/pydatalab/src/pydatalab/errors.py index 5868d9124..0996a54ad 100644 --- a/pydatalab/src/pydatalab/errors.py +++ b/pydatalab/src/pydatalab/errors.py @@ -70,6 +70,7 @@ def handle_pydantic_validation_error(exc: ValidationError) -> tuple[Response, in These always come from malformed data, so should not necessarily trigger the Flask debugger. """ + LOGGER.critical("Pydantic validation error: %s", exc, exc_info=True) response = { "title": exc.__class__.__name__, "message": str(exc.args[:]) if exc.args else "", diff --git a/pydatalab/src/pydatalab/export.py b/pydatalab/src/pydatalab/export.py index 8835ba341..a06551765 100644 --- a/pydatalab/src/pydatalab/export.py +++ b/pydatalab/src/pydatalab/export.py @@ -12,10 +12,10 @@ from pydatalab.config import CONFIG from pydatalab.logger import LOGGER from pydatalab.models import ITEM_MODELS -from pydatalab.mongo import flask_mongo -from pydatalab.routes.v0_1.items import ( +from pydatalab.mongo import ( creators_lookup, files_lookup, + flask_mongo, groups_lookup, ) @@ -112,7 +112,7 @@ def write_eln_file( item_folder = root_folder / item[primary_key] - item_metadata = ITEM_MODELS[item.get("type")](**item).json(indent=2) + item_metadata = ITEM_MODELS[item.get("type")](**item).model_dump_json(indent=2) zipf.writestr(str(item_folder / "metadata.json"), item_metadata) for file in item.get("files", []): @@ -378,7 +378,7 @@ def create_eln_file( item_data = list(cursor)[0] ItemModel = ITEM_MODELS[item_data["type"]] - item_data = ItemModel(**item_data).dict() + item_data = ItemModel(**item_data).model_dump() except IndexError: item_data = None @@ -420,7 +420,7 @@ def create_eln_file( for ind, item in enumerate(all_items): ItemModel = ITEM_MODELS[item["type"]] - _all_items.append(ItemModel(**item).dict()) + _all_items.append(ItemModel(**item).model_dump()) all_items = _all_items diff --git a/pydatalab/src/pydatalab/feature_flags.py b/pydatalab/src/pydatalab/feature_flags.py index 1f56be2e3..09e0ffa8a 100644 --- a/pydatalab/src/pydatalab/feature_flags.py +++ b/pydatalab/src/pydatalab/feature_flags.py @@ -27,6 +27,7 @@ class FeatureFlags(BaseModel): auth_mechanisms: AuthMechanisms = AuthMechanisms() ai_integrations: AIIntegrations = AIIntegrations() email_notifications: bool = False + tags: bool = False FEATURE_FLAGS: FeatureFlags = FeatureFlags() @@ -59,6 +60,8 @@ def check_feature_flags(app): """ + FEATURE_FLAGS.tags = CONFIG.ENABLE_TAGS + if CONFIG.EMAIL_AUTH_SMTP_SETTINGS is None: LOGGER.warning( "No email auth SMTP settings provided, email registration will not be enabled." diff --git a/pydatalab/src/pydatalab/file_utils.py b/pydatalab/src/pydatalab/file_utils.py index 33bb8958b..5bda4eabc 100644 --- a/pydatalab/src/pydatalab/file_utils.py +++ b/pydatalab/src/pydatalab/file_utils.py @@ -336,7 +336,7 @@ def get_file_info_by_id(file_id: str | ObjectId, update_if_live: bool = True) -> if update_if_live and file_info.is_live: file_info = _check_and_sync_file(file_info, file_id) - return file_info.dict() + return file_info.model_dump() def update_uploaded_file(file: FileStorage, file_id: ObjectId, size_bytes: int | None = None): @@ -405,7 +405,7 @@ def update_uploaded_file(file: FileStorage, file_id: ObjectId, size_bytes: int | file.save(updated_file_entry.location) - ret = updated_file_entry.dict() + ret = updated_file_entry.model_dump() ret.update({"_id": file_id}) return ret @@ -498,7 +498,7 @@ def save_uploaded_file( f"Cannot store file: insufficient space available on disk (required: {size_bytes // 1024**3} GB). Please contact your datalab administrator." ) file_collection = client.get_database().files - result = file_collection.insert_one(new_file_document.dict(), session=session) + result = file_collection.insert_one(new_file_document.model_dump(), session=session) if not result.acknowledged: raise RuntimeError( f"db operation failed when trying to insert new file. Result: {result}" @@ -534,7 +534,7 @@ def save_uploaded_file( f"db operation failed when trying to insert new file ObjectId into sample: {item_id}" ) - ret = updated_file_entry.dict() + ret = updated_file_entry.model_dump() ret.update({"_id": inserted_id}) return ret @@ -610,7 +610,7 @@ def add_file_from_remote_directory( creator_ids=creator_ids if creator_ids is not None else [], ) - result = file_collection.insert_one(new_file_document.dict()) + result = file_collection.insert_one(new_file_document.model_dump()) if not result.acknowledged: raise OSError(f"db operation failed when trying to insert new file. Result: {result}") diff --git a/pydatalab/src/pydatalab/main.py b/pydatalab/src/pydatalab/main.py index b4e7cd131..1bfb0f518 100644 --- a/pydatalab/src/pydatalab/main.py +++ b/pydatalab/src/pydatalab/main.py @@ -44,7 +44,7 @@ def create_app( CONFIG.update(config_override) app.config.from_prefixed_env() - app.config.update(CONFIG.dict()) + app.config.update(CONFIG.model_dump()) # This value will still be overwritten by any dotenv values app.config["MAIL_DEBUG"] = app.config.get("MAIL_DEBUG") or CONFIG.TESTING @@ -52,7 +52,7 @@ def create_app( # percolate datalab mail settings up to the `MAIL_` env vars/app config # for use by Flask Mail if CONFIG.EMAIL_AUTH_SMTP_SETTINGS is not None: - mail_settings = CONFIG.EMAIL_AUTH_SMTP_SETTINGS.dict() + mail_settings = CONFIG.EMAIL_AUTH_SMTP_SETTINGS.model_dump() for key in mail_settings: app.config[key] = mail_settings[key] diff --git a/pydatalab/src/pydatalab/models/__init__.py b/pydatalab/src/pydatalab/models/__init__.py index 92db5f475..723a5a999 100644 --- a/pydatalab/src/pydatalab/models/__init__.py +++ b/pydatalab/src/pydatalab/models/__init__.py @@ -1,20 +1,35 @@ -from pydantic import BaseModel +import functools from pydatalab.models.cells import Cell from pydatalab.models.collections import Collection from pydatalab.models.equipment import Equipment from pydatalab.models.files import File +from pydatalab.models.items import Item from pydatalab.models.people import Person from pydatalab.models.samples import Sample from pydatalab.models.starting_materials import StartingMaterial +from pydatalab.models.tags import Tag from pydatalab.models.versions import ItemVersion -ITEM_MODELS: dict[str, type[BaseModel]] = { - "samples": Sample, - "starting_materials": StartingMaterial, - "cells": Cell, - "equipment": Equipment, -} + +@functools.lru_cache(maxsize=1) +def get_item_models() -> dict[str, type[Item]]: + """ + Returns a dictionary of item models keyed by their type. + """ + return { + model.model_json_schema()["properties"]["type"]["default"]: model + for model in Item.__subclasses__() + } + + +@functools.lru_cache(maxsize=1) +def generate_schemas() -> dict[str, dict]: + return {t: model.model_json_schema(by_alias=False) for t, model in get_item_models().items()} + + +ITEM_MODELS: dict[str, type[Item]] = get_item_models() +ITEM_SCHEMAS = generate_schemas() __all__ = ( "File", @@ -24,6 +39,8 @@ "Cell", "Collection", "Equipment", + "Tag", "ItemVersion", "ITEM_MODELS", + "ITEM_SCHEMAS", ) diff --git a/pydatalab/src/pydatalab/models/blocks.py b/pydatalab/src/pydatalab/models/blocks.py index 9e33aec65..e6d9244e3 100644 --- a/pydatalab/src/pydatalab/models/blocks.py +++ b/pydatalab/src/pydatalab/models/blocks.py @@ -1,6 +1,6 @@ -from pydantic import BaseModel, Field +from pydantic import ConfigDict, Field -from pydatalab.models.utils import JSON_ENCODERS, PyObjectId +from pydatalab.models.utils import BaseModel, PyObjectId class DataBlockResponse(BaseModel): @@ -11,6 +11,8 @@ class DataBlockResponse(BaseModel): where possible. """ + model_config = ConfigDict(validate_by_name=True, extra="allow") + blocktype: str """The type of the block.""" @@ -42,26 +44,31 @@ class DataBlockResponse(BaseModel): """Any warnings that occurred during block processing.""" b64_encoded_image: dict[str, str] | None = Field( - datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, + json_schema_extra={"datalab_exclude_from_db": True, "datalab_exclude_from_load": True}, ) """Any base64-encoded image data associated with the block, keyed by `file_id`.""" bokeh_plot_data: dict | None = Field( - datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, + json_schema_extra={"datalab_exclude_from_db": True, "datalab_exclude_from_load": True}, ) """A JSON-encoded string containing the Bokeh plot data, if any.""" - computed: dict | None = Field(default=None, datalab_exclude_from_load=True) + computed: dict | None = Field( + default=None, json_schema_extra={"datalab_exclude_from_load": True} + ) """Any processed or computed data associated with the block, small enough to store and filter directly in the database, i.e., strings or a few hundred numbers not exceeding 16KB in size. Examples could include peak positions, and widths, but not the full spectrum. """ - metadata: dict | None = Field(default=None, datalab_exclude_from_load=True) + processed: dict | None = Field( + default=None, json_schema_extra={"datalab_exclude_from_load": True} + ) + + metadata: dict | None = Field( + default=None, json_schema_extra={"datalab_exclude_from_load": True} + ) """Any structured metadata associated with the block, for example, experimental acquisition parameters.""" - - class Config: - allow_population_by_field_name = True - json_encoders = JSON_ENCODERS - extra = "allow" diff --git a/pydatalab/src/pydatalab/models/cells.py b/pydatalab/src/pydatalab/models/cells.py index 03e23d384..930717a58 100644 --- a/pydatalab/src/pydatalab/models/cells.py +++ b/pydatalab/src/pydatalab/models/cells.py @@ -1,13 +1,16 @@ from enum import Enum +from typing import Literal -from pydantic import Field, root_validator, validator +from pydantic import ( + Field, + field_validator, + model_validator, +) from pydatalab.models.entries import EntryReference from pydatalab.models.items import Item from pydatalab.models.utils import CellStatus, Constituent -# from pydatalab.logger import LOGGER - class CellComponent(Constituent): ... @@ -31,72 +34,75 @@ class Cell(Item): components and the format it was built in. """ - type: str = Field("cells", const="cells", pattern="^cells$") + type: Literal["cells"] = "cells" - cell_format: CellFormat | None + cell_format: CellFormat | None = None """The form factor of the cell, e.g., coin, pouch, in situ or otherwise.""" - cell_format_description: str | None + cell_format_description: str | None = None """Additional human-readable description of the cell form factor, e.g., 18650, AMPIX, CAMPIX""" - cell_preparation_description: str | None + cell_preparation_description: str | None = None + """Description of how the cell was prepared.""" - characteristic_mass: float | None + characteristic_mass: float | None = None """The characteristic mass of the cell in milligrams. Can be used to normalize capacities.""" - characteristic_chemical_formula: str | None + characteristic_chemical_formula: str | None = None """The chemical formula of the active material. Can be used to calculated molar mass in g/mol for normalizing capacities.""" - characteristic_molar_mass: float | None + characteristic_molar_mass: float | None = None """The molar mass of the active material, in g/mol. Will be inferred from the chemical formula, or can be supplied if it cannot be supplied""" - positive_electrode: list[CellComponent] = [] + positive_electrode: list[CellComponent] = Field(default_factory=list) + """The components of the positive electrode (cathode) of the cell.""" + + negative_electrode: list[CellComponent] = Field(default_factory=list) + """The components of the negative electrode (anode) of the cell.""" - negative_electrode: list[CellComponent] = [] + electrolyte: list[CellComponent] = Field(default_factory=list) + """The components of the electrolyte of the cell.""" - electrolyte: list[CellComponent] = [] + active_ion: str | None = None + """The active ion species.""" active_ion_charge: float = 1 + """The charge of the active ion species.""" status: CellStatus = Field(default=CellStatus.ACTIVE) """The status of the cells, indicating its current state.""" - @validator("characteristic_molar_mass", always=True, pre=True) - def set_molar_mass(cls, v, values): + @field_validator("characteristic_molar_mass", mode="before") + @classmethod + def set_molar_mass(cls, v, info): from periodictable import formula - if not v: - chemical_formula = values.get("characteristic_chemical_formula") - + if not v and hasattr(info, "data") and info.data: + chemical_formula = info.data.get("characteristic_chemical_formula") if chemical_formula: try: return formula(chemical_formula).mass except Exception: return None - return v - @root_validator - def add_missing_electrode_relationships(cls, values): + @model_validator(mode="after") + def add_missing_electrode_relationships(self): """Add any missing cell component constituents to parent relationships""" from pydatalab.models.relationships import RelationshipType, TypedRelationship - existing_parthood_relationships = {} - if values.get("relationships") is not None: - # Index by refcode *and* item_id so a stored relationship carrying an - # item_id still matches a refcode-enriched constituent (and vice-versa). - existing_parthood_relationships = { - identifier: relationship - for relationship in values["relationships"] - if relationship.relation == RelationshipType.PARTHOOD - for identifier in (relationship.refcode, relationship.item_id) - if identifier - } - else: - values["relationships"] = [] + # Index by refcode *and* item_id so a stored relationship carrying an + # item_id still matches a refcode-enriched constituent (and vice-versa). + existing_parthood_relationships = { + identifier: relationship + for relationship in self.relationships + if relationship.relation == RelationshipType.PARTHOOD + for identifier in (relationship.refcode, relationship.item_id) + if identifier + } for component in ("positive_electrode", "negative_electrode", "electrolyte"): - for constituent in values.get(component, []): + for constituent in getattr(self, component): if not isinstance(constituent.item, EntryReference): continue @@ -114,7 +120,7 @@ def add_missing_electrode_relationships(cls, values): type=constituent.item.type, description="Is a constituent of", ) - values["relationships"].append(relationship) + self.relationships.append(relationship) else: # Back-fill any identifier missing from the stored relationship relationship.refcode = relationship.refcode or refcode @@ -127,4 +133,4 @@ def add_missing_electrode_relationships(cls, values): if identifier: existing_parthood_relationships[identifier] = relationship - return values + return self diff --git a/pydatalab/src/pydatalab/models/collections.py b/pydatalab/src/pydatalab/models/collections.py index 9d839f25d..9423ef330 100644 --- a/pydatalab/src/pydatalab/models/collections.py +++ b/pydatalab/src/pydatalab/models/collections.py @@ -1,4 +1,8 @@ -from pydantic import Field, root_validator +from typing import Literal + +from pydantic import ( + model_validator, +) from pydatalab.models.entries import Entry from pydatalab.models.traits import HasBlocks, HasOwner @@ -11,21 +15,22 @@ class Collection(Entry, HasOwner, HasBlocks): """ - type: str = Field("collections", const="collections", pattern="^collections$") + type: Literal["collections"] = "collections" - collection_id: HumanReadableIdentifier = Field(None) + collection_id: HumanReadableIdentifier | None = None """A short human-readable/usable name for the collection.""" - title: str | None + title: str | None = None """A descriptive title for the collection.""" - description: str | None + description: str | None = None """A description of the collection, either in plain-text or a markup language.""" - num_items: int | None = Field(None) + num_items: int | None = None """Inlined number of items associated with this collection.""" - @root_validator + @model_validator(mode="before") + @classmethod def check_ids(cls, values): if not any(values.get(k) is not None for k in ("collection_id", "immutable_id")): raise ValueError("Collection must have at least collection_id or immutable_id") diff --git a/pydatalab/src/pydatalab/models/entries.py b/pydatalab/src/pydatalab/models/entries.py index 694a312ad..a5b61f9de 100644 --- a/pydatalab/src/pydatalab/models/entries.py +++ b/pydatalab/src/pydatalab/models/entries.py @@ -1,10 +1,10 @@ import abc -from pydantic import BaseModel, Field, root_validator +from pydantic import ConfigDict, Field, model_validator from pydatalab.models.relationships import TypedRelationship from pydatalab.models.utils import ( - JSON_ENCODERS, + BaseModel, EntryReference, IsoformatDateTime, PyObjectId, @@ -20,21 +20,22 @@ class Entry(BaseModel, abc.ABC): type: str """The resource type of the entry.""" - immutable_id: PyObjectId = Field( + immutable_id: PyObjectId | None = Field( None, title="Immutable ID", alias="_id", - format="uuid", + json_schema_extra={"format": "uuid"}, ) """The immutable database ID of the entry.""" last_modified: IsoformatDateTime | None = None """The timestamp at which the entry was last modified.""" - relationships: list[TypedRelationship] | None = None + relationships: list[TypedRelationship] = Field(default_factory=list) """A list of related entries and their types.""" - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def check_id_names(cls, values): """Slightly upsetting hack: this case *should* be covered by the pydantic setting for populating fields by alias names. @@ -42,6 +43,10 @@ def check_id_names(cls, values): if "_id" in values: values["immutable_id"] = values.pop("_id") + # Coerce a stored/explicit null into an empty list so the field is always a list. + if values.get("relationships") is None: + values["relationships"] = [] + return values def to_reference(self, additional_fields: list[str] | None = None) -> "EntryReference": @@ -63,7 +68,4 @@ def to_reference(self, additional_fields: list[str] | None = None) -> "EntryRefe return EntryReference(**data) - class Config: - allow_population_by_field_name = True - json_encoders = JSON_ENCODERS - extra = "ignore" + model_config = ConfigDict(populate_by_name=True, extra="ignore") diff --git a/pydatalab/src/pydatalab/models/equipment.py b/pydatalab/src/pydatalab/models/equipment.py index 32c8b949c..7164ccb2f 100644 --- a/pydatalab/src/pydatalab/models/equipment.py +++ b/pydatalab/src/pydatalab/models/equipment.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import Field from pydatalab.models.items import Item @@ -13,18 +15,18 @@ class Equipment(Item): the items measured on it. """ - type: str = Field("equipment", const="equipment", pattern="^equipment$") + type: Literal["equipment"] = "equipment" - serial_numbers: str | None + serial_numbers: str | None = None """A string describing one or more serial numbers for the instrument.""" - manufacturer: str | None + manufacturer: str | None = None """The manufacturer of this piece of equipment""" - location: str | None + location: str | None = None """Place where the equipment is located""" - contact: str | None + contact: str | None = None """Contact information for equipment (e.g., email address or phone number).""" status: EquipmentStatus = Field(default=EquipmentStatus.WORKING) diff --git a/pydatalab/src/pydatalab/models/export_task.py b/pydatalab/src/pydatalab/models/export_task.py index 171567038..e2728888d 100644 --- a/pydatalab/src/pydatalab/models/export_task.py +++ b/pydatalab/src/pydatalab/models/export_task.py @@ -1,9 +1,9 @@ from datetime import datetime, timezone from enum import Enum -from pydantic import BaseModel, Field +from pydantic import ConfigDict, Field -from pydatalab.models.utils import PyObjectId +from pydatalab.models.utils import BaseModel, PyObjectId class ExportStatus(str, Enum): @@ -18,23 +18,34 @@ class ExportStatus(str, Enum): class ExportTask(BaseModel): """Model for an export task.""" - task_id: str = Field(..., description="Unique identifier for the export task") - collection_id: str | None = Field(None, description="ID of the collection being exported") - item_id: str | None = Field(None, description="ID of the item being exported") - export_type: str = Field( - default="collection", description="Type of export: 'collection' or 'item' or 'graph'" - ) - status: ExportStatus = Field( - default=ExportStatus.PENDING, description="Current status of the task" - ) - creator_id: PyObjectId = Field(..., description="ID of the user who created the export") - created_at: datetime = Field( - default_factory=lambda: datetime.now(tz=timezone.utc), - description="When the task was created", - ) - completed_at: datetime | None = Field(None, description="When the task was completed") - file_path: str | None = Field(None, description="Path to the generated .eln file") - error_message: str | None = Field(None, description="Error message if status is ERROR") - - class Config: - use_enum_values = True + task_id: str + """Unique identifier for the export task""" + + collection_id: str | None = None + """ID of the collection being exported""" + + item_id: str | None = None + """ID of the item being exported""" + + export_type: str = "collection" + """Type of export: 'collection' or 'item' or 'graph'""" + + status: ExportStatus = ExportStatus.PENDING + """Current status of the task""" + + creator_id: PyObjectId + """ID of the user who created the export""" + + created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc)) + """When the task was created""" + + completed_at: datetime | None = None + """When the task was completed""" + + file_path: str | None = None + """Path to the generated .eln file""" + + error_message: str | None = None + """Error message if status is ERROR""" + + model_config = ConfigDict(use_enum_values=True) diff --git a/pydatalab/src/pydatalab/models/files.py b/pydatalab/src/pydatalab/models/files.py index 1cdc744af..7e965a537 100644 --- a/pydatalab/src/pydatalab/models/files.py +++ b/pydatalab/src/pydatalab/models/files.py @@ -1,10 +1,10 @@ -from typing import Any +from typing import Any, Literal -from pydantic import BaseModel, Field +from pydantic import Field from pydatalab.models.entries import Entry from pydatalab.models.traits import HasOwner, HasRevisionControl -from pydatalab.models.utils import IsoformatDateTime +from pydatalab.models.utils import BaseModel, IsoformatDateTime, PyObjectId class FileChecksums(BaseModel): @@ -20,18 +20,18 @@ class FileChecksums(BaseModel): class File(Entry, HasOwner, HasRevisionControl): """A model for representing a file that has been tracked or uploaded to datalab.""" - type: str = Field("files", const="files", pattern="^files$") + type: Literal["files"] = "files" - size: int | None + size: int | None = None """The size of the file on disk in bytes.""" - last_modified_remote: IsoformatDateTime | None + last_modified_remote: IsoformatDateTime | None = None """The last date/time at which the remote file was modified.""" - item_ids: list[str] + item_ids: list[str] = Field(default_factory=list) """A list of item IDs associated with this file.""" - blocks: list[str] + blocks: list[str] = Field(default_factory=list) """A list of block IDs associated with this file.""" name: str @@ -40,34 +40,44 @@ class File(Entry, HasOwner, HasRevisionControl): extension: str """The file extension that the file was uploaded with.""" - original_name: str | None + original_name: str | None = None """The raw filename as uploaded.""" - location: str | None + location: str | None = None """The location of the file on disk.""" - url_path: str | None + url_path: str | None = None """The path to a remote file.""" - source: str | None + source: str | None = None """The source of the file, e.g. 'remote' or 'uploaded'.""" - time_added: IsoformatDateTime + time_added: IsoformatDateTime | None = None """The timestamp for the original file upload.""" - metadata: dict[Any, Any] | None + metadata: dict[Any, Any] | None = None """Any additional metadata.""" - representation: Any | None + representation: Any | None = None - source_server_name: str | None + source_server_name: str | None = None """The server name at which the file is stored.""" - source_path: str | None + source_path: str | None = None """The path to the file on the remote resource.""" is_live: bool """Whether or not the file should be watched for future updates.""" - checksums: FileChecksums | None + checksums: FileChecksums | None = None """Content checksums (MD5 and SHA-256) of the file.""" + + +class HasFiles(BaseModel): + """Trait mixin for models that can have files attached to them.""" + + files: list["File"] | None = None + """Any files attached to this item.""" + + file_ObjectIds: list[PyObjectId] = Field(default_factory=list) + """Links to object IDs of files stored within the database.""" diff --git a/pydatalab/src/pydatalab/models/items.py b/pydatalab/src/pydatalab/models/items.py index 9ec1646a3..e91ffd525 100644 --- a/pydatalab/src/pydatalab/models/items.py +++ b/pydatalab/src/pydatalab/models/items.py @@ -1,13 +1,14 @@ import abc -from pydantic import validator +from pydantic import field_validator from pydatalab.models.entries import Entry +from pydatalab.models.files import HasFiles from pydatalab.models.traits import ( HasBlocks, - HasFiles, HasOwner, HasRevisionControl, + HasTags, IsCollectable, ) from pydatalab.models.utils import ( @@ -17,7 +18,9 @@ ) -class Item(Entry, HasOwner, HasRevisionControl, IsCollectable, HasBlocks, HasFiles, abc.ABC): +class Item( + Entry, HasOwner, HasRevisionControl, IsCollectable, HasBlocks, HasFiles, HasTags, abc.ABC +): """The generic model for data types that will be exposed with their own named endpoints. `Item` is the abstract base shared by every physical item type: samples, cells, @@ -26,27 +29,26 @@ class Item(Entry, HasOwner, HasRevisionControl, IsCollectable, HasBlocks, HasFil populates itself. """ - refcode: Refcode = None # type: ignore - """A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) - and a locally unique string, ideally created with some consistent scheme. - """ + refcode: Refcode | None = None + """A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.""" item_id: HumanReadableIdentifier """A locally unique, human-readable identifier for the entry. This ID is mutable.""" - description: str | None + description: str | None = None """A description of the item, either in plain-text or a markup language.""" - date: IsoformatDateTime | None + date: IsoformatDateTime | None = None """A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).""" - name: str | None + name: str | None = None """An optional human-readable/usable name for the entry.""" - status: str | None + status: str | None = None """The status of the item, with allowed values defined by the specific item class.""" - @validator("refcode", pre=True, always=True) + @field_validator("refcode", mode="before") + @classmethod def refcode_validator(cls, v): """Generate a refcode if not provided.""" diff --git a/pydatalab/src/pydatalab/models/people.py b/pydatalab/src/pydatalab/models/people.py index a86a2d4a2..5a3897419 100644 --- a/pydatalab/src/pydatalab/models/people.py +++ b/pydatalab/src/pydatalab/models/people.py @@ -1,14 +1,17 @@ from __future__ import annotations from enum import Enum +from typing import Annotated, Literal -import bson -import bson.errors -from pydantic import BaseModel, ConstrainedStr, Field, parse_obj_as, validator from pydantic import EmailStr as PydanticEmailStr +from pydantic import ( + Field, + StringConstraints, + field_validator, +) from pydatalab.models.entries import Entry -from pydatalab.models.utils import HumanReadableIdentifier, PyObjectId, UserRole +from pydatalab.models.utils import BaseModel, HumanReadableIdentifier, PyObjectId, UserRole class IdentityType(str, Enum): @@ -36,29 +39,32 @@ class Identity(BaseModel): name: str """The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.""" - verified: bool = Field(False) + verified: bool = False """Whether the identity has been verified (by some means, e.g., OAuth2 or email)""" - display_name: str | None + display_name: str | None = None """The user's display name associated with the identity, also to be exposed in free text searches.""" - @validator("name", pre=True, always=True) - def add_missing_name(cls, v, values): + @field_validator("name", mode="before") + @classmethod + def add_missing_name(cls, v, info): """If the identity is created without a free-text 'name', then for certain providers, populate this field so that it can appear in the free text index, e.g., an ORCID, or an institutional username from an email address. """ - if v is None: - if values["identity_type"] == IdentityType.ORCID: - return values["identifier"] - if values["identity_type"] == IdentityType.EMAIL: - return values["identifier"].split("@")[0] - + if v is None and hasattr(info, "data") and info.data: + data = info.data + if data.get("identity_type") == IdentityType.ORCID: + return data.get("identifier") + if data.get("identity_type") == IdentityType.EMAIL: + identifier = data.get("identifier", "") + return identifier.split("@")[0] if "@" in identifier else identifier return v - @validator("verified", pre=True, always=True) + @field_validator("verified", mode="before") + @classmethod def add_missing_verification(cls, v): """Fills in missing value for `verified` if not given.""" if not v: @@ -66,18 +72,11 @@ def add_missing_verification(cls, v): return v -class DisplayName(ConstrainedStr): - """A constrained string less than 150 characters long but with - non-empty content, intended to be entered by the user. - - """ - - max_length = 150 - min_length = 1 - strip_whitespace = True - - def __new__(cls, value): - return parse_obj_as(cls, value) +DisplayName = Annotated[ + str, + StringConstraints(min_length=1, max_length=150, strip_whitespace=True), +] +"""A constrained string less than 150 characters long but with non-empty content, intended to be entered by the user.""" class EmailStr(PydanticEmailStr): @@ -112,38 +111,40 @@ class Group(Entry): """ - type: str = Field("groups", const=True) + type: Literal["groups"] = "groups" """The entry type as a string.""" - group_id: HumanReadableIdentifier | None = Field(None) + group_id: HumanReadableIdentifier | None = None """A short, locally-unique ID for the group.""" - members: list[dict] = Field(None) - """A list of people that belong to this group.""" + members: list[dict] = Field(default_factory=list) + """A list of people that belong to this group; stored on the user objects.""" - display_name: DisplayName | None = Field(None) + display_name: DisplayName | None = None """The chosen display name for the group""" - description: str | None = Field(None) + description: str | None = None """A description of the group""" managers: list[PyObjectId | dict] = Field(default_factory=list) - """A list of user IDs that can manage this group.""" + """A list of user IDs that can manage this group; stored in db as list of IDs.""" - @validator("members", pre=True, always=True) + @field_validator("members", mode="before") + @classmethod def cast_members_to_people(cls, v): """Casts members to list of people if not None.""" if v is not None: - return [Person(**member).dict(exclude_unset=True) for member in v] + return [Person(**member).model_dump(exclude_unset=True) for member in v] return v - @validator("managers", pre=True, always=True) + @field_validator("managers", mode="before") + @classmethod def cast_managers_to_people(cls, v): """Casts managers to list of people if not None.""" if v and isinstance(v[0], dict): return [ - Person(**member).dict(exclude_unset=True) + Person(**member).model_dump(exclude_unset=True) for member in v if isinstance(member, dict) ] @@ -154,7 +155,7 @@ def cast_managers_to_people(cls, v): class Person(Entry): """A model that describes an individual and their digital identities.""" - type: str = Field("people", const=True) + type: Literal["people"] = "people" """The entry type as a string.""" identities: list[Identity] = Field(default_factory=list) @@ -181,17 +182,14 @@ class Person(Entry): account_status: AccountStatus = Field(AccountStatus.UNVERIFIED) """The status of the user's account.""" - @validator("type", pre=True, always=True) + @field_validator("type", mode="before") + @classmethod def add_missing_type(cls, v): """Fill in missing `type` field if not provided.""" if v is None: v = "people" return v - @validator("type", pre=True) - def set_default_type(cls, _): - return "people" - @staticmethod def new_user_from_identity( identity: Identity, @@ -214,8 +212,6 @@ def new_user_from_identity( A `Person` object with only the provided identity. """ - user_id = bson.ObjectId() - display_name = None if use_display_name: display_name = identity.display_name @@ -227,7 +223,6 @@ def new_user_from_identity( from pydatalab.mongo import gravatar_hash_for return Person( - immutable_id=user_id, identities=[identity], display_name=display_name, contact_email=contact_email, diff --git a/pydatalab/src/pydatalab/models/relationships.py b/pydatalab/src/pydatalab/models/relationships.py index e844c2498..e4c53ec33 100644 --- a/pydatalab/src/pydatalab/models/relationships.py +++ b/pydatalab/src/pydatalab/models/relationships.py @@ -1,8 +1,13 @@ from enum import Enum -from pydantic import BaseModel, root_validator, validator +from pydantic import ( + ConfigDict, + field_validator, + model_validator, +) from pydatalab.models.utils import ( + BaseModel, HumanReadableIdentifier, KnownType, PyObjectId, @@ -29,46 +34,52 @@ class entryC SIBLING = "sibling" PARTHOOD = "is_part_of" OTHER = "other" + COLLECTIONS = "collections" class TypedRelationship(BaseModel): - description: str | None + description: str | None = None """A description of the relationship.""" - relation: RelationshipType | None + relation: RelationshipType | None = None """The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided.""" type: KnownType """The type of the related resource.""" - immutable_id: PyObjectId | None + immutable_id: PyObjectId | None = None """The immutable ID of the entry that is related to this entry.""" - item_id: HumanReadableIdentifier | None + item_id: HumanReadableIdentifier | None = None """The ID of the entry that is related to this entry.""" - refcode: Refcode | None + refcode: Refcode | None = None """The refcode of the entry that is related to this entry.""" - @validator("relation") - def check_for_description(cls, v, values): - if v == RelationshipType.OTHER and values.get("description") is None: - raise ValueError( - f"A description must be provided if the relationship type is {RelationshipType.OTHER.value!r}." - ) - + @field_validator("relation") + @classmethod + def check_for_description(cls, v, info): + if v == RelationshipType.OTHER: + data = info.data if hasattr(info, "data") and info.data else {} + if data.get("description") is None: + raise ValueError( + f"A description must be provided if the relationship type is {RelationshipType.OTHER.value!r}." + ) return v - @root_validator + @model_validator(mode="before") + @classmethod def check_id_fields(cls, values): """Check that at least one of the possible identifier fields is provided.""" id_fields = ("immutable_id", "item_id", "refcode") - if all(values[f] is None for f in id_fields): + if all(values.get(f) is None for f in id_fields): raise ValueError(f"Must provide at least one of {id_fields!r}") if values.get("refcode") and values.get("item_id"): pass elif values.get("immutable_id") and (values.get("refcode") or values.get("item_id")): - raise ValueError("Must provide only one of {id_fields!r}") + raise ValueError(f"Must provide only one of {id_fields!r}") return values + + model_config = ConfigDict(extra="allow") diff --git a/pydatalab/src/pydatalab/models/samples.py b/pydatalab/src/pydatalab/models/samples.py index c7c891f99..6293f367d 100644 --- a/pydatalab/src/pydatalab/models/samples.py +++ b/pydatalab/src/pydatalab/models/samples.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import Field from pydatalab.models.items import Item @@ -12,7 +14,7 @@ class Sample(Item, HasSynthesisInfo, HasSubstanceInfo): and connected to other items. """ - type: str = Field("samples", const="samples", pattern="^samples$") + type: Literal["samples"] = "samples" status: SampleStatus = Field(default=SampleStatus.ACTIVE) """The status of the sample, indicating its current state.""" diff --git a/pydatalab/src/pydatalab/models/starting_materials.py b/pydatalab/src/pydatalab/models/starting_materials.py index 08655a721..1852a6d0f 100644 --- a/pydatalab/src/pydatalab/models/starting_materials.py +++ b/pydatalab/src/pydatalab/models/starting_materials.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import Field from pydatalab.models.items import Item @@ -13,43 +15,39 @@ class StartingMaterial(Item, HasSynthesisInfo, HasSubstanceInfo): represent either depending on preference. """ - type: str = Field( - "starting_materials", const="starting_materials", pattern="^starting_materials$" - ) + type: Literal["starting_materials"] = "starting_materials" - barcode: str | None = Field( - alias="Barcode", - ) + barcode: str | None = Field(None, alias="Barcode") """A unique barcode provided by an external source, e.g., cheminventory.""" - date: IsoformatDateTime | None = Field(alias="Date Acquired") + date: IsoformatDateTime | None = Field(None, alias="Date Acquired") """The date the item was acquired""" - date_opened: IsoformatDateTime | None = Field(alias="Date opened") + date_opened: IsoformatDateTime | None = Field(None, alias="Date opened") """The date the item was opened""" - chemical_purity: str | None = Field(alias="Chemical purity") + chemical_purity: str | None = Field(None, alias="Chemical purity") """The chemical purity of this container with regards to the defined substance.""" - full_percent: str | None = Field(alias="Full %") + full_percent: str | None = Field(None, alias="Full %") """The amount of the defined substance remaining in the container, expressed as a percentage.""" - name: str | None = Field(alias="Container Name") + name: str | None = Field(None, alias="Container Name") """The name of the substance in the container.""" - size: str | None = Field(alias="Container Size") + size: str | None = Field(None, alias="Container Size") """The total size of the container, in units of `size_unit`.""" - size_unit: str | None = Field(alias="Unit") + size_unit: str | None = Field(None, alias="Unit") """Units for the 'size' field.""" - supplier: str | None = Field(alias="Supplier") + supplier: str | None = Field(None, alias="Supplier") """Supplier or manufacturer of the chemical.""" - location: str | None = Field(alias="Location") + location: str | None = Field(None, alias="Location") """The place where the container is located.""" - comment: str | None = Field(alias="Comments") + comment: str | None = Field(None, alias="Comments") """Any additional comments or notes about the container.""" status: StartingMaterialsStatus = Field(default=StartingMaterialsStatus.AVAILABLE) diff --git a/pydatalab/src/pydatalab/models/tags.py b/pydatalab/src/pydatalab/models/tags.py new file mode 100644 index 000000000..daf178b83 --- /dev/null +++ b/pydatalab/src/pydatalab/models/tags.py @@ -0,0 +1,46 @@ +from typing import Literal + +from pydantic import model_validator + +from pydatalab.models.entries import Entry +from pydatalab.models.utils import AccessScope, PyObjectId + + +class Tag(Entry): + """A tag that can be associated to other Entry entities. + + Tags have a `scope` that controls who can list, use and manage them: + + - `AccessScope.GLOBAL`: available to (and usable by) everyone; created and + managed by administrators only. Global tags have no `owner`. + - `AccessScope.USER`: a user-defined tag owned by exactly one user; only that + user can list, use, edit and delete it. + + Names are only required to be unique within a scope. + """ + + type: Literal["tags"] = "tags" + + name: str + """A short, human-readable label for the tag.""" + + description: str | None = None + """An optional description of the tag, either in plain-text or a markup language.""" + + color: str | None = None + """An optional display color for the tag (e.g. a CSS hex string like `#f1c40f`).""" + + scope: AccessScope + """The scope controlling who can list, use and manage this tag (required).""" + + owner: PyObjectId | None = None + """The database ID of the user that owns this tag.""" + + @model_validator(mode="after") + def _check_scope_owner_consistency(self): + """Ensure `scope` and `owner` are mutually consistent.""" + if self.scope == AccessScope.USER and self.owner is None: + raise ValueError("A user-scoped tag must have an owner.") + if self.scope == AccessScope.GLOBAL and self.owner is not None: + raise ValueError("A global tag cannot have an owner.") + return self diff --git a/pydatalab/src/pydatalab/models/tasks.py b/pydatalab/src/pydatalab/models/tasks.py index fa64714c3..ab21d39c8 100644 --- a/pydatalab/src/pydatalab/models/tasks.py +++ b/pydatalab/src/pydatalab/models/tasks.py @@ -2,9 +2,9 @@ from enum import Enum from typing import Literal -from pydantic import BaseModel, Field, validator +from pydantic import ConfigDict, Field, field_validator -from pydatalab.models.utils import PyObjectId +from pydatalab.models.utils import BaseModel, PyObjectId class TaskStatus(str, Enum): @@ -20,12 +20,17 @@ class TaskType(str, Enum): class TaskStage(BaseModel): - timestamp: datetime = Field(..., description="When this stage occurred") - message: str = Field(..., description="Description of this processing stage") - level: Literal["info", "warning", "error"] = Field( - default="info", description="Severity level of this stage" - ) - detail: str | None = Field(None, description="Optional detailed information about this stage") + timestamp: datetime + """When this stage occurred""" + + message: str + """Description of this processing stage""" + + level: Literal["info", "warning", "error"] = "info" + """Severity level of this stage""" + + detail: str | None = None + """Optional detailed information about this stage""" class TaskSpec(BaseModel): @@ -33,39 +38,62 @@ class TaskSpec(BaseModel): class ExportTaskSpec(TaskSpec): - collection_id: str | None = Field(None, description="Collection ID being exported") - item_id: str | None = Field(None, description="Item ID being exported") - export_type: str = Field(..., description="Type of export: collection/item/graph") - file_path: str | None = Field(None, description="Path to generated .eln file") - stages: list[TaskStage] = Field( - default_factory=list, description="Timestamped processing stages" - ) + collection_id: str | None = None + """Collection ID being exported""" + + item_id: str | None = None + """Item ID being exported""" + + export_type: str + """Type of export: collection/item/graph""" + + file_path: str | None = None + """Path to generated .eln file""" + + stages: list[TaskStage] = Field(default_factory=list) + """Timestamped processing stages""" class BlockProcessingTaskSpec(TaskSpec): - item_id: str = Field(..., description="Item ID containing the block") - block_id: str = Field(..., description="Block ID being processed") - stages: list[TaskStage] = Field( - default_factory=list, description="Timestamped processing stages" - ) + item_id: str + """Item ID containing the block""" + + block_id: str + """Block ID being processed""" + + stages: list[TaskStage] = Field(default_factory=list) + """Timestamped processing stages""" class Task(BaseModel): - task_id: str = Field(..., description="Unique identifier for the task") - type: TaskType = Field(..., description="Type of task") - status: TaskStatus = Field(default=TaskStatus.PENDING, description="Current status") - creator_id: PyObjectId = Field(..., description="ID of the user who created the task") - created_at: datetime = Field( - default_factory=lambda: datetime.now(tz=timezone.utc), - description="When the task was created", - ) - completed_at: datetime | None = Field(None, description="When completed") - error_message: str | None = Field(None, description="Error message if status is ERROR") - spec: ExportTaskSpec | BlockProcessingTaskSpec = Field(..., description="Task-specific data") - - @validator("spec", pre=True, always=True) - def validate_spec_type(cls, v, values): - task_type = values.get("type") + task_id: str + """Unique identifier for the task""" + + type: TaskType + """Type of task""" + + status: TaskStatus = TaskStatus.PENDING + """Current status""" + + creator_id: PyObjectId + """ID of the user who created the task""" + + created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc)) + """When the task was created""" + + completed_at: datetime | None = None + """When completed""" + + error_message: str | None = None + """Error message if status is ERROR""" + + spec: ExportTaskSpec | BlockProcessingTaskSpec + """Task-specific data""" + + @field_validator("spec", mode="before") + @classmethod + def validate_spec_type(cls, v, info): + task_type = info.data.get("type") if task_type == TaskType.EXPORT: if not isinstance(v, ExportTaskSpec): @@ -76,5 +104,4 @@ def validate_spec_type(cls, v, values): return v - class Config: - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) diff --git a/pydatalab/src/pydatalab/models/traits.py b/pydatalab/src/pydatalab/models/traits.py index f13043440..e9ba37300 100644 --- a/pydatalab/src/pydatalab/models/traits.py +++ b/pydatalab/src/pydatalab/models/traits.py @@ -1,10 +1,29 @@ -from typing import Any +from typing import TYPE_CHECKING, Any -from pydantic import BaseModel, Field, root_validator, validator +from pydantic import AliasChoices, ConfigDict, Field, field_validator, model_validator from pydatalab.models.blocks import DataBlockResponse from pydatalab.models.people import Group, Person -from pydatalab.models.utils import Constituent, InlineSubstance, PyObjectId +from pydatalab.models.utils import ( + BaseModel, + Constituent, + EntryReference, + InlineSubstance, + PyObjectId, +) + +if TYPE_CHECKING: + pass + +__all__ = ( + "HasOwner", + "HasRevisionControl", + "HasBlocks", + "IsCollectable", + "HasSynthesisInfo", + "HasSubstanceInfo", + "HasTags", +) class HasOwner(BaseModel): @@ -23,6 +42,57 @@ class HasOwner(BaseModel): """Inlined info for the groups with access to this item.""" +class HasTags(BaseModel): + """Trait mixin for models that can be annotated with tags. + + Note: this mixin only provides the stored `tags` field and its coercion. + Inlining current tag names for display (and dropping references to deleted + tags) is a read-time concern handled by + `pydatalab.mongo.resolve_tags_for_docs`, which each entity's read path must + call explicitly on the docs it returns. + """ + + tags: list[EntryReference] = Field(default_factory=list) + """Tags applied to this entry: references to `tags` entries (by + `immutable_id`).""" + + @field_validator("tags", mode="before") + @classmethod + def coerce_tags(cls, v): + """Coerce raw tag entries into references and de-duplicate. + + A mapping carrying an `immutable_id` becomes an `EntryReference` of type + ``tags``. References are de-duplicated by `immutable_id`. + """ + if v is None: + return [] + if not isinstance(v, list): + raise ValueError("`tags` must be a list") + + coerced: list = [] + seen_refs: set[PyObjectId | None] = set() + + for tag in v: + if isinstance(tag, EntryReference): + if tag.immutable_id not in seen_refs: + seen_refs.add(tag.immutable_id) + coerced.append(tag) + continue + + if isinstance(tag, dict) and tag.get("immutable_id") is not None: + data = dict(tag) + data.setdefault("type", "tags") + ref = EntryReference(**data) + if ref.immutable_id not in seen_refs: + seen_refs.add(ref.immutable_id) + coerced.append(ref) + continue + + raise ValueError(f"Invalid tag entry: {tag!r}") + + return coerced + + class HasRevisionControl(BaseModel): """Trait mixin for models that track a revision history of their own state.""" @@ -36,20 +106,6 @@ class HasRevisionControl(BaseModel): """The version number used by the version control system for tracking snapshots.""" -class HasFiles(BaseModel): - """Trait mixin for models that can have files attached to them.""" - - # Imported here rather than at module level to avoid a circular import, - # as `models.files` itself depends on this module. - from pydatalab.models.files import File - - files: list[File] | None = Field(None) - """Any files attached to this item.""" - - file_ObjectIds: list[PyObjectId] = Field([]) - """Links to object IDs of files stored within the database.""" - - class HasBlocks(BaseModel): """Trait mixin for models that can have data blocks attached to them.""" @@ -60,33 +116,36 @@ class HasBlocks(BaseModel): """The order in which to display block data in the UI.""" +class CollectionReference(BaseModel): + """A reference to a collection, used for inlining collection info within other models.""" + + model_config = ConfigDict(extra="allow", populate_by_name=True) + + immutable_id: PyObjectId = Field(alias="_id") + """The immutable ID of the collection.""" + + class IsCollectable(BaseModel): """Trait mixin for models that can be added to collections. """ - from pydatalab.models.collections import Collection - - collections: list[Collection] = Field([]) + collections: list[CollectionReference] = Field(default_factory=list) """Inlined info for the collections associated with this item.""" - @root_validator - def add_missing_collection_relationships(cls, values): + @model_validator(mode="after") + def add_missing_collection_relationships(self): from pydatalab.models.relationships import TypedRelationship - if values.get("collections") is not None: - new_ids = {coll.immutable_id for coll in values["collections"]} - existing_collection_relationship_ids = set() - if values.get("relationships") is not None: - existing_collection_relationship_ids = { - relationship.immutable_id - for relationship in values["relationships"] - if relationship.type == "collections" - } - else: - values["relationships"] = [] - - for collection in values.get("collections", []): + if self.collections is not None: + new_ids = {coll.immutable_id for coll in self.collections} + existing_collection_relationship_ids = { + relationship.immutable_id + for relationship in self.relationships + if relationship.type == "collections" + } + + for collection in self.collections: if collection.immutable_id not in existing_collection_relationship_ids: relationship = TypedRelationship( relation=None, @@ -94,20 +153,18 @@ def add_missing_collection_relationships(cls, values): type="collections", description="Is a member of", ) - values["relationships"].append(relationship) + self.relationships.append(relationship) - values["relationships"] = [ + self.relationships = [ d - for d in values.get("relationships", []) + for d in self.relationships if d.type != "collections" or d.immutable_id in new_ids ] - if len([d for d in values.get("relationships", []) if d.type == "collections"]) != len( - values.get("collections", []) - ): + if len([d for d in self.relationships if d.type == "collections"]) != len(self.collections): raise RuntimeError("Relationships and collections mismatch") - return values + return self class HasSynthesisInfo(BaseModel): @@ -119,28 +176,24 @@ class HasSynthesisInfo(BaseModel): synthesis_description: str | None = None """Free-text details of the procedure applied to synthesise the sample""" - @root_validator - def add_missing_synthesis_relationships(cls, values): + @model_validator(mode="after") + def add_missing_synthesis_relationships(self): """Add any missing sample synthesis constituents to parent relationships""" from pydatalab.models.relationships import RelationshipType, TypedRelationship constituents_set = set() - if values.get("synthesis_constituents") is not None: - existing_parent_relationships = {} - if values.get("relationships") is not None: - # Index by refcode *and* item_id so a stored relationship carrying an - # item_id still matches a refcode-enriched constituent (and vice-versa). - existing_parent_relationships = { - identifier: relationship - for relationship in values["relationships"] - if relationship.relation == RelationshipType.PARENT - for identifier in (relationship.refcode, relationship.item_id) - if identifier - } - else: - values["relationships"] = [] - - for constituent in values.get("synthesis_constituents", []): + if self.synthesis_constituents is not None: + # Index by refcode *and* item_id so a stored relationship carrying an + # item_id still matches a refcode-enriched constituent (and vice-versa). + existing_parent_relationships = { + identifier: relationship + for relationship in self.relationships + if relationship.relation == RelationshipType.PARENT + for identifier in (relationship.refcode, relationship.item_id) + if identifier + } + + for constituent in self.synthesis_constituents: # If this is an inline relationship, just skip it if isinstance(constituent.item, InlineSubstance): continue @@ -159,7 +212,7 @@ def add_missing_synthesis_relationships(cls, values): type=constituent.item.type, description="Is a constituent of", ) - values["relationships"].append(relationship) + self.relationships.append(relationship) else: # Back-fill any identifier missing from the stored relationship relationship.refcode = relationship.refcode or refcode @@ -177,9 +230,9 @@ def add_missing_synthesis_relationships(cls, values): # Finally, filter out any parent relationships with item that were removed # from the synthesis constituents - values["relationships"] = [ + self.relationships = [ rel - for rel in values["relationships"] + for rel in self.relationships if not ( rel.refcode not in constituents_set and rel.item_id not in constituents_set @@ -188,14 +241,15 @@ def add_missing_synthesis_relationships(cls, values): ) ] - return values + return self class HasSubstanceInfo(BaseModel): """Trait mixin for models that have substance information.""" chemform: str | None = Field( - example=["Na3P", "Na3P", "LiNiO2@C", "Na3+xP", "LiNi1/3Co0.1Mn0.1O2"], + None, + examples=["Na3P", "Na3P", "LiNiO2@C", "Na3+xP", "LiNi1/3Co0.1Mn0.1O2"], ) """A string representation of the chemical formula or composition associated with this sample. @@ -204,7 +258,9 @@ class HasSubstanceInfo(BaseModel): """ - smiles: str | None = Field(None, aliases=["SMILES", "smiles_representation"]) + smiles: str | None = Field( + None, validation_alias=AliasChoices("smiles_representation", "SMILES") + ) """A SMILES string representation of the chemical structure associated with this sample.""" inchi: str | None = Field(None) @@ -214,24 +270,28 @@ class HasSubstanceInfo(BaseModel): """A unique key derived from the InChI.""" GHS_codes: str | None = Field( + None, alias="GHS H-codes", examples=["H224", "H303, H316, H319"], ) """A string describing any GHS hazard codes associated with this item. See https://pubchem.ncbi.nlm.nih.gov/ghs/ for code definitions.""" - molar_mass: float | None = Field(alias="Molecular Weight") + molar_mass: float | None = Field(None, alias="Molecular Weight", validate_default=True) """Mass per formula unit, in g/mol.""" - CAS: str | None = Field(alias="Substance CAS") + CAS: str | None = Field(None, alias="Substance CAS") """The CAS Registry Number for the substance described by this entry.""" - @validator("molar_mass") - def add_molar_mass(cls, v, values): + @field_validator("molar_mass", mode="before") + @classmethod + def add_molar_mass(cls, v, info): + """Fill in the molar mass if not already set and a chemical formula is provided.""" from periodictable import formula - if v is None and values.get("chemform"): + chemform = info.data.get("chemform") + if v is None and chemform: try: - return formula(values.get("chemform")).mass + return formula(chemform).mass except Exception: return None diff --git a/pydatalab/src/pydatalab/models/utils.py b/pydatalab/src/pydatalab/models/utils.py index 58230b184..41c2b40fd 100644 --- a/pydatalab/src/pydatalab/models/utils.py +++ b/pydatalab/src/pydatalab/models/utils.py @@ -4,18 +4,49 @@ from collections.abc import Callable from enum import Enum from functools import partial -from typing import TypeAlias +from typing import Annotated, Any, TypeAlias, get_args import pint -from bson.objectid import ObjectId +from bson import ObjectId from pydantic import ( - BaseModel, - ConstrainedStr, + BaseModel as PydanticBaseModel, +) +from pydantic import ( + ConfigDict, Field, - parse_obj_as, - root_validator, - validator, + StringConstraints, + field_validator, + model_validator, ) +from pydantic_core import core_schema + + +class BaseModel(PydanticBaseModel): + """The base model for all pydatalab models, configured to pull field + descriptions from attribute docstrings into the generated JSON schema. + + """ + + model_config = ConfigDict(use_attribute_docstrings=True) + + @classmethod + def __pydantic_init_subclass__(cls, **kwargs): + """Restore pydantic v1's implicit `None` default for optional fields. + + In pydantic v2 a field annotated ``T | None`` with no assigned default is + *required*, whereas in v1 it implicitly defaulted to ``None``. This back-fills + a ``None`` default for any required field that allows ``None``, so that models + (including those in plugins not yet migrated to v2) keep their v1 semantics. + Fields that already declare a default are untouched. + """ + super().__pydantic_init_subclass__(**kwargs) + changed = False + for field in cls.model_fields.values(): + if field.is_required() and type(None) in get_args(field.annotation): + field.default = None + changed = True + if changed: + cls.model_rebuild(force=True) class ItemType(str, Enum): @@ -92,43 +123,30 @@ class KnownType(str, Enum): """ -class HumanReadableIdentifier(ConstrainedStr): - """Used to constrain human-readable and URL-safe identifiers for items.""" - - min_length = 1 - max_length = 40 - strip_whitespace = True - to_lower = False - strict = False - regex = IDENTIFIER_REGEX - - def __init__(self, value): - self.value = parse_obj_as(type(self), value) - - def __str__(self): - return self.value - - def __repr__(self): - return self.value - - def __bool__(self): - return bool(self.value) - - -class Refcode(HumanReadableIdentifier): - regex = r"^[a-z]{2,10}:" + IDENTIFIER_REGEX[1:] - """A regex to match refcodes that have a lower-case prefix between 2-10 chars, followed by a colon, - and then the normal rules for an ID (url-safe etc.). - - """ - - @property - def prefix(self): - return self.value.split(":")[0] - - @property - def identifier(self): - return self.value.split(":")[1] +HumanReadableIdentifier = Annotated[ + str, + StringConstraints( + min_length=1, + max_length=40, + strip_whitespace=True, + to_lower=False, + strict=False, + pattern=IDENTIFIER_REGEX, + ), +] +"""Used to constrain human-readable and URL-safe identifiers for items.""" + + +Refcode = Annotated[ + str, + StringConstraints( + min_length=1, + max_length=40, + strip_whitespace=True, + pattern=r"^[a-z]{2,10}:" + IDENTIFIER_REGEX[1:], + ), +] +"""A regex to match refcodes that have a lower-case prefix between 2-10 chars, followed by a colon, and then the normal rules for an ID (url-safe etc.).""" class UserRole(str, Enum): @@ -137,6 +155,13 @@ class UserRole(str, Enum): MANAGER = "manager" +class AccessScope(str, Enum): + """The scope that controls who can list, use and manage an entity (e.g. a tag).""" + + GLOBAL = "global" + USER = "user" + + class PintType(str): """A WIP attempt to create a custom pydantic field type for Pint quantities. The idea would eventually be to use TypeAlias to create physical/dimensionful pydantic fields. @@ -149,8 +174,12 @@ def __init__(self, dimensions: str): self._dimensions = dimensions @classmethod - def __get_validators__(self): - yield self.validate + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls.validate, + core_schema.str_schema(), + serialization=core_schema.plain_serializer_function_ser_schema(str, when_used="json"), + ) @classmethod def validate(self, v): @@ -159,77 +188,120 @@ def validate(self, v): raise ValueError("Value {v} must have dimensions of mass, not {v.dimensions}") return q - @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="string") - Mass: TypeAlias = PintType("[mass]") # type: ignore # noqa Volume: TypeAlias = PintType("[volume]") # type: ignore # noqa class PyObjectId(ObjectId): - """A wrapper class for a BSON ObjectId that can be used as a Pydantic field type. - - Modified from "Getting started iwth MongoDB and FastAPI": - https://www.mongodb.com/developer/languages/python/python-quickstart-fastapi/. - - """ + @classmethod + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls.validate, + core_schema.union_schema( + [ + core_schema.str_schema(), + core_schema.is_instance_schema(ObjectId), + core_schema.is_instance_schema(cls), + core_schema.dict_schema(), + core_schema.none_schema(), + ] + ), + serialization=core_schema.plain_serializer_function_ser_schema( + lambda x: str(x) if x else None, when_used="json" + ), + ) @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_json_schema__(cls, core_schema, handler): + return { + "type": "string", + "format": "objectid", + } @classmethod def validate(cls, v): - if isinstance(v, dict) and "$oid" in v: - v = v["$oid"] + if v is None: + return None + if isinstance(v, cls): + return v + if isinstance(v, ObjectId): + return cls(v) - if not ObjectId.is_valid(v): - raise ValueError("Invalid ObjectId") + if isinstance(v, dict): + if "$oid" in v: + return cls(ObjectId(v["$oid"])) + elif "_id" in v and isinstance(v["_id"], (str, ObjectId)): + return cls(ObjectId(v["_id"])) + elif len(v) == 1: + first_val = next(iter(v.values())) + if isinstance(first_val, str) and ObjectId.is_valid(first_val): + return cls(ObjectId(first_val)) + raise ValueError(f"Cannot convert dict to ObjectId: {v}") - return ObjectId(v) + if isinstance(v, str): + if not ObjectId.is_valid(v): + raise ValueError("Invalid ObjectId string") + return cls(ObjectId(v)) - @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="string") + raise ValueError(f"Cannot convert {type(v)} to ObjectId: {v}") class IsoformatDateTime(datetime.datetime): """A datetime container that is more flexible than the pydantic default.""" @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls.validate, + core_schema.union_schema( + [ + core_schema.str_schema(), + core_schema.is_instance_schema(datetime.datetime), + ] + ), + serialization=core_schema.plain_serializer_function_ser_schema( + lambda x: x.isoformat() if x else None, when_used="json" + ), + ) + + @classmethod + def __get_pydantic_json_schema__(cls, core_schema, handler): + return { + "type": "string", + "format": "datetime", + } @classmethod def validate(cls, v) -> datetime.datetime | None: """Cast isoformat strings to datetimes and enforce UTC if tzinfo is missing.""" + if v is None: + return None + + if isinstance(v, datetime.datetime): + if v.tzinfo is None: + v = v.replace(tzinfo=datetime.timezone.utc) + return v + if isinstance(v, str): - if v in ["0", " "]: + if v in ["0", " ", ""]: return None v = datetime.datetime.fromisoformat(v) + if v.tzinfo is None: + v = v.replace(tzinfo=datetime.timezone.utc) + return v - if v.tzinfo is None: - v = v.replace(tzinfo=datetime.timezone.utc) - - return v - - -JSON_ENCODERS = { - pint.Quantity: str, - ObjectId: str, -} + raise ValueError(f"Invalid datetime value: {v}") class RefCodeFactory: refcode_generator: Callable @classmethod - def generate(self): + def generate(cls): from pydatalab.config import CONFIG - return f"{CONFIG.IDENTIFIER_PREFIX}:{self.refcode_generator()}" + return f"{CONFIG.IDENTIFIER_PREFIX}:{cls.refcode_generator()}" def random_uppercase(length: int = 6): @@ -257,7 +329,7 @@ def generate_unique_refcode(): class InlineSubstance(BaseModel): name: str - chemform: str | None + chemform: str | None = None class EntryReference(BaseModel): @@ -269,15 +341,19 @@ class EntryReference(BaseModel): """ type: str - name: str | None - immutable_id: PyObjectId | None - item_id: HumanReadableIdentifier | None - refcode: Refcode | None - chemform: str | None + name: str | None = None + immutable_id: PyObjectId | None = None + item_id: HumanReadableIdentifier | None = None + refcode: Refcode | None = None + chemform: str | None = None - @root_validator + @model_validator(mode="before") + @classmethod def check_id_fields(cls, values): """Check that at least one of the possible identifier fields is provided.""" + if not isinstance(values, dict): + return values + id_fields = ("immutable_id", "item_id", "refcode") if all(values.get(f) is None for f in id_fields): @@ -285,8 +361,7 @@ def check_id_fields(cls, values): return values - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class Constituent(BaseModel): @@ -295,23 +370,22 @@ class Constituent(BaseModel): item: EntryReference | InlineSubstance """A reference to item (sample or starting material) entry for the constituent substance.""" - quantity: float | None = Field(..., ge=0) + quantity: float | None = Field(default=None, ge=0) """The amount of the constituent material used to create the sample.""" - unit: str = Field("g") - """The unit symbol for the value provided in `quantity`, default is mass - in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol). - """ + unit: str = "g" + """The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).""" - @validator("item") + @field_validator("item") + @classmethod def check_itemhood(cls, v): """Check that the reference within the constituent is to an item type.""" - if "type" in (v.value for v in ItemType): - raise ValueError(f"`type` must be one of {ItemType!r}") - + if hasattr(v, "type") and v.type not in [item_type.value for item_type in ItemType]: + raise ValueError(f"`type` must be one of {[t.value for t in ItemType]!r}") return v - @validator("item", pre=True, always=True) + @field_validator("item", mode="before") + @classmethod def coerce_reference(cls, v): if isinstance(v, dict): refcode = v.pop("refcode", None) @@ -327,4 +401,18 @@ def coerce_reference(cls, v): if not name: raise ValueError("Inline substance must have a name!") return InlineSubstance(name=name, chemform=chemform) + elif hasattr(v, "model_dump"): + item_id = getattr(v, "item_id", None) + refcode = getattr(v, "refcode", None) + item_type = getattr(v, "type", None) + name = getattr(v, "name", None) + chemform = getattr(v, "chemform", None) + + if item_id or refcode: + return EntryReference( + item_id=item_id, refcode=refcode, type=item_type, name=name, chemform=chemform + ) + else: + return InlineSubstance(name=name or str(v), chemform=chemform) + return v diff --git a/pydatalab/src/pydatalab/models/versions.py b/pydatalab/src/pydatalab/models/versions.py index cb2644932..a01311424 100644 --- a/pydatalab/src/pydatalab/models/versions.py +++ b/pydatalab/src/pydatalab/models/versions.py @@ -3,9 +3,9 @@ from datetime import datetime from enum import Enum -from pydantic import BaseModel, Field, validator +from pydantic import ConfigDict, Field, model_validator -from pydatalab.models.utils import PyObjectId, Refcode +from pydatalab.models.utils import BaseModel, PyObjectId, Refcode class VersionAction(str, Enum): @@ -27,42 +27,46 @@ class ItemVersion(BaseModel): view history and restore previous states. """ - refcode: Refcode = Field(..., description="The refcode of the item this version belongs to") - version: int = Field(..., ge=1, description="Sequential version number (1-indexed)") - timestamp: datetime = Field( - ..., description="When this version was created (ISO format with timezone)" - ) - action: VersionAction = Field(..., description="The action that triggered this version.") - user_id: PyObjectId | None = Field( - None, description="User's ObjectId for efficient querying and indexing" - ) - creator: dict | None = Field( - None, description="Inlined information about the user who created this version (e.g., name)" - ) - datalab_version: str = Field( - ..., description="Version of datalab-server that created this snapshot" - ) - data: dict = Field(..., description="Complete snapshot of the item data at this version") - restored_from_version: PyObjectId | None = Field( - None, - description="ObjectId of the version that was restored from (only present if action='restored')", - ) - user_agent: str | None = Field( - None, - description="User agent string of the client that triggered this version. Will only be stored if it matches a known value from the datalab ecosystem.", - ) - - @validator("restored_from_version") - def validate_restored_from_version(cls, v, values): + refcode: Refcode + """The refcode of the item this version belongs to""" + + version: int = Field(ge=1) + """Sequential version number (1-indexed)""" + + timestamp: datetime + """When this version was created (ISO format with timezone)""" + + action: VersionAction + """The action that triggered this version: 'created' (item creation), 'manual_save' (user save), 'auto_save' (system save), or 'restored' (version restore)""" + + user_id: PyObjectId | None = None + """User's ObjectId for efficient querying and indexing""" + + creator: dict | None = None + """Inlined information about the user who created this version (e.g., name)""" + + datalab_version: str + """Version of datalab-server that created this snapshot""" + + data: dict + """Complete snapshot of the item data at this version""" + + restored_from_version: PyObjectId | None = None + """ObjectId of the version that was restored from (only present if action='restored')""" + + user_agent: str | None = None + """User agent string of the client that triggered this version. Will only be stored if it matches a known value from the datalab ecosystem.""" + + @model_validator(mode="after") + def validate_restored_from_version(self): """Ensure restored_from_version is only present when action='restored'.""" - action = values.get("action") - if action == VersionAction.RESTORED and v is None: + if self.action == VersionAction.RESTORED and self.restored_from_version is None: raise ValueError("restored_from_version must be provided when action='restored'") - if action != VersionAction.RESTORED and v is not None: + if self.action != VersionAction.RESTORED and self.restored_from_version is not None: raise ValueError( - f"restored_from_version should only be present when action='restored', got action='{action}'" + f"restored_from_version should only be present when action='restored', got action='{self.action}'" ) - return v + return self class VersionCounter(BaseModel): @@ -72,51 +76,31 @@ class VersionCounter(BaseModel): It ensures atomic increment of version numbers to prevent race conditions. """ - refcode: Refcode = Field(..., description="The refcode this counter belongs to") - counter: int = Field( - 1, ge=1, description="Current version counter value (1-indexed, matches version numbers)" - ) + refcode: Refcode + """The refcode this counter belongs to""" + + counter: int = Field(1, ge=1) + """Current version counter value (1-indexed, matches version numbers)""" - class Config: - extra = "ignore" # Allow MongoDB's _id field and other internal fields + model_config = ConfigDict(extra="ignore") class RestoreVersionRequest(BaseModel): """Request body for restoring a version.""" - version_id: str = Field(..., description="ObjectId string of the version to restore to") + version_id: PyObjectId + """ObjectId string of the version to restore to""" - @validator("version_id") - def validate_version_id_format(cls, v): - """Validate that version_id is a valid ObjectId string.""" - try: - from bson import ObjectId - - ObjectId(v) - except Exception as e: - raise ValueError(f"version_id must be a valid ObjectId string: {e}") - return v - - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") class CompareVersionsQuery(BaseModel): """Query parameters for comparing two versions.""" - v1: str = Field(..., description="ObjectId string of the first version") - v2: str = Field(..., description="ObjectId string of the second version") - - @validator("v1", "v2") - def validate_version_ids(cls, v): - """Validate that version IDs are valid ObjectId strings.""" - try: - from bson import ObjectId + v1: PyObjectId + """ObjectId string of the first version""" - ObjectId(v) - except Exception as e: - raise ValueError(f"Version ID must be a valid ObjectId string: {e}") - return v + v2: PyObjectId + """ObjectId string of the second version""" - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") diff --git a/pydatalab/src/pydatalab/mongo.py b/pydatalab/src/pydatalab/mongo.py index b69b51451..6c4fe6c8c 100644 --- a/pydatalab/src/pydatalab/mongo.py +++ b/pydatalab/src/pydatalab/mongo.py @@ -5,12 +5,12 @@ from typing import Any import pymongo +from bson import ObjectId from flask_pymongo import PyMongo from pydantic import BaseModel from pymongo.errors import ConnectionFailure from pydatalab.logger import LOGGER -from pydatalab.models import ITEM_MODELS __all__ = ( "flask_mongo", @@ -20,12 +20,17 @@ "insert_pydantic_model_fork_safe", "gravatar_hash_for", "run_startup_migrations", - "ITEMS_FTS_FIELDS", + "get_items_fts_fields", "USERS_FTS_FIELDS", "COLLECTIONS_FTS_FIELDS", "GROUPS_FTS_FIELDS", + "TAGS_FTS_FIELDS", "generate_heuristic_regex_search", "build_search_pipeline", + "creators_lookup", + "groups_lookup", + "files_lookup", + "resolve_tags_for_docs", ) flask_mongo = PyMongo() @@ -34,20 +39,124 @@ """One-liner that pulls all non-semantic string fields out of all item models implemented for this server. """ -ITEMS_FTS_FIELDS: set[str] = set().union( - *( - { - f - for f, p in model.schema(by_alias=False)["properties"].items() - if ( - p.get("type") == "string" - and p.get("format") not in ("date-time", "uuid") - and f != "type" - ) - } - for model in ITEM_MODELS.values() - ) -) + + +def creators_lookup() -> dict: + return { + "from": "users", + "let": {"creator_ids": "$creator_ids"}, + "pipeline": [ + {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$creator_ids", []]}]}}}, + {"$addFields": {"__order": {"$indexOfArray": ["$$creator_ids", "$_id"]}}}, + {"$sort": {"__order": 1}}, + {"$project": {"_id": 1, "display_name": 1, "gravatar_hash": 1}}, + ], + "as": "creators", + } + + +def groups_lookup() -> dict: + return { + "from": "groups", + "let": {"group_ids": "$group_ids"}, + "pipeline": [ + {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$group_ids", []]}]}}}, + {"$addFields": {"__order": {"$indexOfArray": ["$$group_ids", "$_id"]}}}, + {"$sort": {"__order": 1}}, + {"$project": {"_id": 1, "display_name": 1, "group_id": 1}}, + ], + "as": "groups", + } + + +def files_lookup() -> dict: + return { + "from": "files", + "localField": "file_ObjectIds", + "foreignField": "_id", + "as": "files", + } + + +def resolve_tags_for_docs(docs: list[dict]) -> None: + """Inline tag details into each doc's `tags` field, in place. + + Tag references (mappings carrying an `immutable_id`) are resolved against + the `tags` collection with no permission filter: display access is gated + by the parent entry itself, so every tag on a viewable entry resolves. + References to deleted tags are dropped. + """ + tag_ids: set[ObjectId] = set() + for doc in docs: + for tag in doc.get("tags") or []: + if isinstance(tag, dict): + tag_ids.add(ObjectId(tag["immutable_id"])) + + if not tag_ids: + return + + resolved = { + tag_doc["_id"]: tag_doc + for tag_doc in flask_mongo.db.tags.find( + {"_id": {"$in": list(tag_ids)}}, + projection={"_id": 1, "name": 1, "description": 1, "color": 1, "scope": 1}, + ) + } + + for doc in docs: + tags = doc.get("tags") + if not tags: + continue + resolved_tags: list = [] + for tag in tags: + if isinstance(tag, dict): + match = resolved.get(ObjectId(tag["immutable_id"])) + # Referenced tag no longer exists: drop it silently. + if match is not None: + resolved_tags.append( + { + "type": "tags", + "immutable_id": str(match["_id"]), + "name": match.get("name"), + "description": match.get("description"), + "color": match.get("color"), + # `scope` lets the UI mark user-defined tags distinctly. + "scope": match.get("scope"), + } + ) + doc["tags"] = resolved_tags + + +@lru_cache(maxsize=1) +def get_items_fts_fields() -> set[str]: + """Get all string fields from item models for full-text search.""" + from pydatalab.models import ITEM_MODELS + + fields = set() + + for model_name, model in ITEM_MODELS.items(): + schema = model.model_json_schema(by_alias=False) + + model_fields = set() + for f, p in schema.get("properties", {}).items(): + if f == "type": + continue + + if p.get("type") == "string" and p.get("format") not in ("date-time", "uuid"): + model_fields.add(f) + elif "anyOf" in p: + for option in p["anyOf"]: + if option.get("type") == "string" and option.get("format") not in ( + "date-time", + "uuid", + ): + model_fields.add(f) + break + + fields.update(model_fields) + + return fields + USERS_FTS_FIELDS: set[str] = {"identities.name", "display_name", "contact_email"} """Fields to search for users.""" @@ -58,6 +167,9 @@ GROUPS_FTS_FIELDS: set[str] = {"group_id", "display_name", "description"} """Fields to search for groups.""" +TAGS_FTS_FIELDS: set[str] = {"name", "description"} +"""Fields to search for tags.""" + def generate_heuristic_regex_search( query: str, fields: set[str], part_length: int = 4 @@ -148,7 +260,7 @@ def insert_pydantic_model_fork_safe(model: BaseModel, collection: str) -> str: """Inserts a Pydantic model into chosen collection, returning the inserted ID.""" return ( get_database()[collection] - .insert_one(model.dict(by_alias=True, exclude_none=True)) + .insert_one(model.model_dump(by_alias=False, exclude_none=True)) .inserted_id ) @@ -167,7 +279,6 @@ def _get_active_mongo_client(timeoutMS: int = 1000) -> pymongo.MongoClient: """ from pydatalab.config import CONFIG - from pydatalab.logger import LOGGER try: client = pymongo.MongoClient( @@ -235,6 +346,11 @@ def create_default_indices( """ + items_fts_fields = get_items_fts_fields() + + if not items_fts_fields: + raise ValueError("Cannot create text indices: no fields available for full-text search") + if client is None: client = _get_active_mongo_client() db = client.get_database() @@ -259,7 +375,7 @@ def create_fts(): ret += create_or_recreate_text_index( db.items, - ITEMS_FTS_FIELDS, + items_fts_fields, weights={"refcode": 3, "item_id": 3, "name": 3, "chemform": 3}, ) @@ -269,6 +385,12 @@ def create_fts(): weights={"collection_id": 3, "title": 3, "description": 3}, ) + ret += create_or_recreate_text_index( + db.tags, + ["name", "description"], + weights={"name": 3, "description": 1}, + ) + ret += db.items.create_index("type", name="item type", background=background) ret += db.items.create_index( "item_id", unique=True, name="unique item ID", background=background diff --git a/pydatalab/src/pydatalab/routes/v0_1/__init__.py b/pydatalab/src/pydatalab/routes/v0_1/__init__.py index 93b3a22db..b3a03950a 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/__init__.py +++ b/pydatalab/src/pydatalab/routes/v0_1/__init__.py @@ -13,6 +13,7 @@ from .info import INFO from .items import ITEMS from .remotes import REMOTES +from .tags import TAGS from .users import USERS BLUEPRINTS: tuple[Blueprint, ...] = ( @@ -29,6 +30,7 @@ INFO, GRAPHS, EXPORT, + TAGS, ) __all__ = ("BLUEPRINTS", "OAUTH", "__api_version__", "OAUTH_PROXIES") diff --git a/pydatalab/src/pydatalab/routes/v0_1/admin.py b/pydatalab/src/pydatalab/routes/v0_1/admin.py index 55e886fae..eea5a6478 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/admin.py +++ b/pydatalab/src/pydatalab/routes/v0_1/admin.py @@ -125,7 +125,7 @@ def get_users(): elif not isinstance(user["managers"], list): user["managers"] = [] - return jsonify({"status": "success", "data": [Person(**d).dict() for d in users_list]}) + return jsonify({"status": "success", "data": [Person(**d).model_dump() for d in users_list]}) @ADMIN.route("/roles/", methods=["PATCH"]) @@ -357,7 +357,9 @@ def get_groups(): [{"$match": {}}, {"$lookup": members_lookup}, {"$lookup": managers_lookup}] ) - return jsonify({"status": "success", "data": [Group(**d).dict() for d in group_docs]}), 200 + return jsonify( + {"status": "success", "data": [Group(**d).model_dump() for d in group_docs]} + ), 200 @ADMIN.route("/groups", methods=["PUT"]) @@ -392,7 +394,7 @@ def create_group(): try: group_immutable_id = flask_mongo.db.groups.insert_one( - group.dict(exclude_unset=True) + group.model_dump(exclude_unset=True) ).inserted_id except pymongo.errors.DuplicateKeyError: return jsonify( diff --git a/pydatalab/src/pydatalab/routes/v0_1/auth.py b/pydatalab/src/pydatalab/routes/v0_1/auth.py index 1ded96b38..764a6e664 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/auth.py +++ b/pydatalab/src/pydatalab/routes/v0_1/auth.py @@ -5,7 +5,6 @@ """ import datetime -import json import os import re import secrets @@ -504,7 +503,7 @@ def find_user_with_identity( if verify and not person.identities[identity_index].verified: flask_mongo.db.users.update_one( - {"_id": person.immutable_id}, + {"_id": ObjectId(person.immutable_id)}, {"$set": {f"identities.{identity_index}.verified": True}}, ) @@ -556,7 +555,7 @@ def attach_identity_to_user( RuntimeError: If the update was unsuccessful. """ - update = {"$push": {"identities": identity.dict()}} + update = {"$push": {"identities": identity.model_dump()}} if use_display_name and identity and identity.display_name: update["$set"] = {"display_name": identity.display_name} @@ -1057,7 +1056,7 @@ def redirect_to_ui(blueprint, token): # pylint: disable=unused-argument def get_authenticated_user_info(): """Returns metadata associated with the currently authenticated user.""" if current_user.is_authenticated: - current_user_response = json.loads(current_user.person.json()) + current_user_response = current_user.person.model_dump(mode="json") current_user_response["role"] = current_user.role.value return jsonify(current_user_response), 200 else: @@ -1085,7 +1084,7 @@ def generate_user_api_key(): version=1, ) - flask_mongo.db.api_keys.insert_one(access_key.dict()) + flask_mongo.db.api_keys.insert_one(access_key.model_dump()) return jsonify({"key": new_key, "name": request_json["name"]}), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/blocks.py b/pydatalab/src/pydatalab/routes/v0_1/blocks.py index 6c00d1b2e..8322f1161 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/blocks.py +++ b/pydatalab/src/pydatalab/routes/v0_1/blocks.py @@ -54,7 +54,7 @@ def add_stage(message: str, level: str = "info", traceback: str | None = None): timestamp=datetime.now(tz=timezone.utc), message=message, level=level, detail=traceback ) flask_mongo.db.tasks.update_one( - {"task_id": task_id}, {"$push": {"spec.stages": stage.dict()}} + {"task_id": task_id}, {"$push": {"spec.stages": stage.model_dump()}} ) with app_ctx, req_ctx: @@ -381,7 +381,7 @@ def update_block(): ), ) - flask_mongo.db.tasks.insert_one(block_task.dict()) + flask_mongo.db.tasks.insert_one(block_task.model_dump()) task_scheduler.add_job( func=_process_block_async, diff --git a/pydatalab/src/pydatalab/routes/v0_1/collections.py b/pydatalab/src/pydatalab/routes/v0_1/collections.py index a8afffc3d..a0b3c4f6e 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/collections.py +++ b/pydatalab/src/pydatalab/routes/v0_1/collections.py @@ -1,5 +1,4 @@ import datetime -import json from bson import ObjectId from flask import Blueprint, jsonify, request @@ -87,7 +86,7 @@ def get_collection(collection_id): { "status": "success", "collection_id": collection_id, - "data": json.loads(collection.json(exclude_unset=True)), + "data": collection.model_dump(mode="json", exclude_unset=True), "child_items": list(samples), } ) @@ -168,7 +167,7 @@ def create_collection(): ) result: InsertOneResult = flask_mongo.db.collections.insert_one( - data_model.dict(exclude={"creators", "groups"}) + data_model.model_dump(exclude={"creators", "groups"}) ) if not result.acknowledged: return ( @@ -218,7 +217,7 @@ def create_collection(): response = { "status": "success", - "data": json.loads(data_model.json()), + "data": data_model.model_dump(mode="json"), } if errors: @@ -278,7 +277,7 @@ def save_collection(collection_id): collection.update(updated_data) try: - collection = Collection(**collection).dict() + collection = Collection(**collection).model_dump() except ValidationError as exc: return ( jsonify( @@ -534,7 +533,7 @@ def search_collections(): ) cursor = [ - json.loads(Collection(**doc).json(exclude_unset=True)) + Collection(**doc).model_dump(mode="json", exclude_unset=True) for doc in flask_mongo.db.collections.aggregate(pipeline) ] diff --git a/pydatalab/src/pydatalab/routes/v0_1/export.py b/pydatalab/src/pydatalab/routes/v0_1/export.py index 14f4194d7..fe51ccb0e 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/export.py +++ b/pydatalab/src/pydatalab/routes/v0_1/export.py @@ -134,7 +134,7 @@ def _do_export( def add_stage(message: str, level: str = "info"): stage = TaskStage(timestamp=datetime.now(tz=timezone.utc), message=message, level=level) flask_mongo.db.tasks.update_one( - {"task_id": task_id}, {"$push": {"spec.stages": stage.dict()}} + {"task_id": task_id}, {"$push": {"spec.stages": stage.model_dump()}} ) try: @@ -221,7 +221,7 @@ def start_collection_export(collection_id: str): ), ) - flask_mongo.db.tasks.insert_one(export_task.dict(exclude_none=False)) + flask_mongo.db.tasks.insert_one(export_task.model_dump(exclude_none=False)) task_scheduler.add_job( func=_generate_export_in_background, @@ -390,7 +390,7 @@ def start_item_export(item_id: str): ), ) - flask_mongo.db.tasks.insert_one(export_task.dict(exclude_none=False)) + flask_mongo.db.tasks.insert_one(export_task.model_dump(exclude_none=False)) task_scheduler.add_job( func=_generate_export_in_background, diff --git a/pydatalab/src/pydatalab/routes/v0_1/graphs.py b/pydatalab/src/pydatalab/routes/v0_1/graphs.py index 1299bfd09..72fb68317 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/graphs.py +++ b/pydatalab/src/pydatalab/routes/v0_1/graphs.py @@ -217,7 +217,9 @@ def add_related_items(current_item_id: str, current_depth: int): { "data": { "id": document["item_id"], - "name": document["name"] if document["name"] else document["item_id"], + "name": document.get("name") + if document.get("name") + else document["item_id"], "type": document["type"], "special": document["item_id"] == item_id, } diff --git a/pydatalab/src/pydatalab/routes/v0_1/groups.py b/pydatalab/src/pydatalab/routes/v0_1/groups.py index e1a54d5ef..779166452 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/groups.py +++ b/pydatalab/src/pydatalab/routes/v0_1/groups.py @@ -1,5 +1,3 @@ -import json - from flask import Blueprint, jsonify, request from pydatalab.models.people import Group @@ -42,5 +40,8 @@ def search_groups(): cursor = flask_mongo.db.groups.aggregate(pipeline) return jsonify( - {"status": "success", "data": list(json.loads(Group(**d).json()) for d in cursor)} + { + "status": "success", + "data": [Group(**d).model_dump(mode="json") for d in cursor], + } ), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/info.py b/pydatalab/src/pydatalab/routes/v0_1/info.py index a2212ea87..b2b45aafb 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/info.py +++ b/pydatalab/src/pydatalab/routes/v0_1/info.py @@ -1,20 +1,25 @@ """This submodule defines introspective info endpoints of the API.""" -import json from datetime import datetime from datetime import timedelta as td from datetime import timezone as tz from functools import lru_cache +from typing import Any, Generic, TypeVar from flask import Blueprint, jsonify, request -from pydantic import AnyUrl, BaseModel, Field, validator +from pydantic import ( + AnyUrl, + BaseModel, + ConfigDict, + Field, + field_validator, +) from pydatalab import __version__ from pydatalab.apps import BLOCK_TYPES from pydatalab.config import CONFIG from pydatalab.feature_flags import FEATURE_FLAGS, FeatureFlags -from pydatalab.models import Collection, Person -from pydatalab.models.items import Item +from pydatalab.models import ITEM_SCHEMAS, Person from pydatalab.mongo import flask_mongo from pydatalab.permissions import active_users_or_get_only @@ -24,8 +29,7 @@ class Attributes(BaseModel): - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class Meta(BaseModel): @@ -39,38 +43,42 @@ class Meta(BaseModel): class Links(BaseModel): self: AnyUrl + model_config = ConfigDict(extra="allow") - class Config: - extra = "allow" +AttributesT = TypeVar("AttributesT") -class Data(BaseModel): + +class Data(BaseModel, Generic[AttributesT]): id: str type: str - attributes: Attributes + attributes: AttributesT + """The attributes payload, serialized as whatever concrete type the envelope is + parametrised with (e.g. ``Data[Info]``) so subclass fields are not stripped.""" -class JSONAPIResponse(BaseModel): - data: Data | list[Data] +class JSONAPIResponse(BaseModel, Generic[AttributesT]): + data: Data[AttributesT] | list[Data[AttributesT]] meta: Meta - links: Links | None + links: Links | None = None class MetaPerson(BaseModel): - dislay_name: str | None + display_name: str | None = None contact_email: str class Info(Attributes, Meta): - maintainer: MetaPerson | None - issue_tracker: AnyUrl | None - homepage: AnyUrl | None - source_repository: AnyUrl | None + maintainer: MetaPerson | None = None + issue_tracker: AnyUrl | None = None + homepage: AnyUrl | None = None + source_repository: AnyUrl | None = None identifier_prefix: str - features: FeatureFlags = FEATURE_FLAGS + features: FeatureFlags | None = None max_upload_bytes: int - @validator("maintainer") + @field_validator("maintainer", mode="before") + @classmethod def strip_maintainer_fields(cls, v): if isinstance(v, Person): return MetaPerson(contact_email=v.contact_email, display_name=v.display_name) @@ -81,11 +89,17 @@ def strip_maintainer_fields(cls, v): def _get_deployment_metadata_once() -> dict: identifier_prefix = CONFIG.IDENTIFIER_PREFIX metadata = ( - CONFIG.DEPLOYMENT_METADATA.dict(exclude_none=True) if CONFIG.DEPLOYMENT_METADATA else {} + CONFIG.DEPLOYMENT_METADATA.model_dump(exclude_none=True) + if CONFIG.DEPLOYMENT_METADATA + else {} + ) + metadata.update( + { + "identifier_prefix": identifier_prefix, + "max_upload_bytes": CONFIG.MAX_CONTENT_LENGTH, + "features": FEATURE_FLAGS, + } ) - metadata.update({"identifier_prefix": identifier_prefix}) - metadata.update({"max_upload_bytes": CONFIG.MAX_CONTENT_LENGTH}) - return metadata @@ -95,18 +109,15 @@ def get_info(): versions, features and so on. """ - metadata = _get_deployment_metadata_once() + + response_data = JSONAPIResponse[Info]( + data=Data(id="/", type="info", attributes=Info(**_get_deployment_metadata_once())), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), + links=Links(self=request.url), + ) return ( - jsonify( - json.loads( - JSONAPIResponse( - data=Data(id="/", type="info", attributes=Info(**metadata)), - meta=Meta(query=request.query_string), - links=Links(self=request.url), - ).json() - ) - ), + jsonify(response_data.model_dump(mode="json")), 200, ) @@ -130,97 +141,70 @@ def get_stats(): def list_block_types(): """Returns a list of all blocks implemented in this server.""" return jsonify( - json.loads( - JSONAPIResponse( - data=[ - Data( - id=block_type, - type="block_type", - attributes={ - "name": getattr(block, "name", ""), - "description": getattr(block, "description", ""), - "version": getattr(block, "version", __version__), - "accepted_file_extensions": getattr( - block, "accepted_file_extensions", [] - ), - "multi_file": getattr(block, "multi_file", False), - }, - ) - for block_type, block in BLOCK_TYPES.items() - ], - meta=Meta(query=request.query_string), - ).json() - ) + JSONAPIResponse[dict[str, Any]]( + data=[ + Data( + id=block_type, + type="block_type", + attributes={ + "name": getattr(block, "name", ""), + "description": getattr(block, "description", ""), + "version": getattr(block, "version", __version__), + "accepted_file_extensions": getattr(block, "accepted_file_extensions", []), + "multi_file": getattr(block, "multi_file", False), + }, + ) + for block_type, block in BLOCK_TYPES.items() + ], + meta=Meta(query=request.query_string.decode() if request.query_string else ""), + ).model_dump(mode="json") ) -def get_all_items_models(): - return Item.__subclasses__() - - -def generate_schemas(): - schemas: dict[str, dict] = {} - - for model_class in get_all_items_models() + [Collection]: - model_type = model_class.schema()["properties"]["type"]["default"] - - schemas[model_type] = model_class.schema(by_alias=False) - - return schemas - - -# Generate once on import -SCHEMAS = generate_schemas() - - @INFO.route("/info/types", methods=["GET"]) def list_supported_types(): """Returns a list of supported schemas.""" return jsonify( - json.loads( - JSONAPIResponse( - data=[ - Data( - id=item_type, - type="item_type", - attributes={ - "version": __version__, - "api_version": __api_version__, - "schema": schema, - }, - ) - for item_type, schema in SCHEMAS.items() - ], - meta=Meta(query=request.query_string), - ).json() - ) + JSONAPIResponse[dict[str, Any]]( + data=[ + Data( + id=item_type, + type="item_type", + attributes={ + "version": __version__, + "api_version": __api_version__, + "schema": schema, + }, + ) + for item_type, schema in ITEM_SCHEMAS.items() + ], + meta=Meta(query=request.query_string.decode() if request.query_string else ""), + ).model_dump(mode="json") ) @INFO.route("/info/types/", methods=["GET"]) def get_schema_type(item_type): """Returns the schema of the given type.""" - if item_type not in SCHEMAS: + if item_type not in ITEM_SCHEMAS: return jsonify( {"status": "error", "detail": f"Item type {item_type} not found for this deployment"} ), 404 return jsonify( - json.loads( - JSONAPIResponse( - data=Data( - id=item_type, - type="item_type", - attributes={ - "version": __version__, - "api_version": __api_version__, - "schema": SCHEMAS[item_type], - }, - ), - meta=Meta(query=request.query_string), - ).json() - ) + JSONAPIResponse[dict[str, Any]]( + data=Data( + id=item_type, + type="item_type", + attributes={ + "version": __version__, + "api_version": __api_version__, + "schema": ITEM_SCHEMAS[item_type], + }, + ), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), + ).model_dump(mode="json") ) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index eb14dd5ae..f9d281f52 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -11,21 +11,30 @@ from flask_login import current_user from pydantic import ValidationError from pymongo.errors import DuplicateKeyError -from werkzeug.exceptions import BadRequest, Conflict, NotFound +from werkzeug.exceptions import BadRequest, Conflict, Forbidden, InternalServerError, NotFound from pydatalab.apps import BLOCK_TYPES from pydatalab.config import CONFIG +from pydatalab.feature_flags import FEATURE_FLAGS from pydatalab.logger import LOGGER from pydatalab.models import ITEM_MODELS, ItemVersion from pydatalab.models.items import Item from pydatalab.models.relationships import RelationshipType -from pydatalab.models.utils import InlineSubstance, generate_unique_refcode +from pydatalab.models.utils import AccessScope, InlineSubstance, generate_unique_refcode from pydatalab.models.versions import ( CompareVersionsQuery, RestoreVersionRequest, VersionAction, ) -from pydatalab.mongo import ITEMS_FTS_FIELDS, build_search_pipeline, flask_mongo +from pydatalab.mongo import ( + build_search_pipeline, + creators_lookup, + files_lookup, + flask_mongo, + get_items_fts_fields, + groups_lookup, + resolve_tags_for_docs, +) from pydatalab.permissions import ( PUBLIC_USER_ID, AccessToken, @@ -281,6 +290,8 @@ def get_samples_summary(match: dict | None = None, project: dict | None = None) "refcode": 1, "status": 1, } + if FEATURE_FLAGS.tags: + _project["tags"] = 1 # Cannot mix 0 and 1 keys in MongoDB project so must loop and check if project: @@ -290,7 +301,7 @@ def get_samples_summary(match: dict | None = None, project: dict | None = None) else: _project[key] = 1 - return list( + samples = list( flask_mongo.db.items.aggregate( [ {"$match": match}, @@ -302,34 +313,10 @@ def get_samples_summary(match: dict | None = None, project: dict | None = None) ] ) ) + if FEATURE_FLAGS.tags: + resolve_tags_for_docs(samples) - -def creators_lookup() -> dict: - return { - "from": "users", - "let": {"creator_ids": "$creator_ids"}, - "pipeline": [ - {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$creator_ids", []]}]}}}, - {"$addFields": {"__order": {"$indexOfArray": ["$$creator_ids", "$_id"]}}}, - {"$sort": {"__order": 1}}, - {"$project": {"_id": 1, "display_name": 1, "gravatar_hash": 1}}, - ], - "as": "creators", - } - - -def groups_lookup() -> dict: - return { - "from": "groups", - "let": {"group_ids": "$group_ids"}, - "pipeline": [ - {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$group_ids", []]}]}}}, - {"$addFields": {"__order": {"$indexOfArray": ["$$group_ids", "$_id"]}}}, - {"$sort": {"__order": 1}}, - {"$project": {"_id": 1, "display_name": 1, "group_id": 1}}, - ], - "as": "groups", - } + return samples def entry_reference_lookup(item_doc: dict) -> dict: @@ -411,15 +398,6 @@ def entry_reference_lookup(item_doc: dict) -> dict: return item_doc -def files_lookup() -> dict: - return { - "from": "files", - "localField": "file_ObjectIds", - "foreignField": "_id", - "as": "files", - } - - def collections_lookup() -> dict: """Looks inside the relationships of the item, searches for IDs in the collections table and then projects only the collection ID and name for the response. @@ -516,8 +494,8 @@ def search_items(): Returns: response list of dictionaries containing the matching items in order of descending match score. - """ + """ query = request.args.get("query", type=str) nresults = request.args.get("nresults", default=100, type=int) types = request.args.get("types", default=None) @@ -525,10 +503,10 @@ def search_items(): types = types.split(",") if not query: - return jsonify({"status": "error", "message": "No query provided."}), 400 + raise BadRequest("No query provided.") permissions = get_default_permissions(user_only=False) - pipeline = build_search_pipeline(query, ITEMS_FTS_FIELDS, permissions) + pipeline = build_search_pipeline(query, get_items_fts_fields(), permissions) if types is not None: if pipeline and "$match" in pipeline[0]: @@ -632,6 +610,58 @@ def _copy_sample_from_id(sample_dict: dict, copy_from_item_id: str) -> dict: return sample_dict +def _strip_tag_display_fields(item: dict) -> None: + """Reduce tag references in ``item['tags']`` to the minimal + ``{type, immutable_id}`` link before storage, in place. + + The display fields (name/description/color) are inlined by the client and + re-resolved on every read (`resolve_tags_for_docs`), so persisting them would + be redundant denormalisation. + """ + tags = item.get("tags") + if not isinstance(tags, list): + return + item["tags"] = [ + {"type": "tags", "immutable_id": tag["immutable_id"]} + for tag in tags + if isinstance(tag, dict) and tag.get("immutable_id") is not None + ] + + +def _tag_immutable_ids(tags) -> set[str]: + """Collect the string `immutable_id`s of the tag references in a `tags` list.""" + if not isinstance(tags, list): + return set() + return { + str(tag["immutable_id"]) + for tag in tags + if isinstance(tag, dict) and tag.get("immutable_id") is not None + } + + +def _authorize_added_tags(tags, existing_tag_ids: set[str]) -> None: + """Reject any newly added user-defined tag not owned by the current user. + + A user may add global tags and their own user-defined tags. + """ + # In testing an unauthenticated "public" user can write. + if CONFIG.TESTING and not current_user.is_authenticated: + return + + user_id = current_user.person.immutable_id + + added_ids = _tag_immutable_ids(tags) - existing_tag_ids + if not added_ids: + return + + for tag_doc in flask_mongo.db.tags.find( + {"_id": {"$in": [ObjectId(i) for i in added_ids]}}, + projection={"_id": 1, "scope": 1, "owner": 1}, + ): + if tag_doc.get("scope") == AccessScope.USER.value and tag_doc.get("owner") != user_id: + raise Forbidden("You cannot add a tag that is owned by another user.") + + def _create_sample( sample_dict: dict, copy_from_item_id: str | None = None, @@ -674,7 +704,7 @@ def _create_sample( elif CONFIG.TESTING and not current_user.is_authenticated: # Set fake ID to ObjectId("000000000000000000000000") so a dummy user can be created # locally for testing creator UI elements - new_sample["creator_ids"] = [PUBLIC_USER_ID] + new_sample["creator_ids"] = [str(PUBLIC_USER_ID)] new_sample["creators"] = [ { "display_name": "Public testing user", @@ -718,7 +748,6 @@ def _create_sample( try: # Check on relationship fields and prefill new_sample = entry_reference_lookup(new_sample) - data_model: Item = model(**new_sample) except ValidationError as error: @@ -731,9 +760,10 @@ def _create_sample( # TODO: encode this at the model level, via custom schema properties or hard-coded `.store()` methods # the `Entry` model. try: - result = flask_mongo.db.items.insert_one( - data_model.dict(exclude={"creators", "collections", "groups"}) - ) + to_store = data_model.model_dump(exclude={"creators", "collections", "groups"}) + _authorize_added_tags(to_store.get("tags"), set()) + _strip_tag_display_fields(to_store) + result = flask_mongo.db.items.insert_one(to_store) except DuplicateKeyError as error: raise Conflict(f"Duplicate key error: {str(error)}.") @@ -741,9 +771,13 @@ def _create_sample( raise BadRequest(f"Failed to add new item {new_sample['item_id']!r} to database.") # Save initial version snapshot after successful item creation + refcode = data_model.refcode + if not isinstance(refcode, str): + raise InternalServerError(f"Invalid refcode generated for new item: {refcode!r}") try: version_resp, version_status = save_version_snapshot( - data_model.refcode, action=VersionAction.CREATED + refcode, + action=VersionAction.CREATED, # type: ignore ) if version_status != 200: LOGGER.error( @@ -767,7 +801,7 @@ def _create_sample( data = { "status": "success", "item_id": data_model.item_id, - "sample_list_entry": data_model.dict(), + "sample_list_entry": data_model.model_dump(), } return (data, 201) # 201 Created @@ -799,12 +833,9 @@ def create_samples(): sample_jsons = request_json["new_sample_datas"] if len(sample_jsons) > CONFIG.MAX_BATCH_CREATE_SIZE: - return jsonify( - { - "status": "error", - "message": f"Batch size limit exceeded. Maximum allowed: {CONFIG.MAX_BATCH_CREATE_SIZE}, requested: {len(sample_jsons)}", - } - ), 400 + raise BadRequest( + f"Batch size limit exceeded. Maximum allowed: {CONFIG.MAX_BATCH_CREATE_SIZE}, requested: {len(sample_jsons)}" + ) copy_from_item_ids = request_json.get("copy_from_item_ids") generate_ids_automatically = request_json.get("generate_ids_automatically") @@ -891,13 +922,7 @@ def _process_item_permissions( ] if not groups_requested and not creators_requested: - return ( - { - "status": "error", - "message": "No valid creator or group IDs found in the request.", - }, - 400, - ) + raise BadRequest("No valid creator or group IDs found in the request.") # Validate all creator IDs are present in the database if creator_ids: @@ -905,13 +930,7 @@ def _process_item_permissions( d for d in flask_mongo.db.users.find({"_id": {"$in": creator_ids}}, {"_id": 1}) ] if len(found_creator_ids) != len(creator_ids): - return ( - { - "status": "error", - "message": "One or more creator IDs not found in the database.", - }, - 400, - ) + raise BadRequest("One or more creator IDs not found in the database.") if group_ids: # Validate all group IDs are present in the database @@ -919,13 +938,7 @@ def _process_item_permissions( d for d in flask_mongo.db.groups.find({"_id": {"$in": group_ids}}, {"_id": 1}) ] if len(found_group_ids) != len(group_ids): - return ( - { - "status": "error", - "message": "One or more group IDs not found in the database.", - }, - 400, - ) + raise BadRequest("One or more group IDs not found in the database.") if append_mode: if creators_requested: @@ -987,12 +1000,8 @@ def _process_item_permissions( ) if result.modified_count != 1: - return ( - { - "status": "error", - "message": "Failed to update permissions: you cannot remove yourself or the base owner as a creator.", - }, - 400, + raise BadRequest( + "Failed to update permissions: you cannot remove yourself or the base owner as a creator." ) # Record the new ownership in the item's history as a version @@ -1046,12 +1055,7 @@ def issue_physical_token(refcode: str): ) if not current_item: - return jsonify( - { - "status": "error", - "message": f"No valid item found with the given {refcode=}.", - } - ), 404 + raise NotFound token = secrets.token_urlsafe(16) access_document = AccessToken( @@ -1065,7 +1069,7 @@ def issue_physical_token(refcode: str): ) try: - result = flask_mongo.db.api_keys.insert_one(access_document.dict()) + result = flask_mongo.db.api_keys.insert_one(access_document.model_dump()) if not result.inserted_id: return jsonify( {"status": "error", "message": "Unknown error generating token for item."} @@ -1105,15 +1109,7 @@ def delete_sample(): ) if result.deleted_count != 1: - return ( - jsonify( - { - "status": "error", - "message": f"Failed to delete item with {item_id=}.", - } - ), - 400, - ) + raise BadRequest(f"Failed to delete item with {item_id=}.") flask_mongo.db.api_keys.delete_many({"refcode": item["refcode"], "type": "access_token"}) @@ -1152,7 +1148,6 @@ def get_item_data( elif refcode: if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" - match = {"refcode": refcode} else: raise BadRequest("No item_id or refcode provided.") @@ -1177,19 +1172,12 @@ def get_item_data( try: doc = list(cursor)[0] + except IndexError: doc = None if not doc: - return ( - jsonify( - { - "status": "error", - "message": f"No matching items for {match=} with current authorization.", - } - ), - 404, - ) + raise NotFound # See LAST_MODIFIED_PROJECTION: same backfill, applied outside an aggregation if not doc.get("last_modified") and isinstance(doc.get("_id"), ObjectId): @@ -1206,6 +1194,9 @@ def get_item_data( try: doc = entry_reference_lookup(doc) + # Resolve tag references for display only (a read-time concern): inline + # current tag names and drop references to deleted tags. + resolve_tags_for_docs([doc]) doc = ItemModel(**doc) except ValidationError as error: # The stored document doesn't validate against its declared schema. @@ -1252,7 +1243,7 @@ def get_item_data( ) # loop over and collect all 'outer' relationships presented by other items - incoming_relationships: dict[RelationshipType, set[str]] = {} + incoming_relationships: dict[RelationshipType, set] = {} for d in relationships_query_results: for k in d["relationships"]: if k["relation"] not in incoming_relationships: @@ -1262,7 +1253,7 @@ def get_item_data( ) # loop over and aggregate all 'inner' relationships presented by this item - inlined_relationships: dict[RelationshipType, set[str]] = {} + inlined_relationships: dict[RelationshipType, set] = {} if doc.relationships is not None: inlined_relationships = { relation: { @@ -1282,7 +1273,7 @@ def get_item_data( ) # Must be exported to JSON first to apply the custom pydantic JSON encoders - return_dict = json.loads(doc.json(exclude_unset=True)) + return_dict = doc.model_dump(mode="json", exclude_unset=True) if item_id is None: item_id = return_dict["item_id"] @@ -1307,9 +1298,7 @@ def list_versions(refcode): # Check if user has access to the item (read access) has_access, _ = check_version_access(refcode, user_only=False) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1361,9 +1350,7 @@ def get_version(refcode, version_id): # Check if user has access to the item (read access) has_access, _ = check_version_access(refcode, user_only=False) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1371,7 +1358,7 @@ def get_version(refcode, version_id): try: version_object_id = ObjectId(version_id) except (InvalidId, TypeError): - return jsonify({"status": "error", "message": f"Invalid version_id: {version_id}"}), 400 + raise BadRequest(f"Invalid version_id: {version_id}") version = list( flask_mongo.db.item_versions.aggregate( @@ -1398,7 +1385,7 @@ def get_version(refcode, version_id): version = version[0] if not version: - return jsonify({"status": "error", "message": "Version not found"}), 404 + raise NotFound return jsonify({"status": "success", "version": version}), 200 @@ -1412,9 +1399,7 @@ def compare_versions(refcode): # Check if user has access to the item (read access) has_access, _ = check_version_access(refcode, user_only=False) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1425,20 +1410,12 @@ def compare_versions(refcode): v1=request.args.get("v1", ""), v2=request.args.get("v2", "") ) except ValidationError as exc: - return jsonify( - {"status": "error", "message": "Invalid query parameters", "errors": exc.errors()} - ), 400 - - try: - v1_object_id = ObjectId(query_params.v1) - v2_object_id = ObjectId(query_params.v2) - except (InvalidId, TypeError) as e: - return jsonify({"status": "error", "message": f"Invalid version ID format: {str(e)}"}), 400 + raise BadRequest(f"Invalid query parameters: {exc}") - v1 = flask_mongo.db.item_versions.find_one({"_id": v1_object_id, "refcode": refcode}) - v2 = flask_mongo.db.item_versions.find_one({"_id": v2_object_id, "refcode": refcode}) + v1 = flask_mongo.db.item_versions.find_one({"_id": query_params.v1, "refcode": refcode}) + v2 = flask_mongo.db.item_versions.find_one({"_id": query_params.v2, "refcode": refcode}) if not v1 or not v2: - return jsonify({"status": "error", "message": "One or both versions not found"}), 404 + raise NotFound("One or both versions not found") # Use DeepDiff for proper nested structure comparison # This handles nested dicts, lists, type changes, and provides detailed change information @@ -1483,29 +1460,23 @@ def restore_version(refcode): try: restore_request = RestoreVersionRequest(**request.get_json()) except ValidationError as exc: - return jsonify( - {"status": "error", "message": "Invalid request body", "errors": exc.errors()} - ), 400 + raise BadRequest(f"Invalid request body: {exc}") try: version_object_id = ObjectId(restore_request.version_id) except (InvalidId, TypeError): - return jsonify( - {"status": "error", "message": f"Invalid version_id: {restore_request.version_id}"} - ), 400 + raise BadRequest(f"Invalid version_id: {restore_request.version_id}") # Check permissions - user must have write access current_item = flask_mongo.db.items.find_one( {"refcode": refcode, **get_default_permissions(user_only=True)} ) if not current_item: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound version = flask_mongo.db.item_versions.find_one({"_id": version_object_id, "refcode": refcode}) if not version: - return jsonify({"status": "error", "message": "Version not found"}), 404 + raise NotFound restored_data = version["data"].copy() @@ -1514,12 +1485,9 @@ def restore_version(refcode): # Ensure type consistency if restored_data.get("type") != current_item.get("type"): - return jsonify( - { - "status": "error", - "message": f"Cannot restore version with different type. Current: {current_item.get('type')}, Version: {restored_data.get('type')}", - } - ), 400 + raise BadRequest( + f"Cannot restore version with different type. Current: {current_item.get('type')}, Version: {restored_data.get('type')}" + ) # Atomically get the next version number (used for both version in item_versions and item.version) next_version_number = get_next_version_number(refcode) @@ -1531,19 +1499,15 @@ def restore_version(refcode): # Validate restored data against the item model item_type = current_item["type"] if item_type not in ITEM_MODELS: - return jsonify({"status": "error", "message": f"Invalid item type: {item_type}"}), 400 + raise BadRequest(f"Invalid item type: {item_type}") try: # Validate using the appropriate model ITEM_MODELS[item_type](**restored_data) except ValidationError as exc: - return jsonify( - { - "status": "error", - "message": f"Restored data failed validation: {str(exc)}", - "output": str(exc), - } - ), 400 + raise BadRequest( + f"Restored data failed validation against schema for type {item_type}: {exc}" + ) # Perform the restore first flask_mongo.db.items.update_one({"refcode": refcode}, {"$set": restored_data}) @@ -1580,18 +1544,13 @@ def restore_version(refcode): refcode, str(exc), ) - return jsonify( - { - "status": "error", - "message": f"Restored version data validation failed: {str(exc)}", - "output": str(exc), - } - ), 400 + raise BadRequest(f"Restored version data validation failed: {exc}") - # Insert validated data - flask_mongo.db.item_versions.insert_one( - validated_restored_version.dict(by_alias=True, exclude_none=True) - ) + # Insert validated data, restoring original restored_data into 'data' so None-valued + # fields are not stripped by model_dump(exclude_none=True) recursing into the dict. + restored_version_doc = validated_restored_version.model_dump(exclude_none=True) + restored_version_doc["data"] = restored_data + flask_mongo.db.item_versions.insert_one(restored_version_doc) return jsonify( { @@ -1611,9 +1570,7 @@ def delete_version(refcode, version_id): # Check if user has write access to the item (write access required) has_access, _ = check_version_access(refcode, user_only=True) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1621,13 +1578,13 @@ def delete_version(refcode, version_id): try: version_object_id = ObjectId(version_id) except (InvalidId, TypeError): - return jsonify({"status": "error", "message": f"Invalid version_id: {version_id}"}), 400 + raise BadRequest(f"Invalid version_id: {version_id}") result = flask_mongo.db.item_versions.delete_one({"_id": version_object_id, "refcode": refcode}) if result.deleted_count == 1: return jsonify({"status": "success"}), 200 else: - return jsonify({"status": "error", "message": "Version not found"}), 404 + raise NotFound @ITEMS.route("/items//save-version/", methods=["POST"]) @@ -1670,6 +1627,7 @@ def save_item(): # These keys should not be updated here and cannot be modified by the user through this endpoint for k in ( "_id", + "immutable_id", "file_ObjectIds", "files", "creators", @@ -1691,23 +1649,13 @@ def save_item(): ) if not item: - return ( - jsonify( - status="error", - message=f"Unable to find item with appropriate permissions and {item_id=}.", - ), - 404, - ) + raise NotFound # Store refcode for version saving after successful update refcode = item.get("refcode") if not refcode: - return ( - jsonify( - status="error", - message=f"Item {item_id} does not have a refcode.", - ), - 400, + raise InternalServerError( + f"Item {item_id} does not have a refcode; please report this issue." ) user_only = item["type"] not in ("starting_materials", "equipment") @@ -1717,13 +1665,7 @@ def save_item(): ) if not item: - return ( - jsonify( - status="error", - message=f"Unable to find item with appropriate permissions and {item_id=}.", - ), - 404, - ) + raise NotFound stored_blocks = item.get("blocks_obj", {}) for block_id, block_data in updated_data.get("blocks_obj", {}).items(): @@ -1785,27 +1727,30 @@ def save_item(): preserve_relationships = "collections" not in updated_data original_relationships = item.get("relationships", []) if preserve_relationships else None + # Snapshot the tags already on the item so we only authorize newly added + # tags below. + existing_tag_ids = _tag_immutable_ids(item.get("tags")) + item.update(updated_data) try: item = entry_reference_lookup(item) - item = ITEM_MODELS[item_type](**item).dict() - except ValidationError as exc: - return ( - jsonify( - status="error", - message=f"Unable to update item {item_id=} ({item_type=}) with new data {updated_data}", - output=str(exc), - ), - 400, + item = ITEM_MODELS[item_type](**item).model_dump( + exclude_unset=True, + exclude={"collections", "creators", "immutable_id"}, ) + except ValidationError as exc: + raise BadRequest(f"Unable to update item {item_id=} ({item_type=}): {exc}") + if preserve_relationships and original_relationships is not None: item["relationships"] = original_relationships # remove collections and creators and any other reference fields - item.pop("collections") - item.pop("creators") + item.pop("collections", None) + item.pop("creators", None) + item.pop("immutable_id", None) + item.pop("files", None) # `last_modified` is controlled by the versioning branch below: only bump it when a # snapshot is actually saved, so re-submitting identical data leaves it untouched. @@ -1813,6 +1758,12 @@ def save_item(): if isinstance(existing_last_modified, datetime.datetime): existing_last_modified = existing_last_modified.isoformat() + # A user may not add another user's user-defined tag. + _authorize_added_tags(item.get("tags"), existing_tag_ids) + + # Store tag references minimally; see `_strip_tag_display_fields`. + _strip_tag_display_fields(item) + # Update the item FIRST (transaction safety: item update before version save) result = flask_mongo.db.items.update_one( {"item_id": item_id, **get_default_permissions(user_only=True)}, @@ -1820,14 +1771,7 @@ def save_item(): ) if result.matched_count != 1: - return ( - jsonify( - status="error", - message=f"{item_id} item update failed. no subdocument matched", - output=result.raw_result, - ), - 400, - ) + raise BadRequest(f"{item_id} item update failed. no subdocument matched") # Now save a version AFTER successful item update. # Only increment item.version and bump last_modified when content actually changed @@ -1884,12 +1828,7 @@ def get_access_token_info(refcode: str): ) if not current_item: - return jsonify( - { - "status": "error", - "message": f"No valid item found with the given {refcode=}.", - } - ), 404 + raise NotFound existing_token = flask_mongo.db.api_keys.find_one( {"refcode": refcode, "active": True, "type": "access_token"}, diff --git a/pydatalab/src/pydatalab/routes/v0_1/remotes.py b/pydatalab/src/pydatalab/routes/v0_1/remotes.py index be50502f9..dab3fa00f 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/remotes.py +++ b/pydatalab/src/pydatalab/routes/v0_1/remotes.py @@ -1,4 +1,3 @@ -import json from typing import Any from flask import Blueprint, jsonify, request @@ -76,7 +75,7 @@ def list_remote_directories(): response = {} response["meta"] = {} - response["meta"]["remotes"] = [json.loads(d.json()) for d in CONFIG.REMOTE_FILESYSTEMS] + response["meta"]["remotes"] = [d.model_dump(mode="json") for d in CONFIG.REMOTE_FILESYSTEMS] if all_directory_structures: oldest_update = min(d["last_updated"] for d in all_directory_structures) response["meta"]["oldest_cache_update"] = oldest_update.isoformat() @@ -139,7 +138,7 @@ def get_remote_directory(remote_id: str): response: dict[str, Any] = {} response["meta"] = {} - response["meta"]["remote"] = json.loads(d.json()) + response["meta"]["remote"] = d.model_dump(mode="json") response["data"] = directory_structure return jsonify(response), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/tags.py b/pydatalab/src/pydatalab/routes/v0_1/tags.py new file mode 100644 index 000000000..148024c3c --- /dev/null +++ b/pydatalab/src/pydatalab/routes/v0_1/tags.py @@ -0,0 +1,291 @@ +import datetime +import json + +from bson import ObjectId +from bson.errors import InvalidId +from flask import Blueprint, abort, jsonify, request +from flask_login import current_user +from pydantic import ValidationError +from werkzeug.exceptions import BadRequest, Conflict, Forbidden, NotFound, Unauthorized + +from pydatalab.feature_flags import FEATURE_FLAGS +from pydatalab.logger import logged_route +from pydatalab.models.tags import Tag +from pydatalab.models.utils import AccessScope, UserRole +from pydatalab.mongo import ( + TAGS_FTS_FIELDS, + build_search_pipeline, + flask_mongo, + insert_pydantic_model_fork_safe, +) +from pydatalab.permissions import active_users_or_get_only + +TAGS = Blueprint("tags", __name__) + + +@TAGS.before_request +def _require_tags_feature(): + """Gate the whole blueprint behind the `tags` feature flag.""" + if not FEATURE_FLAGS.tags: + abort(404) + + +@TAGS.before_request +@active_users_or_get_only +def _(): ... + + +def _parse_object_id(raw: str) -> ObjectId | None: + """Parse a string into an ObjectId, returning None if it is not valid.""" + try: + return ObjectId(raw) + except (InvalidId, TypeError): + return None + + +def _is_admin() -> bool: + """Whether the current user is an authenticated administrator.""" + return bool(current_user.is_authenticated and current_user.role == UserRole.ADMIN) + + +def _current_user_id() -> ObjectId | None: + """The immutable ID of the current user, or None if unauthenticated.""" + if current_user.is_authenticated and current_user.person is not None: + return current_user.person.immutable_id + return None + + +def _usable_tags_filter(user_id: ObjectId | None) -> dict: + """The Mongo filter for tags the given user may list and use. + + This is global tags plus the user's own user-defined tags. + """ + if user_id is None: + return {"scope": AccessScope.GLOBAL.value} + return {"$or": [{"scope": AccessScope.GLOBAL.value}, {"owner": user_id}]} + + +def _name_conflict_exists( + name: str, + scope: AccessScope, + owner: ObjectId | None = None, + exclude_id: ObjectId | None = None, +) -> bool: + """Whether a tag with `name` already exists within the given scope. + + The same name may exist across scopes (e.g. a global `x` and a user-defined `x`). + """ + query: dict = {"name": name, "scope": scope.value} + if scope == AccessScope.USER: + query["owner"] = owner + + if exclude_id is not None: + query["_id"] = {"$ne": exclude_id} + + return flask_mongo.db.tags.find_one(query, {"_id": 1}) is not None + + +def _authorize_tag_write(tag_doc: dict) -> bool: + """Whether the current user may edit or delete the given tag document. + + Global tags require an administrator; user-defined tags require the owner. + """ + if tag_doc["scope"] == AccessScope.USER.value: + user_id = _current_user_id() + return user_id is not None and tag_doc.get("owner") == user_id + return _is_admin() + + +@TAGS.route("/tags", methods=["PUT"]) +def create_tag(): + """Create a new tag. + + Anyone logged in can create a user-defined tag. Only administrators can create + global tags. + """ + request_json = request.get_json() + data = request_json.get("data", {}) + + name = data.get("name") + if not name: + raise BadRequest("A tag name is required.") + + try: + scope = AccessScope(data.get("scope") or AccessScope.USER.value) + except ValueError: + raise BadRequest(f"Invalid tag scope {data.get('scope')!r}.") + + if scope == AccessScope.GLOBAL: + if not _is_admin(): + raise Forbidden("Only administrators can create global tags.") + owner = None + else: + owner = _current_user_id() + if owner is None: + raise Unauthorized("You must be logged in to create a user-defined tag.") + + if _name_conflict_exists(name, scope, owner): + raise Conflict(f"A tag named {name!r} already exists.") + + try: + tag = Tag( + name=name, + description=data.get("description"), + color=data.get("color"), + scope=scope, + owner=owner, + last_modified=datetime.datetime.now(datetime.timezone.utc).isoformat(), + ) + except ValidationError as error: + raise BadRequest(f"Unable to create the tag: {error}") + + tag.immutable_id = insert_pydantic_model_fork_safe(tag, "tags") + + return jsonify({"status": "success", "data": json.loads(tag.model_dump_json())}), 201 + + +@TAGS.route("/tags", methods=["GET"]) +def get_tags(): + """Return the tags usable by the current user: global tags plus their own.""" + tags = flask_mongo.db.tags.find(_usable_tags_filter(_current_user_id())).sort("name", 1) + data = [Tag(**doc).model_dump(mode="json") for doc in tags] + return jsonify({"status": "success", "data": data}) + + +@TAGS.route("/search-tags", methods=["GET"]) +def search_tags(): + """Perform a full-text search over the tags usable by the current user. + + GET parameters: + query: String with the search terms. + nresults: Maximum number of results (default 100). + + Returns: + A list of `{type, immutable_id, name, description, color, scope}` + dictionaries in order of descending match score, suitable for use as tag + references. + """ + query = request.args.get("query", type=str) + nresults = request.args.get("nresults", default=100, type=int) + + if not query: + raise BadRequest("No query provided.") + + pipeline = build_search_pipeline( + query, TAGS_FTS_FIELDS, _usable_tags_filter(_current_user_id()) + ) + pipeline.append({"$limit": nresults}) + pipeline.append({"$project": {"_id": 1, "name": 1, "description": 1, "color": 1, "scope": 1}}) + + data = [ + { + "type": "tags", + "immutable_id": str(doc["_id"]), + "name": doc.get("name"), + "description": doc.get("description"), + "color": doc.get("color"), + "scope": doc["scope"], + } + for doc in flask_mongo.db.tags.aggregate(pipeline) + ] + + return jsonify({"status": "success", "data": data}), 200 + + +@TAGS.route("/tags/", methods=["PATCH"]) +@logged_route +def save_tag(tag_id): + """Update a tag's `name`/`description`/`color`. + + Global tags may only be edited by administrators; user-defined tags only by their + owner. + """ + object_id = _parse_object_id(tag_id) + if object_id is None: + raise BadRequest(f"Invalid tag ID {tag_id!r}.") + + request_json = request.get_json() + updated_data = request_json.get("data") + + if not updated_data: + raise BadRequest("No data provided to update the tag with.") + + tag = flask_mongo.db.tags.find_one({"_id": object_id}) + + if not tag: + raise NotFound(f"Unable to find a tag with ID {tag_id!r}.") + + if not _authorize_tag_write(tag): + raise Forbidden("You are not allowed to modify this tag.") + + # Identity, scope and ownership are not editable through this endpoint. + for key in ("_id", "immutable_id", "type", "scope", "owner"): + updated_data.pop(key, None) + + updated_data["last_modified"] = datetime.datetime.now(datetime.timezone.utc).isoformat() + + # Keep names unique within the tag's own scope on rename. + if "name" in updated_data: + scope = AccessScope(tag["scope"]) + if _name_conflict_exists( + updated_data["name"], scope, tag.get("owner"), exclude_id=object_id + ): + raise Conflict(f"A tag named {updated_data['name']!r} already exists.") + + tag.update(updated_data) + + try: + tag = Tag(**tag).model_dump(exclude={"immutable_id"}) + except ValidationError as exc: + raise BadRequest(f"Unable to update tag {tag_id!r} with new data {updated_data}: {exc}") + + result = flask_mongo.db.tags.update_one({"_id": object_id}, {"$set": tag}) + + if result.modified_count != 1: + return ( + jsonify( + status="error", + message=f"Unable to update tag {tag_id!r}.", + output=result.raw_result, + ), + 400, + ) + + return jsonify(status="success"), 200 + + +@TAGS.route("/tags/", methods=["DELETE"]) +def delete_tag(tag_id: str): + """Delete a tag and drop its references from items. + + Global tags may only be deleted by administrators; user-defined tags only by + their owner. + """ + object_id = _parse_object_id(tag_id) + if object_id is None: + raise BadRequest(f"Invalid tag ID {tag_id!r}.") + + tag = flask_mongo.db.tags.find_one({"_id": object_id}) + + if not tag: + raise NotFound(f"No tag found with ID {tag_id!r}.") + + if not _authorize_tag_write(tag): + raise Forbidden("You are not allowed to delete this tag.") + + flask_mongo.db.tags.delete_one({"_id": object_id}) + + # Best-effort cleanup: drop references to the deleted tag from items' `tags` + # arrays. Like collection deletion, this is a raw update that does NOT go + # through the item save route, so it neither bumps `last_modified` nor creates + # a new item version. + # + # This hardcodes `items` as the only `HasTags` collection. Extend it if + # `HasTags` is applied to other entities. Note that references are not deleted + # from item_versions, so a reference to a deleted tag can survive there. + flask_mongo.db.items.update_many( + {"tags": {"$elemMatch": {"immutable_id": object_id, "type": "tags"}}}, + {"$pull": {"tags": {"immutable_id": object_id, "type": "tags"}}}, + ) + + return jsonify(status="success"), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/users.py b/pydatalab/src/pydatalab/routes/v0_1/users.py index b59aca49d..15cc3ca48 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/users.py +++ b/pydatalab/src/pydatalab/routes/v0_1/users.py @@ -1,4 +1,3 @@ -import json from datetime import datetime from datetime import timedelta as td from datetime import timezone as tz @@ -6,11 +5,12 @@ from bson import ObjectId from flask import Blueprint, jsonify, request from flask_login import current_user +from pydantic import ValidationError from werkzeug.exceptions import BadRequest, Forbidden, Unauthorized from pydatalab.config import CONFIG from pydatalab.logger import LOGGER -from pydatalab.models.people import AccountStatus, DisplayName, EmailStr, Person +from pydatalab.models.people import Person from pydatalab.mongo import ( USERS_FTS_FIELDS, build_search_pipeline, @@ -49,22 +49,22 @@ def save_user(user_id): update = {} - try: - if display_name: - update["display_name"] = DisplayName(display_name) + if display_name: + update["display_name"] = display_name - except ValueError: - raise BadRequest(f"Invalid display name {display_name!r} was passed") + if contact_email or contact_email in (None, ""): + if contact_email in ("", None): + update["contact_email"] = None + else: + update["contact_email"] = contact_email - try: - if contact_email or contact_email in (None, ""): - if contact_email in ("", None): - update["contact_email"] = None - else: - update["contact_email"] = EmailStr(contact_email) + if account_status: + update["account_status"] = account_status - except ValueError: - raise BadRequest(f"Invalid email address {contact_email!r} was passed") + try: + _ = Person(**update) + except ValidationError as e: + raise BadRequest(f"Invalid user data: {e.errors()}") from e if "contact_email" in update or "display_name" in update: existing = ( @@ -128,12 +128,6 @@ def save_user(user_id): trigger_email_verification = False LOGGER.critical("Unable to send verification email on this deployment: %s", e) - try: - if account_status: - update["account_status"] = AccountStatus(account_status) - except ValueError: - raise BadRequest(f"Invalid account status {account_status!r} was passed") - if not update: return jsonify({"status": "success", "message": "No update to perform."}), 200 @@ -223,5 +217,5 @@ def search_users(): cursor = flask_mongo.db.users.aggregate(pipeline) return jsonify( - {"status": "success", "users": [json.loads(Person(**d).json()) for d in cursor]} + {"status": "success", "users": [Person(**d).model_dump(mode="json") for d in cursor]} ), 200 diff --git a/pydatalab/src/pydatalab/versioning.py b/pydatalab/src/pydatalab/versioning.py index 351ddf032..7d7916db6 100644 --- a/pydatalab/src/pydatalab/versioning.py +++ b/pydatalab/src/pydatalab/versioning.py @@ -219,10 +219,14 @@ def save_version_snapshot( # permanent gap in the item's version history. next_version_number = get_next_version_number(refcode) - # Insert validated data (convert to dict and exclude None values) - version_doc = validated_version.dict(by_alias=True, exclude_none=True) + # Insert validated data, but restore the original item dict into 'data' so that + # None-valued fields are not stripped by model_dump(exclude_none=True) recursing into it. + version_doc = validated_version.model_dump(exclude_none=True) + version_doc["data"] = item version_doc["version"] = next_version_number + flask_mongo.db.item_versions.insert_one(version_doc) + return ( {"status": "success", "message": "Version saved.", "version": next_version_number}, 200, diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index 6ba1387be..888ceda35 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -1,3 +1,4 @@ +# This file was edited with the assistance of an AI model and requires human review from the contributor. import json import os import pathlib @@ -5,14 +6,10 @@ import shutil import subprocess import time -import typing import tomlkit from invoke import Collection, task -if typing.TYPE_CHECKING: - from pydatalab.models.utils import UserRole - ns = Collection() dev = Collection("dev") admin = Collection("admin") @@ -33,11 +30,13 @@ def load_plugin_schema(): need them. A JSON Schema generated from the returned model is emitted to `pydatalab/schemas/plugin_config.json` by `invoke dev.generate-schemas`. """ - from pydantic import BaseModel, root_validator + from pydantic import BaseModel, ConfigDict, model_validator class UvSource(BaseModel): """A single entry under `[tool.uv.sources]` in plugins.toml.""" + model_config = ConfigDict(extra="forbid") + git: str | None = None rev: str | None = None branch: str | None = None @@ -45,10 +44,8 @@ class UvSource(BaseModel): path: str | None = None editable: bool | None = None - class Config: - extra = "forbid" - - @root_validator + @model_validator(mode="before") + @classmethod def _exactly_one_source(cls, values): has_git = values.get("git") is not None has_path = values.get("path") is not None @@ -59,36 +56,33 @@ def _exactly_one_source(cls, values): return values class UvSection(BaseModel): - sources: dict[str, UvSource] = {} + model_config = ConfigDict(extra="forbid") - class Config: - extra = "forbid" + sources: dict[str, UvSource] = {} class ToolSection(BaseModel): - uv: UvSection = UvSection() + model_config = ConfigDict(extra="forbid") - class Config: - extra = "forbid" + uv: UvSection = UvSection() class PluginConfigModel(BaseModel): """The schema for the top-level plugins.toml file.""" + model_config = ConfigDict(extra="forbid") + dependencies: list[str] = [] tool: ToolSection = ToolSection() - class Config: - extra = "forbid" - - @root_validator - def _sources_must_match_dependencies(cls, values): - deps = {d.split("[")[0].strip() for d in values.get("dependencies", [])} - sources = values.get("tool", ToolSection()).uv.sources + @model_validator(mode="after") + def _sources_must_match_dependencies(self): + deps = {d.split("[")[0].strip() for d in self.dependencies} + sources = self.tool.uv.sources orphans = sorted(set(sources) - deps) if orphans: raise ValueError( f"[tool.uv.sources] entries have no matching dependency: {orphans}" ) - return values + return self return PluginConfigModel @@ -115,7 +109,7 @@ def generate_schemas(_): schemas_path = pathlib.Path(__file__).parent / "schemas" for model in ITEM_MODELS.values(): - schema = model.schema(by_alias=False) + schema = model.model_json_schema(by_alias=False) with open(schemas_path / f"{model.__name__.lower()}.json", "w") as f: json.dump(schema, f, indent=2) @@ -286,7 +280,7 @@ def create_mongo_indices(_): @task -def change_user_role(_, display_name: str, role: "UserRole"): +def change_user_role(_, display_name: str, role: str): """This task takes a user's name and gives them the desired role.""" from bson import ObjectId @@ -294,8 +288,8 @@ def change_user_role(_, display_name: str, role: "UserRole"): from pydatalab.mongo import _get_active_mongo_client try: - role = getattr(UserRole, role.upper()) - except AttributeError: + role = UserRole(role.lower()) + except ValueError: raise SystemExit(f"Invalid role: {role!r}. Must be one of {UserRole.__members__}") from None matches = list( @@ -371,6 +365,58 @@ def manually_register_user( admin.add_task(manually_register_user) +@task +def seed_e2e_admin( + _, + display_name: str = "Test Admin", + contact_email: str = "admin-user@example.com", +): + """Ensure a predefined, active admin user exists, for end-to-end testing. + + The tests may need an administrator, but the first admin cannot be created + over the API. This task creates the user with an email identity if it does + not already exist, marks it active, and grants it the admin role. + """ + from pydatalab.models.people import AccountStatus, Identity, Person + from pydatalab.models.utils import UserRole + from pydatalab.mongo import get_database, insert_pydantic_model_fork_safe + + db = get_database() + + user = db.users.find_one( + {"identities.identity_type": "email", "identities.identifier": contact_email} + ) + + if user is None: + new_user = Person( + display_name=display_name, + contact_email=contact_email, + account_status=AccountStatus.ACTIVE, + identities=[ + Identity( + identity_type="email", + identifier=contact_email, + name=contact_email, + verified=True, + ) + ], + ) + user_id = insert_pydantic_model_fork_safe(new_user, "users") + print(f"Created active user {display_name!r} <{contact_email}> ({user_id}).") + else: + user_id = user["_id"] + db.users.update_one( + {"_id": user_id}, {"$set": {"account_status": AccountStatus.ACTIVE.value}} + ) + print(f"User <{contact_email}> already exists ({user_id}); ensured active.") + + db.roles.update_one({"_id": user_id}, {"$set": {"role": UserRole.ADMIN.value}}, upsert=True) + print(f"Granted the admin role to <{contact_email}>.") + + +admin.add_task(seed_e2e_admin) + + @task def repair_files(_, resync: bool = True): """Loop through samples and find any with attached files @@ -427,11 +473,12 @@ def add_missing_refcodes(_): def _check_id(id=None, base_url=None, api_key=None): - from pydatalab.logger import setup_log - """Checks the given item ID served at the base URL and logs the result.""" + import requests + from pydatalab.logger import setup_log + log = setup_log("check_item_validity") response = requests.get( f"{base_url}/get-item-data/{id}", headers={"DATALAB-API-KEY": api_key}, timeout=30 diff --git a/pydatalab/tests/server/conftest.py b/pydatalab/tests/server/conftest.py index 5be100adc..c5ac697cf 100644 --- a/pydatalab/tests/server/conftest.py +++ b/pydatalab/tests/server/conftest.py @@ -58,6 +58,7 @@ def app_config(secret_key, files_directory): "REMOTE_FILESYSTEMS": example_remotes, "FILE_DIRECTORY": str(files_directory), "TESTING": False, + "ENABLE_TAGS": True, "ROOT_PATH": "/", "SECRET_KEY": secret_key, "AUTO_ACTIVATE_ACCOUNTS": False, @@ -515,7 +516,7 @@ def fixture_insert_complicated_sample_constituents(user_id): creator_ids=[user_id], refcode=generate_unique_refcode(), ) - flask_mongo.db.items.insert_one(sm.dict(exclude_unset=False)) + flask_mongo.db.items.insert_one(sm.model_dump(exclude_unset=False)) items.append(sm) yield items @@ -528,7 +529,7 @@ def fixture_insert_complicated_sample_constituents(user_id): def example_items(user_id, admin_user_id): """Create a collection of samples with mixed ownership between the user and admin.""" return [ - d.dict(exclude_unset=False) + d.model_dump(exclude_unset=False) for d in [ Sample( **{ @@ -615,23 +616,23 @@ def example_items(user_id, admin_user_id): @pytest.fixture(scope="module", name="default_sample_dict") -def fixture_default_sample_dict(default_sample): - return default_sample.dict(exclude_unset=True) +def fixture_default_sample_model_dump(default_sample): + return default_sample.model_dump(exclude_unset=True) @pytest.fixture(scope="module", name="default_cell_dict") -def fixture_default_cell_dict(default_cell): - return default_cell.dict(exclude_unset=True) +def fixture_default_cell_model_dump(default_cell): + return default_cell.model_dump(exclude_unset=True) @pytest.fixture(scope="module", name="default_starting_material_dict") -def fixture_default_starting_material_dict(default_starting_material): - return default_starting_material.dict(exclude_unset=True) +def fixture_default_starting_material_model_dump(default_starting_material): + return default_starting_material.model_dump(exclude_unset=True) @pytest.fixture(scope="module", name="default_equipment_dict") -def fixture_default_equipment_dict(default_equipment): - return default_equipment.dict(exclude_unset=True) +def fixture_default_equipment_model_dump(default_equipment): + return default_equipment.model_dump(exclude_unset=True) def _insert_and_cleanup_item_from_model(model): @@ -640,7 +641,7 @@ def _insert_and_cleanup_item_from_model(model): refcode = generate_unique_refcode() model.refcode = refcode - flask_mongo.db.items.insert_one(model.dict(exclude_unset=False)) + flask_mongo.db.items.insert_one(model.model_dump(exclude_unset=False)) yield model flask_mongo.db.items.delete_one({"refcode": model.refcode}) diff --git a/pydatalab/tests/server/test_async_blocks.py b/pydatalab/tests/server/test_async_blocks.py index 6e85ff4a2..62c59859c 100644 --- a/pydatalab/tests/server/test_async_blocks.py +++ b/pydatalab/tests/server/test_async_blocks.py @@ -147,7 +147,7 @@ def test_status_pending(self, client, user_id, database): block_id="test_block", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -182,7 +182,7 @@ def test_status_processing_with_stages(self, client, user_id, database): stages=stages, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -226,7 +226,7 @@ def test_status_ready_with_gridfs_data(self, client, user_id, database, app): block_id=block_id, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) # Write block data to GridFS from pydatalab.mongo import get_database @@ -276,7 +276,7 @@ def test_status_ready_no_gridfs_data(self, client, user_id, database): block_id=block_id, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -304,7 +304,7 @@ def test_status_error(self, client, user_id, database): block_id="test_block", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -334,7 +334,7 @@ def test_status_ignores_export_tasks(self, client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 404 @@ -369,7 +369,7 @@ def test_status_ready_respects_permissions(self, another_client, user_id, databa block_id=block_id, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) from pydatalab.mongo import get_database @@ -428,7 +428,7 @@ def test_successful_processing(self, app, user_id, database): status=TaskStatus.PENDING, spec=BlockProcessingTaskSpec(item_id=item_id, block_id=block_id), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) block_data = { "blocktype": "comment", @@ -470,7 +470,7 @@ def test_processing_error_handling(self, app, user_id, database): status=TaskStatus.PENDING, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) # Pass an invalid block type to trigger an error block_data = { @@ -509,7 +509,7 @@ def test_marks_timed_out_tasks_as_error(self, app, user_id, database): status=TaskStatus.PROCESSING, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - task_dict = task.dict() + task_dict = task.model_dump() task_dict["created_at"] = old_time database.tasks.insert_one(task_dict) @@ -534,7 +534,7 @@ def test_does_not_timeout_recent_tasks(self, app, user_id, database): status=TaskStatus.PROCESSING, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _cleanup_stale_tasks() @@ -558,7 +558,7 @@ def test_purges_old_completed_tasks_and_gridfs(self, app, user_id, database): completed_at=old_time, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - task_dict = task.dict() + task_dict = task.model_dump() task_dict["created_at"] = old_time database.tasks.insert_one(task_dict) @@ -586,7 +586,7 @@ def test_does_not_purge_recent_completed_tasks(self, app, user_id, database): completed_at=datetime.now(tz=timezone.utc), spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _cleanup_stale_tasks() diff --git a/pydatalab/tests/server/test_export.py b/pydatalab/tests/server/test_export.py index 9ef6ef757..7e8d4c7d3 100644 --- a/pydatalab/tests/server/test_export.py +++ b/pydatalab/tests/server/test_export.py @@ -78,7 +78,7 @@ def test_get_export_status_pending(client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/status") assert response.status_code == 200 @@ -107,7 +107,7 @@ def test_get_export_status_ready(client, user_id, database, tmp_path): file_path=str(file_path), ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/status") assert response.status_code == 200 @@ -136,7 +136,7 @@ def test_get_export_status_error(client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/status") assert response.status_code == 200 @@ -180,7 +180,7 @@ def test_download_export_success(client, user_id, database, tmp_path, monkeypatc file_path=str(file_path), ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/download") assert response.status_code == 200 @@ -203,7 +203,7 @@ def test_not_users_export_download(client, user_id, another_client, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = another_client.get(f"/exports/{task_id}/download") assert response.status_code == 404 @@ -223,7 +223,7 @@ def test_download_export_not_ready(client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/download") assert response.status_code == 400 @@ -248,7 +248,7 @@ def test_download_export_file_missing(client, user_id, database): file_path="/nonexistent/path.eln.zip", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/download") assert response.status_code == 404 @@ -275,7 +275,7 @@ def test_do_export_success(database, sample_collection, insert_default_sample, u export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _do_export(task_id, collection_id=collection_id, export_type="collection") @@ -299,7 +299,7 @@ def test_do_export_success(database, sample_collection, insert_default_sample, u export_type="item", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _do_export(task_id, item_id=item_id, export_type="item") @@ -326,7 +326,7 @@ def test_do_export_error_handling(database, user_id): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _do_export(task_id, collection_id="nonexistent_collection", export_type="collection") @@ -367,7 +367,7 @@ def test_cleanup_old_exports(database, user_id, tmp_path, monkeypatch): file_path=str(old_file), ), ) - database.tasks.insert_one(old_task.dict()) + database.tasks.insert_one(old_task.model_dump()) # A recent export: created within the window, must be retained. recent_task_id = "recent-export-to-keep" @@ -385,7 +385,7 @@ def test_cleanup_old_exports(database, user_id, tmp_path, monkeypatch): file_path=str(recent_file), ), ) - database.tasks.insert_one(recent_task.dict()) + database.tasks.insert_one(recent_task.model_dump()) try: _cleanup_old_exports() @@ -422,7 +422,7 @@ def test_cleanup_old_exports_missing_file(database, user_id): file_path="/nonexistent/already-deleted.eln.zip", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) try: _cleanup_old_exports() @@ -447,7 +447,7 @@ def test_do_export_status_transitions(database, sample_collection, user_id): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) status_during_export = [] diff --git a/pydatalab/tests/server/test_graph.py b/pydatalab/tests/server/test_graph.py index 3e10a9dc9..9c1f30e3d 100644 --- a/pydatalab/tests/server/test_graph.py +++ b/pydatalab/tests/server/test_graph.py @@ -61,7 +61,7 @@ def test_simple_graph(admin_client): ) new_samples = [ - json.loads(d.json()) + json.loads(d.model_dump_json()) for d in [parent, child_1, child_2, child_3, child_4, missing_child, cell] ] diff --git a/pydatalab/tests/server/test_item_graph.py b/pydatalab/tests/server/test_item_graph.py index 08ffa7db4..f04169aa6 100644 --- a/pydatalab/tests/server/test_item_graph.py +++ b/pydatalab/tests/server/test_item_graph.py @@ -15,7 +15,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(material.json())}, + json={"new_sample_data": json.loads(material.model_dump_json())}, ) assert creation.status_code == 201 @@ -39,7 +39,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(parent.json())}, + json={"new_sample_data": json.loads(parent.model_dump_json())}, ) assert creation.status_code == 201 @@ -68,7 +68,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(child.json())}, + json={"new_sample_data": json.loads(child.model_dump_json())}, ) grandchild = Sample( @@ -80,7 +80,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(grandchild.json())}, + json={"new_sample_data": json.loads(grandchild.model_dump_json())}, ) great_grandchild = Sample( @@ -92,7 +92,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(great_grandchild.json())}, + json={"new_sample_data": json.loads(great_grandchild.model_dump_json())}, ) graph = client.get("/item-graph").json @@ -114,7 +114,8 @@ def test_single_starting_material(admin_client, client): ) creation = admin_client.post( - "/new-sample/", json={"new_sample_data": json.loads(admin_great_great_grandchild.json())} + "/new-sample/", + json={"new_sample_data": json.loads(admin_great_great_grandchild.model_dump_json())}, ) graph = admin_client.get("/item-graph/great-grandchild?sudo=1").json diff --git a/pydatalab/tests/server/test_item_versions.py b/pydatalab/tests/server/test_item_versions.py index d99af7d1f..8a3eb20b3 100644 --- a/pydatalab/tests/server/test_item_versions.py +++ b/pydatalab/tests/server/test_item_versions.py @@ -44,7 +44,7 @@ def sample_with_version(client, user_id): "synthesis_description": "Initial synthesis", } ) - flask_mongo.db.items.insert_one(sample.dict(exclude_unset=False)) + flask_mongo.db.items.insert_one(sample.model_dump(exclude_unset=False)) yield sample @@ -108,7 +108,6 @@ def test_save_version_nonexistent_item(self, client): response = client.post("/items/nonexistent/save-version/") assert response.status_code == 404 - assert response.json["status"] == "error" class TestListVersions: @@ -251,8 +250,6 @@ def test_get_version_nonexistent(self, client, sample_with_version): response = client.get(f"/items/{refcode}/versions/{fake_id}/") assert response.status_code == 404 - assert response.json["status"] == "error" - assert "not found" in response.json["message"].lower() class TestCompareVersions: @@ -301,24 +298,12 @@ def test_compare_versions_missing_parameters(self, client, sample_with_version): # Missing v2 - request.args.get() returns "" for missing params, which fails ObjectId validation response = client.get(f"/items/{refcode}/compare-versions/?v1=some_id") assert response.status_code == 400 - assert response.json["message"] == "Invalid query parameters" - assert "errors" in response.json - errors = response.json["errors"] - # Should have error for v2 (empty string is invalid ObjectId) - v2_errors = [e for e in errors if "v2" in str(e["loc"])] - assert len(v2_errors) == 1 - assert "valid objectid" in v2_errors[0]["msg"].lower() + assert "Invalid query parameters" in response.json["message"] # Missing v1 - same behavior response = client.get(f"/items/{refcode}/compare-versions/?v2=some_id") assert response.status_code == 400 - assert response.json["message"] == "Invalid query parameters" - assert "errors" in response.json - errors = response.json["errors"] - # Should have error for v1 (empty string is invalid ObjectId) - v1_errors = [e for e in errors if "v1" in str(e["loc"])] - assert len(v1_errors) == 1 - assert "valid objectid" in v1_errors[0]["msg"].lower() + assert "Invalid query parameters" in response.json["message"] def test_compare_versions_invalid_id(self, client, sample_with_version): """Test comparing versions with invalid ID format.""" @@ -326,15 +311,7 @@ def test_compare_versions_invalid_id(self, client, sample_with_version): response = client.get(f"/items/{refcode}/compare-versions/?v1=invalid&v2=invalid") assert response.status_code == 400 - assert response.json["message"] == "Invalid query parameters" - # Check Pydantic's structured error response - assert "errors" in response.json - errors = response.json["errors"] - # Should have errors for both v1 and v2 - assert len(errors) == 2 - for error in errors: - assert error["loc"][0] in ["v1", "v2"] - assert "valid ObjectId" in error["msg"] + assert "Invalid query parameters" in response.json["message"] def test_compare_versions_detects_changes(self, client, sample_with_version): """Test that compare_versions properly detects changes using DeepDiff.""" @@ -502,13 +479,7 @@ def test_restore_version_missing_version_id(self, client, sample_with_version): response = client.post(f"/items/{refcode}/restore-version/", json={}) assert response.status_code == 400 - assert response.json["message"] == "Invalid request body" - # Check Pydantic's structured error response - assert "errors" in response.json - errors = response.json["errors"] - assert len(errors) == 1 - assert errors[0]["loc"] == ["version_id"] - assert "required" in errors[0]["msg"].lower() + assert "Invalid request body" in response.json["message"] def test_restore_version_invalid_id(self, client, sample_with_version): """Test restoring with invalid version ID.""" @@ -516,13 +487,7 @@ def test_restore_version_invalid_id(self, client, sample_with_version): response = client.post(f"/items/{refcode}/restore-version/", json={"version_id": "invalid"}) assert response.status_code == 400 - assert response.json["message"] == "Invalid request body" - # Check Pydantic's structured error response - assert "errors" in response.json - errors = response.json["errors"] - assert len(errors) == 1 - assert errors[0]["loc"] == ["version_id"] - assert "valid ObjectId" in errors[0]["msg"] + assert "Invalid request body" in response.json["message"] def test_restore_version_nonexistent(self, client, sample_with_version): """Test restoring non-existent version.""" @@ -531,7 +496,6 @@ def test_restore_version_nonexistent(self, client, sample_with_version): response = client.post(f"/items/{refcode}/restore-version/", json={"version_id": fake_id}) assert response.status_code == 404 - assert "not found" in response.json["message"].lower() def test_restore_version_increments_version_number(self, client, sample_with_version): """Test that restore increments the item version number.""" @@ -594,7 +558,6 @@ def test_delete_version_nonexistent(self, client, sample_with_version): response = client.delete(f"/items/{refcode}/versions/{fake_id}/") assert response.status_code == 404 - assert response.json["status"] == "error" def test_delete_version_invalid_id(self, client, sample_with_version): """Test deleting with invalid ID format.""" @@ -615,7 +578,7 @@ def test_save_item_creates_version(self, client, sample_with_version): refcode_short = sample_with_version.refcode.split(":")[1] # Modify and save the item using save-item endpoint - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "Updated via save-item" response = client.post("/save-item/", json={"item_id": item_id, "data": item_data}) @@ -639,7 +602,7 @@ def test_save_item_increments_version(self, client, sample_with_version): refcode = sample_with_version.refcode - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "First real change" client.post("/save-item/", json={"item_id": sample_with_version.item_id, "data": item_data}) @@ -707,7 +670,7 @@ def test_save_item_endpoint_creates_manual_save_action(self, client, sample_with refcode_short = sample_with_version.refcode.split(":")[1] # Save via save-item endpoint (user clicking save button) - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "Updated via save-item" client.post("/save-item/", json={"item_id": sample_with_version.item_id, "data": item_data}) @@ -875,7 +838,7 @@ def test_save_item_identical_does_not_create_version(self, client, sample_with_v refcode = sample_with_version.refcode.split(":")[1] full_refcode = sample_with_version.refcode - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "A new description" client.post("/save-item/", json={"item_id": item_id, "data": item_data}) @@ -895,7 +858,7 @@ def test_save_item_with_change_creates_new_version(self, client, sample_with_ver refcode = sample_with_version.refcode.split(":")[1] full_refcode = sample_with_version.refcode - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "First real change" client.post("/save-item/", json={"item_id": item_id, "data": item_data}) @@ -974,7 +937,6 @@ def test_restore_requires_write_permission(self, another_client, sample_with_ver # Should fail because another_client doesn't have access to this item assert response.status_code == 404 - assert "not found or insufficient permissions" in response.json["message"].lower() class TestEdgeCases: @@ -1551,3 +1513,132 @@ def test_unchanged_permissions_mint_nothing(self, client, api_sample, user_id): versions = client.get(f"/items/{api_sample['refcode']}/versions/").json["versions"] assert len(versions) == before + + +class TestNoneFieldPreservation: + """Tests that None-valued fields are preserved through item creation, versioning, and restore. + + Covers two bugs introduced during the pydantic v1->v2 migration: + + Bug 1 (items.py): model_dump(exclude_none=True) on item creation strips None fields from + MongoDB entirely, so newly created items are missing keys like 'description', 'chemform', etc. + + Bug 2 (versioning.py): model_dump(exclude_none=True) on ItemVersion now recursively strips + None values from inside the 'data' dict (pydantic v2 changed behaviour vs v1), so version + snapshots also lose those keys. When restoring, $set cannot clear a key that is absent + from the restore payload. + """ + + def test_item_creation_preserves_none_fields_in_db(self, client): + """Creating an item with no description should store description=None in MongoDB, + not omit the key entirely.""" + from pydatalab.mongo import flask_mongo + + item_id = "none_field_test_create" + response = client.post( + "/new-sample/", + json={"name": "None field test", "item_id": item_id, "type": "samples"}, + ) + assert response.status_code == 201, response.json + + refcode = response.json["sample_list_entry"]["refcode"] + item = flask_mongo.db.items.find_one({"refcode": refcode}) + + # 'description' is a defined field on Sample — it must be present as a key + # even when the user did not supply a value. + assert "description" in item, ( + "'description' key missing from newly-created item; " + "model_dump(exclude_none=True) is incorrectly stripping None fields on creation" + ) + + # Cleanup + flask_mongo.db.items.delete_one({"refcode": refcode}) + flask_mongo.db.item_versions.delete_many({"refcode": refcode}) + flask_mongo.db.version_counters.delete_one({"refcode": refcode}) + + def test_version_snapshot_preserves_none_fields(self, client): + """Version snapshot stored in item_versions must retain None-valued fields from the + item document; pydantic v2 model_dump(exclude_none=True) must not strip them from + inside the nested 'data' dict.""" + from pydatalab.mongo import flask_mongo + + item_id = "none_field_test_version" + response = client.post( + "/new-sample/", + json={"name": "None field version test", "item_id": item_id, "type": "samples"}, + ) + assert response.status_code == 201, response.json + refcode = response.json["sample_list_entry"]["refcode"] + refcode_short = refcode.split(":")[-1] + + # The initial version is created on item creation; retrieve it. + list_resp = client.get(f"/items/{refcode_short}/versions/") + assert len(list_resp.json["versions"]) == 1 + version_id = list_resp.json["versions"][0]["_id"] + + version_resp = client.get(f"/items/{refcode_short}/versions/{version_id}/") + snapshot_data = version_resp.json["version"]["data"] + + assert "description" in snapshot_data, ( + "'description' key missing from version snapshot 'data'; " + "model_dump(exclude_none=True) on ItemVersion is recursively stripping None " + "values from the nested 'data' dict (pydantic v2 regression)" + ) + + # Cleanup + flask_mongo.db.items.delete_one({"refcode": refcode}) + flask_mongo.db.item_versions.delete_many({"refcode": refcode}) + flask_mongo.db.version_counters.delete_one({"refcode": refcode}) + + def test_restore_to_version_with_none_field_clears_populated_field(self, client): + """Restoring to a version where a field was None must clear that field in the current + item, even if the field has since been populated. + + Failure mode: the version snapshot is missing the key entirely (due to Bug 1/2), so + $set has nothing to write and the field retains its populated value after restore. + """ + from pydatalab.mongo import flask_mongo + + item_id = "none_field_test_restore" + create_resp = client.post( + "/new-sample/", + json={"name": "Restore none test", "item_id": item_id, "type": "samples"}, + ) + assert create_resp.status_code == 201, create_resp.json + refcode = create_resp.json["sample_list_entry"]["refcode"] + refcode_short = refcode.split(":")[-1] + + # Confirm the initial version has no description set (None / absent). + list_resp = client.get(f"/items/{refcode_short}/versions/") + v1_id = list_resp.json["versions"][0]["_id"] + + # Now populate description and save — this creates version 2. + get_resp = client.get(f"/get-item-data/{item_id}") + item_data = get_resp.json["item_data"] + item_data["description"] = "A description added later" + save_resp = client.post("/save-item/", json={"item_id": item_id, "data": item_data}) + assert save_resp.status_code == 200, save_resp.json + + # Verify description is now set in the DB. + current = flask_mongo.db.items.find_one({"refcode": refcode}) + assert current.get("description") == "A description added later" + + # Restore to version 1 (description was None/absent). + restore_resp = client.post( + f"/items/{refcode_short}/restore-version/", json={"version_id": v1_id} + ) + assert restore_resp.status_code == 200, restore_resp.json + + # After restore, description must be None (or absent), not the later value. + restored = flask_mongo.db.items.find_one({"refcode": refcode}) + assert restored.get("description") is None, ( + f"description should be None after restoring to the version where it was unset, " + f"but got {restored.get('description')!r}. " + "This happens because the version snapshot is missing the 'description' key, " + "so $set does not overwrite the populated value." + ) + + # Cleanup + flask_mongo.db.items.delete_one({"refcode": refcode}) + flask_mongo.db.item_versions.delete_many({"refcode": refcode}) + flask_mongo.db.version_counters.delete_one({"refcode": refcode}) diff --git a/pydatalab/tests/server/test_items.py b/pydatalab/tests/server/test_items.py index e58bafd2d..9c9ad1598 100644 --- a/pydatalab/tests/server/test_items.py +++ b/pydatalab/tests/server/test_items.py @@ -22,7 +22,7 @@ def test_single_item_endpoints(client, inserted_default_items): def test_fts_fields(): """Test non-exhaustively that certain fields make it into the fts index.""" - from pydatalab.mongo import ITEMS_FTS_FIELDS + from pydatalab.mongo import get_items_fts_fields fields = ("item_id", "name", "description", "refcode", "synthesis_description", "supplier") - assert all(field in ITEMS_FTS_FIELDS for field in fields) + assert all(field in get_items_fts_fields() for field in fields) diff --git a/pydatalab/tests/server/test_samples.py b/pydatalab/tests/server/test_samples.py index 6e55a35e5..15b3920e7 100644 --- a/pydatalab/tests/server/test_samples.py +++ b/pydatalab/tests/server/test_samples.py @@ -360,7 +360,7 @@ def test_item_regex_search( def test_new_sample_with_relationships( client, complicated_sample, insert_complicated_sample_constituents ): - complicated_sample_json = json.loads(complicated_sample.json()) + complicated_sample_json = json.loads(complicated_sample.model_dump_json()) response = client.post("/new-sample/", json=complicated_sample_json) # Test that 201: Created is emitted assert response.status_code == 201, response.json @@ -424,7 +424,7 @@ def test_new_sample_with_relationships( description="This is a new relationship", ) ) - derived_sample_json = json.loads(derived_sample.json()) + derived_sample_json = json.loads(derived_sample.model_dump_json()) response = client.post("/new-sample/", json=derived_sample_json) # Test that 201: Created is emitted @@ -528,7 +528,7 @@ def test_copy_from_sample(client, complicated_sample): """ complicated_sample.item_id = "new_complicated_sample" - complicated_sample_json = json.loads(complicated_sample.json()) + complicated_sample_json = json.loads(complicated_sample.model_dump_json()) response = client.post("/new-sample/", json=complicated_sample_json) # Test that 201: Created is emitted @@ -558,12 +558,13 @@ def test_copy_from_sample(client, complicated_sample): @pytest.mark.dependency(depends=["test_copy_from_sample"]) def test_create_multiple_samples(client, complicated_sample): - samples = [complicated_sample, complicated_sample.copy()] + samples = [complicated_sample, complicated_sample.model_copy()] samples[0].item_id = "another_new_complicated_sample" samples[1].item_id = "additional_new_complicated_sample" response = client.post( - "/new-samples/", json={"new_sample_datas": [json.loads(s.json()) for s in samples]} + "/new-samples/", + json={"new_sample_datas": [json.loads(s.model_dump_json()) for s in samples]}, ) assert response.status_code == 207, response.json assert response.json["nsuccess"] == 2, response.json @@ -577,7 +578,7 @@ def test_create_multiple_samples(client, complicated_sample): response = client.post( "/new-samples/", json={ - "new_sample_datas": [json.loads(s.json()) for s in samples], + "new_sample_datas": [json.loads(s.model_dump_json()) for s in samples], "copy_from_item_ids": [ "another_new_complicated_sample", "additional_new_complicated_sample", @@ -599,7 +600,7 @@ def test_create_multiple_samples(client, complicated_sample): @pytest.mark.dependency(depends=["test_create_multiple_samples"]) def test_create_cell(client, default_cell): - response = client.post("/new-sample/", json=json.loads(default_cell.json())) + response = client.post("/new-sample/", json=json.loads(default_cell.model_dump_json())) assert response.status_code == 201, response.json assert response.json["status"] == "success" @@ -624,8 +625,8 @@ def test_create_cell(client, default_cell): assert cell["electrolyte"][1]["item"]["chemform"] == "NaCl" assert ( - cell["positive_electrode"][0]["item"]["name"] - == default_cell.positive_electrode[0].item.name + cell["positive_electrode"][0]["item"]["item_id"] + == default_cell.positive_electrode[0].item.item_id ) # The anode has a "real" entry in the db, so it should be looked up properly # with dereferenced chemical formula and name, even if it was not provided at link time @@ -672,7 +673,9 @@ def test_create_collections(client, default_collection, database): assert response.status_code == 200 # Create an empty collection - response = client.put("/collections", json={"data": json.loads(default_collection.json())}) + response = client.put( + "/collections", json={"data": json.loads(default_collection.model_dump_json())} + ) assert response.status_code == 201, response.json assert response.json["status"] == "success" assert response.json["data"]["collection_id"] == "test_collection" @@ -691,7 +694,7 @@ def test_create_collections(client, default_collection, database): new_collection = copy.deepcopy(default_collection) new_collection.collection_id = "test_collection_2" - data = json.loads(new_collection.json()) + data = json.loads(new_collection.model_dump_json()) data.update( { "starting_members": [ @@ -924,7 +927,7 @@ def test_remove_items_from_collection_success( response = client.post("/new-sample/", json=sample_dict) assert response.status_code == 201 - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_remove" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -996,7 +999,7 @@ def test_remove_items_from_collection_not_found(client): @pytest.mark.dependency() def test_remove_items_from_collection_no_items_provided(client, default_collection): """Test removing with no item IDs provided.""" - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_empty_items" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1012,7 +1015,7 @@ def test_remove_items_from_collection_no_items_provided(client, default_collecti @pytest.mark.dependency() def test_remove_items_from_collection_no_matching_items(client, default_collection): """Test removing items that don't exist.""" - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_no_match" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1041,7 +1044,7 @@ def test_remove_items_from_collection_partial_success( response = client.post("/new-sample/", json=sample_dict) assert response.status_code == 201 - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_partial" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1100,7 +1103,7 @@ def test_copy_sample_and_add_to_collection(client, default_sample_dict, default_ assert response.status_code == 201 assert response.json["status"] == "success" - collection_dict = default_collection.dict().copy() + collection_dict = default_collection.model_dump().copy() collection_dict["collection_id"] = "test_copy_collection" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1132,12 +1135,12 @@ def test_copy_sample_and_add_to_collection(client, default_sample_dict, default_ def test_copy_sample_from_collection_to_different_collection( client, default_sample_dict, default_collection ): - collection1_dict = default_collection.dict().copy() + collection1_dict = default_collection.model_dump().copy() collection1_dict["collection_id"] = "collection_1" response = client.put("/collections", json={"data": collection1_dict}) assert response.status_code == 201 - collection2_dict = default_collection.dict().copy() + collection2_dict = default_collection.model_dump().copy() collection2_dict["collection_id"] = "collection_2" response = client.put("/collections", json={"data": collection2_dict}) assert response.status_code == 201 @@ -1177,7 +1180,7 @@ def test_copy_sample_from_collection_to_different_collection( @pytest.mark.dependency(depends=["test_copy_sample_from_collection_to_different_collection"]) def test_copy_sample_without_copying_collections(client, default_sample_dict, default_collection): - collection_dict = default_collection.dict().copy() + collection_dict = default_collection.model_dump().copy() collection_dict["collection_id"] = "test_no_auto_copy_collection" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1217,7 +1220,7 @@ def test_collections_permissions(client, admin_client, default_sample_dict, defa assert response.json["status"] == "success" refcode = response.json["sample_list_entry"]["refcode"] - admin_collection = default_collection.dict().copy() + admin_collection = default_collection.model_dump().copy() admin_collection["collection_id"] = "admin_only_collection" response = admin_client.put("/collections", json={"data": admin_collection}) @@ -1290,6 +1293,40 @@ def test_save_item_with_malformed_constituent_returns_400(client, default_sample assert response.json["status"] == "error" +def test_clearing_field_via_save_item_persists_to_db(client): + """Setting a field then clearing it (setting to None) via save-item must + actually clear it in MongoDB. Fails if exclude_none=True strips the None + from the $set payload, leaving the old value untouched.""" + item_id = "clear_field_test" + response = client.post( + "/new-sample/", + json={"name": "Clear field test", "item_id": item_id, "type": "samples"}, + ) + assert response.status_code == 201, response.json + + # Set molar_mass and save. + item_data = client.get(f"/get-item-data/{item_id}").json["item_data"] + item_data["molar_mass"] = 123.45 + assert ( + client.post("/save-item/", json={"item_id": item_id, "data": item_data}).status_code == 200 + ) + + assert client.get(f"/get-item-data/{item_id}").json["item_data"]["molar_mass"] == 123.45 + + # Now clear molar_mass and save again. + item_data = client.get(f"/get-item-data/{item_id}").json["item_data"] + item_data["molar_mass"] = None + assert ( + client.post("/save-item/", json={"item_id": item_id, "data": item_data}).status_code == 200 + ) + + result = client.get(f"/get-item-data/{item_id}").json["item_data"].get("molar_mass") + assert result is None, ( + f"molar_mass should be None after being cleared but got {result!r}; " + "exclude_none=True in save_item is stripping the None from the $set payload" + ) + + def test_get_item_with_malformed_stored_constituent_returns_500(client, database, user_id): """If an item with a malformed constituent ended up in the database (e.g. from an older write path), GET should surface it as a server-side diff --git a/pydatalab/tests/server/test_search.py b/pydatalab/tests/server/test_search.py index 40e4771dd..d9b0cabe9 100644 --- a/pydatalab/tests/server/test_search.py +++ b/pydatalab/tests/server/test_search.py @@ -52,7 +52,7 @@ def test_collection_regex_search(query, should_find, real_mongo_client, client, if real_mongo_client is None: pytest.skip("Skipping search tests, not connected to real MongoDB") - test_collection = default_collection.dict() + test_collection = default_collection.model_dump() clean_query = query.replace("=", "_").replace("%23", "hash_").replace("'", "") test_collection["collection_id"] = f"test_coll_{clean_query}" test_collection["title"] = "test collection" diff --git a/pydatalab/tests/server/test_starting_materials.py b/pydatalab/tests/server/test_starting_materials.py index 3ee28080a..080559e5d 100644 --- a/pydatalab/tests/server/test_starting_materials.py +++ b/pydatalab/tests/server/test_starting_materials.py @@ -133,7 +133,7 @@ def test_starting_materials_list_includes_collections( client, default_starting_material_dict, default_collection ): """GET /starting-materials/ should include a collections array with collection_id for each item.""" - collection_data = json.loads(default_collection.json()) + collection_data = json.loads(default_collection.model_dump_json()) collection_data["starting_members"] = [{"item_id": default_starting_material_dict["item_id"]}] response = client.put("/collections", json={"data": collection_data}) assert response.status_code == 201, response.json diff --git a/pydatalab/tests/server/test_tags.py b/pydatalab/tests/server/test_tags.py new file mode 100644 index 000000000..86bcafe69 --- /dev/null +++ b/pydatalab/tests/server/test_tags.py @@ -0,0 +1,419 @@ +"""Tests for the scoped tags routes. + +Tags have scopes: + +- **global**: available to (and usable by) everyone; only administrators can + create, edit or delete them. +- **user**: a user-defined tag owned by exactly one user; only that user can list, + use, edit or delete it (though it is still *displayed* on any item the viewer + can access). + +Names are unique *within a scope*; the stable identity of a tag is its +`immutable_id`. +""" + +import pytest +from bson import ObjectId + + +@pytest.fixture(autouse=True) +def _isolate_tags(database): + """Isolate each test. + + The test database is only dropped per-module, so the `tags` collection would + otherwise leak between tests. Clear it before and after each test. + """ + database.tags.delete_many({}) + yield + database.tags.delete_many({}) + + +def _create_tag(client, name, scope=None, description=None, color=None): + """Helper to PUT a tag and return the response.""" + data = {"name": name} + if scope is not None: + data["scope"] = scope + if description is not None: + data["description"] = description + if color is not None: + data["color"] = color + return client.put("/tags", json={"data": data}) + + +# --- creation & authoring policy ------------------------------------------------- + + +def test_create_user_defined_tag(client, unauthenticated_client, user_id): + """Any logged-in user can create a user-defined tag they own; the default scope is + ``user``. (Previously tag creation was admin-only; users can now self-serve.)""" + # An unauthenticated user is rejected. + assert _create_tag(unauthenticated_client, "unauth-tag").status_code == 401 + + # A normal user can create a user-defined tag (scope defaults to "user"). + response = _create_tag(client, "my-tag", description="mine", color="#f1c40f") + assert response.status_code == 201, response.json + tag = response.json["data"] + assert tag["type"] == "tags" + assert tag["name"] == "my-tag" + assert tag["color"] == "#f1c40f" + assert tag["scope"] == "user" + assert tag["owner"] == str(user_id) + assert tag["immutable_id"] + + # Scope is modelled without the HasOwner mixin. + assert "creator_ids" not in tag + assert "group_ids" not in tag + + +def test_create_global_tag_admin_only(client, admin_client): + """Only an administrator can create a global tag.""" + # A normal user cannot create a global tag. + assert _create_tag(client, "user-global", scope="global").status_code == 403 + + # An admin can; it has no owner. + response = _create_tag(admin_client, "admin-global", scope="global") + assert response.status_code == 201, response.json + tag = response.json["data"] + assert tag["scope"] == "global" + assert tag["owner"] is None + + +def test_invalid_scope_rejected(client): + """An unknown scope value is a 400.""" + assert _create_tag(client, "weird", scope="team").status_code == 400 + + +# --- visibility ------------------------------------------------------------------ + + +def test_user_defined_tag_visibility(client, another_client, admin_client, user_id): + """A user-defined tag is listed/searchable only by its owner.""" + tag_id = _create_tag(client, "secret-tag", color="#abcdef").json["data"]["immutable_id"] + + # The owner sees it in the listing and search, marked as a user-scoped tag. + listed = {t["name"]: t for t in client.get("/tags").json["data"]} + assert "secret-tag" in listed + assert listed["secret-tag"]["scope"] == "user" + assert listed["secret-tag"]["owner"] == str(user_id) + + found = { + r["name"]: r + for r in client.get("/search-tags", query_string={"query": "secret"}).json["data"] + } + assert found["secret-tag"]["scope"] == "user" + + # Another user (and an admin, without sudo) does not see it at all. + for other in (another_client, admin_client): + names = {t["name"] for t in other.get("/tags").json["data"]} + assert "secret-tag" not in names + search = other.get("/search-tags", query_string={"query": "secret"}).json["data"] + assert all(r["immutable_id"] != tag_id for r in search) + + +def test_global_tag_visible_to_all(client, another_client, admin_client): + """A global tag is listed and searchable by everyone, with scope ``global``.""" + assert ( + _create_tag(admin_client, "shared-tag", scope="global", color="#abcdef").status_code == 201 + ) + + for c in (client, another_client, admin_client): + listed = {t["name"]: t for t in c.get("/tags").json["data"]} + assert "shared-tag" in listed + assert listed["shared-tag"]["scope"] == "global" + + found = { + r["name"]: r + for r in c.get("/search-tags", query_string={"query": "shared"}).json["data"] + } + assert found["shared-tag"]["scope"] == "global" + assert found["shared-tag"]["type"] == "tags" + + # The empty-query case is still rejected. + assert client.get("/search-tags", query_string={"query": ""}).status_code == 400 + + +# --- scope-based name uniqueness ------------------------------------------------- + + +def test_scope_based_name_uniqueness(client, another_client, admin_client): + """Names are unique within a scope, but may repeat across scopes/owners.""" + # A global and a user-defined tag may share a name. + assert _create_tag(admin_client, "flammable", scope="global").status_code == 201 + assert _create_tag(client, "flammable").status_code == 201 + + # Two different users may each own a user-defined "flammable". + assert _create_tag(another_client, "flammable").status_code == 201 + + # The same owner cannot create a second user-defined tag with the same name. + assert _create_tag(client, "flammable").status_code == 409 + + # A second global with the same name is rejected. + assert _create_tag(admin_client, "flammable", scope="global").status_code == 409 + + +# --- editing & deletion ---------------------------------------------------------- + + +def test_patch_tag_owner_only(client, another_client, admin_client): + """User-defined tags are editable only by their owner; global tags only by admins.""" + user_defined_id = _create_tag(client, "editable", description="first").json["data"][ + "immutable_id" + ] + global_id = _create_tag(admin_client, "global-editable", scope="global").json["data"][ + "immutable_id" + ] + + # Another user cannot edit someone else's user-defined tag. + assert ( + another_client.patch( + f"/tags/{user_defined_id}", json={"data": {"description": "nope"}} + ).status_code + == 403 + ) + + # The owner can. + assert ( + client.patch( + f"/tags/{user_defined_id}", json={"data": {"description": "updated"}} + ).status_code + == 200 + ) + patched = next( + t for t in client.get("/tags").json["data"] if t["immutable_id"] == user_defined_id + ) + assert patched["description"] == "updated" + + # A non-admin cannot edit a global tag; an admin can. + assert ( + client.patch(f"/tags/{global_id}", json={"data": {"description": "x"}}).status_code == 403 + ) + assert ( + admin_client.patch(f"/tags/{global_id}", json={"data": {"description": "y"}}).status_code + == 200 + ) + + # Scope/owner are immutable through PATCH. + assert ( + client.patch(f"/tags/{user_defined_id}", json={"data": {"scope": "global"}}).status_code + == 200 + ) + still = next( + t for t in client.get("/tags").json["data"] if t["immutable_id"] == user_defined_id + ) + assert still["scope"] == "user" + + # An invalid ID is a 400; a missing tag is a 404. + assert client.patch("/tags/not-an-object-id", json={"data": {"name": "x"}}).status_code == 400 + assert client.patch(f"/tags/{ObjectId()}", json={"data": {"name": "x"}}).status_code == 404 + + +def test_patch_tag_rename_unique_within_scope(client): + """Renaming a user-defined tag onto another of the owner's names is rejected.""" + tag_id = _create_tag(client, "one").json["data"]["immutable_id"] + assert _create_tag(client, "two").status_code == 201 + + assert client.patch(f"/tags/{tag_id}", json={"data": {"name": "two"}}).status_code == 409 + + +def test_delete_tag_owner_only(client, another_client, admin_client, database): + """User-defined tags are deletable only by their owner; global only by admins; refs + are pulled from items on delete.""" + user_defined_id = _create_tag(client, "deletable", color="#abcdef").json["data"]["immutable_id"] + + # Apply the tag to an item to check the reference cleanup on delete. + assert ( + client.post("/new-sample/", json={"type": "samples", "item_id": "tag-delete"}).status_code + == 201 + ) + assert ( + client.post( + "/save-item/", + json={ + "item_id": "tag-delete", + "data": {"tags": [{"type": "tags", "immutable_id": user_defined_id}]}, + }, + ).status_code + == 200 + ) + + # Another user cannot delete someone else's user-defined tag. + assert another_client.delete(f"/tags/{user_defined_id}").status_code == 403 + + # An admin cannot delete another user's user-defined tag either (admins manage global). + assert admin_client.delete(f"/tags/{user_defined_id}").status_code == 403 + + # The owner can. + assert client.delete(f"/tags/{user_defined_id}").status_code == 200 + names = {t["name"] for t in client.get("/tags").json["data"]} + assert "deletable" not in names + + # The reference is pulled from the item document in the database. + stored = database.items.find_one({"item_id": "tag-delete"}) + assert [t for t in stored.get("tags", []) if isinstance(t, dict)] == [] + + # Deleting a non-existent tag is a 404. + assert client.delete(f"/tags/{user_defined_id}").status_code == 404 + + # Global tags: only admins can delete. + global_id = _create_tag(admin_client, "global-del", scope="global").json["data"]["immutable_id"] + assert client.delete(f"/tags/{global_id}").status_code == 403 + assert admin_client.delete(f"/tags/{global_id}").status_code == 200 + + +# --- applying tags to items (add-authorization) ---------------------------------- + + +def _make_shared_sample(client, database, item_id, extra_owner_id): + """Create a sample via the API, then share write access with a second user by + adding them to `creator_ids` directly (mirrors co-ownership).""" + assert ( + client.post("/new-sample/", json={"type": "samples", "item_id": item_id}).status_code == 201 + ) + database.items.update_one({"item_id": item_id}, {"$addToSet": {"creator_ids": extra_owner_id}}) + + +def _item_tag_ids(client, item_id): + resp = client.get(f"/get-item-data/{item_id}") + assert resp.status_code == 200, resp.json + return [t["immutable_id"] for t in resp.json["item_data"]["tags"] if isinstance(t, dict)] + + +def test_cannot_add_others_user_defined_tag( + client, another_client, admin_client, database, another_user_id +): + """A user may add global and own user-defined tags, but not another user's + user-defined tag; they may keep and remove an already-present foreign tag.""" + a_tag = _create_tag(client, "a-user-defined").json["data"]["immutable_id"] + b_tag = _create_tag(another_client, "b-user-defined").json["data"]["immutable_id"] + g_tag = _create_tag(admin_client, "g-global", scope="global").json["data"]["immutable_id"] + + # A shared sample both users can write. + _make_shared_sample(client, database, "shared-item", another_user_id) + + def save(c, tags): + return c.post( + "/save-item/", + json={"item_id": "shared-item", "data": {"tags": tags}}, + ) + + def ref(tid): + return {"type": "tags", "immutable_id": tid} + + # B can add a global tag and their own user-defined tag. + assert save(another_client, [ref(g_tag), ref(b_tag)]).status_code == 200 + assert set(_item_tag_ids(another_client, "shared-item")) == {g_tag, b_tag} + + # B cannot *introduce* A's user-defined tag (a new addition B does not own). + assert save(another_client, [ref(g_tag), ref(b_tag), ref(a_tag)]).status_code == 403 + # The rejected save left the stored tags unchanged. + assert set(_item_tag_ids(another_client, "shared-item")) == {g_tag, b_tag} + + # A (the owner) can add their own user-defined tag; the item now carries it. + assert save(client, [ref(g_tag), ref(b_tag), ref(a_tag)]).status_code == 200 + assert set(_item_tag_ids(client, "shared-item")) == {g_tag, b_tag, a_tag} + + # B may *keep* A's already-present tag through an unrelated re-save (no new + # foreign tag is introduced), and may *remove* it (removal is never blocked). + assert save(another_client, [ref(g_tag), ref(b_tag), ref(a_tag)]).status_code == 200 + assert save(another_client, [ref(g_tag), ref(b_tag)]).status_code == 200 + assert set(_item_tag_ids(another_client, "shared-item")) == {g_tag, b_tag} + + +def test_cannot_add_foreign_tag_on_creation(client, another_client): + """User-defined tags owned by another user are rejected at item creation too.""" + b_tag = _create_tag(another_client, "b-only").json["data"]["immutable_id"] + + response = client.post( + "/new-sample/", + json={ + "type": "samples", + "item_id": "create-foreign", + "tags": [{"type": "tags", "immutable_id": b_tag}], + }, + ) + assert response.status_code == 403, response.json + + +# --- read-time resolution -------------------------------------------------------- + + +def test_item_tag_resolution_includes_scope( + client, another_client, admin_client, database, another_user_id +): + """Resolved item tags carry `scope`, and another user viewing a shared item + sees the owner's user-defined tag (display is gated by the item, not the tag).""" + user_defined_id = _create_tag(client, "resolve-user-defined", color="#abcdef").json["data"][ + "immutable_id" + ] + global_id = _create_tag(admin_client, "resolve-global", scope="global").json["data"][ + "immutable_id" + ] + + _make_shared_sample(client, database, "resolve-item", another_user_id) + + save = client.post( + "/save-item/", + json={ + "item_id": "resolve-item", + "data": { + "tags": [ + {"type": "tags", "immutable_id": user_defined_id, "name": "stale"}, + {"type": "tags", "immutable_id": global_id}, + ] + }, + }, + ) + assert save.status_code == 200, save.json + + # Stored references are minimal (no display fields, incl. scope). + stored = database.items.find_one({"item_id": "resolve-item"}) + assert {(t["type"], t["immutable_id"]) for t in stored["tags"]} == { + ("tags", ObjectId(user_defined_id)), + ("tags", ObjectId(global_id)), + } + + # Another user viewing the shared item sees both tags, with resolved scope. + tags = { + t["immutable_id"]: t + for t in another_client.get("/get-item-data/resolve-item").json["item_data"]["tags"] + if isinstance(t, dict) + } + assert tags[user_defined_id]["scope"] == "user" + assert tags[user_defined_id]["name"] == "resolve-user-defined" # re-resolved, not stale + assert tags[user_defined_id]["color"] == "#abcdef" + assert tags[global_id]["scope"] == "global" + + +# --- feature flag ---------------------------------------------------------------- + + +def test_tags_feature_flag_gate(client, admin_client, monkeypatch): + """When the `tags` feature flag is off, the whole blueprint 404s.""" + from pydatalab.feature_flags import FEATURE_FLAGS + + monkeypatch.setattr(FEATURE_FLAGS, "tags", False) + assert client.get("/tags").status_code == 404 + assert client.get("/search-tags", query_string={"query": "x"}).status_code == 404 + assert _create_tag(client, "flagged-off").status_code == 404 + + +def test_tags_stripped_on_creation(client, admin_client, database): + """Tags provided directly at item creation are stored as minimal references.""" + tag_id = _create_tag(admin_client, "create-global", scope="global", color="#abcdef").json[ + "data" + ]["immutable_id"] + + response = client.post( + "/new-sample/", + json={ + "type": "samples", + "item_id": "tag-on-create", + "tags": [ + {"type": "tags", "immutable_id": tag_id, "name": "stale", "color": "#abcdef"}, + ], + }, + ) + assert response.status_code == 201, response.json + + stored = database.items.find_one({"item_id": "tag-on-create"}) + assert stored["tags"] == [{"type": "tags", "immutable_id": ObjectId(tag_id)}] diff --git a/pydatalab/tests/test_models.py b/pydatalab/tests/test_models.py index 1e59bfbce..aeaf6bcc3 100644 --- a/pydatalab/tests/test_models.py +++ b/pydatalab/tests/test_models.py @@ -52,7 +52,7 @@ def test_sample_with_inlined_reference(): assert parents[0].item_id == a.item_id assert parents[0].refcode == a.refcode - b_both = Sample(**json.loads(b_both.json())) + b_both = Sample(**json.loads(b_both.model_dump_json())) parents = [r for r in b_both.relationships if r.relation == RelationshipType.PARENT] assert len(parents) == 1 @@ -81,7 +81,28 @@ def test_sample_with_inlined_reference(): @pytest.mark.parametrize("model", ITEM_MODELS.values()) def test_generate_schemas(model): """Test that all item model schemas can be generated.""" - assert model.schema() + assert model.model_json_schema() + + +def test_attribute_docstrings_in_schema(): + """Field docstrings should be pulled into the JSON schema as descriptions. + + Relies on ``use_attribute_docstrings`` being set on the shared ``BaseModel`` + in ``pydatalab.models.utils``; covers fields defined across several different + trait mixins to confirm the config propagates through inheritance. + """ + properties = Sample.model_json_schema(by_alias=False)["properties"] + + expected = { + "creators": "Inlined info for the people associated with this item.", # HasOwner + "blocks_obj": "A mapping from block ID to block data.", # HasBlocks + "revision": "The revision number of the entry.", # HasRevisionControl + "collections": "Inlined info for the collections associated with this item.", # IsCollectable + "name": "An optional human-readable/usable name for the entry.", # Item + } + + for field, description in expected.items(): + assert properties[field]["description"] == description def test_relationship_with_custom_type(): @@ -163,9 +184,111 @@ def test_file(): assert sample.files[1].type == "files" +def test_tag_model(): + from pydatalab.models.tags import Tag + from pydatalab.models.utils import AccessScope + + tag = Tag(name="test_tag", description="This is an example", color="#f1c40f", scope="global") + assert tag.type == "tags" + assert tag.name == "test_tag" + assert tag.description == "This is an example" + assert tag.color == "#f1c40f" + assert tag.scope == AccessScope.GLOBAL + assert tag.owner is None + + # Scope is modelled explicitly via `scope`/`owner` (not the `HasOwner` mixin). + assert not hasattr(tag, "creator_ids") + assert not hasattr(tag, "group_ids") + + oid = ObjectId("0123456789ab0123456789ab") + doc = {"_id": oid, "type": "tags", "name": "glovebox", "scope": "global"} + stored_tag = Tag(**doc) + assert stored_tag.immutable_id == oid + assert stored_tag.description is None + assert stored_tag.color is None + assert stored_tag.model_dump()["immutable_id"] == oid + assert stored_tag.scope == AccessScope.GLOBAL + + # Both `name` and `scope` are required. + with pytest.raises(pydantic.ValidationError): + Tag(scope="global", description="missing a name") + with pytest.raises(pydantic.ValidationError): + Tag(name="missing-a-scope") + + +def test_tag_scope_owner_consistency(): + """A user-scoped tag must have an owner; a global tag must not.""" + from pydatalab.models.tags import Tag + from pydatalab.models.utils import AccessScope + + owner = ObjectId() + + # A valid user-defined tag. + user_defined = Tag(name="mine", scope="user", owner=owner) + assert user_defined.scope == AccessScope.USER + assert user_defined.owner == owner + # `owner` is preserved as an ObjectId in the stored (python-mode) dump. + assert isinstance(user_defined.model_dump(exclude_none=True)["owner"], ObjectId) + # ... and stringified in the JSON dump sent to clients. + assert json.loads(user_defined.model_dump_json())["owner"] == str(owner) + + # A valid global tag has no owner. + glob = Tag(name="shared", scope="global") + assert glob.owner is None + + # A user-scoped tag without an owner is rejected. + with pytest.raises(pydantic.ValidationError): + Tag(name="bad", scope="user") + + # A global tag with an owner is rejected. + with pytest.raises(pydantic.ValidationError): + Tag(name="bad", scope="global", owner=owner) + + +def test_item_tags_coercion(): + """The `HasTags` mixin coerces references and de-duplicates tags on items.""" + from pydatalab.models.samples import Sample + from pydatalab.models.utils import EntryReference + + oid = ObjectId("0123456789ab0123456789ab") + + sample = Sample( + item_id="tagged", + tags=[ + {"type": "tags", "immutable_id": str(oid), "name": "Curated"}, + {"type": "tags", "immutable_id": str(oid)}, # same reference by id -> dropped + ], + ) + + assert len(sample.tags) == 1 + ref = sample.tags[0] + assert isinstance(ref, EntryReference) + assert ref.type == "tags" + assert ref.immutable_id == oid + assert ref.name == "Curated" + + # Default is an empty list, so existing tag-less documents stay valid. + assert Sample(item_id="untagged").tags == [] + + # A reference to a (possibly deleted) tag still validates. + dangling = Sample(item_id="dangling", tags=[{"type": "tags", "immutable_id": str(ObjectId())}]) + assert len(dangling.tags) == 1 + + # Bare string tags are not allowed: only references to tags entries. + with pytest.raises(pydantic.ValidationError): + Sample(item_id="string-tag", tags=["custom"]) + + # Re-validating a dumped item round-trips the reference tags list. + roundtrip = Sample(**json.loads(sample.model_dump_json())) + assert [type(t).__name__ for t in roundtrip.tags] == ["EntryReference"] + + def test_custom_and_inherited_items(): class TestItem(Item): type: str = "items_custom" + new_field: str + + TestItem.model_rebuild() item = TestItem( type="items_custom", @@ -174,9 +297,10 @@ class TestItem(Item): creators=None, date="2020-01-01 00:00", item_id="1234", + new_field="This is a new field", ) - item_dict = item.dict() + item_dict = item.model_dump() assert item_dict["type"] == "items_custom" assert item_dict["creator_ids"][0] == ObjectId("0123456789ab0123456789ab") assert item_dict["creator_ids"][1] == ObjectId("1023456789ab0123456789ab") @@ -184,10 +308,10 @@ class TestItem(Item): tzinfo=datetime.timezone.utc ) - item_json = json.loads(item.json()) + item_json = json.loads(item.model_dump_json()) assert item_json["type"] == "items_custom" - assert item_json["creator_ids"][0] == str(ObjectId("0123456789ab0123456789ab")) - assert item_json["creator_ids"][1] == str(ObjectId("1023456789ab0123456789ab")) + assert item_json["creator_ids"][0] == "0123456789ab0123456789ab" + assert item_json["creator_ids"][1] == "1023456789ab0123456789ab" assert ( item_json["date"] == datetime.datetime.fromisoformat("2020-01-01 00:00") @@ -203,7 +327,7 @@ class TestItem(Item): item_id="1234", ) - sample_dict = sample.dict() + sample_dict = sample.model_dump() assert sample_dict["type"] == "samples" assert sample_dict["creator_ids"][0] == ObjectId("0123456789ab0123456789ab") assert sample_dict["creator_ids"][1] == ObjectId("1023456789ab0123456789ab") @@ -214,7 +338,7 @@ class TestItem(Item): "2020-01-01 00:00" ).replace(tzinfo=datetime.timezone.utc) - sample_json = json.loads(sample.json()) + sample_json = json.loads(sample.model_dump_json()) assert sample_json["type"] == "samples" assert sample_json["creator_ids"][0] == str(ObjectId("0123456789ab0123456789ab")) assert sample_json["creator_ids"][1] == str(ObjectId("1023456789ab0123456789ab")) @@ -240,13 +364,17 @@ class TestItem(Item): "MP2018_TEST_COMMERCIAL", "MP2018_TEST_COMMERCIAL_4.5V_hold", "AAAAAA", - 111111111, + "111111111", ], ) def test_good_ids(id): """Test good human-readable IDs for validity.""" - assert HumanReadableIdentifier(id) + class TestModel(pydantic.BaseModel): + test_id: HumanReadableIdentifier + + model = TestModel(test_id=id) + assert model.test_id == id @pytest.mark.parametrize( @@ -256,6 +384,7 @@ def test_good_ids(id): "mp 1 2 3 4 5 6", "lithium & sodium", "me388-123456789-123456789-really-long-descriptive-identifier-that-should-be-the-name-but-is-otherwise-valid", + 111111111, 1111111111111111111111111111111111111111111111111, "_AAAA", "AAA_", @@ -267,8 +396,11 @@ def test_good_ids(id): def test_bad_ids(id): """Test bad human-readable IDs for invalidity.""" + class TestModel(pydantic.BaseModel): + test_id: HumanReadableIdentifier + with pytest.raises(pydantic.ValidationError): - HumanReadableIdentifier(id) + TestModel(test_id=id) def test_cell_with_inlined_reference(): @@ -291,7 +423,7 @@ def test_cell_with_inlined_reference(): assert cell assert len(cell.relationships) == 1 - cell = Cell(**json.loads(cell.json())) + cell = Cell(**json.loads(cell.model_dump_json())) assert cell assert len(cell.relationships) == 1 @@ -411,7 +543,7 @@ def test_cell_relationship_deduplication(): assert parthood[0].item_id == "test_cathode" # Re-validating an already-clean cell must not grow the relationships list. - cell = Cell(**json.loads(cell.json())) + cell = Cell(**json.loads(cell.model_dump_json())) parthood = [r for r in cell.relationships if r.relation == RelationshipType.PARTHOOD] assert len(parthood) == 1 assert parthood[0].refcode == "grey:ABCDEF" @@ -506,7 +638,7 @@ def test_sample_synthesis_relationship_deduplication(): assert parents[0].item_id == "sm_1" # Re-validating an already-clean sample must not grow the relationships list. - sample = Sample(**json.loads(sample.json())) + sample = Sample(**json.loads(sample.model_dump_json())) parents = [r for r in sample.relationships if r.relation == RelationshipType.PARENT] assert len(parents) == 1 assert parents[0].refcode == "grey:ABCDEF" @@ -569,8 +701,11 @@ def test_good_refcodes(refcode): def test_bad_refcodes(refcode): """Test bad refcodes for invalidity.""" + class TestModel(pydantic.BaseModel): + test_refcode: Refcode + with pytest.raises(pydantic.ValidationError): - Refcode(refcode) + TestModel(test_refcode=refcode) @pytest.mark.parametrize( @@ -585,7 +720,10 @@ def test_bad_refcodes(refcode): def test_good_display_name(display_name): """Test good display name for validity.""" - assert DisplayName(display_name) + class TestModel(pydantic.BaseModel): + name: DisplayName + + assert TestModel(name=display_name) @pytest.mark.parametrize( @@ -599,8 +737,11 @@ def test_good_display_name(display_name): def test_bad_display_name(display_name): """Test bad display_name for invalidity.""" + class TestModel(pydantic.BaseModel): + name: DisplayName + with pytest.raises(ValueError): - DisplayName(display_name) + TestModel(name=display_name) @pytest.mark.parametrize( @@ -610,7 +751,11 @@ def test_bad_display_name(display_name): ], ) def test_good_email(contact_email): - assert EmailStr(contact_email) + + class TestModel(pydantic.BaseModel): + email: EmailStr + + assert TestModel(email=contact_email) @pytest.mark.parametrize( @@ -623,5 +768,9 @@ def test_good_email(contact_email): ], ) def test_bad_email(contact_email): + + class TestModel(pydantic.BaseModel): + email: EmailStr + with pytest.raises(ValueError): - assert EmailStr(contact_email) + TestModel(email=contact_email) diff --git a/pydatalab/uv.lock b/pydatalab/uv.lock index de5bb6833..5158e7b8e 100644 --- a/pydatalab/uv.lock +++ b/pydatalab/uv.lock @@ -81,6 +81,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, ] +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + [[package]] name = "anthropic" version = "0.120.2" @@ -498,8 +507,10 @@ dependencies = [ { name = "pandas", extra = ["excel"] }, { name = "periodictable" }, { name = "pint" }, - { name = "pydantic", extra = ["dotenv", "email"] }, + { name = "pydantic", extra = ["email"] }, + { name = "pydantic-settings" }, { name = "pymongo" }, + { name = "python-dotenv" }, { name = "tomlkit" }, ] @@ -619,10 +630,12 @@ requires-dist = [ { name = "pint", specifier = "~=0.24" }, { name = "pyarrow", marker = "extra == 'apps'", specifier = "~=23.0.1" }, { name = "pybaselines", marker = "extra == 'apps'", specifier = "~=1.1" }, - { name = "pydantic", extras = ["dotenv", "email"], specifier = "<2.0" }, + { name = "pydantic", extras = ["email"], specifier = ">=2.11,<3.0" }, + { name = "pydantic-settings", specifier = "~=2.10" }, { name = "pyjwt", marker = "extra == 'server'", specifier = "~=2.9" }, { name = "pymongo", specifier = "~=4.7" }, { name = "python-dateutil", marker = "extra == 'apps'", specifier = "~=2.9" }, + { name = "python-dotenv", specifier = "~=1.1" }, { name = "python-dotenv", marker = "extra == 'server'", specifier = "~=1.0" }, { name = "renishawwire", marker = "extra == 'apps'", specifier = ">=0.1.16" }, { name = "scipy", marker = "extra == 'apps'", specifier = "~=1.13" }, @@ -2020,29 +2033,76 @@ wheels = [ [[package]] name = "pydantic" -version = "1.10.26" +version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, { name = "typing-extensions" }, + { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7b/da/fd89f987a376c807cd81ea0eff4589aade783bbb702637b4734ef2c743a2/pydantic-1.10.26.tar.gz", hash = "sha256:8c6aa39b494c5af092e690127c283d84f363ac36017106a9e66cb33a22ac412e", size = 357906, upload-time = "2025-12-18T15:47:46.557Z" } +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/c1/d521e64c8130e1ad9d22c270bed3fabcc0940c9539b076b639c88fd32a8d/pydantic-1.10.26-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:116233e53889bcc536f617e38c1b8337d7fa9c280f0fd7a4045947515a785637", size = 2428347, upload-time = "2025-12-18T15:46:39.41Z" }, - { url = "https://files.pythonhosted.org/packages/2c/08/f4b804a00c16e3ea994cb640a7c25c579b4f1fa674cde6a19fa0dfb0ae4f/pydantic-1.10.26-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c3cfdd361addb6eb64ccd26ac356ad6514cee06a61ab26b27e16b5ed53108f77", size = 2212605, upload-time = "2025-12-18T15:46:41.006Z" }, - { url = "https://files.pythonhosted.org/packages/5d/78/0df4b9efef29bbc5e39f247fcba99060d15946b4463d82a5589cf7923d71/pydantic-1.10.26-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e4451951a9a93bf9a90576f3e25240b47ee49ab5236adccb8eff6ac943adf0f", size = 2753560, upload-time = "2025-12-18T15:46:43.215Z" }, - { url = "https://files.pythonhosted.org/packages/68/66/6ab6c1d3a116d05d2508fce64f96e35242938fac07544d611e11d0d363a0/pydantic-1.10.26-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9858ed44c6bea5f29ffe95308db9e62060791c877766c67dd5f55d072c8612b5", size = 2859235, upload-time = "2025-12-18T15:46:45.112Z" }, - { url = "https://files.pythonhosted.org/packages/61/4e/f1676bb0fcdf6ed2ce4670d7d1fc1d6c3a06d84497644acfbe02649503f1/pydantic-1.10.26-cp311-cp311-win_amd64.whl", hash = "sha256:ac1089f723e2106ebde434377d31239e00870a7563245072968e5af5cc4d33df", size = 2066646, upload-time = "2025-12-18T15:46:46.816Z" }, - { url = "https://files.pythonhosted.org/packages/1f/98/556e82f00b98486def0b8af85da95e69d2be7e367cf2431408e108bc3095/pydantic-1.10.26-py3-none-any.whl", hash = "sha256:c43ad70dc3ce7787543d563792426a16fd7895e14be4b194b5665e36459dd917", size = 166975, upload-time = "2025-12-18T15:47:44.927Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, ] [package.optional-dependencies] -dotenv = [ - { name = "python-dotenv" }, -] email = [ { name = "email-validator" }, ] +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/ca/31c57507b13119d7d3cfa1576dad2911a4861e3be07b579395f4e9d393f9/pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117", size = 261253, upload-time = "2026-08-07T09:24:57.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, +] + [[package]] name = "pygments" version = "2.20.0" @@ -2645,6 +2705,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + [[package]] name = "tzdata" version = "2026.3" diff --git a/webapp/cypress/component/TagColorPickerTest.cy.jsx b/webapp/cypress/component/TagColorPickerTest.cy.jsx new file mode 100644 index 000000000..ceb976bb4 --- /dev/null +++ b/webapp/cypress/component/TagColorPickerTest.cy.jsx @@ -0,0 +1,33 @@ +import TagColorPicker from "@/components/TagColorPicker.vue"; +import { TAG_COLOR_PALETTE } from "@/resources.js"; + +describe("TagColorPicker.vue", () => { + it("renders the preset palette", () => { + cy.mount(TagColorPicker, { props: { modelValue: null } }); + cy.get(".swatch").should("have.length", TAG_COLOR_PALETTE.length); + }); + + it("emits the chosen preset color", () => { + cy.mount(TagColorPicker, { + props: { modelValue: null, "onUpdate:modelValue": cy.spy().as("update") }, + }); + const color = TAG_COLOR_PALETTE[0]; + cy.get(`.swatch[title="${color}"]`).click(); + cy.get("@update").should("have.been.calledWith", color); + }); + + it("marks the active preset as selected", () => { + const color = TAG_COLOR_PALETTE[1]; + cy.mount(TagColorPicker, { props: { modelValue: color } }); + cy.get(`.swatch[title="${color}"]`).should("have.class", "selected"); + }); + + it("emits a custom color from the native picker", () => { + cy.mount(TagColorPicker, { + props: { modelValue: null, "onUpdate:modelValue": cy.spy().as("update") }, + }); + // The native color input normalises to a lowercase 6-digit hex. + cy.get('input[type="color"]').invoke("val", "#123456").trigger("input"); + cy.get("@update").should("have.been.calledWith", "#123456"); + }); +}); diff --git a/webapp/cypress/component/TagFormModalTest.cy.jsx b/webapp/cypress/component/TagFormModalTest.cy.jsx new file mode 100644 index 000000000..f3d0ccdda --- /dev/null +++ b/webapp/cypress/component/TagFormModalTest.cy.jsx @@ -0,0 +1,124 @@ +import TagFormModal from "@/components/TagFormModal.vue"; +import { DEFAULT_TAG_COLOR } from "@/resources.js"; +import { createStore } from "vuex"; + +// Mount the modal closed, then open it by flipping `modelValue` so the Modal's open watcher +// (and the form's populate/reset watcher) fire as they do in the app. Returns the test-utils +// wrapper aliased as "wrapper" for emitted assertions. `role` sets the current user's role +// (the modal offers the "global" scope only to admins). +// +// Note: component tests run without the app's global Bootstrap CSS, so the Modal's backdrop +// overlays the dialog (no .modal z-index). We use { force: true } on interactions to bypass +// that purely-visual actionability check; the request-body and emit assertions are unaffected. +function mountAndOpen({ tag = null, role = "user" } = {}) { + const store = createStore({ + state() { + return { currentUserRole: role }; + }, + }); + return cy + .mount(TagFormModal, { + props: { modelValue: false, tag }, + global: { plugins: [store] }, + }) + .then(({ wrapper }) => { + cy.wrap(wrapper).as("wrapper"); + return wrapper.setProps({ modelValue: true }); + }); +} + +describe("TagFormModal.vue", () => { + describe("create mode", () => { + it("creates a user-defined tag by default (non-admin has no scope choice)", () => { + cy.intercept("PUT", "**/tags", { statusCode: 201, body: { status: "success", data: {} } }).as( + "create", + ); + mountAndOpen(); + + // A non-admin cannot choose a scope; the scope field is a disabled "User-defined". + cy.get('[data-testid="tag-scope-select"]').should("not.exist"); + cy.get("#tag-scope").should("be.disabled").and("have.value", "User-defined"); + + cy.get("#tag-name").type("flammable", { force: true }); + cy.get("#tag-description").type("burns", { force: true }); + cy.get('input[type="submit"]').click({ force: true }); + + // The color picker is left untouched, so the payload carries the default tag color, + // and the scope defaults to a user-defined ("user") tag. + cy.wait("@create") + .its("request.body") + .should("deep.equal", { + data: { + name: "flammable", + description: "burns", + color: DEFAULT_TAG_COLOR, + scope: "user", + }, + }); + cy.get("@wrapper").should((wrapper) => { + expect(wrapper.emitted("tag-created")).to.have.length(1); + // The modal asks its parent to close on success. + expect(wrapper.emitted("update:modelValue").at(-1)).to.deep.equal([false]); + }); + }); + + it("lets an admin create a global tag", () => { + cy.intercept("PUT", "**/tags", { statusCode: 201, body: { status: "success", data: {} } }).as( + "create", + ); + mountAndOpen({ role: "admin" }); + + // An admin can choose the scope. (force: true — see mount note: the Modal + // backdrop overlays the dialog without the app's global Bootstrap CSS.) + cy.get('[data-testid="tag-scope-select"]').select("global", { force: true }); + cy.get("#tag-name").type("corrosive", { force: true }); + cy.get('input[type="submit"]').click({ force: true }); + + cy.wait("@create").its("request.body.data.scope").should("equal", "global"); + }); + + it("shows a name conflict (409) inline instead of an error dialog", () => { + cy.intercept("PUT", "**/tags", { + statusCode: 409, + body: { status: "error", message: "A tag named 'dup' already exists." }, + }).as("create"); + mountAndOpen(); + + cy.get("#tag-name").type("dup", { force: true }); + cy.get('input[type="submit"]').click({ force: true }); + + cy.wait("@create"); + cy.get(".form-error").should("contain", "already exists"); + // The modal stays open on a conflict. + cy.get("@wrapper").should((wrapper) => { + expect(wrapper.emitted("tag-created")).to.be.undefined; + }); + }); + }); + + describe("edit mode", () => { + const existingTag = { + immutable_id: "tag-1", + name: "old-name", + description: "desc", + color: "#abcdef", + }; + + it("pre-fills fields and updates metadata via PATCH /tags/", () => { + cy.intercept("PATCH", "**/tags/*", { statusCode: 200, body: { status: "success" } }).as( + "updateTag", + ); + mountAndOpen({ tag: existingTag }); + + cy.get("#tag-name").should("have.value", "old-name"); + cy.get("#tag-name").clear({ force: true }); + cy.get("#tag-name").type("new-name", { force: true }); + cy.get('input[type="submit"]').click({ force: true }); + + cy.wait("@updateTag").its("request.body.data.name").should("equal", "new-name"); + cy.get("@wrapper").should((wrapper) => { + expect(wrapper.emitted("tag-updated")).to.have.length(1); + }); + }); + }); +}); diff --git a/webapp/cypress/component/TagManagementTableTest.cy.jsx b/webapp/cypress/component/TagManagementTableTest.cy.jsx new file mode 100644 index 000000000..a0478184c --- /dev/null +++ b/webapp/cypress/component/TagManagementTableTest.cy.jsx @@ -0,0 +1,117 @@ +import TagManagementTable from "@/components/TagManagementTable.vue"; +import StyledTooltip from "@/components/StyledTooltip.vue"; +import PrimeVue from "primevue/config"; +import { createStore } from "vuex"; + +const TAGS = [ + { + immutable_id: "t1", + type: "tags", + name: "flammable", + description: "burns", + color: "#f1c40f", + // A user-defined tag owned by the current user ("self"). + scope: "user", + owner: "self", + }, + { + immutable_id: "t2", + type: "tags", + name: "global-tag", + description: null, + color: null, + scope: "global", + owner: null, + }, +]; + +function mountTable(role) { + const store = createStore({ + state() { + return { + currentUserID: "self", + currentUserRole: role, + datatablePaginationSettings: { + tags: { page: 0, rows: 20 }, + }, + tag_list: TAGS, + }; + }, + }); + + cy.mount(TagManagementTable, { + global: { + plugins: [store, PrimeVue], + components: { + StyledTooltip, + }, + }, + }); +} + +describe("TagManagementTable Component Tests", () => { + it("renders the expected columns (including Scope)", () => { + mountTable("admin"); + const headers = ["", "Tag", "Description", "Scope", "Actions"]; + cy.get(".p-datatable-column-header-content").should("have.length", headers.length); + cy.get(".p-datatable-column-header-content").each((header, index) => { + cy.wrap(header).should("contain.text", headers[index]); + }); + }); + + it("shows a scope badge per tag", () => { + mountTable("admin"); + cy.get(".p-datatable-tbody") + .find("tr") + .eq(0) + .find('[data-testid="tag-scope-badge"]') + .should("contain.text", "User-defined"); + cy.get(".p-datatable-tbody") + .find("tr") + .eq(1) + .find('[data-testid="tag-scope-badge"]') + .should("contain.text", "Global"); + }); + + it("displays a badge per tag from the store", () => { + mountTable("admin"); + cy.get(".p-datatable-tbody") + .find("tr") + .eq(0) + .within(() => { + cy.get("td").eq(1).find(".badge").should("contain.text", "flammable"); + }); + cy.get(".p-datatable-tbody") + .find("tr") + .eq(1) + .within(() => { + cy.get("td").eq(1).find(".badge").should("contain.text", "global-tag"); + }); + }); + + it("shows the create button and Edit/Delete on every tag for an admin", () => { + // An admin owns the user-defined tag ("self") and manages the global tag, so both rows. + mountTable("admin"); + cy.get('[data-testid="add-tag-button"]').should("exist"); + cy.get('button[title="Edit tag"]').should("have.length", TAGS.length); + cy.get('button[title="Delete tag"]').should("have.length", TAGS.length); + }); + + it("lets a non-admin create tags and manage only their own user-defined tags", () => { + // A non-admin can create (user-defined) tags, and manage their own user-defined tag, + // but not the global tag (which only admins manage). + mountTable("user"); + cy.get('[data-testid="add-tag-button"]').should("exist"); + cy.get('button[title="Edit tag"]').should("have.length", 1); + cy.get('button[title="Delete tag"]').should("have.length", 1); + // The controls sit on the user-defined (own) tag row, not the global one. + cy.get(".p-datatable-tbody") + .find("tr") + .eq(0) + .within(() => cy.get('button[title="Edit tag"]').should("exist")); + cy.get(".p-datatable-tbody") + .find("tr") + .eq(1) + .within(() => cy.get('button[title="Edit tag"]').should("not.exist")); + }); +}); diff --git a/webapp/cypress/component/TagSelectTest.cy.jsx b/webapp/cypress/component/TagSelectTest.cy.jsx new file mode 100644 index 000000000..daab36af7 --- /dev/null +++ b/webapp/cypress/component/TagSelectTest.cy.jsx @@ -0,0 +1,71 @@ +import TagSelect from "@/components/TagSelect.vue"; + +describe("TagSelect.vue", () => { + const tag = { + type: "tags", + immutable_id: "0123456789ab0123456789ab", + name: "test-tag", + description: "reacts with air", + color: "#f1c40f", + scope: "user", + }; + + beforeEach(() => { + cy.intercept("GET", "**/search-tags*", { + body: { status: "success", data: [tag] }, + }).as("searchTags"); + }); + + it("searches and emits a reference object when a tag is selected", () => { + cy.mount(TagSelect, { + props: { modelValue: [], "onUpdate:modelValue": cy.spy().as("update") }, + }); + + cy.get(".vs__search").type("test-man"); + cy.wait("@searchTags"); + // The option shows the tag name, a color swatch and a scope pill. + cy.get(".vs__dropdown-option").contains("test-tag").should("exist"); + cy.get(".vs__dropdown-option .color-swatch").should("exist"); + cy.get('.vs__dropdown-option [data-testid="tag-scope-badge"]').should( + "contain", + "user-defined", + ); + cy.get(".vs__dropdown-option").contains("test-tag").click(); + + // The reference preserves display fields (color/description/scope). + cy.get("@update").should("have.been.calledWith", [ + { + type: "tags", + immutable_id: tag.immutable_id, + name: "test-tag", + color: "#f1c40f", + description: "reacts with air", + scope: "user", + }, + ]); + }); + + it("does not offer to create a tag for a typed value with no match", () => { + cy.intercept("GET", "**/search-tags*", { + body: { status: "success", data: [] }, + }).as("searchTagsEmpty"); + + cy.mount(TagSelect, { + props: { modelValue: [], "onUpdate:modelValue": cy.spy().as("update") }, + }); + + cy.get(".vs__search").type("brand-new-tag"); + cy.wait("@searchTagsEmpty"); + // A typed value matching no tag is not selectable (no ad-hoc tag creation). + cy.get(".vs__dropdown-option").should("not.exist"); + cy.get(".vs__no-options").should("contain", "No matching tags"); + }); + + it("renders existing reference tags as selected chips", () => { + cy.mount(TagSelect, { + props: { modelValue: [tag] }, + }); + + cy.get(".vs__selected").should("contain", "test-tag"); + }); +}); diff --git a/webapp/cypress/e2e/tagsManagement.cy.js b/webapp/cypress/e2e/tagsManagement.cy.js new file mode 100644 index 000000000..25b8cbb2e --- /dev/null +++ b/webapp/cypress/e2e/tagsManagement.cy.js @@ -0,0 +1,198 @@ +// E2e tests for the tag management page (/tags). Needs the dev server + API (:5001) running +// with testing auth AND the tags feature enabled (PYDATALAB_ENABLE_TAGS). +// +// Tags have two scopes: "global" (admin-managed, usable by everyone) and "user" (user-defined, +// owned and managed by a single user). admin-user@example.com is an admin by the same +// convention as authenticatedSampleTests.cy.js. + +// Role is determined server-side by the email (see authenticatedSampleTests.cy.js): +// `admin-user@example.com` is an admin, `test-user@example.com` is a plain user. +const adminEmail = "admin-user@example.com"; +const userEmail = "test-user@example.com"; // a non-admin user + +describe("Tag management page (admin, global tags)", () => { + // Names must not be substrings of each other: cy.contains matches substrings, so a + // "not.exist" check on the original would still match the renamed badge otherwise. + const tagName = "e2e-create-tag"; + const renamedTag = "e2e-renamed-tag"; + + beforeEach(() => { + cy.loginViaTestMagicLink(adminEmail); + cy.deleteTagByNameViaAPI(tagName); + cy.deleteTagByNameViaAPI(renamedTag); + }); + + after(() => { + cy.loginViaTestMagicLink(adminEmail); + cy.deleteTagByNameViaAPI(tagName); + cy.deleteTagByNameViaAPI(renamedTag); + }); + + it("creates, edits and deletes a global tag", () => { + cy.visit("/tags"); + + // Create (an admin picks the "global" scope). + cy.get('[data-testid="add-tag-button"]').click(); + cy.get('[data-testid="tag-scope-select"]').select("global"); + cy.get("#tag-name").type(tagName); + cy.get("#tag-description").type("created in an e2e test"); + cy.get(".swatch").first().click(); + cy.get(".modal-footer input[type=submit]:visible").click(); + // Scope badge assertions to the table: the (closed) edit/create modal keeps a hidden + // TagBadge preview in the DOM (Modal uses display:none), which a document-wide + // `.badge` match would pick up and break the `not.exist` checks below. + cy.get('[data-testid="tags-table"]').contains(".badge", tagName).should("exist"); + // The row is marked as a global tag. + cy.get('[data-testid="tags-table"]') + .contains("tr", tagName) + .find('[data-testid="tag-scope-badge"]') + .should("contain.text", "Global"); + + // Edit (rename) + cy.contains("tr", tagName).find('button[title="Edit tag"]').click(); + cy.get("#tag-name").clear(); + cy.get("#tag-name").type(renamedTag); + cy.get(".modal-footer input[type=submit]:visible").click(); + cy.get('[data-testid="tags-table"]').contains(".badge", renamedTag).should("exist"); + cy.get('[data-testid="tags-table"]').contains(".badge", tagName).should("not.exist"); + + // Delete + cy.contains("tr", renamedTag).find('button[title="Delete tag"]').click(); + cy.get('[data-testid="dialog-modal-confirm-button"]').click(); + cy.get('[data-testid="tags-table"]').contains(".badge", renamedTag).should("not.exist"); + }); +}); + +describe("Tag management page (user, user-defined tags)", () => { + const tagName = "e2e-user-defined-tag"; + const renamedTag = "e2e-user-defined-renamed"; + + beforeEach(() => { + cy.loginViaTestMagicLink(userEmail); + cy.deleteTagByNameViaAPI(tagName); + cy.deleteTagByNameViaAPI(renamedTag); + }); + + after(() => { + cy.loginViaTestMagicLink(userEmail); + cy.deleteTagByNameViaAPI(tagName); + cy.deleteTagByNameViaAPI(renamedTag); + }); + + it("lets a non-admin create, edit and delete their own user-defined tag", () => { + cy.visit("/tags"); + + // Create. A non-admin has no scope choice; the tag is user-defined by default. + cy.get('[data-testid="add-tag-button"]').click(); + cy.get('[data-testid="tag-scope-select"]').should("not.exist"); + cy.get("#tag-name").type(tagName); + cy.get(".modal-footer input[type=submit]:visible").click(); + + cy.get('[data-testid="tags-table"]').contains(".badge", tagName).should("exist"); + cy.get('[data-testid="tags-table"]') + .contains("tr", tagName) + .find('[data-testid="tag-scope-badge"]') + .should("contain.text", "User-defined"); + + // The owner can edit and delete their own user-defined tag. + cy.contains("tr", tagName).find('button[title="Edit tag"]').click(); + cy.get("#tag-name").clear(); + cy.get("#tag-name").type(renamedTag); + cy.get(".modal-footer input[type=submit]:visible").click(); + cy.get('[data-testid="tags-table"]').contains(".badge", renamedTag).should("exist"); + + cy.contains("tr", renamedTag).find('button[title="Delete tag"]').click(); + cy.get('[data-testid="dialog-modal-confirm-button"]').click(); + cy.get('[data-testid="tags-table"]').contains(".badge", renamedTag).should("not.exist"); + }); +}); + +describe("Tag management permissions", () => { + const tagName = "e2e-perm-tag"; + + before(() => { + cy.loginViaTestMagicLink(adminEmail); + cy.deleteTagByNameViaAPI(tagName); + // A global tag: everyone can see it, but only admins can edit/delete it. + cy.createTagViaAPI({ name: tagName, scope: "global" }); + }); + + after(() => { + cy.loginViaTestMagicLink(adminEmail); + cy.deleteTagByNameViaAPI(tagName); + }); + + it("lets a non-admin create tags but not manage a global tag", () => { + cy.loginViaTestMagicLink(userEmail); + cy.visit("/tags"); + cy.contains("tr", tagName).should("exist"); // the global tag is visible to everyone + // A non-admin can now create their own (user-defined) tags. + cy.get('[data-testid="add-tag-button"]').should("exist"); + // ... but cannot edit or delete a global tag. + cy.contains("tr", tagName).within(() => { + cy.get('button[title="Edit tag"]').should("not.exist"); + cy.get('button[title="Delete tag"]').should("not.exist"); + }); + }); + + it("shows edit/delete controls on a global tag for an admin", () => { + cy.loginViaTestMagicLink(adminEmail); + cy.visit("/tags"); + cy.get('[data-testid="add-tag-button"]').should("exist"); + cy.contains("tr", tagName).within(() => { + cy.get('button[title="Edit tag"]').should("exist"); + }); + }); +}); + +describe("Applying a tag to an item", () => { + const intTag = "e2e-applied-tag"; + const sampleId = "e2e-tag-sample"; + + before(() => { + // A global tag so any user can apply it. + cy.loginViaTestMagicLink(adminEmail); + cy.deleteTagByNameViaAPI(intTag); + cy.createTagViaAPI({ name: intTag, scope: "global" }); + }); + + beforeEach(() => { + cy.loginViaTestMagicLink(userEmail); + cy.deleteSampleViaAPI(sampleId); + cy.visit("/samples"); + cy.createSample(sampleId, "Tag e2e sample"); + }); + + after(() => { + cy.loginViaTestMagicLink(adminEmail); + cy.deleteSampleViaAPI(sampleId); + cy.deleteTagByNameViaAPI(intTag); + }); + + it("applies a tag and drops it from the item when the tag is deleted", () => { + cy.intercept("GET", "**/search-tags*").as("searchTags"); + cy.intercept("POST", "**/save-item/").as("save"); + + cy.visit(`/edit/${sampleId}`); + + // Enter edit mode on the Tags field (click the label text, away from the cog link), + // then pick the tag from the TagSelect dropdown. + cy.get("#tags").click("left"); + cy.get("#tags").parent().find(".vs__search").type(intTag); + cy.wait("@searchTags"); + cy.get("#tags").parent().contains(".vs__dropdown-option", intTag).click(); + + // Save (Ctrl/Cmd+S) and confirm the tag survives a reload. + cy.get("body").type("{ctrl}s"); + cy.wait("@save"); + cy.reload(); + cy.contains(".badge", intTag).should("exist"); + + // Deleting the tag (as admin) removes the reference from the item on the next read. + cy.loginViaTestMagicLink(adminEmail); + cy.deleteTagByNameViaAPI(intTag); + cy.loginViaTestMagicLink(userEmail); + cy.reload(); + cy.contains(".badge", intTag).should("not.exist"); + }); +}); diff --git a/webapp/cypress/support/commands.js b/webapp/cypress/support/commands.js index f50c47a7e..dfc66624e 100644 --- a/webapp/cypress/support/commands.js +++ b/webapp/cypress/support/commands.js @@ -115,6 +115,38 @@ Cypress.Commands.add("deleteSampleViaAPI", (item_id) => { }); }); +Cypress.Commands.add("createTagViaAPI", (data) => { + // data: { name, description?, color?, scope? }. `scope` defaults to "user" + // (a user-defined tag owned by the logged-in user); pass scope: "global" (as an + // admin) for a tag every user can use. Returns the new tag's id. + return cy + .request({ + method: "PUT", + url: API_URL + "/tags", + body: { data }, + failOnStatusCode: false, + }) + .then((response) => response.body?.data?.immutable_id ?? null); +}); + +Cypress.Commands.add("deleteTagByNameViaAPI", (name) => { + // Best-effort cleanup: delete every tag with this name (requires admin auth). + cy.request({ method: "GET", url: API_URL + "/tags", failOnStatusCode: false }).then( + (response) => { + const tags = response.body?.data ?? []; + tags + .filter((tag) => tag.name === name) + .forEach((tag) => { + cy.request({ + method: "DELETE", + url: API_URL + "/tags/" + tag.immutable_id, + failOnStatusCode: false, + }); + }); + }, + ); +}); + Cypress.Commands.add("uploadFileViaAPI", (itemId, path) => { cy.log("Upload a test file via the API: " + path); cy.fixture(path, "binary") diff --git a/webapp/src/components/BaseIconCounter.vue b/webapp/src/components/BaseIconCounter.vue index 2800e39f8..c74b823f6 100644 --- a/webapp/src/components/BaseIconCounter.vue +++ b/webapp/src/components/BaseIconCounter.vue @@ -4,7 +4,7 @@
- {{ displayCount }} + {{ prefix }}{{ displayCount }}
@@ -14,9 +14,7 @@
- - {{ displayCount }} - + {{ prefix }}{{ displayCount }}
@@ -33,6 +31,10 @@ export default { type: Number, default: 0, }, + prefix: { + type: String, + default: "", + }, showIcon: { type: Boolean, default: false, diff --git a/webapp/src/components/CellInformation.vue b/webapp/src/components/CellInformation.vue index 1d1416675..24455375c 100644 --- a/webapp/src/components/CellInformation.vue +++ b/webapp/src/components/CellInformation.vue @@ -43,6 +43,11 @@ +
+
+ +
+
@@ -126,6 +131,7 @@ import ToggleableCollectionFormGroup from "@/components/ToggleableCollectionForm import ToggleableCreatorsFormGroup from "@/components/ToggleableCreatorsFormGroup"; import ToggleableItemStatusFormGroup from "@/components/ToggleableItemStatusFormGroup"; import ToggleableGroupsFormGroup from "@/components/ToggleableGroupsFormGroup"; +import ToggleableTagsFormGroup from "@/components/ToggleableTagsFormGroup"; import { cellFormats } from "@/resources.js"; export default { @@ -140,6 +146,7 @@ export default { ToggleableCreatorsFormGroup, ToggleableItemStatusFormGroup, ToggleableGroupsFormGroup, + ToggleableTagsFormGroup, }, props: { item_id: { @@ -174,12 +181,16 @@ export default { CellFormatDescription: createComputedSetterForItemField("cell_format_description"), CharacteristicMass: createComputedSetterForItemField("characteristic_mass"), Collections: createComputedSetterForItemField("collections"), + Tags: createComputedSetterForItemField("tags"), Status: createComputedSetterForItemField("status"), + enableTags() { + return this.$store.state.serverInfo?.features?.tags ?? false; + }, schema() { return this.$store.state.schemas[this.item?.type]; }, possibleItemStatuses() { - return this.schema?.attributes?.schema?.definitions?.CellStatus?.enum; + return this.schema?.attributes?.schema?.["$defs"]?.CellStatus?.enum; }, }, }; diff --git a/webapp/src/components/DynamicDataTable.vue b/webapp/src/components/DynamicDataTable.vue index f73e387b3..f8d97874a 100644 --- a/webapp/src/components/DynamicDataTable.vue +++ b/webapp/src/components/DynamicDataTable.vue @@ -54,6 +54,7 @@ @open-qr-scanner-modal="qrScannerModalIsOpen = true" @open-create-collection-modal="createCollectionModalIsOpen = true" @open-create-equipment-modal="createEquipmentModalIsOpen = true" + @open-create-tag-modal="$emit('open-create-tag-modal')" @open-add-to-collection-modal="addToCollectionModalIsOpen = true" @open-batch-share-modal="batchShareModalIsOpen = true" @delete-selected-items="deleteSelectedItems" @@ -217,6 +218,41 @@ + + @@ -23,7 +15,7 @@ export default { components: { DynamicDataTable }, data() { return { - sampleColumns: [ + baseSampleColumns: [ { field: "item_id", header: "ID", @@ -75,6 +67,39 @@ export default { }; }, computed: { + enableTags() { + // Tag column only if enabled globally. + return this.$store.state.serverInfo?.features?.tags ?? false; + }, + sampleColumns() { + const columns = [...this.baseSampleColumns]; + if (this.enableTags) { + const insertBeforeBlocks = columns.findIndex((column) => column.field === "blocks"); + columns.splice(insertBeforeBlocks, 0, { + field: "tags", + header: "Tags", + body: "TagList", + filter: true, + label: "Tags", + }); + } + return columns; + }, + sampleGlobalFilterFields() { + const fields = [ + "item_id", + "name", + "refcode", + "chemform", + "creatorsList", + "blocks", + "characteristic_chemical_formula", + ]; + if (this.enableTags) { + fields.push("tagsList"); + } + return fields; + }, samples() { if (!this.$store.state.sample_list) { return null; @@ -90,6 +115,10 @@ export default { .map((collection) => collection.collection_id) .join(", "), creatorsList: sample.creators.map((creator) => creator.display_name).join(", "), + tagsList: (sample.tags || []) + .map((tag) => tag.name) + .filter(Boolean) + .join(", "), }; }); }, diff --git a/webapp/src/components/StartingMaterialInformation.vue b/webapp/src/components/StartingMaterialInformation.vue index 26a78494f..3fae27e91 100644 --- a/webapp/src/components/StartingMaterialInformation.vue +++ b/webapp/src/components/StartingMaterialInformation.vue @@ -47,6 +47,12 @@
+
+
+ +
+
+
@@ -127,6 +133,7 @@ import ItemRelationshipVisualization from "@/components/ItemRelationshipVisualiz import ToggleableCreatorsFormGroup from "@/components/ToggleableCreatorsFormGroup"; import ToggleableGroupsFormGroup from "@/components/ToggleableGroupsFormGroup"; import LocationInput from "@/components/LocationInput"; +import ToggleableTagsFormGroup from "@/components/ToggleableTagsFormGroup"; import AutoComplete from "primevue/autocomplete"; import { getStartingMaterialList, getEquipmentList } from "@/server_fetch_utils.js"; @@ -148,6 +155,7 @@ export default { ToggleableCreatorsFormGroup, ToggleableGroupsFormGroup, LocationInput, + ToggleableTagsFormGroup, }, props: { item_id: { type: String, required: true }, @@ -176,15 +184,19 @@ export default { Location: createComputedSetterForItemField("location"), ItemDescription: createComputedSetterForItemField("description"), Collections: createComputedSetterForItemField("collections"), + Tags: createComputedSetterForItemField("tags"), Refcode: createComputedSetterForItemField("refcode"), Status: createComputedSetterForItemField("status"), ItemCreators: createComputedSetterForItemField("creators"), ItemGroups: createComputedSetterForItemField("groups"), + enableTags() { + return this.$store.state.serverInfo?.features?.tags ?? false; + }, schema() { return this.$store.state.schemas[this.item?.type]; }, possibleItemStatuses() { - return this.schema?.attributes?.schema?.definitions?.StartingMaterialsStatus?.enum; + return this.schema?.attributes?.schema?.["$defs"]?.StartingMaterialsStatus?.enum; }, Barcode: createComputedSetterForItemField("barcode"), uniqueSuppliers() { diff --git a/webapp/src/components/TagActionsCell.vue b/webapp/src/components/TagActionsCell.vue new file mode 100644 index 000000000..3b22c4d92 --- /dev/null +++ b/webapp/src/components/TagActionsCell.vue @@ -0,0 +1,88 @@ + + + + + diff --git a/webapp/src/components/TagBadge.vue b/webapp/src/components/TagBadge.vue new file mode 100644 index 000000000..10e6d6faf --- /dev/null +++ b/webapp/src/components/TagBadge.vue @@ -0,0 +1,90 @@ + + + + + diff --git a/webapp/src/components/TagColorPicker.vue b/webapp/src/components/TagColorPicker.vue new file mode 100644 index 000000000..03a82704b --- /dev/null +++ b/webapp/src/components/TagColorPicker.vue @@ -0,0 +1,106 @@ + + + + + diff --git a/webapp/src/components/TagFormModal.vue b/webapp/src/components/TagFormModal.vue new file mode 100644 index 000000000..11a1397c8 --- /dev/null +++ b/webapp/src/components/TagFormModal.vue @@ -0,0 +1,207 @@ + + + + + diff --git a/webapp/src/components/TagList.vue b/webapp/src/components/TagList.vue new file mode 100644 index 000000000..b36a34941 --- /dev/null +++ b/webapp/src/components/TagList.vue @@ -0,0 +1,123 @@ + + + + + diff --git a/webapp/src/components/TagManagementTable.vue b/webapp/src/components/TagManagementTable.vue new file mode 100644 index 000000000..a29d8565d --- /dev/null +++ b/webapp/src/components/TagManagementTable.vue @@ -0,0 +1,85 @@ + + + diff --git a/webapp/src/components/TagScopeBadge.vue b/webapp/src/components/TagScopeBadge.vue new file mode 100644 index 000000000..303a6066a --- /dev/null +++ b/webapp/src/components/TagScopeBadge.vue @@ -0,0 +1,28 @@ + + + diff --git a/webapp/src/components/TagSelect.vue b/webapp/src/components/TagSelect.vue new file mode 100644 index 000000000..0d13b5305 --- /dev/null +++ b/webapp/src/components/TagSelect.vue @@ -0,0 +1,178 @@ + + + + + diff --git a/webapp/src/components/ToggleableTagsFormGroup.vue b/webapp/src/components/ToggleableTagsFormGroup.vue new file mode 100644 index 000000000..88cc0e039 --- /dev/null +++ b/webapp/src/components/ToggleableTagsFormGroup.vue @@ -0,0 +1,82 @@ + + + + + diff --git a/webapp/src/field_utils.js b/webapp/src/field_utils.js index f17aaa1f1..dffef9fd6 100644 --- a/webapp/src/field_utils.js +++ b/webapp/src/field_utils.js @@ -185,3 +185,27 @@ export function validateEntryID(id, takenIds = [], existingIds = []) { } return ""; } + +export function readableTextColor(hexColor) { + // Return a readable text color ("#000" or "#fff") for a given background hex + // color, based on its perceptual luminance. Falls back to black for invalid input. + if (!hexColor || typeof hexColor !== "string") { + return "#000"; + } + let hex = hexColor.trim().replace(/^#/, ""); + if (hex.length === 3) { + hex = hex + .split("") + .map((c) => c + c) + .join(""); + } + if (hex.length !== 6 || /[^0-9a-fA-F]/.test(hex)) { + return "#000"; + } + const r = parseInt(hex.slice(0, 2), 16); + const g = parseInt(hex.slice(2, 4), 16); + const b = parseInt(hex.slice(4, 6), 16); + // Perceptual luminance (sRGB weights), normalised to [0, 1]. + const luminance = (0.299 * r + 0.587 * g + 0.114 * b) / 255; + return luminance > 0.6 ? "#000" : "#fff"; +} diff --git a/webapp/src/main.js b/webapp/src/main.js index f75d0aa99..545dade39 100644 --- a/webapp/src/main.js +++ b/webapp/src/main.js @@ -77,6 +77,7 @@ import { faCaretDown, faLock, faClock, + faUser, } from "@fortawesome/free-solid-svg-icons"; import { faPlusSquare } from "@fortawesome/free-regular-svg-icons"; import { faGithub, faOrcid, faGoogle, faMicrosoft } from "@fortawesome/free-brands-svg-icons"; @@ -154,6 +155,7 @@ library.add( faCaretDown, faLock, faClock, + faUser, ); // import "@uppy/vue" diff --git a/webapp/src/resources.js b/webapp/src/resources.js index e4ed4d6d1..73a0a7948 100644 --- a/webapp/src/resources.js +++ b/webapp/src/resources.js @@ -143,6 +143,25 @@ export const SAMPLE_TABLE_TYPES = ["samples", "cells"]; export const INVENTORY_TABLE_TYPES = ["starting_materials"]; export const EQUIPMENT_TABLE_TYPES = ["equipment"]; +// Curated palette of distinguishable preset colors offered for tag colors. +export const TAG_COLOR_PALETTE = [ + "#e74c3c", + "#e67e22", + "#f1c40f", + "#2ecc71", + "#1abc9c", + "#3498db", + "#9b59b6", + "#34495e", + "#95a5a6", + "#e84393", + "#00b894", + "#fdcb6e", +]; + +// The color assigned to a newly created tag. +export const DEFAULT_TAG_COLOR = "#95a5a6"; + export const cellFormats = { coin: "coin", pouch: "pouch", diff --git a/webapp/src/router/index.js b/webapp/src/router/index.js index 3b96fc9a7..80a500c59 100644 --- a/webapp/src/router/index.js +++ b/webapp/src/router/index.js @@ -3,6 +3,7 @@ import Samples from "../views/Samples.vue"; import Equipment from "../views/Equipment.vue"; import StartingMaterials from "../views/StartingMaterials.vue"; import Collections from "@/views/Collections.vue"; +import Tags from "@/views/Tags.vue"; import NotFound from "../views/NotFound.vue"; import EditPage from "../views/EditPage.vue"; import CollectionPage from "../views/CollectionPage.vue"; @@ -13,6 +14,8 @@ import Login from "../views/Login.vue"; import Login2 from "../views/Login2.vue"; import Login3 from "../views/Login3.vue"; import { API_URL } from "@/resources.js"; +import { getInfo } from "@/server_fetch_utils.js"; +import store from "@/store/index.js"; const routes = [ { @@ -73,6 +76,20 @@ const routes = [ name: "collections", component: Collections, }, + { + path: "/tags", + name: "tags", + component: Tags, + // Only reachable when the backend reports the tags feature as enabled. + beforeEnter: async (to, from, next) => { + const serverInfo = store.state.serverInfo ?? (await getInfo()); + if (serverInfo.features?.tags) { + next(); + } else { + next({ path: "/" }); + } + }, + }, { path: "/collections/:id", name: "Collection", diff --git a/webapp/src/server_fetch_utils.js b/webapp/src/server_fetch_utils.js index d8865df3c..efaf9b556 100644 --- a/webapp/src/server_fetch_utils.js +++ b/webapp/src/server_fetch_utils.js @@ -561,6 +561,61 @@ export function searchCollections(query, nresults = 100) { }); } +export function createTag(data) { + // data: { name, description?, color?, scope? }. `scope` is "user" (user-defined, + // default) or "global" (admins only). The caller refreshes the list via + // getTags(). Rejects with the server message on error (e.g. 409 duplicate name). + return fetch_put(`${API_URL}/tags`, { data }).then(function (response_json) { + return response_json.data; + }); +} + +export function updateTag(tagId, data) { + // Update a tag's metadata (name/description/color). Rejects with the server message (e.g. 409). + return fetch_patch(`${API_URL}/tags/${tagId}`, { data }); +} + +export function deleteTag(tagId) { + return fetch_delete(`${API_URL}/tags/${tagId}`) + .then(function (response_json) { + if (response_json.status !== "success") { + throw new Error("Failed to delete tag: " + response_json.message); + } + store.commit("deleteFromTagList", tagId); + }) + .catch((error) => { + DialogService.error({ + title: "Unable to delete tag", + message: `Failed to delete tag: ${error}`, + }); + throw error; + }); +} + +export function getTags() { + return fetch_get(`${API_URL}/tags`) + .then(function (response_json) { + store.commit("setTagList", response_json.data); + }) + .catch((error) => { + if (error === "UNAUTHORIZED") { + store.commit("setTagList", []); + } else { + throw error; + } + }); +} + +export function searchTags(query, nresults = 100) { + // construct a url with parameters: + var url = new URL(`${API_URL}/search-tags`); + var params = { query: query, nresults: nresults }; + Object.keys(params).forEach((key) => url.searchParams.append(key, params[key])); + return fetch_get(url).then(function (response_json) { + return response_json.data; + }); +} + export function searchGroups(query, nresults = 100) { // construct a url with parameters: var url = new URL(`${API_URL}/search/groups`); diff --git a/webapp/src/store/index.js b/webapp/src/store/index.js index 8dd7cf01e..7ec2e38f4 100644 --- a/webapp/src/store/index.js +++ b/webapp/src/store/index.js @@ -18,6 +18,7 @@ export default createStore({ equipment_list: null, starting_material_list: null, collection_list: null, + tag_list: null, groups_list: null, saved_status_items: {}, saved_status_blocks: {}, @@ -72,6 +73,10 @@ export default createStore({ page: 0, rows: 10, }, + tags: { + page: 0, + rows: 20, + }, }, block_errors: {}, block_infos: {}, @@ -98,6 +103,20 @@ export default createStore({ // collectionSummaries is an array of json objects summarizing the available collections state.collection_list = collectionSummaries || []; }, + setTagList(state, tags) { + // tags is an array of tag objects + state.tag_list = tags || []; + }, + deleteFromTagList(state, tagId) { + if (state.tag_list === null) return; + + const index = state.tag_list.map((t) => t.immutable_id).indexOf(tagId); + if (index > -1) { + state.tag_list.splice(index, 1); + } else { + console.warn(`deleteFromTagList couldn't find the tag with id ${tagId}`); + } + }, setGroupsList(state, groups) { state.groups_list = groups; }, diff --git a/webapp/src/views/Tags.vue b/webapp/src/views/Tags.vue new file mode 100644 index 000000000..77ebe9a67 --- /dev/null +++ b/webapp/src/views/Tags.vue @@ -0,0 +1,36 @@ + + + + +