From f7b5bbef84a809da158e28f4feac02a14a0140ac Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Sun, 7 Jun 2026 15:40:18 +0100 Subject: [PATCH 01/40] Update pydantic version across various files --- .github/dependabot.yml | 2 - .pre-commit-config.yaml | 2 +- pydatalab/pyproject.toml | 4 +- pydatalab/uv.lock | 98 ++++++++++++++++++++++++++++++++++------ 4 files changed, 89 insertions(+), 17 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7bc4c2488..bbd34c4f3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,8 +29,6 @@ updates: - minor - patch ignore: - - dependency-name: "pydantic" - versions: [ ">=2" ] - dependency-name: "bokeh" versions: [ ">=3" ] - dependency-name: "langchain" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aeb026a94..f359d5dc3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,7 +67,7 @@ repos: hooks: - id: mypy additional_dependencies: - ["types-python-dateutil", "types-requests", "types-paramiko", "pydantic~=1.10"] + ["types-python-dateutil", "types-requests", "types-paramiko", "pydantic", "pydantic-settings"] args: ["--config-file", "pydatalab/pyproject.toml"] - repo: local diff --git a/pydatalab/pyproject.toml b/pydatalab/pyproject.toml index e8ac75338..ce8c670c8 100644 --- a/pydatalab/pyproject.toml +++ b/pydatalab/pyproject.toml @@ -20,7 +20,9 @@ dependencies = [ "bokeh ~= 2.4, < 3.0", "matplotlib ~= 3.8", "periodictable ~= 2.1", - "pydantic[email, dotenv] < 2.0", + "pydantic[email] >= 2.11, < 3.0", + "pydantic-settings ~= 2.10", + "python-dotenv ~= 1.1", "pint ~= 0.24", "pandas[excel] ~= 2.2", "pymongo ~= 4.7", diff --git a/pydatalab/uv.lock b/pydatalab/uv.lock index de5bb6833..5158e7b8e 100644 --- a/pydatalab/uv.lock +++ b/pydatalab/uv.lock @@ -81,6 +81,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, ] +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + [[package]] name = "anthropic" version = "0.120.2" @@ -498,8 +507,10 @@ dependencies = [ { name = "pandas", extra = ["excel"] }, { name = "periodictable" }, { name = "pint" }, - { name = "pydantic", extra = ["dotenv", "email"] }, + { name = "pydantic", extra = ["email"] }, + { name = "pydantic-settings" }, { name = "pymongo" }, + { name = "python-dotenv" }, { name = "tomlkit" }, ] @@ -619,10 +630,12 @@ requires-dist = [ { name = "pint", specifier = "~=0.24" }, { name = "pyarrow", marker = "extra == 'apps'", specifier = "~=23.0.1" }, { name = "pybaselines", marker = "extra == 'apps'", specifier = "~=1.1" }, - { name = "pydantic", extras = ["dotenv", "email"], specifier = "<2.0" }, + { name = "pydantic", extras = ["email"], specifier = ">=2.11,<3.0" }, + { name = "pydantic-settings", specifier = "~=2.10" }, { name = "pyjwt", marker = "extra == 'server'", specifier = "~=2.9" }, { name = "pymongo", specifier = "~=4.7" }, { name = "python-dateutil", marker = "extra == 'apps'", specifier = "~=2.9" }, + { name = "python-dotenv", specifier = "~=1.1" }, { name = "python-dotenv", marker = "extra == 'server'", specifier = "~=1.0" }, { name = "renishawwire", marker = "extra == 'apps'", specifier = ">=0.1.16" }, { name = "scipy", marker = "extra == 'apps'", specifier = "~=1.13" }, @@ -2020,29 +2033,76 @@ wheels = [ [[package]] name = "pydantic" -version = "1.10.26" +version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, { name = "typing-extensions" }, + { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7b/da/fd89f987a376c807cd81ea0eff4589aade783bbb702637b4734ef2c743a2/pydantic-1.10.26.tar.gz", hash = "sha256:8c6aa39b494c5af092e690127c283d84f363ac36017106a9e66cb33a22ac412e", size = 357906, upload-time = "2025-12-18T15:47:46.557Z" } +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/c1/d521e64c8130e1ad9d22c270bed3fabcc0940c9539b076b639c88fd32a8d/pydantic-1.10.26-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:116233e53889bcc536f617e38c1b8337d7fa9c280f0fd7a4045947515a785637", size = 2428347, upload-time = "2025-12-18T15:46:39.41Z" }, - { url = "https://files.pythonhosted.org/packages/2c/08/f4b804a00c16e3ea994cb640a7c25c579b4f1fa674cde6a19fa0dfb0ae4f/pydantic-1.10.26-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c3cfdd361addb6eb64ccd26ac356ad6514cee06a61ab26b27e16b5ed53108f77", size = 2212605, upload-time = "2025-12-18T15:46:41.006Z" }, - { url = "https://files.pythonhosted.org/packages/5d/78/0df4b9efef29bbc5e39f247fcba99060d15946b4463d82a5589cf7923d71/pydantic-1.10.26-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e4451951a9a93bf9a90576f3e25240b47ee49ab5236adccb8eff6ac943adf0f", size = 2753560, upload-time = "2025-12-18T15:46:43.215Z" }, - { url = "https://files.pythonhosted.org/packages/68/66/6ab6c1d3a116d05d2508fce64f96e35242938fac07544d611e11d0d363a0/pydantic-1.10.26-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9858ed44c6bea5f29ffe95308db9e62060791c877766c67dd5f55d072c8612b5", size = 2859235, upload-time = "2025-12-18T15:46:45.112Z" }, - { url = "https://files.pythonhosted.org/packages/61/4e/f1676bb0fcdf6ed2ce4670d7d1fc1d6c3a06d84497644acfbe02649503f1/pydantic-1.10.26-cp311-cp311-win_amd64.whl", hash = "sha256:ac1089f723e2106ebde434377d31239e00870a7563245072968e5af5cc4d33df", size = 2066646, upload-time = "2025-12-18T15:46:46.816Z" }, - { url = "https://files.pythonhosted.org/packages/1f/98/556e82f00b98486def0b8af85da95e69d2be7e367cf2431408e108bc3095/pydantic-1.10.26-py3-none-any.whl", hash = "sha256:c43ad70dc3ce7787543d563792426a16fd7895e14be4b194b5665e36459dd917", size = 166975, upload-time = "2025-12-18T15:47:44.927Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, ] [package.optional-dependencies] -dotenv = [ - { name = "python-dotenv" }, -] email = [ { name = "email-validator" }, ] +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/ca/31c57507b13119d7d3cfa1576dad2911a4861e3be07b579395f4e9d393f9/pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117", size = 261253, upload-time = "2026-08-07T09:24:57.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/a4/2bffa9f8e804325a09867f0e9d30795c80ea9f8d62560bd1b6ad6220eb2f/pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42", size = 69413, upload-time = "2026-08-07T09:24:55.839Z" }, +] + [[package]] name = "pygments" version = "2.20.0" @@ -2645,6 +2705,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + [[package]] name = "tzdata" version = "2026.3" From 7238da25fdb1c3b61019fba9ad13d51d531353a4 Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Tue, 11 Nov 2025 01:06:15 +0000 Subject: [PATCH 02/40] Refactor server config for pydantic v2 --- pydatalab/src/pydatalab/config.py | 114 ++++++++++++++++++------------ 1 file changed, 70 insertions(+), 44 deletions(-) diff --git a/pydatalab/src/pydatalab/config.py b/pydatalab/src/pydatalab/config.py index 74c6b8c20..793d371d6 100644 --- a/pydatalab/src/pydatalab/config.py +++ b/pydatalab/src/pydatalab/config.py @@ -9,12 +9,13 @@ from pydantic import ( AnyUrl, BaseModel, - BaseSettings, + ConfigDict, Field, ValidationError, - root_validator, - validator, + field_validator, + model_validator, ) +from pydantic_settings import BaseSettings, SettingsConfigDict from pydatalab.models import Person from pydatalab.models.utils import RandomAlphabeticalRefcodeFactory, RefCodeFactory @@ -24,7 +25,7 @@ config_logger = logging.getLogger("pydatalab.config") -def config_file_settings(settings: BaseSettings) -> dict[str, Any]: +def config_file_settings(settings_cls: type[BaseSettings] | None = None) -> dict[str, Any]: """Returns a dictionary of server settings loaded from the default or specified JSON config file location (via the env var `PYDATALAB_CONFIG_FILE`). @@ -34,7 +35,7 @@ def config_file_settings(settings: BaseSettings) -> dict[str, Any]: res = {} if config_file.is_file(): config_logger.debug("Loading from config file at %s", config_file) - config_file_content = config_file.read_text(encoding=settings.__config__.env_file_encoding) + config_file_content = config_file.read_text(encoding="utf-8") try: res = json.loads(config_file_content) @@ -51,20 +52,20 @@ def config_file_settings(settings: BaseSettings) -> dict[str, Any]: class DeploymentMetadata(BaseModel): """A model for specifying metadata about a datalab deployment.""" - maintainer: Person | None + maintainer: Person | None = None issue_tracker: AnyUrl | None = Field("https://github.com/datalab-org/datalab/issues") - homepage: AnyUrl | None + homepage: AnyUrl | None = None source_repository: AnyUrl | None = Field("https://github.com/datalab-org/datalab") - @validator("maintainer") + @field_validator("maintainer") + @classmethod def strip_fields_from_person(cls, v): if not v.contact_email: raise ValueError("Must provide contact email for maintainer.") return Person(contact_email=v.contact_email, display_name=v.display_name) - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class BackupStrategy(BaseModel): @@ -75,7 +76,8 @@ class BackupStrategy(BaseModel): description="Whether this backup strategy is active; i.e., whether it is actually used. All strategies will be disabled in testing scenarios.", ) hostname: str | None = Field( - description="The hostname of the SSH-accessible server on which to store the backup (`None` indicates local backups)." + None, + description="The hostname of the SSH-accessible server on which to store the backup (`None` indicates local backups).", ) location: Path = Field( description="The location under which to store the backups on the host. Each backup will be date-stamped and stored in a subdirectory of this location." @@ -87,7 +89,7 @@ class BackupStrategy(BaseModel): frequency: str | None = Field( None, description="The frequency of the backup, described in the crontab syntax.", - pattern=r"^(?:\*|\d+(?:-\d+)?)(?:\/\d+)?(?:,\d+(?:-\d+)?(?:\/\d+)?)*$", + examples=["5 4 * * *", "5 2 1 1,4,7,10 *"], ) notification_email_address: str | None = Field( None, description="An email address to send backup notifications to." @@ -143,7 +145,7 @@ class ServerConfig(BaseSettings): False, description="Whether to run the server in testing mode, i.e., without user auth." ) - SECRET_KEY: str = Field( + SECRET_KEY: str | None = Field( None, description="The secret key to use for Flask. This value should be changed and/or loaded from an environment variable for production deployments.", ) @@ -280,18 +282,24 @@ class ServerConfig(BaseSettings): description="The desired backup configuration.", ) - @root_validator + @model_validator(mode="before") + @classmethod def validate_cache_ages(cls, values): - if values.get("REMOTE_CACHE_MIN_AGE") > values.get("REMOTE_CACHE_MAX_AGE"): + min_age = values.get("REMOTE_CACHE_MIN_AGE") + max_age = values.get("REMOTE_CACHE_MAX_AGE") + + if min_age is not None and max_age is not None and min_age > max_age: raise RuntimeError( - f"The maximum cache age must be greater than the minimum cache age: min {values.get('REMOTE_CACHE_MIN_AGE')=}, max {values.get('REMOTE_CACHE_MAX_AGE')=}" + f"The maximum cache age must be greater than the minimum cache age: min {min_age=}, max {max_age=}" ) return values - @validator("SECRET_KEY", pre=True, always=True) - def validate_secret_key(cls, v, values): + @field_validator("SECRET_KEY", mode="before") + @classmethod + def validate_secret_key(cls, v, info): if v is None: - if values.get("TESTING"): + data = info.data if hasattr(info, "data") else {} + if data.get("TESTING"): config_logger.error( "`CONFIG.TESTING` is enabled - generating a deterministic secret key for testing purposes. This MUST be updated for production deployments." ) @@ -301,7 +309,8 @@ def validate_secret_key(cls, v, values): return v - @validator("ROOT_PATH") + @field_validator("ROOT_PATH", mode="before") + @classmethod def validate_root_path(cls, v): if not v.startswith("/"): v = "/" + v @@ -311,20 +320,22 @@ def validate_root_path(cls, v): return v - @validator("IDENTIFIER_PREFIX", pre=True, always=True) - def validate_identifier_prefix(cls, v, values): + @field_validator("IDENTIFIER_PREFIX", mode="before") + @classmethod + def validate_identifier_prefix(cls, v, info): """Make sure that the identifier prefix is set and is valid, raising clear error messages if not. If in testing mode, then set the prefix to 'test' too. The app startup will test for this value and should also warn aggressively that this is unset. """ - if values.get("TESTING") or v is None: + data = info.data if hasattr(info, "data") else {} + if data.get("TESTING") or v is None: return "test" if len(v) > 12: raise RuntimeError( - "Identifier prefix must be less than 12 characters long, received {v=}" + f"Identifier prefix must be less than 12 characters long, received {v=}" ) # test a trial refcode @@ -336,18 +347,18 @@ def validate_identifier_prefix(cls, v, values): raise RuntimeError( f"Invalid identifier prefix: {v}. Validation with refcode `AAAAAA` returned error: {exc}" ) - return v - @root_validator + @model_validator(mode="before") + @classmethod def deactivate_backup_strategies_during_testing(cls, values): if values.get("TESTING"): for name in values.get("BACKUP_STRATEGIES", {}): values["BACKUP_STRATEGIES"][name].active = False - return values - @validator("LOG_FILE") + @field_validator("LOG_FILE", mode="before") + @classmethod def make_missing_log_directory(cls, v): """Make sure that the log directory exists and is writable.""" if v is None: @@ -360,25 +371,40 @@ def make_missing_log_directory(cls, v): raise RuntimeError(f"Unable to create log file at {v}") from exc return v - class Config: - env_prefix = "pydatalab_" - extra = "allow" - env_file = ".env" - env_file_encoding = "utf-8" - validate_assignment = True - - @classmethod - def customise_sources( - cls, + def update(self, values: dict): + """Update the configuration with new values, following Pydantic v1 behavior.""" + for key, value in values.items(): + key_upper = key.upper() + if hasattr(self, key_upper): + setattr(self, key_upper, value) + else: + setattr(self, key_upper, value) + + model_config = SettingsConfigDict( + env_prefix="PYDATALAB_", + extra="allow", + env_file=".env", + env_file_encoding="utf-8", + validate_assignment=True, + case_sensitive=False, + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings, + env_settings, + dotenv_settings, + file_secret_settings, + ): + return ( init_settings, env_settings, + dotenv_settings, + config_file_settings, file_secret_settings, - ): - return (init_settings, env_settings, config_file_settings, file_secret_settings) - - def update(self, mapping): - for key in mapping: - setattr(self, key.upper(), mapping[key]) + ) CONFIG: ServerConfig = ServerConfig() From 5f4315b03eb2e21cf894b84aba6f18367efa2b38 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 15:51:35 +0100 Subject: [PATCH 03/40] Update traits module to pydantic v2 --- pydatalab/src/pydatalab/models/traits.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pydatalab/src/pydatalab/models/traits.py b/pydatalab/src/pydatalab/models/traits.py index f13043440..c53ce5a07 100644 --- a/pydatalab/src/pydatalab/models/traits.py +++ b/pydatalab/src/pydatalab/models/traits.py @@ -1,11 +1,20 @@ from typing import Any -from pydantic import BaseModel, Field, root_validator, validator +from pydantic import BaseModel, Field, field_validator, model_validator from pydatalab.models.blocks import DataBlockResponse from pydatalab.models.people import Group, Person from pydatalab.models.utils import Constituent, InlineSubstance, PyObjectId +__all__ = ( + "HasOwner", + "HasRevisionControl", + "HasBlocks", + "IsCollectable", + "HasSynthesisInfo", + "HasSubstanceInfo", +) + class HasOwner(BaseModel): """Trait mixin for models that record who created them and which groups can access them.""" @@ -70,7 +79,8 @@ class IsCollectable(BaseModel): collections: list[Collection] = Field([]) """Inlined info for the collections associated with this item.""" - @root_validator + @model_validator(mode="before") + @classmethod def add_missing_collection_relationships(cls, values): from pydatalab.models.relationships import TypedRelationship @@ -119,7 +129,8 @@ class HasSynthesisInfo(BaseModel): synthesis_description: str | None = None """Free-text details of the procedure applied to synthesise the sample""" - @root_validator + @model_validator(mode="before") + @classmethod def add_missing_synthesis_relationships(cls, values): """Add any missing sample synthesis constituents to parent relationships""" from pydatalab.models.relationships import RelationshipType, TypedRelationship @@ -225,7 +236,8 @@ class HasSubstanceInfo(BaseModel): CAS: str | None = Field(alias="Substance CAS") """The CAS Registry Number for the substance described by this entry.""" - @validator("molar_mass") + @field_validator("molar_mass", mode="before") + @classmethod def add_molar_mass(cls, v, values): from periodictable import formula From c424096ee8f4bc2cbf6179acca901810897e9a70 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 11 Nov 2025 01:06:39 +0000 Subject: [PATCH 04/40] Refactor model imports to avoid circular deps --- pydatalab/src/pydatalab/models/__init__.py | 29 ++++++++++++++++------ 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/pydatalab/src/pydatalab/models/__init__.py b/pydatalab/src/pydatalab/models/__init__.py index 92db5f475..ea76f4514 100644 --- a/pydatalab/src/pydatalab/models/__init__.py +++ b/pydatalab/src/pydatalab/models/__init__.py @@ -1,20 +1,34 @@ -from pydantic import BaseModel +import functools from pydatalab.models.cells import Cell from pydatalab.models.collections import Collection from pydatalab.models.equipment import Equipment from pydatalab.models.files import File +from pydatalab.models.items import Item from pydatalab.models.people import Person from pydatalab.models.samples import Sample from pydatalab.models.starting_materials import StartingMaterial from pydatalab.models.versions import ItemVersion -ITEM_MODELS: dict[str, type[BaseModel]] = { - "samples": Sample, - "starting_materials": StartingMaterial, - "cells": Cell, - "equipment": Equipment, -} + +@functools.lru_cache(maxsize=1) +def get_item_models() -> dict[str, type[Item]]: + """ + Returns a dictionary of item models keyed by their type. + """ + return { + model.model_json_schema()["properties"]["type"]["default"]: model + for model in Item.__subclasses__() + } + + +@functools.lru_cache(maxsize=1) +def generate_schemas() -> dict[str, dict]: + return {t: model.model_json_schema(by_alias=False) for t, model in get_item_models().items()} + + +ITEM_MODELS: dict[str, type[Item]] = get_item_models() +ITEM_SCHEMAS = generate_schemas() __all__ = ( "File", @@ -26,4 +40,5 @@ "Equipment", "ItemVersion", "ITEM_MODELS", + "ITEM_SCHEMAS", ) From 18ab61e841c7c44f909f4468240f6ced6cee8e8a Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Tue, 11 Nov 2025 01:07:23 +0000 Subject: [PATCH 05/40] Rework all models and tasks into pydantic v2 amenable form Move back to descriptions in field docstrings and configure this with customised BaseModel More model updates for pydantic v2 --- pydatalab/schemas/cell.json | 2193 ++++++++++------ pydatalab/schemas/equipment.json | 1777 ++++++++----- pydatalab/schemas/plugin_config.json | 176 +- pydatalab/schemas/sample.json | 2116 +++++++++------ pydatalab/schemas/startingmaterial.json | 2269 +++++++++++------ pydatalab/src/pydatalab/models/blocks.py | 11 +- pydatalab/src/pydatalab/models/cells.py | 79 +- pydatalab/src/pydatalab/models/collections.py | 19 +- pydatalab/src/pydatalab/models/entries.py | 22 +- pydatalab/src/pydatalab/models/equipment.py | 10 +- pydatalab/src/pydatalab/models/export_task.py | 50 +- pydatalab/src/pydatalab/models/files.py | 39 +- pydatalab/src/pydatalab/models/items.py | 23 +- pydatalab/src/pydatalab/models/people.py | 87 +- .../src/pydatalab/models/relationships.py | 43 +- pydatalab/src/pydatalab/models/samples.py | 4 +- .../pydatalab/models/starting_materials.py | 24 +- pydatalab/src/pydatalab/models/tasks.py | 89 +- pydatalab/src/pydatalab/models/traits.py | 131 +- pydatalab/src/pydatalab/models/utils.py | 262 +- pydatalab/src/pydatalab/models/versions.py | 76 +- pydatalab/tasks.py | 8 +- pydatalab/tests/test_models.py | 21 + 23 files changed, 6045 insertions(+), 3484 deletions(-) diff --git a/pydatalab/schemas/cell.json b/pydatalab/schemas/cell.json index 17104a678..9922552af 100644 --- a/pydatalab/schemas/cell.json +++ b/pydatalab/schemas/cell.json @@ -1,958 +1,1547 @@ { - "title": "Cell", - "description": "A model for representing electrochemical cells.\n\nA cell is an electrochemical device assembled from other items, recording its\ncomponents and the format it was built in.", - "type": "object", - "properties": { - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "cells", - "const": "cells", - "pattern": "^cells$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Name", + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" + ], + "title": "AccountStatus", "type": "string" }, - "status": { - "default": "active", - "allOf": [ - { - "$ref": "#/definitions/CellStatus" + "CellComponent": { + "properties": { + "item": { + "anyOf": [ + { + "$ref": "#/$defs/EntryReference" + }, + { + "$ref": "#/$defs/InlineSubstance" + } + ], + "description": "A reference to item (sample or starting material) entry for the constituent substance.", + "title": "Item" + }, + "quantity": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the constituent material used to create the sample.", + "title": "Quantity" + }, + "unit": { + "default": "g", + "description": "The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).", + "title": "Unit", + "type": "string" } - ] - }, - "cell_format": { - "$ref": "#/definitions/CellFormat" - }, - "cell_format_description": { - "title": "Cell Format Description", - "type": "string" - }, - "cell_preparation_description": { - "title": "Cell Preparation Description", - "type": "string" - }, - "characteristic_mass": { - "title": "Characteristic Mass", - "type": "number" - }, - "characteristic_chemical_formula": { - "title": "Characteristic Chemical Formula", - "type": "string" - }, - "characteristic_molar_mass": { - "title": "Characteristic Molar Mass", - "type": "number" - }, - "positive_electrode": { - "title": "Positive Electrode", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/CellComponent" - } - }, - "negative_electrode": { - "title": "Negative Electrode", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/CellComponent" - } - }, - "electrolyte": { - "title": "Electrolyte", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/CellComponent" - } + }, + "required": [ + "item" + ], + "title": "CellComponent", + "type": "object" }, - "active_ion_charge": { - "title": "Active Ion Charge", - "default": 1, - "type": "number" - } - }, - "required": [ - "item_id" - ], - "definitions": { - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "CellFormat": { "enum": [ - "parent", - "child", - "sibling", - "is_part_of", + "coin", + "pouch", + "in situ (XRD)", + "in situ (NMR)", + "in situ (SQUID)", + "in situ (optical)", + "swagelok", + "cylindrical", "other" ], + "title": "CellFormat", "type": "string" }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "CellStatus": { + "description": "An enumeration of the status of cells", "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" + "active", + "planned", + "disposed", + "cycled", + "shorted", + "dismantled", + "other" ], + "title": "CellStatus", "type": "string" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "description": { - "title": "Description", - "type": "string" - }, - "relation": { - "$ref": "#/definitions/RelationshipType" - }, - "type": { - "$ref": "#/definitions/KnownType" - }, - "immutable_id": { - "title": "Immutable Id", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "collection_id": { + "description": "The immutable ID of the collection.", + "title": "Collection Id", "type": "string" } }, "required": [ - "type" - ] - }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" + "collection_id" ], - "type": "string" + "title": "CollectionReference", + "type": "object" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "DataBlockResponse": { + "additionalProperties": true, + "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" + "blocktype": { + "description": "The type of the block.", + "title": "Blocktype", + "type": "string" }, - "identifier": { - "title": "Identifier", + "block_id": { + "description": "A shorthand random ID for the block.", + "title": "Block Id", "type": "string" }, - "name": { + "item_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" + }, + "collection_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" + }, + "freeform_comment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" + }, + "file_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" + }, + "file_ids": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" + }, + "errors": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" + }, + "warnings": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" + }, + "b64_encoded_image": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" + }, + "bokeh_plot_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" + }, + "computed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" + } + }, + "required": [ + "blocktype", + "block_id", + "b64_encoded_image", + "bokeh_plot_data" + ], + "title": "DataBlockResponse", + "type": "object" + }, + "EntryReference": { + "additionalProperties": true, + "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", + "properties": { + "type": { + "title": "Type", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Immutable Id" + }, + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Item Id" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Refcode" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "type" + ], + "title": "EntryReference", + "type": "object" + }, + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", + "properties": { + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "files", + "default": "files", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" + }, + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" + }, + "item_ids": { + "description": "A list of item IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Item Ids", + "type": "array" + }, + "blocks": { + "description": "A list of block IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Blocks", + "type": "array" + }, + "name": { + "description": "The filename on disk.", "title": "Name", "type": "string" }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" - }, - "display_name": { - "title": "Display Name", + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", + "type": "string" + }, + "original_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" + }, + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" + }, + "time_added": { + "description": "The timestamp for the original file upload.", + "format": "datetime", + "title": "Time Added", + "type": "string" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" + }, + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" + }, + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" + }, + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", + "type": "boolean" + }, + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "description": "Content checksums (MD5 and SHA-256) of the file." + } + }, + "required": [ + "name", + "extension", + "time_added", + "is_live", + "checksums" + ], + "title": "File", + "type": "object" + }, + "FileChecksums": { + "description": "Content checksums for a file.", + "properties": { + "md5": { + "description": "The MD5 hex digest of the file contents.", + "title": "Md5", + "type": "string" + }, + "sha256": { + "description": "The SHA-256 hex digest of the file contents.", + "title": "Sha256", "type": "string" } }, "required": [ - "identity_type", - "identifier", - "name" - ] - }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" + "md5", + "sha256" ], - "type": "string" + "title": "FileChecksums", + "type": "object" }, "Group": { - "title": "Group", "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", "properties": { "type": { - "title": "Type", - "default": "groups", "const": "groups", + "default": "groups", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, "members": { - "title": "Members", - "type": "array", + "description": "A list of people that belong to this group; stored on the user objects.", "items": { + "additionalProperties": true, "type": "object" - } + }, + "title": "Members", + "type": "array" }, "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" }, "description": { - "title": "Description", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" }, "managers": { - "title": "Managers", - "type": "array", + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", "items": { "anyOf": [ { + "format": "objectid", "type": "string" }, { + "additionalProperties": true, "type": "object" } ] - } + }, + "title": "Managers", + "type": "array" } - } - }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", - "enum": [ - "active", - "unverified", - "deactivated" - ], - "type": "string" + }, + "title": "Group", + "type": "object" }, - "Person": { - "title": "Person", - "description": "A model that describes an individual and their digital identities.", - "type": "object", + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", "properties": { - "type": { - "title": "Type", - "default": "people", - "const": "people", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "identities": { - "title": "Identities", - "type": "array", - "items": { - "$ref": "#/definitions/Identity" - } + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", "type": "string" }, - "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" - }, - "gravatar_hash": { - "title": "Gravatar Hash", + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", "type": "string" }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" }, - "role": { - "default": "user", - "allOf": [ + "display_name": { + "anyOf": [ { - "$ref": "#/definitions/UserRole" - } - ] - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "account_status": { - "default": "unverified", - "allOf": [ + "type": "string" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] - } - } - }, - "FileChecksums": { - "title": "FileChecksums", - "description": "Content checksums for a file.", - "type": "object", - "properties": { - "md5": { - "title": "Md5", - "type": "string" - }, - "sha256": { - "title": "Sha256", - "type": "string" + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" } }, "required": [ - "md5", - "sha256" - ] + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "InlineSubstance": { "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "size": { - "title": "Size", - "type": "integer" - }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" - }, - "item_ids": { - "title": "Item Ids", - "type": "array", - "items": { - "type": "string" - } - }, - "blocks": { - "title": "Blocks", - "type": "array", - "items": { - "type": "string" - } - }, "name": { "title": "Name", "type": "string" }, - "extension": { - "title": "Extension", + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "name" + ], + "title": "InlineSubstance", + "type": "object" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" + ], + "title": "KnownType", + "type": "string" + }, + "Person": { + "description": "A model that describes an individual and their digital identities.", + "properties": { + "type": { + "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, - "original_name": { - "title": "Original Name", - "type": "string" + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" }, - "location": { - "title": "Location", - "type": "string" + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, - "url_path": { - "title": "Url Path", - "type": "string" + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "source": { - "title": "Source", - "type": "string" + "identities": { + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", + "items": { + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" }, - "metadata": { - "title": "Metadata", - "type": "object" + "contact_email": { + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" }, - "representation": { - "title": "Representation" + "gravatar_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" }, - "source_server_name": { - "title": "Source Server Name", - "type": "string" + "managers": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" }, - "source_path": { - "title": "Source Path", - "type": "string" + "role": { + "$ref": "#/$defs/UserRole", + "default": "user", + "description": "The role assigned to this person." }, - "is_live": { - "title": "Is Live", - "type": "boolean" + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" }, - "checksums": { - "$ref": "#/definitions/FileChecksums" + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." } }, - "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + "title": "Person", + "type": "object" }, - "DataBlockResponse": { - "title": "DataBlockResponse", - "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" + }, + "TypedRelationship": { + "additionalProperties": true, "properties": { - "blocktype": { - "title": "Blocktype", - "type": "string" - }, - "block_id": { - "title": "Block Id", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "type": "string" - }, - "collection_id": { - "title": "Collection Id", - "type": "string" - }, - "title": { - "title": "Title", - "type": "string" + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" }, - "freeform_comment": { - "title": "Freeform Comment", - "type": "string" + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." }, - "file_id": { - "title": "File Id", - "type": "string" + "type": { + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." }, - "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" }, - "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" }, - "warnings": { - "title": "Warnings", - "type": "array", + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" + } + }, + "required": [ + "type" + ], + "title": "TypedRelationship", + "type": "object" + }, + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing electrochemical cells.\n\nA cell is an electrochemical device assembled from other items, recording its\ncomponents and the format it was built in.", + "properties": { + "files": { + "anyOf": [ + { "items": { - "type": "string" - } - }, - "b64_encoded_image": { - "title": "B64 Encoded Image", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "bokeh_plot_data": { - "title": "Bokeh Plot Data", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object" + "$ref": "#/$defs/File" + }, + "type": "array" }, - "computed": { - "title": "Computed", - "datalab_exclude_from_load": true, + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this item.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, "type": "object" }, - "metadata": { - "title": "Metadata", - "datalab_exclude_from_load": true, - "type": "object" + { + "type": "null" } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" }, - "required": [ - "blocktype", - "block_id" - ] + "title": "Creator Ids", + "type": "array" }, - "Collection": { - "title": "Collection", - "description": "A model for representing a group of related items, for example all the samples\nbelonging to one batch or project.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", + "creators": { + "anyOf": [ + { "items": { - "type": "string" - } + "$ref": "#/$defs/Person" + }, + "type": "array" }, - "groups": { - "title": "Groups", - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { "items": { - "$ref": "#/definitions/Group" - } + "$ref": "#/$defs/Group" + }, + "type": "array" }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "cells", + "default": "cells", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", "type": "string" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "title": { - "title": "Title", - "type": "string" - }, - "description": { - "title": "Description", + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", "type": "string" }, - "num_items": { - "title": "Num Items", - "type": "integer" + { + "type": "null" } - } - }, - "CellStatus": { - "title": "CellStatus", - "description": "An enumeration of the status of cells", - "enum": [ - "active", - "planned", - "disposed", - "cycled", - "shorted", - "dismantled", - "other" ], - "type": "string" + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" }, - "CellFormat": { - "title": "CellFormat", - "description": "An enumeration.", - "enum": [ - "coin", - "pouch", - "in situ (XRD)", - "in situ (NMR)", - "in situ (SQUID)", - "in situ (optical)", - "swagelok", - "cylindrical", - "other" - ], + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", "type": "string" }, - "EntryReference": { - "title": "EntryReference", - "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", - "type": "object", - "properties": { - "type": { - "title": "Type", + "description": { + "anyOf": [ + { "type": "string" }, - "name": { - "title": "Name", + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "immutable_id": { - "title": "Immutable Id", + { + "type": "null" + } + ], + "default": null, + "description": "A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).", + "title": "Date" + }, + "name": { + "anyOf": [ + { "type": "string" }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + { + "type": "null" + } + ], + "default": null, + "description": "An optional human-readable/usable name for the entry.", + "title": "Name" + }, + "status": { + "$ref": "#/$defs/CellStatus", + "default": "active", + "description": "The status of the cells, indicating its current state." + }, + "cell_format": { + "anyOf": [ + { + "$ref": "#/$defs/CellFormat" }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + { + "type": "null" + } + ], + "default": null, + "description": "The form factor of the cell, e.g., coin, pouch, in situ or otherwise." + }, + "cell_format_description": { + "anyOf": [ + { "type": "string" }, - "chemform": { - "title": "Chemform", - "type": "string" + { + "type": "null" } - }, - "required": [ - "type" - ] + ], + "default": null, + "description": "Additional human-readable description of the cell form factor, e.g., 18650, AMPIX, CAMPIX", + "title": "Cell Format Description" }, - "InlineSubstance": { - "title": "InlineSubstance", - "type": "object", - "properties": { - "name": { - "title": "Name", + "cell_preparation_description": { + "anyOf": [ + { "type": "string" }, - "chemform": { - "title": "Chemform", - "type": "string" + { + "type": "null" } - }, - "required": [ - "name" - ] + ], + "default": null, + "description": "Description of how the cell was prepared.", + "title": "Cell Preparation Description" }, - "CellComponent": { - "title": "CellComponent", - "description": "A constituent of a sample.", - "type": "object", - "properties": { - "item": { - "title": "Item", - "anyOf": [ - { - "$ref": "#/definitions/EntryReference" - }, - { - "$ref": "#/definitions/InlineSubstance" - } - ] - }, - "quantity": { - "title": "Quantity", - "minimum": 0, + "characteristic_mass": { + "anyOf": [ + { "type": "number" }, - "unit": { - "title": "Unit", - "default": "g", + { + "type": "null" + } + ], + "default": null, + "description": "The characteristic mass of the cell in milligrams. Can be used to normalize capacities.", + "title": "Characteristic Mass" + }, + "characteristic_chemical_formula": { + "anyOf": [ + { "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chemical formula of the active material. Can be used to calculated molar mass in g/mol for normalizing capacities.", + "title": "Characteristic Chemical Formula" + }, + "characteristic_molar_mass": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" } + ], + "default": null, + "description": "The molar mass of the active material, in g/mol. Will be inferred from the chemical formula, or can be supplied if it cannot be supplied", + "title": "Characteristic Molar Mass" + }, + "positive_electrode": { + "items": { + "$ref": "#/$defs/CellComponent" }, - "required": [ - "item", - "quantity" - ] + "title": "Positive Electrode", + "type": "array" + }, + "negative_electrode": { + "items": { + "$ref": "#/$defs/CellComponent" + }, + "title": "Negative Electrode", + "type": "array" + }, + "electrolyte": { + "items": { + "$ref": "#/$defs/CellComponent" + }, + "title": "Electrolyte", + "type": "array" + }, + "active_ion": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The active ion species.", + "title": "Active Ion" + }, + "active_ion_charge": { + "default": 1, + "title": "Active Ion Charge", + "type": "number" } - } + }, + "required": [ + "item_id" + ], + "title": "Cell", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/equipment.json b/pydatalab/schemas/equipment.json index 33d5fca20..25a0fe91f 100644 --- a/pydatalab/schemas/equipment.json +++ b/pydatalab/schemas/equipment.json @@ -1,816 +1,1319 @@ { - "title": "Equipment", - "description": "A model for representing a piece of equipment.\n\nEquipment represents an instrument or apparatus in the lab, which can be linked to\nthe items measured on it.", - "type": "object", - "properties": { - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "equipment", - "const": "equipment", - "pattern": "^equipment$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Name", - "type": "string" - }, - "status": { - "default": "working", - "allOf": [ - { - "$ref": "#/definitions/EquipmentStatus" - } - ] - }, - "serial_numbers": { - "title": "Serial Numbers", - "type": "string" - }, - "manufacturer": { - "title": "Manufacturer", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" - }, - "contact": { - "title": "Contact", - "type": "string" - } - }, - "required": [ - "item_id" - ], - "definitions": { - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" - ], - "type": "string" - }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", - "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" - ], - "type": "string" - }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", - "properties": { - "description": { - "title": "Description", - "type": "string" - }, - "relation": { - "$ref": "#/definitions/RelationshipType" - }, - "type": { - "$ref": "#/definitions/KnownType" - }, - "immutable_id": { - "title": "Immutable Id", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - } - }, - "required": [ - "type" - ] - }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" + "active", + "unverified", + "deactivated" ], + "title": "AccountStatus", "type": "string" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" - }, - "identifier": { - "title": "Identifier", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" - }, - "display_name": { - "title": "Display Name", + "collection_id": { + "description": "The immutable ID of the collection.", + "title": "Collection Id", "type": "string" } }, "required": [ - "identity_type", - "identifier", - "name" - ] - }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" + "collection_id" ], - "type": "string" + "title": "CollectionReference", + "type": "object" }, - "Group": { - "title": "Group", - "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", + "DataBlockResponse": { + "additionalProperties": true, + "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", "properties": { - "type": { - "title": "Type", - "default": "groups", - "const": "groups", + "blocktype": { + "description": "The type of the block.", + "title": "Blocktype", "type": "string" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + "block_id": { + "description": "A shorthand random ID for the block.", + "title": "Block Id", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } + "item_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" }, - "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "collection_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" }, - "members": { - "title": "Members", - "type": "array", - "items": { - "type": "object" - } + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "freeform_comment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" }, - "description": { - "title": "Description", - "type": "string" + "file_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "anyOf": [ - { + "file_ids": { + "anyOf": [ + { + "items": { + "format": "objectid", "type": "string" }, - { - "type": "object" - } - ] - } - } - } - }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", - "enum": [ - "active", - "unverified", - "deactivated" - ], - "type": "string" - }, - "Person": { - "title": "Person", - "description": "A model that describes an individual and their digital identities.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "default": "people", - "const": "people", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "identities": { - "title": "Identities", - "type": "array", - "items": { - "$ref": "#/definitions/Identity" - } - }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" - }, - "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" }, - "gravatar_hash": { - "title": "Gravatar Hash", - "type": "string" + "errors": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } + "warnings": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" }, - "role": { - "default": "user", - "allOf": [ + "b64_encoded_image": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, { - "$ref": "#/definitions/UserRole" + "type": "null" } - ] + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "bokeh_plot_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" }, - "account_status": { - "default": "unverified", - "allOf": [ + "computed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] - } - } - }, - "FileChecksums": { - "title": "FileChecksums", - "description": "Content checksums for a file.", - "type": "object", - "properties": { - "md5": { - "title": "Md5", - "type": "string" + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" }, - "sha256": { - "title": "Sha256", - "type": "string" + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" } }, "required": [ - "md5", - "sha256" - ] + "blocktype", + "block_id", + "b64_encoded_image", + "bokeh_plot_data" + ], + "title": "DataBlockResponse", + "type": "object" + }, + "EquipmentStatus": { + "description": "An enumeration of the status of equipments", + "enum": [ + "working", + "broken", + "being_fixed", + "defunct", + "not_being_fixed", + "other" + ], + "title": "EquipmentStatus", + "type": "string" }, "File": { - "title": "File", "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", "properties": { "revision": { - "title": "Revision", "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", "type": "integer" }, "revisions": { - "title": "Revisions", - "type": "object" + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" }, "version": { - "title": "Version", "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", "type": "integer" }, "creator_ids": { - "title": "Creator Ids", "default": [], - "type": "array", + "description": "The database IDs of the user(s) who created the item.", "items": { + "format": "objectid", "type": "string" - } + }, + "title": "Creator Ids", + "type": "array" }, "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" }, "group_ids": { - "title": "Group Ids", "default": [], - "type": "array", + "description": "The database IDs of the group(s) that have read-access to this item.", "items": { + "format": "objectid", "type": "string" - } + }, + "title": "Group Ids", + "type": "array" }, "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" }, "type": { - "title": "Type", - "default": "files", "const": "files", - "pattern": "^files$", + "default": "files", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "size": { - "title": "Size", - "type": "integer" + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" }, "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" }, "item_ids": { - "title": "Item Ids", - "type": "array", + "description": "A list of item IDs associated with this file.", "items": { "type": "string" - } + }, + "title": "Item Ids", + "type": "array" }, "blocks": { - "title": "Blocks", - "type": "array", + "description": "A list of block IDs associated with this file.", "items": { "type": "string" - } + }, + "title": "Blocks", + "type": "array" }, "name": { + "description": "The filename on disk.", "title": "Name", "type": "string" }, "extension": { + "description": "The file extension that the file was uploaded with.", "title": "Extension", "type": "string" }, "original_name": { - "title": "Original Name", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" }, "location": { - "title": "Location", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" }, "url_path": { - "title": "Url Path", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" }, "source": { - "title": "Source", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" }, "time_added": { + "description": "The timestamp for the original file upload.", + "format": "datetime", "title": "Time Added", - "type": "string", - "format": "date-time" + "type": "string" }, "metadata": { - "title": "Metadata", - "type": "object" + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" }, "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, "title": "Representation" }, "source_server_name": { - "title": "Source Server Name", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" }, "source_path": { - "title": "Source Path", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" }, "is_live": { + "description": "Whether or not the file should be watched for future updates.", "title": "Is Live", "type": "boolean" }, "checksums": { - "$ref": "#/definitions/FileChecksums" + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ - "item_ids", - "blocks", "name", "extension", "time_added", - "is_live" - ] + "is_live", + "checksums" + ], + "title": "File", + "type": "object" }, - "DataBlockResponse": { - "title": "DataBlockResponse", - "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", + "FileChecksums": { + "description": "Content checksums for a file.", "properties": { - "blocktype": { - "title": "Blocktype", + "md5": { + "description": "The MD5 hex digest of the file contents.", + "title": "Md5", "type": "string" }, - "block_id": { - "title": "Block Id", + "sha256": { + "description": "The SHA-256 hex digest of the file contents.", + "title": "Sha256", "type": "string" - }, - "item_id": { - "title": "Item Id", + } + }, + "required": [ + "md5", + "sha256" + ], + "title": "FileChecksums", + "type": "object" + }, + "Group": { + "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", + "properties": { + "type": { + "const": "groups", + "default": "groups", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, - "collection_id": { - "title": "Collection Id", - "type": "string" + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" }, - "title": { - "title": "Title", + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "group_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" + }, + "members": { + "description": "A list of people that belong to this group; stored on the user objects.", + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Members", + "type": "array" + }, + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" + }, + "managers": { + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", + "items": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + } + ] + }, + "title": "Managers", + "type": "array" + } + }, + "title": "Group", + "type": "object" + }, + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", + "properties": { + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." + }, + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", "type": "string" }, - "freeform_comment": { - "title": "Freeform Comment", + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", "type": "string" }, - "file_id": { - "title": "File Id", + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" + }, + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" + } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" + ], + "title": "KnownType", + "type": "string" + }, + "Person": { + "description": "A model that describes an individual and their digital identities.", + "properties": { + "type": { + "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, - "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" }, - "errors": { - "title": "Errors", - "type": "array", + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", "items": { - "type": "string" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "warnings": { - "title": "Warnings", - "type": "array", + "identities": { + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", "items": { - "type": "string" - } + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" + }, + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" + }, + "contact_email": { + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" + }, + "gravatar_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" + }, + "managers": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" + }, + "role": { + "$ref": "#/$defs/UserRole", + "default": "user", + "description": "The role assigned to this person." + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" + }, + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." + } + }, + "title": "Person", + "type": "object" + }, + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" + }, + "TypedRelationship": { + "additionalProperties": true, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" }, - "b64_encoded_image": { - "title": "B64 Encoded Image", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." }, - "bokeh_plot_data": { - "title": "Bokeh Plot Data", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object" + "type": { + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." }, - "computed": { - "title": "Computed", - "datalab_exclude_from_load": true, - "type": "object" + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" }, - "metadata": { - "title": "Metadata", - "datalab_exclude_from_load": true, - "type": "object" + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" } }, "required": [ - "blocktype", - "block_id" - ] + "type" + ], + "title": "TypedRelationship", + "type": "object" }, - "Collection": { - "title": "Collection", - "description": "A model for representing a group of related items, for example all the samples\nbelonging to one batch or project.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing a piece of equipment.\n\nEquipment represents an instrument or apparatus in the lab, which can be linked to\nthe items measured on it.", + "properties": { + "files": { + "anyOf": [ + { "items": { - "type": "string" - } + "$ref": "#/$defs/File" + }, + "type": "array" }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this item.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" }, - "creators": { - "title": "Creators", - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { "items": { - "$ref": "#/definitions/Person" - } + "$ref": "#/$defs/Person" + }, + "type": "array" }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { "items": { - "type": "string" - } + "$ref": "#/$defs/Group" + }, + "type": "array" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "equipment", + "default": "equipment", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { + "type": "string" }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", + "type": "string" }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + { + "type": "null" + } + ], + "default": null, + "description": "A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).", + "title": "Date" + }, + "name": { + "anyOf": [ + { "type": "string" }, - "title": { - "title": "Title", + { + "type": "null" + } + ], + "default": null, + "description": "An optional human-readable/usable name for the entry.", + "title": "Name" + }, + "status": { + "$ref": "#/$defs/EquipmentStatus", + "default": "working", + "description": "The status of the equipment, indicating its current state." + }, + "serial_numbers": { + "anyOf": [ + { "type": "string" }, - "description": { - "title": "Description", + { + "type": "null" + } + ], + "default": null, + "description": "A string describing one or more serial numbers for the instrument.", + "title": "Serial Numbers" + }, + "manufacturer": { + "anyOf": [ + { "type": "string" }, - "num_items": { - "title": "Num Items", - "type": "integer" + { + "type": "null" } - } + ], + "default": null, + "description": "The manufacturer of this piece of equipment", + "title": "Manufacturer" }, - "EquipmentStatus": { - "title": "EquipmentStatus", - "description": "An enumeration of the status of equipments", - "enum": [ - "working", - "broken", - "being_fixed", - "defunct", - "not_being_fixed", - "other" + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "type": "string" + "default": null, + "description": "Place where the equipment is located", + "title": "Location" + }, + "contact": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Contact information for equipment (e.g., email address or phone number).", + "title": "Contact" } - } + }, + "required": [ + "item_id", + "contact" + ], + "title": "Equipment", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/plugin_config.json b/pydatalab/schemas/plugin_config.json index ae64bcb3c..71db64db6 100644 --- a/pydatalab/schemas/plugin_config.json +++ b/pydatalab/schemas/plugin_config.json @@ -1,96 +1,134 @@ { - "title": "PluginConfigModel", - "description": "The schema for the top-level plugins.toml file.", - "type": "object", - "properties": { - "dependencies": { - "title": "Dependencies", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "tool": { - "title": "Tool", - "default": { + "$defs": { + "ToolSection": { + "additionalProperties": false, + "properties": { "uv": { - "sources": {} + "$ref": "#/$defs/UvSection", + "default": { + "sources": {} + } } }, - "allOf": [ - { - "$ref": "#/definitions/ToolSection" + "title": "ToolSection", + "type": "object" + }, + "UvSection": { + "additionalProperties": false, + "properties": { + "sources": { + "additionalProperties": { + "$ref": "#/$defs/UvSource" + }, + "default": {}, + "title": "Sources", + "type": "object" } - ] - } - }, - "additionalProperties": false, - "definitions": { + }, + "title": "UvSection", + "type": "object" + }, "UvSource": { - "title": "UvSource", + "additionalProperties": false, "description": "A single entry under `[tool.uv.sources]` in plugins.toml.", - "type": "object", "properties": { "git": { - "title": "Git", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Git" }, "rev": { - "title": "Rev", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Rev" }, "branch": { - "title": "Branch", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Branch" }, "tag": { - "title": "Tag", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tag" }, "path": { - "title": "Path", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" }, "editable": { - "title": "Editable", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Editable" } }, - "additionalProperties": false - }, - "UvSection": { - "title": "UvSection", - "type": "object", - "properties": { - "sources": { - "title": "Sources", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/UvSource" - } - } + "title": "UvSource", + "type": "object" + } + }, + "additionalProperties": false, + "description": "The schema for the top-level plugins.toml file.", + "properties": { + "dependencies": { + "default": [], + "items": { + "type": "string" }, - "additionalProperties": false + "title": "Dependencies", + "type": "array" }, - "ToolSection": { - "title": "ToolSection", - "type": "object", - "properties": { + "tool": { + "$ref": "#/$defs/ToolSection", + "default": { "uv": { - "title": "Uv", - "default": { - "sources": {} - }, - "allOf": [ - { - "$ref": "#/definitions/UvSection" - } - ] + "sources": {} } - }, - "additionalProperties": false + } } - } + }, + "title": "PluginConfigModel", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/sample.json b/pydatalab/schemas/sample.json index 66fe4e4d9..727e23f89 100644 --- a/pydatalab/schemas/sample.json +++ b/pydatalab/schemas/sample.json @@ -1,944 +1,1540 @@ { - "title": "Sample", - "description": "A model for representing an experimental sample.\n\nA physical thing in the lab that can be created, characterised\nand connected to other items.", - "type": "object", - "properties": { - "chemform": { - "title": "Chemform", - "example": [ - "Na3P", - "Na3P", - "LiNiO2@C", - "Na3+xP", - "LiNi1/3Co0.1Mn0.1O2" - ], - "type": "string" - }, - "smiles": { - "title": "Smiles", - "aliases": [ - "SMILES", - "smiles_representation" - ], - "type": "string" - }, - "inchi": { - "title": "Inchi", - "type": "string" - }, - "inchi_key": { - "title": "Inchi Key", - "type": "string" - }, - "GHS_codes": { - "title": "Ghs H-Codes", - "examples": [ - "H224", - "H303, H316, H319" + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" ], + "title": "AccountStatus", "type": "string" }, - "molar_mass": { - "title": "Molecular Weight", - "type": "number" - }, - "CAS": { - "title": "Substance Cas", - "type": "string" - }, - "synthesis_constituents": { - "title": "Synthesis Constituents", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Constituent" - } - }, - "synthesis_description": { - "title": "Synthesis Description", - "type": "string" - }, - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "samples", - "const": "samples", - "pattern": "^samples$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Name", - "type": "string" - }, - "status": { - "default": "active", - "allOf": [ - { - "$ref": "#/definitions/ItemStatus" - } - ] - } - }, - "required": [ - "item_id" - ], - "definitions": { - "EntryReference": { - "title": "EntryReference", - "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "immutable_id": { - "title": "Immutable Id", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "chemform": { - "title": "Chemform", - "type": "string" - } - }, - "required": [ - "type" - ] - }, - "InlineSubstance": { - "title": "InlineSubstance", - "type": "object", + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "name": { - "title": "Name", - "type": "string" - }, - "chemform": { - "title": "Chemform", + "collection_id": { + "description": "The immutable ID of the collection.", + "title": "Collection Id", "type": "string" } }, "required": [ - "name" - ] + "collection_id" + ], + "title": "CollectionReference", + "type": "object" }, "Constituent": { - "title": "Constituent", "description": "A constituent of a sample.", - "type": "object", "properties": { "item": { - "title": "Item", "anyOf": [ { - "$ref": "#/definitions/EntryReference" + "$ref": "#/$defs/EntryReference" }, { - "$ref": "#/definitions/InlineSubstance" + "$ref": "#/$defs/InlineSubstance" } - ] + ], + "description": "A reference to item (sample or starting material) entry for the constituent substance.", + "title": "Item" }, "quantity": { - "title": "Quantity", - "minimum": 0, - "type": "number" + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the constituent material used to create the sample.", + "title": "Quantity" }, "unit": { - "title": "Unit", "default": "g", + "description": "The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).", + "title": "Unit", "type": "string" } }, "required": [ - "item", - "quantity" - ] - }, - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" - ], - "type": "string" - }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", - "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" + "item" ], - "type": "string" + "title": "Constituent", + "type": "object" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "DataBlockResponse": { + "additionalProperties": true, + "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", "properties": { - "description": { - "title": "Description", + "blocktype": { + "description": "The type of the block.", + "title": "Blocktype", "type": "string" }, - "relation": { - "$ref": "#/definitions/RelationshipType" - }, - "type": { - "$ref": "#/definitions/KnownType" - }, - "immutable_id": { - "title": "Immutable Id", + "block_id": { + "description": "A shorthand random ID for the block.", + "title": "Block Id", "type": "string" }, "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - } - }, + "collection_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" + }, + "freeform_comment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" + }, + "file_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" + }, + "file_ids": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" + }, + "errors": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" + }, + "warnings": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" + }, + "b64_encoded_image": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" + }, + "bokeh_plot_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" + }, + "computed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" + } + }, "required": [ - "type" - ] + "blocktype", + "block_id", + "b64_encoded_image", + "bokeh_plot_data" + ], + "title": "DataBlockResponse", + "type": "object" }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" + "EntryReference": { + "additionalProperties": true, + "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", + "properties": { + "type": { + "title": "Type", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Immutable Id" + }, + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Item Id" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Refcode" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "type" ], - "type": "string" + "title": "EntryReference", + "type": "object" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "files", + "default": "files", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" + }, + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" + }, + "item_ids": { + "description": "A list of item IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Item Ids", + "type": "array" + }, + "blocks": { + "description": "A list of block IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Blocks", + "type": "array" + }, + "name": { + "description": "The filename on disk.", + "title": "Name", + "type": "string" + }, + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", + "type": "string" + }, + "original_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" }, - "identifier": { - "title": "Identifier", - "type": "string" + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" }, - "name": { - "title": "Name", + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" + }, + "time_added": { + "description": "The timestamp for the original file upload.", + "format": "datetime", + "title": "Time Added", "type": "string" }, - "verified": { - "title": "Verified", - "default": false, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" + }, + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" + }, + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" + }, + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", "type": "boolean" }, - "display_name": { - "title": "Display Name", - "type": "string" + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ - "identity_type", - "identifier", - "name" - ] + "name", + "extension", + "time_added", + "is_live", + "checksums" + ], + "title": "File", + "type": "object" }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" + "FileChecksums": { + "description": "Content checksums for a file.", + "properties": { + "md5": { + "description": "The MD5 hex digest of the file contents.", + "title": "Md5", + "type": "string" + }, + "sha256": { + "description": "The SHA-256 hex digest of the file contents.", + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "md5", + "sha256" ], - "type": "string" + "title": "FileChecksums", + "type": "object" }, "Group": { - "title": "Group", "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", "properties": { "type": { - "title": "Type", - "default": "groups", "const": "groups", + "default": "groups", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, "members": { - "title": "Members", - "type": "array", + "description": "A list of people that belong to this group; stored on the user objects.", "items": { + "additionalProperties": true, "type": "object" - } + }, + "title": "Members", + "type": "array" }, "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" }, "description": { - "title": "Description", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" }, "managers": { - "title": "Managers", - "type": "array", + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", "items": { "anyOf": [ { + "format": "objectid", "type": "string" }, { + "additionalProperties": true, "type": "object" } ] - } + }, + "title": "Managers", + "type": "array" } - } + }, + "title": "Group", + "type": "object" }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", + "properties": { + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." + }, + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", + "type": "string" + }, + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", + "type": "string" + }, + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" + }, + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" + } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "InlineSubstance": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "name" + ], + "title": "InlineSubstance", + "type": "object" + }, + "ItemStatus": { + "description": "An enumeration of the status of items", "enum": [ "active", - "unverified", - "deactivated" + "planned", + "disposed", + "completed", + "failed", + "other" + ], + "title": "ItemStatus", + "type": "string" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" ], + "title": "KnownType", "type": "string" }, "Person": { - "title": "Person", "description": "A model that describes an individual and their digital identities.", - "type": "object", "properties": { "type": { - "title": "Type", - "default": "people", "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "identities": { - "title": "Identities", - "type": "array", + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", "items": { - "$ref": "#/definitions/Identity" - } + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" }, "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" }, "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" }, "gravatar_hash": { - "title": "Gravatar Hash", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" }, "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" }, "role": { + "$ref": "#/$defs/UserRole", "default": "user", - "allOf": [ - { - "$ref": "#/definitions/UserRole" - } - ] + "description": "The role assigned to this person." }, "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "account_status": { - "default": "unverified", - "allOf": [ + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] - } - } - }, - "FileChecksums": { - "title": "FileChecksums", - "description": "Content checksums for a file.", - "type": "object", - "properties": { - "md5": { - "title": "Md5", - "type": "string" + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" }, - "sha256": { - "title": "Sha256", - "type": "string" + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." } }, - "required": [ - "md5", - "sha256" - ] + "title": "Person", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" + }, + "TypedRelationship": { + "additionalProperties": true, "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." }, "type": { - "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", - "type": "string" + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." }, "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "size": { - "title": "Size", - "type": "integer" - }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" - }, - "item_ids": { - "title": "Item Ids", - "type": "array", - "items": { - "type": "string" - } - }, - "blocks": { - "title": "Blocks", - "type": "array", - "items": { - "type": "string" - } - }, - "name": { - "title": "Name", - "type": "string" - }, - "extension": { - "title": "Extension", - "type": "string" - }, - "original_name": { - "title": "Original Name", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" - }, - "url_path": { - "title": "Url Path", - "type": "string" - }, - "source": { - "title": "Source", - "type": "string" - }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" - }, - "metadata": { - "title": "Metadata", - "type": "object" - }, - "representation": { - "title": "Representation" - }, - "source_server_name": { - "title": "Source Server Name", - "type": "string" - }, - "source_path": { - "title": "Source Path", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" }, - "is_live": { - "title": "Is Live", - "type": "boolean" + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" }, - "checksums": { - "$ref": "#/definitions/FileChecksums" + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" } }, "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + "type" + ], + "title": "TypedRelationship", + "type": "object" }, - "DataBlockResponse": { - "title": "DataBlockResponse", - "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", - "properties": { - "blocktype": { - "title": "Blocktype", - "type": "string" - }, - "block_id": { - "title": "Block Id", + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing an experimental sample.\n\nA physical thing in the lab that can be created, characterised\nand connected to other items.", + "properties": { + "chemform": { + "anyOf": [ + { "type": "string" }, - "item_id": { - "title": "Item Id", + { + "type": "null" + } + ], + "default": null, + "description": "A string representation of the chemical formula or composition associated with this sample.\n\nThe representation is relatively free-form; clients are expected parse and interpret HTML markup for subscripts\nand accept unicode characters for greek letters.", + "examples": [ + "Na3P", + "Na3P", + "LiNiO2@C", + "Na3+xP", + "LiNi1/3Co0.1Mn0.1O2" + ], + "title": "Chemform" + }, + "smiles": { + "anyOf": [ + { "type": "string" }, - "collection_id": { - "title": "Collection Id", + { + "type": "null" + } + ], + "default": null, + "description": "A SMILES string representation of the chemical structure associated with this sample.", + "title": "Smiles" + }, + "inchi": { + "anyOf": [ + { "type": "string" }, - "title": { - "title": "Title", + { + "type": "null" + } + ], + "default": null, + "description": "An International Chemical Identifier (InChI) string representation of chemicals/molecules associated with this sample.", + "title": "Inchi" + }, + "inchi_key": { + "anyOf": [ + { "type": "string" }, - "freeform_comment": { - "title": "Freeform Comment", + { + "type": "null" + } + ], + "default": null, + "description": "A unique key derived from the InChI.", + "title": "Inchi Key" + }, + "GHS_codes": { + "anyOf": [ + { "type": "string" }, - "file_id": { - "title": "File Id", - "type": "string" + { + "type": "null" + } + ], + "default": null, + "description": "A string describing any GHS hazard codes associated with this item. See https://pubchem.ncbi.nlm.nih.gov/ghs/ for code definitions.", + "examples": [ + "H224", + "H303, H316, H319" + ], + "title": "Ghs Codes" + }, + "molar_mass": { + "anyOf": [ + { + "type": "number" }, - "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } + { + "type": "null" + } + ], + "default": null, + "description": "Mass per formula unit, in g/mol.", + "title": "Molar Mass" + }, + "CAS": { + "anyOf": [ + { + "type": "string" }, - "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } + { + "type": "null" + } + ], + "default": null, + "description": "The CAS Registry Number for the substance described by this entry.", + "title": "Cas" + }, + "synthesis_constituents": { + "default": [], + "description": "A list of references to constituent materials giving the amount and relevant inlined details of consituent items.", + "items": { + "$ref": "#/$defs/Constituent" + }, + "title": "Synthesis Constituents", + "type": "array" + }, + "synthesis_description": { + "anyOf": [ + { + "type": "string" }, - "warnings": { - "title": "Warnings", - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "Free-text details of the procedure applied to synthesise the sample", + "title": "Synthesis Description" + }, + "files": { + "anyOf": [ + { "items": { - "type": "string" - } - }, - "b64_encoded_image": { - "title": "B64 Encoded Image", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "bokeh_plot_data": { - "title": "Bokeh Plot Data", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object" + "$ref": "#/$defs/File" + }, + "type": "array" }, - "computed": { - "title": "Computed", - "datalab_exclude_from_load": true, + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this item.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, "type": "object" }, - "metadata": { - "title": "Metadata", - "datalab_exclude_from_load": true, - "type": "object" + { + "type": "null" } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" }, - "required": [ - "blocktype", - "block_id" - ] + "title": "Creator Ids", + "type": "array" }, - "Collection": { - "title": "Collection", - "description": "A model for representing a group of related items, for example all the samples\nbelonging to one batch or project.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", + "creators": { + "anyOf": [ + { "items": { - "type": "string" - } + "$ref": "#/$defs/Person" + }, + "type": "array" }, - "groups": { - "title": "Groups", - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { "items": { - "$ref": "#/definitions/Group" - } + "$ref": "#/$defs/Group" + }, + "type": "array" }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "samples", + "default": "samples", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", "type": "string" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", "type": "string" }, - "title": { - "title": "Title", + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { "type": "string" }, - "description": { - "title": "Description", + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "num_items": { - "title": "Num Items", - "type": "integer" + { + "type": "null" } - } + ], + "default": null, + "description": "A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).", + "title": "Date" }, - "ItemStatus": { - "title": "ItemStatus", - "description": "An enumeration of the status of items", - "enum": [ - "active", - "planned", - "disposed", - "completed", - "failed", - "other" + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "type": "string" + "default": null, + "description": "An optional human-readable/usable name for the entry.", + "title": "Name" + }, + "status": { + "$ref": "#/$defs/ItemStatus", + "default": "active", + "description": "The status of the sample, indicating its current state." } - } + }, + "required": [ + "item_id" + ], + "title": "Sample", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/startingmaterial.json b/pydatalab/schemas/startingmaterial.json index 6e5306c02..2a779a084 100644 --- a/pydatalab/schemas/startingmaterial.json +++ b/pydatalab/schemas/startingmaterial.json @@ -1,982 +1,1659 @@ { - "title": "StartingMaterial", - "description": "A model for representing a starting material, i.e., a chemical or precursor held\nin the lab's inventory, from which samples are made.\n\nThe model mixes container-level and substance-level information and can be used to\nrepresent either depending on preference.", - "type": "object", - "properties": { - "chemform": { - "title": "Chemform", - "example": [ - "Na3P", - "Na3P", - "LiNiO2@C", - "Na3+xP", - "LiNi1/3Co0.1Mn0.1O2" + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" ], + "title": "AccountStatus", "type": "string" }, - "smiles": { - "title": "Smiles", - "aliases": [ - "SMILES", - "smiles_representation" + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", + "properties": { + "collection_id": { + "description": "The immutable ID of the collection.", + "title": "Collection Id", + "type": "string" + } + }, + "required": [ + "collection_id" ], - "type": "string" - }, - "inchi": { - "title": "Inchi", - "type": "string" - }, - "inchi_key": { - "title": "Inchi Key", - "type": "string" + "title": "CollectionReference", + "type": "object" }, - "GHS_codes": { - "title": "Ghs H-Codes", - "examples": [ - "H224", - "H303, H316, H319" + "Constituent": { + "description": "A constituent of a sample.", + "properties": { + "item": { + "anyOf": [ + { + "$ref": "#/$defs/EntryReference" + }, + { + "$ref": "#/$defs/InlineSubstance" + } + ], + "description": "A reference to item (sample or starting material) entry for the constituent substance.", + "title": "Item" + }, + "quantity": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the constituent material used to create the sample.", + "title": "Quantity" + }, + "unit": { + "default": "g", + "description": "The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).", + "title": "Unit", + "type": "string" + } + }, + "required": [ + "item" ], - "type": "string" - }, - "molar_mass": { - "title": "Molecular Weight", - "type": "number" - }, - "CAS": { - "title": "Substance Cas", - "type": "string" - }, - "synthesis_constituents": { - "title": "Synthesis Constituents", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Constituent" - } - }, - "synthesis_description": { - "title": "Synthesis Description", - "type": "string" - }, - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", + "title": "Constituent", "type": "object" }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "starting_materials", - "const": "starting_materials", - "pattern": "^starting_materials$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date Acquired", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Container Name", - "type": "string" - }, - "status": { - "default": "available", - "allOf": [ - { - "$ref": "#/definitions/StartingMaterialsStatus" + "DataBlockResponse": { + "additionalProperties": true, + "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", + "properties": { + "blocktype": { + "description": "The type of the block.", + "title": "Blocktype", + "type": "string" + }, + "block_id": { + "description": "A shorthand random ID for the block.", + "title": "Block Id", + "type": "string" + }, + "item_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" + }, + "collection_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" + }, + "title": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" + }, + "freeform_comment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" + }, + "file_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" + }, + "file_ids": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" + }, + "errors": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" + }, + "warnings": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" + }, + "b64_encoded_image": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" + }, + "bokeh_plot_data": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" + }, + "computed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" } - ] - }, - "barcode": { - "title": "Barcode", - "type": "string" - }, - "date_opened": { - "title": "Date Opened", - "type": "string", - "format": "date-time" - }, - "chemical_purity": { - "title": "Chemical Purity", - "type": "string" - }, - "full_percent": { - "title": "Full %", - "type": "string" - }, - "size": { - "title": "Container Size", - "type": "string" - }, - "size_unit": { - "title": "Unit", - "type": "string" - }, - "supplier": { - "title": "Supplier", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" + }, + "required": [ + "blocktype", + "block_id", + "b64_encoded_image", + "bokeh_plot_data" + ], + "title": "DataBlockResponse", + "type": "object" }, - "comment": { - "title": "Comments", - "type": "string" - } - }, - "required": [ - "item_id" - ], - "definitions": { "EntryReference": { - "title": "EntryReference", + "additionalProperties": true, "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", - "type": "object", "properties": { "type": { "title": "Type", "type": "string" }, - "name": { - "title": "Name", - "type": "string" - }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Immutable Id" + }, + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Item Id" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Refcode" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "type" + ], + "title": "EntryReference", + "type": "object" + }, + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", + "properties": { + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "files", + "default": "files", + "title": "Type", + "type": "string" + }, "immutable_id": { - "title": "Immutable Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" + }, + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" + }, + "item_ids": { + "description": "A list of item IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Item Ids", + "type": "array" + }, + "blocks": { + "description": "A list of block IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Blocks", + "type": "array" }, - "chemform": { - "title": "Chemform", - "type": "string" - } - }, - "required": [ - "type" - ] - }, - "InlineSubstance": { - "title": "InlineSubstance", - "type": "object", - "properties": { "name": { + "description": "The filename on disk.", "title": "Name", "type": "string" }, - "chemform": { - "title": "Chemform", + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", "type": "string" - } - }, - "required": [ - "name" - ] - }, - "Constituent": { - "title": "Constituent", - "description": "A constituent of a sample.", - "type": "object", - "properties": { - "item": { - "title": "Item", + }, + "original_name": { "anyOf": [ { - "$ref": "#/definitions/EntryReference" + "type": "string" }, { - "$ref": "#/definitions/InlineSubstance" + "type": "null" } - ] + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" }, - "quantity": { - "title": "Quantity", - "minimum": 0, - "type": "number" + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" }, - "unit": { - "title": "Unit", - "default": "g", - "type": "string" - } - }, - "required": [ - "item", - "quantity" - ] - }, - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" - ], - "type": "string" - }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", - "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" - ], - "type": "string" - }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", - "properties": { - "description": { - "title": "Description", + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" + }, + "time_added": { + "description": "The timestamp for the original file upload.", + "format": "datetime", + "title": "Time Added", "type": "string" }, - "relation": { - "$ref": "#/definitions/RelationshipType" + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" }, - "type": { - "$ref": "#/definitions/KnownType" + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" }, - "immutable_id": { - "title": "Immutable Id", - "type": "string" + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", + "type": "boolean" + }, + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ - "type" - ] - }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" + "name", + "extension", + "time_added", + "is_live", + "checksums" ], - "type": "string" + "title": "File", + "type": "object" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "FileChecksums": { + "description": "Content checksums for a file.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" - }, - "identifier": { - "title": "Identifier", - "type": "string" - }, - "name": { - "title": "Name", + "md5": { + "description": "The MD5 hex digest of the file contents.", + "title": "Md5", "type": "string" }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" - }, - "display_name": { - "title": "Display Name", + "sha256": { + "description": "The SHA-256 hex digest of the file contents.", + "title": "Sha256", "type": "string" } }, "required": [ - "identity_type", - "identifier", - "name" - ] - }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" + "md5", + "sha256" ], - "type": "string" + "title": "FileChecksums", + "type": "object" }, "Group": { - "title": "Group", "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", "properties": { "type": { - "title": "Type", - "default": "groups", "const": "groups", + "default": "groups", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, "members": { - "title": "Members", - "type": "array", + "description": "A list of people that belong to this group; stored on the user objects.", "items": { + "additionalProperties": true, "type": "object" - } + }, + "title": "Members", + "type": "array" }, "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" }, "description": { - "title": "Description", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" }, "managers": { - "title": "Managers", - "type": "array", + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", "items": { "anyOf": [ { + "format": "objectid", "type": "string" }, { + "additionalProperties": true, "type": "object" } ] - } + }, + "title": "Managers", + "type": "array" } - } + }, + "title": "Group", + "type": "object" }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", + "properties": { + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." + }, + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", + "type": "string" + }, + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", + "type": "string" + }, + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" + }, + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" + } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", "enum": [ - "active", - "unverified", - "deactivated" + "email", + "orcid", + "github", + "google", + "microsoft" ], + "title": "IdentityType", + "type": "string" + }, + "InlineSubstance": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "name" + ], + "title": "InlineSubstance", + "type": "object" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" + ], + "title": "KnownType", "type": "string" }, "Person": { - "title": "Person", "description": "A model that describes an individual and their digital identities.", - "type": "object", "properties": { "type": { - "title": "Type", - "default": "people", "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "identities": { - "title": "Identities", - "type": "array", + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", "items": { - "$ref": "#/definitions/Identity" - } + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" }, "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" }, "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" + }, + "gravatar_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" + }, + "managers": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" + }, + "role": { + "$ref": "#/$defs/UserRole", + "default": "user", + "description": "The role assigned to this person." + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" + }, + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." + } + }, + "title": "Person", + "type": "object" + }, + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" + }, + "StartingMaterialsStatus": { + "description": "An enumeration of the status of starting materials", + "enum": [ + "ordered", + "disposed", + "planned", + "available", + "unavailable", + "exhausted", + "other" + ], + "title": "StartingMaterialsStatus", + "type": "string" + }, + "TypedRelationship": { + "additionalProperties": true, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" }, - "gravatar_hash": { - "title": "Gravatar Hash", - "type": "string" + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } + "type": { + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." }, - "role": { - "default": "user", - "allOf": [ + "immutable_id": { + "anyOf": [ { - "$ref": "#/definitions/UserRole" + "format": "objectid", + "type": "string" + }, + { + "type": "null" } - ] - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" }, - "account_status": { - "default": "unverified", - "allOf": [ + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] - } - } - }, - "FileChecksums": { - "title": "FileChecksums", - "description": "Content checksums for a file.", - "type": "object", - "properties": { - "md5": { - "title": "Md5", - "type": "string" + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" }, - "sha256": { - "title": "Sha256", - "type": "string" + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" } }, "required": [ - "md5", - "sha256" - ] + "type" + ], + "title": "TypedRelationship", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", - "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing a starting material, i.e., a chemical or precursor held\nin the lab's inventory, from which samples are made.\n\nThe model mixes container-level and substance-level information and can be used to\nrepresent either depending on preference.", + "properties": { + "chemform": { + "anyOf": [ + { "type": "string" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + { + "type": "null" + } + ], + "default": null, + "description": "A string representation of the chemical formula or composition associated with this sample.\n\nThe representation is relatively free-form; clients are expected parse and interpret HTML markup for subscripts\nand accept unicode characters for greek letters.", + "examples": [ + "Na3P", + "Na3P", + "LiNiO2@C", + "Na3+xP", + "LiNi1/3Co0.1Mn0.1O2" + ], + "title": "Chemform" + }, + "smiles": { + "anyOf": [ + { "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "size": { - "title": "Size", - "type": "integer" - }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" - }, - "item_ids": { - "title": "Item Ids", - "type": "array", - "items": { - "type": "string" - } - }, - "blocks": { - "title": "Blocks", - "type": "array", - "items": { - "type": "string" - } - }, - "name": { - "title": "Name", + { + "type": "null" + } + ], + "default": null, + "description": "A SMILES string representation of the chemical structure associated with this sample.", + "title": "Smiles" + }, + "inchi": { + "anyOf": [ + { "type": "string" }, - "extension": { - "title": "Extension", + { + "type": "null" + } + ], + "default": null, + "description": "An International Chemical Identifier (InChI) string representation of chemicals/molecules associated with this sample.", + "title": "Inchi" + }, + "inchi_key": { + "anyOf": [ + { "type": "string" }, - "original_name": { - "title": "Original Name", + { + "type": "null" + } + ], + "default": null, + "description": "A unique key derived from the InChI.", + "title": "Inchi Key" + }, + "GHS_codes": { + "anyOf": [ + { "type": "string" }, - "location": { - "title": "Location", - "type": "string" + { + "type": "null" + } + ], + "default": null, + "description": "A string describing any GHS hazard codes associated with this item. See https://pubchem.ncbi.nlm.nih.gov/ghs/ for code definitions.", + "examples": [ + "H224", + "H303, H316, H319" + ], + "title": "Ghs Codes" + }, + "molar_mass": { + "anyOf": [ + { + "type": "number" }, - "url_path": { - "title": "Url Path", + { + "type": "null" + } + ], + "default": null, + "description": "Mass per formula unit, in g/mol.", + "title": "Molar Mass" + }, + "CAS": { + "anyOf": [ + { "type": "string" }, - "source": { - "title": "Source", + { + "type": "null" + } + ], + "default": null, + "description": "The CAS Registry Number for the substance described by this entry.", + "title": "Cas" + }, + "synthesis_constituents": { + "default": [], + "description": "A list of references to constituent materials giving the amount and relevant inlined details of consituent items.", + "items": { + "$ref": "#/$defs/Constituent" + }, + "title": "Synthesis Constituents", + "type": "array" + }, + "synthesis_description": { + "anyOf": [ + { "type": "string" }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" + { + "type": "null" + } + ], + "default": null, + "description": "Free-text details of the procedure applied to synthesise the sample", + "title": "Synthesis Description" + }, + "files": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" }, - "metadata": { - "title": "Metadata", + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this item.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, "type": "object" }, - "representation": { - "title": "Representation" + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" }, - "source_server_name": { - "title": "Source Server Name", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "starting_materials", + "default": "starting_materials", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", "type": "string" }, - "source_path": { - "title": "Source Path", + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "is_live": { - "title": "Is Live", - "type": "boolean" - }, - "checksums": { - "$ref": "#/definitions/FileChecksums" + { + "type": "null" } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" }, - "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + "title": "Relationships", + "type": "array" }, - "DataBlockResponse": { - "title": "DataBlockResponse", - "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", - "properties": { - "blocktype": { - "title": "Blocktype", + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", "type": "string" }, - "block_id": { - "title": "Block Id", + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { "type": "string" }, - "item_id": { - "title": "Item Id", + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "collection_id": { - "title": "Collection Id", + { + "type": "null" + } + ], + "default": null, + "description": "The date the item was acquired", + "title": "Date" + }, + "name": { + "anyOf": [ + { "type": "string" }, - "title": { - "title": "Title", + { + "type": "null" + } + ], + "description": "The name of the substance in the container.", + "title": "Name" + }, + "status": { + "$ref": "#/$defs/StartingMaterialsStatus", + "default": "available", + "description": "The status of the starting materials, indicating its current state." + }, + "barcode": { + "anyOf": [ + { "type": "string" }, - "freeform_comment": { - "title": "Freeform Comment", + { + "type": "null" + } + ], + "default": null, + "description": "A unique barcode provided by an external source, e.g., cheminventory.", + "title": "Barcode" + }, + "date_opened": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "file_id": { - "title": "File Id", + { + "type": "null" + } + ], + "default": null, + "description": "The date the item was opened", + "title": "Date Opened" + }, + "chemical_purity": { + "anyOf": [ + { "type": "string" }, - "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } - }, - "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } - }, - "warnings": { - "title": "Warnings", - "type": "array", - "items": { - "type": "string" - } - }, - "b64_encoded_image": { - "title": "B64 Encoded Image", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } - }, - "bokeh_plot_data": { - "title": "Bokeh Plot Data", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object" - }, - "computed": { - "title": "Computed", - "datalab_exclude_from_load": true, - "type": "object" - }, - "metadata": { - "title": "Metadata", - "datalab_exclude_from_load": true, - "type": "object" + { + "type": "null" } - }, - "required": [ - "blocktype", - "block_id" - ] + ], + "description": "The chemical purity of this container with regards to the defined substance.", + "title": "Chemical Purity" }, - "Collection": { - "title": "Collection", - "description": "A model for representing a group of related items, for example all the samples\nbelonging to one batch or project.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", + "full_percent": { + "anyOf": [ + { "type": "string" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the defined substance remaining in the container, expressed as a percentage.", + "title": "Full Percent" + }, + "size": { + "anyOf": [ + { "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + { + "type": "null" + } + ], + "default": null, + "description": "The total size of the container, in units of `size_unit`.", + "title": "Size" + }, + "size_unit": { + "anyOf": [ + { "type": "string" }, - "title": { - "title": "Title", + { + "type": "null" + } + ], + "default": null, + "description": "Units for the 'size' field.", + "title": "Size Unit" + }, + "supplier": { + "anyOf": [ + { "type": "string" }, - "description": { - "title": "Description", + { + "type": "null" + } + ], + "description": "Supplier or manufacturer of the chemical.", + "title": "Supplier" + }, + "location": { + "anyOf": [ + { "type": "string" }, - "num_items": { - "title": "Num Items", - "type": "integer" + { + "type": "null" } - } + ], + "default": null, + "description": "The place where the container is located.", + "title": "Location" }, - "StartingMaterialsStatus": { - "title": "StartingMaterialsStatus", - "description": "An enumeration of the status of starting materials", - "enum": [ - "ordered", - "disposed", - "planned", - "available", - "unavailable", - "exhausted", - "other" + "comment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "type": "string" + "default": null, + "description": "Any additional comments or notes about the container.", + "title": "Comment" } - } + }, + "required": [ + "item_id", + "name", + "chemical_purity", + "supplier" + ], + "title": "StartingMaterial", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/src/pydatalab/models/blocks.py b/pydatalab/src/pydatalab/models/blocks.py index 9e33aec65..a89af0ffc 100644 --- a/pydatalab/src/pydatalab/models/blocks.py +++ b/pydatalab/src/pydatalab/models/blocks.py @@ -1,6 +1,6 @@ -from pydantic import BaseModel, Field +from pydantic import ConfigDict, Field -from pydatalab.models.utils import JSON_ENCODERS, PyObjectId +from pydatalab.models.utils import BaseModel, PyObjectId class DataBlockResponse(BaseModel): @@ -11,6 +11,8 @@ class DataBlockResponse(BaseModel): where possible. """ + model_config = ConfigDict(validate_by_name=True, extra="allow") + blocktype: str """The type of the block.""" @@ -60,8 +62,3 @@ class DataBlockResponse(BaseModel): metadata: dict | None = Field(default=None, datalab_exclude_from_load=True) """Any structured metadata associated with the block, for example, experimental acquisition parameters.""" - - class Config: - allow_population_by_field_name = True - json_encoders = JSON_ENCODERS - extra = "allow" diff --git a/pydatalab/src/pydatalab/models/cells.py b/pydatalab/src/pydatalab/models/cells.py index 03e23d384..54461ce52 100644 --- a/pydatalab/src/pydatalab/models/cells.py +++ b/pydatalab/src/pydatalab/models/cells.py @@ -1,13 +1,16 @@ from enum import Enum +from typing import Literal -from pydantic import Field, root_validator, validator +from pydantic import ( + Field, + field_validator, + model_validator, +) from pydatalab.models.entries import EntryReference from pydatalab.models.items import Item from pydatalab.models.utils import CellStatus, Constituent -# from pydatalab.logger import LOGGER - class CellComponent(Constituent): ... @@ -31,72 +34,66 @@ class Cell(Item): components and the format it was built in. """ - type: str = Field("cells", const="cells", pattern="^cells$") + type: Literal["cells"] = "cells" - cell_format: CellFormat | None + cell_format: CellFormat | None = None """The form factor of the cell, e.g., coin, pouch, in situ or otherwise.""" - cell_format_description: str | None + cell_format_description: str | None = None """Additional human-readable description of the cell form factor, e.g., 18650, AMPIX, CAMPIX""" - cell_preparation_description: str | None + cell_preparation_description: str | None = None + """Description of how the cell was prepared.""" - characteristic_mass: float | None + characteristic_mass: float | None = None """The characteristic mass of the cell in milligrams. Can be used to normalize capacities.""" - characteristic_chemical_formula: str | None + characteristic_chemical_formula: str | None = None """The chemical formula of the active material. Can be used to calculated molar mass in g/mol for normalizing capacities.""" - characteristic_molar_mass: float | None + characteristic_molar_mass: float | None = None """The molar mass of the active material, in g/mol. Will be inferred from the chemical formula, or can be supplied if it cannot be supplied""" - positive_electrode: list[CellComponent] = [] - - negative_electrode: list[CellComponent] = [] - - electrolyte: list[CellComponent] = [] - + positive_electrode: list[CellComponent] = Field(default_factory=list) + negative_electrode: list[CellComponent] = Field(default_factory=list) + electrolyte: list[CellComponent] = Field(default_factory=list) + active_ion: str | None = None + """The active ion species.""" active_ion_charge: float = 1 - status: CellStatus = Field(default=CellStatus.ACTIVE) """The status of the cells, indicating its current state.""" - @validator("characteristic_molar_mass", always=True, pre=True) - def set_molar_mass(cls, v, values): + @field_validator("characteristic_molar_mass", mode="before") + @classmethod + def set_molar_mass(cls, v, info): from periodictable import formula - if not v: - chemical_formula = values.get("characteristic_chemical_formula") - + if not v and hasattr(info, "data") and info.data: + chemical_formula = info.data.get("characteristic_chemical_formula") if chemical_formula: try: return formula(chemical_formula).mass except Exception: return None - return v - @root_validator - def add_missing_electrode_relationships(cls, values): + @model_validator(mode="after") + def add_missing_electrode_relationships(self): """Add any missing cell component constituents to parent relationships""" from pydatalab.models.relationships import RelationshipType, TypedRelationship - existing_parthood_relationships = {} - if values.get("relationships") is not None: - # Index by refcode *and* item_id so a stored relationship carrying an - # item_id still matches a refcode-enriched constituent (and vice-versa). - existing_parthood_relationships = { - identifier: relationship - for relationship in values["relationships"] - if relationship.relation == RelationshipType.PARTHOOD - for identifier in (relationship.refcode, relationship.item_id) - if identifier - } - else: - values["relationships"] = [] + # Index by refcode *and* item_id so a stored relationship carrying an + # item_id still matches a refcode-enriched constituent (and vice-versa). + existing_parthood_relationships = { + identifier: relationship + for relationship in self.relationships + if relationship.relation == RelationshipType.PARTHOOD + for identifier in (relationship.refcode, relationship.item_id) + if identifier + } for component in ("positive_electrode", "negative_electrode", "electrolyte"): - for constituent in values.get(component, []): + for constituent in getattr(self, component): if not isinstance(constituent.item, EntryReference): continue @@ -114,7 +111,7 @@ def add_missing_electrode_relationships(cls, values): type=constituent.item.type, description="Is a constituent of", ) - values["relationships"].append(relationship) + self.relationships.append(relationship) else: # Back-fill any identifier missing from the stored relationship relationship.refcode = relationship.refcode or refcode @@ -127,4 +124,4 @@ def add_missing_electrode_relationships(cls, values): if identifier: existing_parthood_relationships[identifier] = relationship - return values + return self diff --git a/pydatalab/src/pydatalab/models/collections.py b/pydatalab/src/pydatalab/models/collections.py index 9d839f25d..139798eba 100644 --- a/pydatalab/src/pydatalab/models/collections.py +++ b/pydatalab/src/pydatalab/models/collections.py @@ -1,4 +1,8 @@ -from pydantic import Field, root_validator +from typing import Literal + +from pydantic import ( + model_validator, +) from pydatalab.models.entries import Entry from pydatalab.models.traits import HasBlocks, HasOwner @@ -11,21 +15,22 @@ class Collection(Entry, HasOwner, HasBlocks): """ - type: str = Field("collections", const="collections", pattern="^collections$") + type: Literal["collections"] = "collections" - collection_id: HumanReadableIdentifier = Field(None) + collection_id: HumanReadableIdentifier """A short human-readable/usable name for the collection.""" - title: str | None + title: str | None = None """A descriptive title for the collection.""" - description: str | None + description: str | None = None """A description of the collection, either in plain-text or a markup language.""" - num_items: int | None = Field(None) + num_items: int | None = None """Inlined number of items associated with this collection.""" - @root_validator + @model_validator(mode="before") + @classmethod def check_ids(cls, values): if not any(values.get(k) is not None for k in ("collection_id", "immutable_id")): raise ValueError("Collection must have at least collection_id or immutable_id") diff --git a/pydatalab/src/pydatalab/models/entries.py b/pydatalab/src/pydatalab/models/entries.py index 694a312ad..a5b61f9de 100644 --- a/pydatalab/src/pydatalab/models/entries.py +++ b/pydatalab/src/pydatalab/models/entries.py @@ -1,10 +1,10 @@ import abc -from pydantic import BaseModel, Field, root_validator +from pydantic import ConfigDict, Field, model_validator from pydatalab.models.relationships import TypedRelationship from pydatalab.models.utils import ( - JSON_ENCODERS, + BaseModel, EntryReference, IsoformatDateTime, PyObjectId, @@ -20,21 +20,22 @@ class Entry(BaseModel, abc.ABC): type: str """The resource type of the entry.""" - immutable_id: PyObjectId = Field( + immutable_id: PyObjectId | None = Field( None, title="Immutable ID", alias="_id", - format="uuid", + json_schema_extra={"format": "uuid"}, ) """The immutable database ID of the entry.""" last_modified: IsoformatDateTime | None = None """The timestamp at which the entry was last modified.""" - relationships: list[TypedRelationship] | None = None + relationships: list[TypedRelationship] = Field(default_factory=list) """A list of related entries and their types.""" - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def check_id_names(cls, values): """Slightly upsetting hack: this case *should* be covered by the pydantic setting for populating fields by alias names. @@ -42,6 +43,10 @@ def check_id_names(cls, values): if "_id" in values: values["immutable_id"] = values.pop("_id") + # Coerce a stored/explicit null into an empty list so the field is always a list. + if values.get("relationships") is None: + values["relationships"] = [] + return values def to_reference(self, additional_fields: list[str] | None = None) -> "EntryReference": @@ -63,7 +68,4 @@ def to_reference(self, additional_fields: list[str] | None = None) -> "EntryRefe return EntryReference(**data) - class Config: - allow_population_by_field_name = True - json_encoders = JSON_ENCODERS - extra = "ignore" + model_config = ConfigDict(populate_by_name=True, extra="ignore") diff --git a/pydatalab/src/pydatalab/models/equipment.py b/pydatalab/src/pydatalab/models/equipment.py index 32c8b949c..b86e2e897 100644 --- a/pydatalab/src/pydatalab/models/equipment.py +++ b/pydatalab/src/pydatalab/models/equipment.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import Field from pydatalab.models.items import Item @@ -13,15 +15,15 @@ class Equipment(Item): the items measured on it. """ - type: str = Field("equipment", const="equipment", pattern="^equipment$") + type: Literal["equipment"] = "equipment" - serial_numbers: str | None + serial_numbers: str | None = None """A string describing one or more serial numbers for the instrument.""" - manufacturer: str | None + manufacturer: str | None = None """The manufacturer of this piece of equipment""" - location: str | None + location: str | None = None """Place where the equipment is located""" contact: str | None diff --git a/pydatalab/src/pydatalab/models/export_task.py b/pydatalab/src/pydatalab/models/export_task.py index 171567038..d2085ffde 100644 --- a/pydatalab/src/pydatalab/models/export_task.py +++ b/pydatalab/src/pydatalab/models/export_task.py @@ -1,9 +1,9 @@ from datetime import datetime, timezone from enum import Enum -from pydantic import BaseModel, Field +from pydantic import Field -from pydatalab.models.utils import PyObjectId +from pydatalab.models.utils import BaseModel, PyObjectId class ExportStatus(str, Enum): @@ -18,23 +18,35 @@ class ExportStatus(str, Enum): class ExportTask(BaseModel): """Model for an export task.""" - task_id: str = Field(..., description="Unique identifier for the export task") - collection_id: str | None = Field(None, description="ID of the collection being exported") - item_id: str | None = Field(None, description="ID of the item being exported") - export_type: str = Field( - default="collection", description="Type of export: 'collection' or 'item' or 'graph'" - ) - status: ExportStatus = Field( - default=ExportStatus.PENDING, description="Current status of the task" - ) - creator_id: PyObjectId = Field(..., description="ID of the user who created the export") - created_at: datetime = Field( - default_factory=lambda: datetime.now(tz=timezone.utc), - description="When the task was created", - ) - completed_at: datetime | None = Field(None, description="When the task was completed") - file_path: str | None = Field(None, description="Path to the generated .eln file") - error_message: str | None = Field(None, description="Error message if status is ERROR") + task_id: str + """Unique identifier for the export task""" + + collection_id: str | None = None + """ID of the collection being exported""" + + item_id: str | None = None + """ID of the item being exported""" + + export_type: str = "collection" + """Type of export: 'collection' or 'item' or 'graph'""" + + status: ExportStatus = ExportStatus.PENDING + """Current status of the task""" + + creator_id: PyObjectId + """ID of the user who created the export""" + + created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc)) + """When the task was created""" + + completed_at: datetime | None = None + """When the task was completed""" + + file_path: str | None = None + """Path to the generated .eln file""" + + error_message: str | None = None + """Error message if status is ERROR""" class Config: use_enum_values = True diff --git a/pydatalab/src/pydatalab/models/files.py b/pydatalab/src/pydatalab/models/files.py index 1cdc744af..edf092fa9 100644 --- a/pydatalab/src/pydatalab/models/files.py +++ b/pydatalab/src/pydatalab/models/files.py @@ -1,10 +1,11 @@ -from typing import Any +from typing import Any, Literal -from pydantic import BaseModel, Field +from pydantic import Field +from pydatalab.models import traits from pydatalab.models.entries import Entry from pydatalab.models.traits import HasOwner, HasRevisionControl -from pydatalab.models.utils import IsoformatDateTime +from pydatalab.models.utils import BaseModel, IsoformatDateTime class FileChecksums(BaseModel): @@ -20,18 +21,18 @@ class FileChecksums(BaseModel): class File(Entry, HasOwner, HasRevisionControl): """A model for representing a file that has been tracked or uploaded to datalab.""" - type: str = Field("files", const="files", pattern="^files$") + type: Literal["files"] = "files" - size: int | None + size: int | None = None """The size of the file on disk in bytes.""" - last_modified_remote: IsoformatDateTime | None + last_modified_remote: IsoformatDateTime | None = None """The last date/time at which the remote file was modified.""" - item_ids: list[str] + item_ids: list[str] = Field(default_factory=list) """A list of item IDs associated with this file.""" - blocks: list[str] + blocks: list[str] = Field(default_factory=list) """A list of block IDs associated with this file.""" name: str @@ -40,30 +41,30 @@ class File(Entry, HasOwner, HasRevisionControl): extension: str """The file extension that the file was uploaded with.""" - original_name: str | None + original_name: str | None = None """The raw filename as uploaded.""" - location: str | None + location: str | None = None """The location of the file on disk.""" - url_path: str | None + url_path: str | None = None """The path to a remote file.""" - source: str | None + source: str | None = None """The source of the file, e.g. 'remote' or 'uploaded'.""" time_added: IsoformatDateTime """The timestamp for the original file upload.""" - metadata: dict[Any, Any] | None + metadata: dict[Any, Any] | None = None """Any additional metadata.""" - representation: Any | None + representation: Any | None = None - source_server_name: str | None + source_server_name: str | None = None """The server name at which the file is stored.""" - source_path: str | None + source_path: str | None = None """The path to the file on the remote resource.""" is_live: bool @@ -71,3 +72,9 @@ class File(Entry, HasOwner, HasRevisionControl): checksums: FileChecksums | None """Content checksums (MD5 and SHA-256) of the file.""" + + +# `traits.HasFiles` cannot import `File` at module scope (this module needs the +# traits at class-definition time), so inject it and resolve the forward ref here. +traits.File = File # type: ignore[misc] +traits.HasFiles.model_rebuild() diff --git a/pydatalab/src/pydatalab/models/items.py b/pydatalab/src/pydatalab/models/items.py index 9ec1646a3..326946f84 100644 --- a/pydatalab/src/pydatalab/models/items.py +++ b/pydatalab/src/pydatalab/models/items.py @@ -1,8 +1,12 @@ import abc -from pydantic import validator +from pydantic import field_validator from pydatalab.models.entries import Entry + +# Importing `files` here resolves the deferred `File` annotation on `HasFiles` +# before any `Item` subclass is built. +from pydatalab.models.files import File # noqa: F401 from pydatalab.models.traits import ( HasBlocks, HasFiles, @@ -26,27 +30,26 @@ class Item(Entry, HasOwner, HasRevisionControl, IsCollectable, HasBlocks, HasFil populates itself. """ - refcode: Refcode = None # type: ignore - """A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) - and a locally unique string, ideally created with some consistent scheme. - """ + refcode: Refcode | None = None + """A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.""" item_id: HumanReadableIdentifier """A locally unique, human-readable identifier for the entry. This ID is mutable.""" - description: str | None + description: str | None = None """A description of the item, either in plain-text or a markup language.""" - date: IsoformatDateTime | None + date: IsoformatDateTime | None = None """A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).""" - name: str | None + name: str | None = None """An optional human-readable/usable name for the entry.""" - status: str | None + status: str | None = None """The status of the item, with allowed values defined by the specific item class.""" - @validator("refcode", pre=True, always=True) + @field_validator("refcode", mode="before") + @classmethod def refcode_validator(cls, v): """Generate a refcode if not provided.""" diff --git a/pydatalab/src/pydatalab/models/people.py b/pydatalab/src/pydatalab/models/people.py index a86a2d4a2..ec3485ade 100644 --- a/pydatalab/src/pydatalab/models/people.py +++ b/pydatalab/src/pydatalab/models/people.py @@ -1,14 +1,17 @@ from __future__ import annotations from enum import Enum +from typing import Annotated, Literal -import bson -import bson.errors -from pydantic import BaseModel, ConstrainedStr, Field, parse_obj_as, validator from pydantic import EmailStr as PydanticEmailStr +from pydantic import ( + Field, + StringConstraints, + field_validator, +) from pydatalab.models.entries import Entry -from pydatalab.models.utils import HumanReadableIdentifier, PyObjectId, UserRole +from pydatalab.models.utils import BaseModel, HumanReadableIdentifier, PyObjectId, UserRole class IdentityType(str, Enum): @@ -36,29 +39,32 @@ class Identity(BaseModel): name: str """The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.""" - verified: bool = Field(False) + verified: bool = False """Whether the identity has been verified (by some means, e.g., OAuth2 or email)""" - display_name: str | None + display_name: str | None = None """The user's display name associated with the identity, also to be exposed in free text searches.""" - @validator("name", pre=True, always=True) - def add_missing_name(cls, v, values): + @field_validator("name", mode="before") + @classmethod + def add_missing_name(cls, v, info): """If the identity is created without a free-text 'name', then for certain providers, populate this field so that it can appear in the free text index, e.g., an ORCID, or an institutional username from an email address. """ - if v is None: - if values["identity_type"] == IdentityType.ORCID: - return values["identifier"] - if values["identity_type"] == IdentityType.EMAIL: - return values["identifier"].split("@")[0] - + if v is None and hasattr(info, "data") and info.data: + data = info.data + if data.get("identity_type") == IdentityType.ORCID: + return data.get("identifier") + if data.get("identity_type") == IdentityType.EMAIL: + identifier = data.get("identifier", "") + return identifier.split("@")[0] if "@" in identifier else identifier return v - @validator("verified", pre=True, always=True) + @field_validator("verified", mode="before") + @classmethod def add_missing_verification(cls, v): """Fills in missing value for `verified` if not given.""" if not v: @@ -66,18 +72,11 @@ def add_missing_verification(cls, v): return v -class DisplayName(ConstrainedStr): - """A constrained string less than 150 characters long but with - non-empty content, intended to be entered by the user. - - """ - - max_length = 150 - min_length = 1 - strip_whitespace = True - - def __new__(cls, value): - return parse_obj_as(cls, value) +DisplayName = Annotated[ + str, + StringConstraints(min_length=1, max_length=150, strip_whitespace=True), +] +"""A constrained string less than 150 characters long but with non-empty content, intended to be entered by the user.""" class EmailStr(PydanticEmailStr): @@ -112,38 +111,40 @@ class Group(Entry): """ - type: str = Field("groups", const=True) + type: Literal["groups"] = "groups" """The entry type as a string.""" - group_id: HumanReadableIdentifier | None = Field(None) + group_id: HumanReadableIdentifier | None = None """A short, locally-unique ID for the group.""" - members: list[dict] = Field(None) - """A list of people that belong to this group.""" + members: list[dict] = Field(default_factory=list) + """A list of people that belong to this group; stored on the user objects.""" - display_name: DisplayName | None = Field(None) + display_name: DisplayName | None = None """The chosen display name for the group""" - description: str | None = Field(None) + description: str | None = None """A description of the group""" managers: list[PyObjectId | dict] = Field(default_factory=list) - """A list of user IDs that can manage this group.""" + """A list of user IDs that can manage this group; stored in db as list of IDs.""" - @validator("members", pre=True, always=True) + @field_validator("members", mode="before") + @classmethod def cast_members_to_people(cls, v): """Casts members to list of people if not None.""" if v is not None: - return [Person(**member).dict(exclude_unset=True) for member in v] + return [Person(**member).model_dump(exclude_unset=True) for member in v] return v - @validator("managers", pre=True, always=True) + @field_validator("managers", mode="before") + @classmethod def cast_managers_to_people(cls, v): """Casts managers to list of people if not None.""" if v and isinstance(v[0], dict): return [ - Person(**member).dict(exclude_unset=True) + Person(**member).model_dump(exclude_unset=True) for member in v if isinstance(member, dict) ] @@ -154,7 +155,7 @@ def cast_managers_to_people(cls, v): class Person(Entry): """A model that describes an individual and their digital identities.""" - type: str = Field("people", const=True) + type: Literal["people"] = "people" """The entry type as a string.""" identities: list[Identity] = Field(default_factory=list) @@ -181,14 +182,15 @@ class Person(Entry): account_status: AccountStatus = Field(AccountStatus.UNVERIFIED) """The status of the user's account.""" - @validator("type", pre=True, always=True) + @field_validator("type", mode="before") + @classmethod def add_missing_type(cls, v): """Fill in missing `type` field if not provided.""" if v is None: v = "people" return v - @validator("type", pre=True) + @classmethod def set_default_type(cls, _): return "people" @@ -214,8 +216,6 @@ def new_user_from_identity( A `Person` object with only the provided identity. """ - user_id = bson.ObjectId() - display_name = None if use_display_name: display_name = identity.display_name @@ -227,7 +227,6 @@ def new_user_from_identity( from pydatalab.mongo import gravatar_hash_for return Person( - immutable_id=user_id, identities=[identity], display_name=display_name, contact_email=contact_email, diff --git a/pydatalab/src/pydatalab/models/relationships.py b/pydatalab/src/pydatalab/models/relationships.py index e844c2498..e4c53ec33 100644 --- a/pydatalab/src/pydatalab/models/relationships.py +++ b/pydatalab/src/pydatalab/models/relationships.py @@ -1,8 +1,13 @@ from enum import Enum -from pydantic import BaseModel, root_validator, validator +from pydantic import ( + ConfigDict, + field_validator, + model_validator, +) from pydatalab.models.utils import ( + BaseModel, HumanReadableIdentifier, KnownType, PyObjectId, @@ -29,46 +34,52 @@ class entryC SIBLING = "sibling" PARTHOOD = "is_part_of" OTHER = "other" + COLLECTIONS = "collections" class TypedRelationship(BaseModel): - description: str | None + description: str | None = None """A description of the relationship.""" - relation: RelationshipType | None + relation: RelationshipType | None = None """The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided.""" type: KnownType """The type of the related resource.""" - immutable_id: PyObjectId | None + immutable_id: PyObjectId | None = None """The immutable ID of the entry that is related to this entry.""" - item_id: HumanReadableIdentifier | None + item_id: HumanReadableIdentifier | None = None """The ID of the entry that is related to this entry.""" - refcode: Refcode | None + refcode: Refcode | None = None """The refcode of the entry that is related to this entry.""" - @validator("relation") - def check_for_description(cls, v, values): - if v == RelationshipType.OTHER and values.get("description") is None: - raise ValueError( - f"A description must be provided if the relationship type is {RelationshipType.OTHER.value!r}." - ) - + @field_validator("relation") + @classmethod + def check_for_description(cls, v, info): + if v == RelationshipType.OTHER: + data = info.data if hasattr(info, "data") and info.data else {} + if data.get("description") is None: + raise ValueError( + f"A description must be provided if the relationship type is {RelationshipType.OTHER.value!r}." + ) return v - @root_validator + @model_validator(mode="before") + @classmethod def check_id_fields(cls, values): """Check that at least one of the possible identifier fields is provided.""" id_fields = ("immutable_id", "item_id", "refcode") - if all(values[f] is None for f in id_fields): + if all(values.get(f) is None for f in id_fields): raise ValueError(f"Must provide at least one of {id_fields!r}") if values.get("refcode") and values.get("item_id"): pass elif values.get("immutable_id") and (values.get("refcode") or values.get("item_id")): - raise ValueError("Must provide only one of {id_fields!r}") + raise ValueError(f"Must provide only one of {id_fields!r}") return values + + model_config = ConfigDict(extra="allow") diff --git a/pydatalab/src/pydatalab/models/samples.py b/pydatalab/src/pydatalab/models/samples.py index c7c891f99..6293f367d 100644 --- a/pydatalab/src/pydatalab/models/samples.py +++ b/pydatalab/src/pydatalab/models/samples.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import Field from pydatalab.models.items import Item @@ -12,7 +14,7 @@ class Sample(Item, HasSynthesisInfo, HasSubstanceInfo): and connected to other items. """ - type: str = Field("samples", const="samples", pattern="^samples$") + type: Literal["samples"] = "samples" status: SampleStatus = Field(default=SampleStatus.ACTIVE) """The status of the sample, indicating its current state.""" diff --git a/pydatalab/src/pydatalab/models/starting_materials.py b/pydatalab/src/pydatalab/models/starting_materials.py index 08655a721..5167a1bb5 100644 --- a/pydatalab/src/pydatalab/models/starting_materials.py +++ b/pydatalab/src/pydatalab/models/starting_materials.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import Field from pydatalab.models.items import Item @@ -13,43 +15,39 @@ class StartingMaterial(Item, HasSynthesisInfo, HasSubstanceInfo): represent either depending on preference. """ - type: str = Field( - "starting_materials", const="starting_materials", pattern="^starting_materials$" - ) + type: Literal["starting_materials"] = "starting_materials" - barcode: str | None = Field( - alias="Barcode", - ) + barcode: str | None = Field(None, alias="Barcode") """A unique barcode provided by an external source, e.g., cheminventory.""" - date: IsoformatDateTime | None = Field(alias="Date Acquired") + date: IsoformatDateTime | None = Field(None, alias="Date Acquired") """The date the item was acquired""" - date_opened: IsoformatDateTime | None = Field(alias="Date opened") + date_opened: IsoformatDateTime | None = Field(None, alias="Date opened") """The date the item was opened""" chemical_purity: str | None = Field(alias="Chemical purity") """The chemical purity of this container with regards to the defined substance.""" - full_percent: str | None = Field(alias="Full %") + full_percent: str | None = Field(None, alias="Full %") """The amount of the defined substance remaining in the container, expressed as a percentage.""" name: str | None = Field(alias="Container Name") """The name of the substance in the container.""" - size: str | None = Field(alias="Container Size") + size: str | None = Field(None, alias="Container Size") """The total size of the container, in units of `size_unit`.""" - size_unit: str | None = Field(alias="Unit") + size_unit: str | None = Field(None, alias="Unit") """Units for the 'size' field.""" supplier: str | None = Field(alias="Supplier") """Supplier or manufacturer of the chemical.""" - location: str | None = Field(alias="Location") + location: str | None = Field(None, alias="Location") """The place where the container is located.""" - comment: str | None = Field(alias="Comments") + comment: str | None = Field(None, alias="Comments") """Any additional comments or notes about the container.""" status: StartingMaterialsStatus = Field(default=StartingMaterialsStatus.AVAILABLE) diff --git a/pydatalab/src/pydatalab/models/tasks.py b/pydatalab/src/pydatalab/models/tasks.py index fa64714c3..14728df26 100644 --- a/pydatalab/src/pydatalab/models/tasks.py +++ b/pydatalab/src/pydatalab/models/tasks.py @@ -2,9 +2,9 @@ from enum import Enum from typing import Literal -from pydantic import BaseModel, Field, validator +from pydantic import Field, validator -from pydatalab.models.utils import PyObjectId +from pydatalab.models.utils import BaseModel, PyObjectId class TaskStatus(str, Enum): @@ -20,12 +20,17 @@ class TaskType(str, Enum): class TaskStage(BaseModel): - timestamp: datetime = Field(..., description="When this stage occurred") - message: str = Field(..., description="Description of this processing stage") - level: Literal["info", "warning", "error"] = Field( - default="info", description="Severity level of this stage" - ) - detail: str | None = Field(None, description="Optional detailed information about this stage") + timestamp: datetime + """When this stage occurred""" + + message: str + """Description of this processing stage""" + + level: Literal["info", "warning", "error"] = "info" + """Severity level of this stage""" + + detail: str | None = None + """Optional detailed information about this stage""" class TaskSpec(BaseModel): @@ -33,35 +38,57 @@ class TaskSpec(BaseModel): class ExportTaskSpec(TaskSpec): - collection_id: str | None = Field(None, description="Collection ID being exported") - item_id: str | None = Field(None, description="Item ID being exported") - export_type: str = Field(..., description="Type of export: collection/item/graph") - file_path: str | None = Field(None, description="Path to generated .eln file") - stages: list[TaskStage] = Field( - default_factory=list, description="Timestamped processing stages" - ) + collection_id: str | None = None + """Collection ID being exported""" + + item_id: str | None = None + """Item ID being exported""" + + export_type: str + """Type of export: collection/item/graph""" + + file_path: str | None = None + """Path to generated .eln file""" + + stages: list[TaskStage] = Field(default_factory=list) + """Timestamped processing stages""" class BlockProcessingTaskSpec(TaskSpec): - item_id: str = Field(..., description="Item ID containing the block") - block_id: str = Field(..., description="Block ID being processed") - stages: list[TaskStage] = Field( - default_factory=list, description="Timestamped processing stages" - ) + item_id: str + """Item ID containing the block""" + + block_id: str + """Block ID being processed""" + + stages: list[TaskStage] = Field(default_factory=list) + """Timestamped processing stages""" class Task(BaseModel): - task_id: str = Field(..., description="Unique identifier for the task") - type: TaskType = Field(..., description="Type of task") - status: TaskStatus = Field(default=TaskStatus.PENDING, description="Current status") - creator_id: PyObjectId = Field(..., description="ID of the user who created the task") - created_at: datetime = Field( - default_factory=lambda: datetime.now(tz=timezone.utc), - description="When the task was created", - ) - completed_at: datetime | None = Field(None, description="When completed") - error_message: str | None = Field(None, description="Error message if status is ERROR") - spec: ExportTaskSpec | BlockProcessingTaskSpec = Field(..., description="Task-specific data") + task_id: str + """Unique identifier for the task""" + + type: TaskType + """Type of task""" + + status: TaskStatus = TaskStatus.PENDING + """Current status""" + + creator_id: PyObjectId + """ID of the user who created the task""" + + created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc)) + """When the task was created""" + + completed_at: datetime | None = None + """When completed""" + + error_message: str | None = None + """Error message if status is ERROR""" + + spec: ExportTaskSpec | BlockProcessingTaskSpec + """Task-specific data""" @validator("spec", pre=True, always=True) def validate_spec_type(cls, v, values): diff --git a/pydatalab/src/pydatalab/models/traits.py b/pydatalab/src/pydatalab/models/traits.py index c53ce5a07..e369a92c6 100644 --- a/pydatalab/src/pydatalab/models/traits.py +++ b/pydatalab/src/pydatalab/models/traits.py @@ -1,15 +1,19 @@ -from typing import Any +from typing import TYPE_CHECKING, Any -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import AliasChoices, ConfigDict, Field, field_validator, model_validator from pydatalab.models.blocks import DataBlockResponse from pydatalab.models.people import Group, Person -from pydatalab.models.utils import Constituent, InlineSubstance, PyObjectId +from pydatalab.models.utils import BaseModel, Constituent, InlineSubstance, PyObjectId + +if TYPE_CHECKING: + from pydatalab.models.files import File __all__ = ( "HasOwner", "HasRevisionControl", "HasBlocks", + "HasFiles", "IsCollectable", "HasSynthesisInfo", "HasSubstanceInfo", @@ -48,14 +52,10 @@ class HasRevisionControl(BaseModel): class HasFiles(BaseModel): """Trait mixin for models that can have files attached to them.""" - # Imported here rather than at module level to avoid a circular import, - # as `models.files` itself depends on this module. - from pydatalab.models.files import File - - files: list[File] | None = Field(None) + files: list["File"] | None = None """Any files attached to this item.""" - file_ObjectIds: list[PyObjectId] = Field([]) + file_ObjectIds: list[PyObjectId] = Field(default_factory=list) """Links to object IDs of files stored within the database.""" @@ -69,34 +69,36 @@ class HasBlocks(BaseModel): """The order in which to display block data in the UI.""" +class CollectionReference(BaseModel): + """A reference to a collection, used for inlining collection info within other models.""" + + model_config = ConfigDict(extra="allow") + + collection_id: str + """The immutable ID of the collection.""" + + class IsCollectable(BaseModel): """Trait mixin for models that can be added to collections. """ - from pydatalab.models.collections import Collection - - collections: list[Collection] = Field([]) + collections: list[CollectionReference] = Field(default_factory=list) """Inlined info for the collections associated with this item.""" - @model_validator(mode="before") - @classmethod - def add_missing_collection_relationships(cls, values): + @model_validator(mode="after") + def add_missing_collection_relationships(self): from pydatalab.models.relationships import TypedRelationship - if values.get("collections") is not None: - new_ids = {coll.immutable_id for coll in values["collections"]} - existing_collection_relationship_ids = set() - if values.get("relationships") is not None: - existing_collection_relationship_ids = { - relationship.immutable_id - for relationship in values["relationships"] - if relationship.type == "collections" - } - else: - values["relationships"] = [] - - for collection in values.get("collections", []): + if self.collections is not None: + new_ids = {coll.immutable_id for coll in self.collections} + existing_collection_relationship_ids = { + relationship.immutable_id + for relationship in self.relationships + if relationship.type == "collections" + } + + for collection in self.collections: if collection.immutable_id not in existing_collection_relationship_ids: relationship = TypedRelationship( relation=None, @@ -104,20 +106,18 @@ def add_missing_collection_relationships(cls, values): type="collections", description="Is a member of", ) - values["relationships"].append(relationship) + self.relationships.append(relationship) - values["relationships"] = [ + self.relationships = [ d - for d in values.get("relationships", []) + for d in self.relationships if d.type != "collections" or d.immutable_id in new_ids ] - if len([d for d in values.get("relationships", []) if d.type == "collections"]) != len( - values.get("collections", []) - ): + if len([d for d in self.relationships if d.type == "collections"]) != len(self.collections): raise RuntimeError("Relationships and collections mismatch") - return values + return self class HasSynthesisInfo(BaseModel): @@ -129,29 +129,24 @@ class HasSynthesisInfo(BaseModel): synthesis_description: str | None = None """Free-text details of the procedure applied to synthesise the sample""" - @model_validator(mode="before") - @classmethod - def add_missing_synthesis_relationships(cls, values): + @model_validator(mode="after") + def add_missing_synthesis_relationships(self): """Add any missing sample synthesis constituents to parent relationships""" from pydatalab.models.relationships import RelationshipType, TypedRelationship constituents_set = set() - if values.get("synthesis_constituents") is not None: - existing_parent_relationships = {} - if values.get("relationships") is not None: - # Index by refcode *and* item_id so a stored relationship carrying an - # item_id still matches a refcode-enriched constituent (and vice-versa). - existing_parent_relationships = { - identifier: relationship - for relationship in values["relationships"] - if relationship.relation == RelationshipType.PARENT - for identifier in (relationship.refcode, relationship.item_id) - if identifier - } - else: - values["relationships"] = [] - - for constituent in values.get("synthesis_constituents", []): + if self.synthesis_constituents is not None: + # Index by refcode *and* item_id so a stored relationship carrying an + # item_id still matches a refcode-enriched constituent (and vice-versa). + existing_parent_relationships = { + identifier: relationship + for relationship in self.relationships + if relationship.relation == RelationshipType.PARENT + for identifier in (relationship.refcode, relationship.item_id) + if identifier + } + + for constituent in self.synthesis_constituents: # If this is an inline relationship, just skip it if isinstance(constituent.item, InlineSubstance): continue @@ -170,7 +165,7 @@ def add_missing_synthesis_relationships(cls, values): type=constituent.item.type, description="Is a constituent of", ) - values["relationships"].append(relationship) + self.relationships.append(relationship) else: # Back-fill any identifier missing from the stored relationship relationship.refcode = relationship.refcode or refcode @@ -188,9 +183,9 @@ def add_missing_synthesis_relationships(cls, values): # Finally, filter out any parent relationships with item that were removed # from the synthesis constituents - values["relationships"] = [ + self.relationships = [ rel - for rel in values["relationships"] + for rel in self.relationships if not ( rel.refcode not in constituents_set and rel.item_id not in constituents_set @@ -199,14 +194,15 @@ def add_missing_synthesis_relationships(cls, values): ) ] - return values + return self class HasSubstanceInfo(BaseModel): """Trait mixin for models that have substance information.""" chemform: str | None = Field( - example=["Na3P", "Na3P", "LiNiO2@C", "Na3+xP", "LiNi1/3Co0.1Mn0.1O2"], + None, + examples=["Na3P", "Na3P", "LiNiO2@C", "Na3+xP", "LiNi1/3Co0.1Mn0.1O2"], ) """A string representation of the chemical formula or composition associated with this sample. @@ -215,7 +211,9 @@ class HasSubstanceInfo(BaseModel): """ - smiles: str | None = Field(None, aliases=["SMILES", "smiles_representation"]) + smiles: str | None = Field( + None, validation_alias=AliasChoices("smiles_representation", "SMILES") + ) """A SMILES string representation of the chemical structure associated with this sample.""" inchi: str | None = Field(None) @@ -225,25 +223,28 @@ class HasSubstanceInfo(BaseModel): """A unique key derived from the InChI.""" GHS_codes: str | None = Field( + None, alias="GHS H-codes", examples=["H224", "H303, H316, H319"], ) """A string describing any GHS hazard codes associated with this item. See https://pubchem.ncbi.nlm.nih.gov/ghs/ for code definitions.""" - molar_mass: float | None = Field(alias="Molecular Weight") + molar_mass: float | None = Field(None, alias="Molecular Weight", validate_default=True) """Mass per formula unit, in g/mol.""" - CAS: str | None = Field(alias="Substance CAS") + CAS: str | None = Field(None, alias="Substance CAS") """The CAS Registry Number for the substance described by this entry.""" @field_validator("molar_mass", mode="before") @classmethod - def add_molar_mass(cls, v, values): + def add_molar_mass(cls, v, info): + """Fill in the molar mass if not already set and a chemical formula is provided.""" from periodictable import formula - if v is None and values.get("chemform"): + chemform = info.data.get("chemform") + if v is None and chemform: try: - return formula(values.get("chemform")).mass + return formula(chemform).mass except Exception: return None diff --git a/pydatalab/src/pydatalab/models/utils.py b/pydatalab/src/pydatalab/models/utils.py index 58230b184..96df03119 100644 --- a/pydatalab/src/pydatalab/models/utils.py +++ b/pydatalab/src/pydatalab/models/utils.py @@ -4,18 +4,30 @@ from collections.abc import Callable from enum import Enum from functools import partial -from typing import TypeAlias +from typing import Annotated, Any, TypeAlias import pint -from bson.objectid import ObjectId +from bson import ObjectId from pydantic import ( - BaseModel, - ConstrainedStr, + BaseModel as PydanticBaseModel, +) +from pydantic import ( + ConfigDict, Field, - parse_obj_as, - root_validator, - validator, + StringConstraints, + field_validator, + model_validator, ) +from pydantic_core import core_schema + + +class BaseModel(PydanticBaseModel): + """The base model for all pydatalab models, configured to pull field + descriptions from attribute docstrings into the generated JSON schema. + + """ + + model_config = ConfigDict(use_attribute_docstrings=True) class ItemType(str, Enum): @@ -92,43 +104,30 @@ class KnownType(str, Enum): """ -class HumanReadableIdentifier(ConstrainedStr): - """Used to constrain human-readable and URL-safe identifiers for items.""" - - min_length = 1 - max_length = 40 - strip_whitespace = True - to_lower = False - strict = False - regex = IDENTIFIER_REGEX - - def __init__(self, value): - self.value = parse_obj_as(type(self), value) - - def __str__(self): - return self.value - - def __repr__(self): - return self.value - - def __bool__(self): - return bool(self.value) - - -class Refcode(HumanReadableIdentifier): - regex = r"^[a-z]{2,10}:" + IDENTIFIER_REGEX[1:] - """A regex to match refcodes that have a lower-case prefix between 2-10 chars, followed by a colon, - and then the normal rules for an ID (url-safe etc.). - - """ - - @property - def prefix(self): - return self.value.split(":")[0] - - @property - def identifier(self): - return self.value.split(":")[1] +HumanReadableIdentifier = Annotated[ + str, + StringConstraints( + min_length=1, + max_length=40, + strip_whitespace=True, + to_lower=False, + strict=False, + pattern=IDENTIFIER_REGEX, + ), +] +"""Used to constrain human-readable and URL-safe identifiers for items.""" + + +Refcode = Annotated[ + str, + StringConstraints( + min_length=1, + max_length=40, + strip_whitespace=True, + pattern=r"^[a-z]{2,10}:" + IDENTIFIER_REGEX[1:], + ), +] +"""A regex to match refcodes that have a lower-case prefix between 2-10 chars, followed by a colon, and then the normal rules for an ID (url-safe etc.).""" class UserRole(str, Enum): @@ -149,8 +148,12 @@ def __init__(self, dimensions: str): self._dimensions = dimensions @classmethod - def __get_validators__(self): - yield self.validate + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls.validate, + core_schema.str_schema(), + serialization=core_schema.plain_serializer_function_ser_schema(str, when_used="json"), + ) @classmethod def validate(self, v): @@ -159,77 +162,120 @@ def validate(self, v): raise ValueError("Value {v} must have dimensions of mass, not {v.dimensions}") return q - @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="string") - Mass: TypeAlias = PintType("[mass]") # type: ignore # noqa Volume: TypeAlias = PintType("[volume]") # type: ignore # noqa class PyObjectId(ObjectId): - """A wrapper class for a BSON ObjectId that can be used as a Pydantic field type. - - Modified from "Getting started iwth MongoDB and FastAPI": - https://www.mongodb.com/developer/languages/python/python-quickstart-fastapi/. - - """ + @classmethod + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls.validate, + core_schema.union_schema( + [ + core_schema.str_schema(), + core_schema.is_instance_schema(ObjectId), + core_schema.is_instance_schema(cls), + core_schema.dict_schema(), + core_schema.none_schema(), + ] + ), + serialization=core_schema.plain_serializer_function_ser_schema( + lambda x: str(x) if x else None, when_used="json" + ), + ) @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_json_schema__(cls, core_schema, handler): + return { + "type": "string", + "format": "objectid", + } @classmethod def validate(cls, v): - if isinstance(v, dict) and "$oid" in v: - v = v["$oid"] + if v is None: + return None + if isinstance(v, cls): + return v + if isinstance(v, ObjectId): + return cls(v) - if not ObjectId.is_valid(v): - raise ValueError("Invalid ObjectId") + if isinstance(v, dict): + if "$oid" in v: + return cls(ObjectId(v["$oid"])) + elif "_id" in v and isinstance(v["_id"], (str, ObjectId)): + return cls(ObjectId(v["_id"])) + elif len(v) == 1: + first_val = next(iter(v.values())) + if isinstance(first_val, str) and ObjectId.is_valid(first_val): + return cls(ObjectId(first_val)) + raise ValueError(f"Cannot convert dict to ObjectId: {v}") - return ObjectId(v) + if isinstance(v, str): + if not ObjectId.is_valid(v): + raise ValueError("Invalid ObjectId string") + return cls(ObjectId(v)) - @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="string") + raise ValueError(f"Cannot convert {type(v)} to ObjectId: {v}") class IsoformatDateTime(datetime.datetime): """A datetime container that is more flexible than the pydantic default.""" @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls.validate, + core_schema.union_schema( + [ + core_schema.str_schema(), + core_schema.is_instance_schema(datetime.datetime), + ] + ), + serialization=core_schema.plain_serializer_function_ser_schema( + lambda x: x.isoformat() if x else None, when_used="json" + ), + ) + + @classmethod + def __get_pydantic_json_schema__(cls, core_schema, handler): + return { + "type": "string", + "format": "datetime", + } @classmethod def validate(cls, v) -> datetime.datetime | None: """Cast isoformat strings to datetimes and enforce UTC if tzinfo is missing.""" + if v is None: + return None + + if isinstance(v, datetime.datetime): + if v.tzinfo is None: + v = v.replace(tzinfo=datetime.timezone.utc) + return v + if isinstance(v, str): - if v in ["0", " "]: + if v in ["0", " ", ""]: return None v = datetime.datetime.fromisoformat(v) + if v.tzinfo is None: + v = v.replace(tzinfo=datetime.timezone.utc) + return v - if v.tzinfo is None: - v = v.replace(tzinfo=datetime.timezone.utc) - - return v - - -JSON_ENCODERS = { - pint.Quantity: str, - ObjectId: str, -} + raise ValueError(f"Invalid datetime value: {v}") class RefCodeFactory: refcode_generator: Callable @classmethod - def generate(self): + def generate(cls): from pydatalab.config import CONFIG - return f"{CONFIG.IDENTIFIER_PREFIX}:{self.refcode_generator()}" + return f"{CONFIG.IDENTIFIER_PREFIX}:{cls.refcode_generator()}" def random_uppercase(length: int = 6): @@ -257,7 +303,7 @@ def generate_unique_refcode(): class InlineSubstance(BaseModel): name: str - chemform: str | None + chemform: str | None = None class EntryReference(BaseModel): @@ -269,15 +315,19 @@ class EntryReference(BaseModel): """ type: str - name: str | None - immutable_id: PyObjectId | None - item_id: HumanReadableIdentifier | None - refcode: Refcode | None - chemform: str | None + name: str | None = None + immutable_id: PyObjectId | None = None + item_id: HumanReadableIdentifier | None = None + refcode: Refcode | None = None + chemform: str | None = None - @root_validator + @model_validator(mode="before") + @classmethod def check_id_fields(cls, values): """Check that at least one of the possible identifier fields is provided.""" + if not isinstance(values, dict): + return values + id_fields = ("immutable_id", "item_id", "refcode") if all(values.get(f) is None for f in id_fields): @@ -285,8 +335,7 @@ def check_id_fields(cls, values): return values - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class Constituent(BaseModel): @@ -295,23 +344,22 @@ class Constituent(BaseModel): item: EntryReference | InlineSubstance """A reference to item (sample or starting material) entry for the constituent substance.""" - quantity: float | None = Field(..., ge=0) + quantity: float | None = Field(default=None, ge=0) """The amount of the constituent material used to create the sample.""" - unit: str = Field("g") - """The unit symbol for the value provided in `quantity`, default is mass - in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol). - """ + unit: str = "g" + """The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).""" - @validator("item") + @field_validator("item") + @classmethod def check_itemhood(cls, v): """Check that the reference within the constituent is to an item type.""" - if "type" in (v.value for v in ItemType): - raise ValueError(f"`type` must be one of {ItemType!r}") - + if hasattr(v, "type") and v.type not in [item_type.value for item_type in ItemType]: + raise ValueError(f"`type` must be one of {[t.value for t in ItemType]!r}") return v - @validator("item", pre=True, always=True) + @field_validator("item", mode="before") + @classmethod def coerce_reference(cls, v): if isinstance(v, dict): refcode = v.pop("refcode", None) @@ -327,4 +375,18 @@ def coerce_reference(cls, v): if not name: raise ValueError("Inline substance must have a name!") return InlineSubstance(name=name, chemform=chemform) + elif hasattr(v, "model_dump"): + item_id = getattr(v, "item_id", None) + refcode = getattr(v, "refcode", None) + item_type = getattr(v, "type", None) + name = getattr(v, "name", None) + chemform = getattr(v, "chemform", None) + + if item_id or refcode: + return EntryReference( + item_id=item_id, refcode=refcode, type=item_type, name=name, chemform=chemform + ) + else: + return InlineSubstance(name=name or str(v), chemform=chemform) + return v diff --git a/pydatalab/src/pydatalab/models/versions.py b/pydatalab/src/pydatalab/models/versions.py index cb2644932..318561fe8 100644 --- a/pydatalab/src/pydatalab/models/versions.py +++ b/pydatalab/src/pydatalab/models/versions.py @@ -3,9 +3,9 @@ from datetime import datetime from enum import Enum -from pydantic import BaseModel, Field, validator +from pydantic import Field, validator -from pydatalab.models.utils import PyObjectId, Refcode +from pydatalab.models.utils import BaseModel, PyObjectId, Refcode class VersionAction(str, Enum): @@ -27,30 +27,35 @@ class ItemVersion(BaseModel): view history and restore previous states. """ - refcode: Refcode = Field(..., description="The refcode of the item this version belongs to") - version: int = Field(..., ge=1, description="Sequential version number (1-indexed)") - timestamp: datetime = Field( - ..., description="When this version was created (ISO format with timezone)" - ) - action: VersionAction = Field(..., description="The action that triggered this version.") - user_id: PyObjectId | None = Field( - None, description="User's ObjectId for efficient querying and indexing" - ) - creator: dict | None = Field( - None, description="Inlined information about the user who created this version (e.g., name)" - ) - datalab_version: str = Field( - ..., description="Version of datalab-server that created this snapshot" - ) - data: dict = Field(..., description="Complete snapshot of the item data at this version") - restored_from_version: PyObjectId | None = Field( - None, - description="ObjectId of the version that was restored from (only present if action='restored')", - ) - user_agent: str | None = Field( - None, - description="User agent string of the client that triggered this version. Will only be stored if it matches a known value from the datalab ecosystem.", - ) + refcode: Refcode + """The refcode of the item this version belongs to""" + + version: int = Field(ge=1) + """Sequential version number (1-indexed)""" + + timestamp: datetime + """When this version was created (ISO format with timezone)""" + + action: VersionAction + """The action that triggered this version: 'created' (item creation), 'manual_save' (user save), 'auto_save' (system save), or 'restored' (version restore)""" + + user_id: PyObjectId | None = None + """User's ObjectId for efficient querying and indexing""" + + creator: dict | None = None + """Inlined information about the user who created this version (e.g., name)""" + + datalab_version: str + """Version of datalab-server that created this snapshot""" + + data: dict + """Complete snapshot of the item data at this version""" + + restored_from_version: PyObjectId | None = None + """ObjectId of the version that was restored from (only present if action='restored')""" + + user_agent: str | None = None + """User agent string of the client that triggered this version. Will only be stored if it matches a known value from the datalab ecosystem.""" @validator("restored_from_version") def validate_restored_from_version(cls, v, values): @@ -72,10 +77,11 @@ class VersionCounter(BaseModel): It ensures atomic increment of version numbers to prevent race conditions. """ - refcode: Refcode = Field(..., description="The refcode this counter belongs to") - counter: int = Field( - 1, ge=1, description="Current version counter value (1-indexed, matches version numbers)" - ) + refcode: Refcode + """The refcode this counter belongs to""" + + counter: int = Field(1, ge=1) + """Current version counter value (1-indexed, matches version numbers)""" class Config: extra = "ignore" # Allow MongoDB's _id field and other internal fields @@ -84,7 +90,8 @@ class Config: class RestoreVersionRequest(BaseModel): """Request body for restoring a version.""" - version_id: str = Field(..., description="ObjectId string of the version to restore to") + version_id: str + """ObjectId string of the version to restore to""" @validator("version_id") def validate_version_id_format(cls, v): @@ -104,8 +111,11 @@ class Config: class CompareVersionsQuery(BaseModel): """Query parameters for comparing two versions.""" - v1: str = Field(..., description="ObjectId string of the first version") - v2: str = Field(..., description="ObjectId string of the second version") + v1: str + """ObjectId string of the first version""" + + v2: str + """ObjectId string of the second version""" @validator("v1", "v2") def validate_version_ids(cls, v): diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index 6ba1387be..be3868b45 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -33,7 +33,7 @@ def load_plugin_schema(): need them. A JSON Schema generated from the returned model is emitted to `pydatalab/schemas/plugin_config.json` by `invoke dev.generate-schemas`. """ - from pydantic import BaseModel, root_validator + from pydantic import BaseModel, model_validator class UvSource(BaseModel): """A single entry under `[tool.uv.sources]` in plugins.toml.""" @@ -48,7 +48,8 @@ class UvSource(BaseModel): class Config: extra = "forbid" - @root_validator + @model_validator(mode="before") + @classmethod def _exactly_one_source(cls, values): has_git = values.get("git") is not None has_path = values.get("path") is not None @@ -79,7 +80,8 @@ class PluginConfigModel(BaseModel): class Config: extra = "forbid" - @root_validator + @model_validator(mode="before") + @classmethod def _sources_must_match_dependencies(cls, values): deps = {d.split("[")[0].strip() for d in values.get("dependencies", [])} sources = values.get("tool", ToolSection()).uv.sources diff --git a/pydatalab/tests/test_models.py b/pydatalab/tests/test_models.py index 1e59bfbce..c2ba5959b 100644 --- a/pydatalab/tests/test_models.py +++ b/pydatalab/tests/test_models.py @@ -84,6 +84,27 @@ def test_generate_schemas(model): assert model.schema() +def test_attribute_docstrings_in_schema(): + """Field docstrings should be pulled into the JSON schema as descriptions. + + Relies on ``use_attribute_docstrings`` being set on the shared ``BaseModel`` + in ``pydatalab.models.utils``; covers fields defined across several different + trait mixins to confirm the config propagates through inheritance. + """ + properties = Sample.model_json_schema(by_alias=False)["properties"] + + expected = { + "creators": "Inlined info for the people associated with this item.", # HasOwner + "blocks_obj": "A mapping from block ID to block data.", # HasBlocks + "revision": "The revision number of the entry.", # HasRevisionControl + "collections": "Inlined info for the collections associated with this item.", # IsCollectable + "name": "An optional human-readable/usable name for the entry.", # Item + } + + for field, description in expected.items(): + assert properties[field]["description"] == description + + def test_relationship_with_custom_type(): """Test that a relationship with a custom type can be created.""" relationship = TypedRelationship( From 86134fdf9de7545abdae1e4c9195ef77cdc0dd8b Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Tue, 11 Nov 2025 01:08:22 +0000 Subject: [PATCH 06/40] Load FTS fields for MongoDB from model schemas directly Rework ITEMS_FTS_FIELDS lookup --- pydatalab/src/pydatalab/mongo.py | 60 ++++++++++++++------ pydatalab/src/pydatalab/routes/v0_1/items.py | 4 +- pydatalab/tests/server/test_items.py | 4 +- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/pydatalab/src/pydatalab/mongo.py b/pydatalab/src/pydatalab/mongo.py index b69b51451..a8b7334af 100644 --- a/pydatalab/src/pydatalab/mongo.py +++ b/pydatalab/src/pydatalab/mongo.py @@ -10,7 +10,6 @@ from pymongo.errors import ConnectionFailure from pydatalab.logger import LOGGER -from pydatalab.models import ITEM_MODELS __all__ = ( "flask_mongo", @@ -34,20 +33,40 @@ """One-liner that pulls all non-semantic string fields out of all item models implemented for this server. """ -ITEMS_FTS_FIELDS: set[str] = set().union( - *( - { - f - for f, p in model.schema(by_alias=False)["properties"].items() - if ( - p.get("type") == "string" - and p.get("format") not in ("date-time", "uuid") - and f != "type" - ) - } - for model in ITEM_MODELS.values() - ) -) + + +@lru_cache(maxsize=1) +def get_items_fts_fields() -> set[str]: + """Get all string fields from item models for full-text search.""" + from pydatalab.models import ITEM_MODELS + + fields = set() + + for model_name, model in ITEM_MODELS.items(): + schema = model.model_json_schema(by_alias=False) + + model_fields = set() + for f, p in schema.get("properties", {}).items(): + if f == "type": + continue + + if p.get("type") == "string" and p.get("format") not in ("date-time", "uuid"): + model_fields.add(f) + elif "anyOf" in p: + for option in p["anyOf"]: + if option.get("type") == "string" and option.get("format") not in ( + "date-time", + "uuid", + ): + model_fields.add(f) + break + + fields.update(model_fields) + + return fields + + +ITEMS_FTS_FIELDS: set[str] = set() USERS_FTS_FIELDS: set[str] = {"identities.name", "display_name", "contact_email"} """Fields to search for users.""" @@ -148,7 +167,7 @@ def insert_pydantic_model_fork_safe(model: BaseModel, collection: str) -> str: """Inserts a Pydantic model into chosen collection, returning the inserted ID.""" return ( get_database()[collection] - .insert_one(model.dict(by_alias=True, exclude_none=True)) + .insert_one(model.model_dump(by_alias=True, exclude_none=True)) .inserted_id ) @@ -167,7 +186,6 @@ def _get_active_mongo_client(timeoutMS: int = 1000) -> pymongo.MongoClient: """ from pydatalab.config import CONFIG - from pydatalab.logger import LOGGER try: client = pymongo.MongoClient( @@ -235,6 +253,14 @@ def create_default_indices( """ + global ITEMS_FTS_FIELDS + + if not ITEMS_FTS_FIELDS: + ITEMS_FTS_FIELDS = get_items_fts_fields() + + if not ITEMS_FTS_FIELDS: + raise ValueError("Cannot create text indices: no fields available for full-text search") + if client is None: client = _get_active_mongo_client() db = client.get_database() diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index eb14dd5ae..c469190fd 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -25,7 +25,7 @@ RestoreVersionRequest, VersionAction, ) -from pydatalab.mongo import ITEMS_FTS_FIELDS, build_search_pipeline, flask_mongo +from pydatalab.mongo import build_search_pipeline, flask_mongo, get_items_fts_fields from pydatalab.permissions import ( PUBLIC_USER_ID, AccessToken, @@ -528,7 +528,7 @@ def search_items(): return jsonify({"status": "error", "message": "No query provided."}), 400 permissions = get_default_permissions(user_only=False) - pipeline = build_search_pipeline(query, ITEMS_FTS_FIELDS, permissions) + pipeline = build_search_pipeline(query, get_items_fts_fields(), permissions) if types is not None: if pipeline and "$match" in pipeline[0]: diff --git a/pydatalab/tests/server/test_items.py b/pydatalab/tests/server/test_items.py index e58bafd2d..9c9ad1598 100644 --- a/pydatalab/tests/server/test_items.py +++ b/pydatalab/tests/server/test_items.py @@ -22,7 +22,7 @@ def test_single_item_endpoints(client, inserted_default_items): def test_fts_fields(): """Test non-exhaustively that certain fields make it into the fts index.""" - from pydatalab.mongo import ITEMS_FTS_FIELDS + from pydatalab.mongo import get_items_fts_fields fields = ("item_id", "name", "description", "refcode", "synthesis_description", "supplier") - assert all(field in ITEMS_FTS_FIELDS for field in fields) + assert all(field in get_items_fts_fields() for field in fields) From f06642fc33b6300c846d1a368bc36d87e1aaf861 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 11 Nov 2025 01:09:59 +0000 Subject: [PATCH 07/40] Misc pydantic v2 bumps around code --- pydatalab/src/pydatalab/backups.py | 2 +- pydatalab/src/pydatalab/errors.py | 1 + pydatalab/src/pydatalab/file_utils.py | 10 +++++----- pydatalab/src/pydatalab/main.py | 4 ++-- pydatalab/tasks.py | 23 +++++------------------ 5 files changed, 14 insertions(+), 26 deletions(-) diff --git a/pydatalab/src/pydatalab/backups.py b/pydatalab/src/pydatalab/backups.py index bea81e84d..f058a62a6 100644 --- a/pydatalab/src/pydatalab/backups.py +++ b/pydatalab/src/pydatalab/backups.py @@ -92,7 +92,7 @@ def take_snapshot(snapshot_path: Path, encrypt: bool = False) -> None: LOGGER.debug("Dumping server config.") with tempfile.TemporaryDirectory() as temp_dir: with open(tmp_config := Path(temp_dir) / "config.json", "w") as f: - data = CONFIG.json(indent=2, exclude_unset=True) + data = CONFIG.model_dump_json(indent=2, exclude_unset=True) f.write(data) tar.add( diff --git a/pydatalab/src/pydatalab/errors.py b/pydatalab/src/pydatalab/errors.py index 5868d9124..0996a54ad 100644 --- a/pydatalab/src/pydatalab/errors.py +++ b/pydatalab/src/pydatalab/errors.py @@ -70,6 +70,7 @@ def handle_pydantic_validation_error(exc: ValidationError) -> tuple[Response, in These always come from malformed data, so should not necessarily trigger the Flask debugger. """ + LOGGER.critical("Pydantic validation error: %s", exc, exc_info=True) response = { "title": exc.__class__.__name__, "message": str(exc.args[:]) if exc.args else "", diff --git a/pydatalab/src/pydatalab/file_utils.py b/pydatalab/src/pydatalab/file_utils.py index 33bb8958b..5bda4eabc 100644 --- a/pydatalab/src/pydatalab/file_utils.py +++ b/pydatalab/src/pydatalab/file_utils.py @@ -336,7 +336,7 @@ def get_file_info_by_id(file_id: str | ObjectId, update_if_live: bool = True) -> if update_if_live and file_info.is_live: file_info = _check_and_sync_file(file_info, file_id) - return file_info.dict() + return file_info.model_dump() def update_uploaded_file(file: FileStorage, file_id: ObjectId, size_bytes: int | None = None): @@ -405,7 +405,7 @@ def update_uploaded_file(file: FileStorage, file_id: ObjectId, size_bytes: int | file.save(updated_file_entry.location) - ret = updated_file_entry.dict() + ret = updated_file_entry.model_dump() ret.update({"_id": file_id}) return ret @@ -498,7 +498,7 @@ def save_uploaded_file( f"Cannot store file: insufficient space available on disk (required: {size_bytes // 1024**3} GB). Please contact your datalab administrator." ) file_collection = client.get_database().files - result = file_collection.insert_one(new_file_document.dict(), session=session) + result = file_collection.insert_one(new_file_document.model_dump(), session=session) if not result.acknowledged: raise RuntimeError( f"db operation failed when trying to insert new file. Result: {result}" @@ -534,7 +534,7 @@ def save_uploaded_file( f"db operation failed when trying to insert new file ObjectId into sample: {item_id}" ) - ret = updated_file_entry.dict() + ret = updated_file_entry.model_dump() ret.update({"_id": inserted_id}) return ret @@ -610,7 +610,7 @@ def add_file_from_remote_directory( creator_ids=creator_ids if creator_ids is not None else [], ) - result = file_collection.insert_one(new_file_document.dict()) + result = file_collection.insert_one(new_file_document.model_dump()) if not result.acknowledged: raise OSError(f"db operation failed when trying to insert new file. Result: {result}") diff --git a/pydatalab/src/pydatalab/main.py b/pydatalab/src/pydatalab/main.py index b4e7cd131..1bfb0f518 100644 --- a/pydatalab/src/pydatalab/main.py +++ b/pydatalab/src/pydatalab/main.py @@ -44,7 +44,7 @@ def create_app( CONFIG.update(config_override) app.config.from_prefixed_env() - app.config.update(CONFIG.dict()) + app.config.update(CONFIG.model_dump()) # This value will still be overwritten by any dotenv values app.config["MAIL_DEBUG"] = app.config.get("MAIL_DEBUG") or CONFIG.TESTING @@ -52,7 +52,7 @@ def create_app( # percolate datalab mail settings up to the `MAIL_` env vars/app config # for use by Flask Mail if CONFIG.EMAIL_AUTH_SMTP_SETTINGS is not None: - mail_settings = CONFIG.EMAIL_AUTH_SMTP_SETTINGS.dict() + mail_settings = CONFIG.EMAIL_AUTH_SMTP_SETTINGS.model_dump() for key in mail_settings: app.config[key] = mail_settings[key] diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index be3868b45..aeedcd452 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -5,13 +5,11 @@ import shutil import subprocess import time -import typing import tomlkit from invoke import Collection, task -if typing.TYPE_CHECKING: - from pydatalab.models.utils import UserRole +from pydatalab.models.utils import UserRole ns = Collection() dev = Collection("dev") @@ -45,9 +43,6 @@ class UvSource(BaseModel): path: str | None = None editable: bool | None = None - class Config: - extra = "forbid" - @model_validator(mode="before") @classmethod def _exactly_one_source(cls, values): @@ -62,24 +57,15 @@ def _exactly_one_source(cls, values): class UvSection(BaseModel): sources: dict[str, UvSource] = {} - class Config: - extra = "forbid" - class ToolSection(BaseModel): uv: UvSection = UvSection() - class Config: - extra = "forbid" - class PluginConfigModel(BaseModel): """The schema for the top-level plugins.toml file.""" dependencies: list[str] = [] tool: ToolSection = ToolSection() - class Config: - extra = "forbid" - @model_validator(mode="before") @classmethod def _sources_must_match_dependencies(cls, values): @@ -117,7 +103,7 @@ def generate_schemas(_): schemas_path = pathlib.Path(__file__).parent / "schemas" for model in ITEM_MODELS.values(): - schema = model.schema(by_alias=False) + schema = model.model_json_schema(by_alias=False) with open(schemas_path / f"{model.__name__.lower()}.json", "w") as f: json.dump(schema, f, indent=2) @@ -429,11 +415,12 @@ def add_missing_refcodes(_): def _check_id(id=None, base_url=None, api_key=None): - from pydatalab.logger import setup_log - """Checks the given item ID served at the base URL and logs the result.""" + import requests + from pydatalab.logger import setup_log + log = setup_log("check_item_validity") response = requests.get( f"{base_url}/get-item-data/{id}", headers={"DATALAB-API-KEY": api_key}, timeout=30 From dd1b28ead3f8adb30fc7f966fb3b14d69b6280a8 Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Tue, 11 Nov 2025 01:12:51 +0000 Subject: [PATCH 08/40] Update various routes for pydantic v2 Fixes for routes Chatblock patch; set default values in block More updates for blocks --- pydatalab/src/pydatalab/apps/chat/blocks.py | 15 +-- pydatalab/src/pydatalab/apps/chat/models.py | 2 +- pydatalab/src/pydatalab/apps/nmr/models.py | 5 +- pydatalab/src/pydatalab/apps/xrd/blocks.py | 2 +- pydatalab/src/pydatalab/blocks/base.py | 6 +- pydatalab/src/pydatalab/routes/v0_1/auth.py | 8 +- .../src/pydatalab/routes/v0_1/collections.py | 10 +- pydatalab/src/pydatalab/routes/v0_1/graphs.py | 4 +- pydatalab/src/pydatalab/routes/v0_1/info.py | 108 +++++++++--------- pydatalab/src/pydatalab/routes/v0_1/items.py | 72 +++++++++--- .../src/pydatalab/routes/v0_1/remotes.py | 6 +- pydatalab/src/pydatalab/routes/v0_1/users.py | 35 +++--- 12 files changed, 159 insertions(+), 114 deletions(-) diff --git a/pydatalab/src/pydatalab/apps/chat/blocks.py b/pydatalab/src/pydatalab/apps/chat/blocks.py index c2bce7df3..b95294faa 100644 --- a/pydatalab/src/pydatalab/apps/chat/blocks.py +++ b/pydatalab/src/pydatalab/apps/chat/blocks.py @@ -2,7 +2,7 @@ import warnings from langchain_core.messages import AIMessage, HumanMessage, SystemMessage -from pydantic import Field, validator +from pydantic import Field, field_validator from pydatalab.blocks.base import DataBlock from pydatalab.models import ITEM_MODELS @@ -16,15 +16,16 @@ class ChatBlockResponse(DataBlockResponse): messages: list[dict] = Field(default_factory=list) - prompt: str | None + prompt: str | None = None model: str available_models: dict[str, ModelCard] | None = Field( - datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, validate_default=True, datalab_exclude_from_db=True, datalab_exclude_from_load=True ) - token_count: int | None + token_count: int | None = None temperature: float - @validator("available_models", pre=True, always=True) + @field_validator("available_models", mode="before") + @classmethod def set_available_models(cls, _): return AVAILABLE_MODELS @@ -204,8 +205,8 @@ def _prepare_item_json_for_chat(self, item_id: str, item_data: dict | None = Non item_model.blocks_obj = { k: block for k, block in item_model.blocks_obj.items() if block.blocktype != "chat" } - item_data = item_model.dict(exclude_none=True, exclude_unset=True) - item_data["type"] = item_model.type + item_info = item_model.model_dump(exclude_none=True, exclude_unset=True) + item_info["type"] = item_model.type # strip irrelevant or large fields item_filenames = { diff --git a/pydatalab/src/pydatalab/apps/chat/models.py b/pydatalab/src/pydatalab/apps/chat/models.py index 03efc7246..756ca1970 100644 --- a/pydatalab/src/pydatalab/apps/chat/models.py +++ b/pydatalab/src/pydatalab/apps/chat/models.py @@ -9,7 +9,7 @@ class ModelCard(BaseModel): context_window: int input_cost_usd_per_MTok: float output_cost_usd_per_MTok: float - chat_client: type[BaseChatModel] | None = Field(exclude=True) + chat_client: type[BaseChatModel] | None = Field(None, exclude=True) __all__ = ("AVAILABLE_MODELS", "ModelCard") diff --git a/pydatalab/src/pydatalab/apps/nmr/models.py b/pydatalab/src/pydatalab/apps/nmr/models.py index 269684c26..73a981b4a 100644 --- a/pydatalab/src/pydatalab/apps/nmr/models.py +++ b/pydatalab/src/pydatalab/apps/nmr/models.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from pydatalab.models.blocks import DataBlockResponse @@ -46,8 +46,7 @@ class NMRMetadata(BaseModel): title: str | None = None """Title or description of the NMR experiment.""" - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") class NMRModel(DataBlockResponse): diff --git a/pydatalab/src/pydatalab/apps/xrd/blocks.py b/pydatalab/src/pydatalab/apps/xrd/blocks.py index cf7f327a3..6393464f4 100644 --- a/pydatalab/src/pydatalab/apps/xrd/blocks.py +++ b/pydatalab/src/pydatalab/apps/xrd/blocks.py @@ -264,7 +264,7 @@ def generate_xrd_plot(self, filenames: list[str | Path] | None = None) -> None: pattern_df.attrs["wavelength"] = ( f"{self.data.get('wavelength', self.defaults['wavelength'])} Å" ) - peak_information[str(f["immutable_id"])] = PeakInformation(**peak_data).dict() + peak_information[str(f["immutable_id"])] = PeakInformation(**peak_data).model_dump() if len(all_files) > 1: pattern_df["normalized intensity (staggered)"] += ind pattern_dfs.append(pattern_df) diff --git a/pydatalab/src/pydatalab/blocks/base.py b/pydatalab/src/pydatalab/blocks/base.py index 3d88e1961..36ea3aac7 100644 --- a/pydatalab/src/pydatalab/blocks/base.py +++ b/pydatalab/src/pydatalab/blocks/base.py @@ -198,7 +198,7 @@ def to_db(self) -> dict: for (f, s) in self.block_db_model.schema()["properties"].items() if s.get("datalab_exclude_from_db") } - return self.block_db_model(**self.data).dict( + return self.block_db_model(**self.data).model_dump( exclude=exclude_fields, exclude_unset=True, exclude_none=True, @@ -254,7 +254,7 @@ def to_web(self) -> dict[str, Any]: else: self.data.pop("warnings", None) - return self.block_db_model(**self.data).dict(exclude_unset=True, exclude_none=True) + return self.block_db_model(**self.data).model_dump(exclude_unset=True, exclude_none=True) def process_events(self, events: list[dict] | dict): """Handle any supported events passed to the block.""" @@ -354,7 +354,7 @@ def update_from_web(self, data: dict, stored_data: dict | None = None): if s.get("datalab_exclude_from_load") } [data.pop(f, None) for f in exclude_fields] - self.data.update(self.block_db_model(**data).dict()) + self.data.update(self.block_db_model(**data).model_dump(exclude_unset=True)) # Fields stripped above (e.g., `metadata`, `computed`) are server-authoritative: # writable only via block events or block code, never from the web payload. # Without restoring them from the stored state here, the model defaults diff --git a/pydatalab/src/pydatalab/routes/v0_1/auth.py b/pydatalab/src/pydatalab/routes/v0_1/auth.py index 1ded96b38..f14efe6ad 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/auth.py +++ b/pydatalab/src/pydatalab/routes/v0_1/auth.py @@ -504,7 +504,7 @@ def find_user_with_identity( if verify and not person.identities[identity_index].verified: flask_mongo.db.users.update_one( - {"_id": person.immutable_id}, + {"_id": ObjectId(person.immutable_id)}, {"$set": {f"identities.{identity_index}.verified": True}}, ) @@ -556,7 +556,7 @@ def attach_identity_to_user( RuntimeError: If the update was unsuccessful. """ - update = {"$push": {"identities": identity.dict()}} + update = {"$push": {"identities": identity.model_dump()}} if use_display_name and identity and identity.display_name: update["$set"] = {"display_name": identity.display_name} @@ -1057,7 +1057,7 @@ def redirect_to_ui(blueprint, token): # pylint: disable=unused-argument def get_authenticated_user_info(): """Returns metadata associated with the currently authenticated user.""" if current_user.is_authenticated: - current_user_response = json.loads(current_user.person.json()) + current_user_response = json.loads(current_user.person.model_dump_json()) current_user_response["role"] = current_user.role.value return jsonify(current_user_response), 200 else: @@ -1085,7 +1085,7 @@ def generate_user_api_key(): version=1, ) - flask_mongo.db.api_keys.insert_one(access_key.dict()) + flask_mongo.db.api_keys.insert_one(access_key.model_dump()) return jsonify({"key": new_key, "name": request_json["name"]}), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/collections.py b/pydatalab/src/pydatalab/routes/v0_1/collections.py index a8afffc3d..fd3f6dc6f 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/collections.py +++ b/pydatalab/src/pydatalab/routes/v0_1/collections.py @@ -87,7 +87,7 @@ def get_collection(collection_id): { "status": "success", "collection_id": collection_id, - "data": json.loads(collection.json(exclude_unset=True)), + "data": json.loads(collection.model_dump_json(exclude_unset=True)), "child_items": list(samples), } ) @@ -168,7 +168,7 @@ def create_collection(): ) result: InsertOneResult = flask_mongo.db.collections.insert_one( - data_model.dict(exclude={"creators", "groups"}) + data_model.model_dump(exclude={"creators", "groups"}) ) if not result.acknowledged: return ( @@ -218,7 +218,7 @@ def create_collection(): response = { "status": "success", - "data": json.loads(data_model.json()), + "data": json.loads(data_model.model_dump_json()), } if errors: @@ -278,7 +278,7 @@ def save_collection(collection_id): collection.update(updated_data) try: - collection = Collection(**collection).dict() + collection = Collection(**collection).model_dump() except ValidationError as exc: return ( jsonify( @@ -534,7 +534,7 @@ def search_collections(): ) cursor = [ - json.loads(Collection(**doc).json(exclude_unset=True)) + json.loads(Collection(**doc).model_dump_json(exclude_unset=True)) for doc in flask_mongo.db.collections.aggregate(pipeline) ] diff --git a/pydatalab/src/pydatalab/routes/v0_1/graphs.py b/pydatalab/src/pydatalab/routes/v0_1/graphs.py index 1299bfd09..72fb68317 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/graphs.py +++ b/pydatalab/src/pydatalab/routes/v0_1/graphs.py @@ -217,7 +217,9 @@ def add_related_items(current_item_id: str, current_depth: int): { "data": { "id": document["item_id"], - "name": document["name"] if document["name"] else document["item_id"], + "name": document.get("name") + if document.get("name") + else document["item_id"], "type": document["type"], "special": document["item_id"] == item_id, } diff --git a/pydatalab/src/pydatalab/routes/v0_1/info.py b/pydatalab/src/pydatalab/routes/v0_1/info.py index a2212ea87..40e4fc6a5 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/info.py +++ b/pydatalab/src/pydatalab/routes/v0_1/info.py @@ -7,14 +7,20 @@ from functools import lru_cache from flask import Blueprint, jsonify, request -from pydantic import AnyUrl, BaseModel, Field, validator +from pydantic import ( + AnyUrl, + BaseModel, + ConfigDict, + Field, + field_validator, + model_validator, +) from pydatalab import __version__ from pydatalab.apps import BLOCK_TYPES from pydatalab.config import CONFIG from pydatalab.feature_flags import FEATURE_FLAGS, FeatureFlags -from pydatalab.models import Collection, Person -from pydatalab.models.items import Item +from pydatalab.models import ITEM_SCHEMAS, Person from pydatalab.mongo import flask_mongo from pydatalab.permissions import active_users_or_get_only @@ -24,8 +30,7 @@ class Attributes(BaseModel): - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class Meta(BaseModel): @@ -39,9 +44,7 @@ class Meta(BaseModel): class Links(BaseModel): self: AnyUrl - - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class Data(BaseModel): @@ -53,39 +56,58 @@ class Data(BaseModel): class JSONAPIResponse(BaseModel): data: Data | list[Data] meta: Meta - links: Links | None + links: Links | None = None class MetaPerson(BaseModel): - dislay_name: str | None + display_name: str | None = None contact_email: str class Info(Attributes, Meta): - maintainer: MetaPerson | None - issue_tracker: AnyUrl | None - homepage: AnyUrl | None - source_repository: AnyUrl | None + maintainer: MetaPerson | None = None + issue_tracker: AnyUrl | None = None + homepage: AnyUrl | None = None + source_repository: AnyUrl | None = None identifier_prefix: str features: FeatureFlags = FEATURE_FLAGS max_upload_bytes: int - @validator("maintainer") + @field_validator("maintainer", mode="before") + @classmethod def strip_maintainer_fields(cls, v): if isinstance(v, Person): return MetaPerson(contact_email=v.contact_email, display_name=v.display_name) return v + @model_validator(mode="after") + def ensure_features_serialization(self): + """Ensure features are properly serialized for frontend consumption.""" + if hasattr(self.features, "model_dump"): + features_dict = self.features.model_dump() + else: + features_dict = self.features + + if not isinstance(self.features, FeatureFlags): + self.features = FeatureFlags(**features_dict) + return self + @lru_cache(maxsize=1) def _get_deployment_metadata_once() -> dict: identifier_prefix = CONFIG.IDENTIFIER_PREFIX metadata = ( - CONFIG.DEPLOYMENT_METADATA.dict(exclude_none=True) if CONFIG.DEPLOYMENT_METADATA else {} + CONFIG.DEPLOYMENT_METADATA.model_dump(exclude_none=True) + if CONFIG.DEPLOYMENT_METADATA + else {} + ) + metadata.update( + { + "identifier_prefix": identifier_prefix, + "max_upload_bytes": CONFIG.MAX_CONTENT_LENGTH, + "features": FEATURE_FLAGS, + } ) - metadata.update({"identifier_prefix": identifier_prefix}) - metadata.update({"max_upload_bytes": CONFIG.MAX_CONTENT_LENGTH}) - return metadata @@ -95,18 +117,15 @@ def get_info(): versions, features and so on. """ - metadata = _get_deployment_metadata_once() + + response_data = JSONAPIResponse( + data=Data(id="/", type="info", attributes=Info(**_get_deployment_metadata_once())), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), + links=Links(self=request.url), + ) return ( - jsonify( - json.loads( - JSONAPIResponse( - data=Data(id="/", type="info", attributes=Info(**metadata)), - meta=Meta(query=request.query_string), - links=Links(self=request.url), - ).json() - ) - ), + jsonify(json.loads(response_data.model_dump_json())), 200, ) @@ -149,30 +168,11 @@ def list_block_types(): for block_type, block in BLOCK_TYPES.items() ], meta=Meta(query=request.query_string), - ).json() + ).model_dump_json() ) ) -def get_all_items_models(): - return Item.__subclasses__() - - -def generate_schemas(): - schemas: dict[str, dict] = {} - - for model_class in get_all_items_models() + [Collection]: - model_type = model_class.schema()["properties"]["type"]["default"] - - schemas[model_type] = model_class.schema(by_alias=False) - - return schemas - - -# Generate once on import -SCHEMAS = generate_schemas() - - @INFO.route("/info/types", methods=["GET"]) def list_supported_types(): """Returns a list of supported schemas.""" @@ -190,10 +190,10 @@ def list_supported_types(): "schema": schema, }, ) - for item_type, schema in SCHEMAS.items() + for item_type, schema in ITEM_SCHEMAS.items() ], meta=Meta(query=request.query_string), - ).json() + ).model_dump_json() ) ) @@ -201,7 +201,7 @@ def list_supported_types(): @INFO.route("/info/types/", methods=["GET"]) def get_schema_type(item_type): """Returns the schema of the given type.""" - if item_type not in SCHEMAS: + if item_type not in ITEM_SCHEMAS: return jsonify( {"status": "error", "detail": f"Item type {item_type} not found for this deployment"} ), 404 @@ -215,11 +215,11 @@ def get_schema_type(item_type): attributes={ "version": __version__, "api_version": __api_version__, - "schema": SCHEMAS[item_type], + "schema": ITEM_SCHEMAS[item_type], }, ), meta=Meta(query=request.query_string), - ).json() + ).model_dump_json() ) ) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index c469190fd..9a58a237b 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -516,8 +516,8 @@ def search_items(): Returns: response list of dictionaries containing the matching items in order of descending match score. - """ + """ query = request.args.get("query", type=str) nresults = request.args.get("nresults", default=100, type=int) types = request.args.get("types", default=None) @@ -674,7 +674,7 @@ def _create_sample( elif CONFIG.TESTING and not current_user.is_authenticated: # Set fake ID to ObjectId("000000000000000000000000") so a dummy user can be created # locally for testing creator UI elements - new_sample["creator_ids"] = [PUBLIC_USER_ID] + new_sample["creator_ids"] = [str(PUBLIC_USER_ID)] new_sample["creators"] = [ { "display_name": "Public testing user", @@ -718,7 +718,6 @@ def _create_sample( try: # Check on relationship fields and prefill new_sample = entry_reference_lookup(new_sample) - data_model: Item = model(**new_sample) except ValidationError as error: @@ -732,7 +731,9 @@ def _create_sample( # the `Entry` model. try: result = flask_mongo.db.items.insert_one( - data_model.dict(exclude={"creators", "collections", "groups"}) + data_model.model_dump( + exclude={"creators", "collections", "groups"}, exclude_none=True, by_alias=True + ) ) except DuplicateKeyError as error: raise Conflict(f"Duplicate key error: {str(error)}.") @@ -767,7 +768,7 @@ def _create_sample( data = { "status": "success", "item_id": data_model.item_id, - "sample_list_entry": data_model.dict(), + "sample_list_entry": data_model.model_dump(), } return (data, 201) # 201 Created @@ -1152,7 +1153,6 @@ def get_item_data( elif refcode: if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" - match = {"refcode": refcode} else: raise BadRequest("No item_id or refcode provided.") @@ -1177,6 +1177,7 @@ def get_item_data( try: doc = list(cursor)[0] + except IndexError: doc = None @@ -1252,7 +1253,7 @@ def get_item_data( ) # loop over and collect all 'outer' relationships presented by other items - incoming_relationships: dict[RelationshipType, set[str]] = {} + incoming_relationships: dict[RelationshipType, set] = {} for d in relationships_query_results: for k in d["relationships"]: if k["relation"] not in incoming_relationships: @@ -1262,7 +1263,7 @@ def get_item_data( ) # loop over and aggregate all 'inner' relationships presented by this item - inlined_relationships: dict[RelationshipType, set[str]] = {} + inlined_relationships: dict[RelationshipType, set] = {} if doc.relationships is not None: inlined_relationships = { relation: { @@ -1282,7 +1283,7 @@ def get_item_data( ) # Must be exported to JSON first to apply the custom pydantic JSON encoders - return_dict = json.loads(doc.json(exclude_unset=True)) + return_dict = doc.model_dump(mode="json", exclude_unset=True) if item_id is None: item_id = return_dict["item_id"] @@ -1590,7 +1591,7 @@ def restore_version(refcode): # Insert validated data flask_mongo.db.item_versions.insert_one( - validated_restored_version.dict(by_alias=True, exclude_none=True) + validated_restored_version.model_dump(by_alias=True, exclude_none=True) ) return jsonify( @@ -1670,6 +1671,7 @@ def save_item(): # These keys should not be updated here and cannot be modified by the user through this endpoint for k in ( "_id", + "immutable_id", "file_ObjectIds", "files", "creators", @@ -1780,6 +1782,42 @@ def save_item(): ] updated_data["collections"] = inaccessible + existing_item = flask_mongo.db.items.find_one({"item_id": item_id}) + if existing_item: + existing_relationships = existing_item.get("relationships", []) + non_collection_relationships = [ + rel for rel in existing_relationships if rel.get("type") != "collections" + ] + + collection_relationships = [] + for coll in updated_data.get("collections", []): + immutable_id = coll.get("immutable_id") + collection_id = coll.get("collection_id") + + if immutable_id: + if isinstance(immutable_id, str): + from bson import ObjectId + + immutable_id = ObjectId(immutable_id) + elif collection_id: + collection_doc = flask_mongo.db.collections.find_one( + {"collection_id": collection_id} + ) + if collection_doc: + immutable_id = collection_doc["_id"] + + if immutable_id: + collection_relationships.append( + { + "relation": None, + "immutable_id": immutable_id, + "type": "collections", + "description": "Is a member of", + } + ) + + updated_data["relationships"] = non_collection_relationships + collection_relationships + item_type = item["type"] preserve_relationships = "collections" not in updated_data @@ -1789,7 +1827,13 @@ def save_item(): try: item = entry_reference_lookup(item) - item = ITEM_MODELS[item_type](**item).dict() + item = ITEM_MODELS[item_type](**item).model_dump( + exclude_none=True, + exclude_unset=True, + by_alias=True, + exclude={"collections", "creators", "immutable_id"}, + ) + except ValidationError as exc: return ( jsonify( @@ -1804,8 +1848,10 @@ def save_item(): item["relationships"] = original_relationships # remove collections and creators and any other reference fields - item.pop("collections") - item.pop("creators") + item.pop("collections", None) + item.pop("creators", None) + item.pop("immutable_id", None) + item.pop("files", None) # `last_modified` is controlled by the versioning branch below: only bump it when a # snapshot is actually saved, so re-submitting identical data leaves it untouched. diff --git a/pydatalab/src/pydatalab/routes/v0_1/remotes.py b/pydatalab/src/pydatalab/routes/v0_1/remotes.py index be50502f9..4e69952a2 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/remotes.py +++ b/pydatalab/src/pydatalab/routes/v0_1/remotes.py @@ -76,7 +76,9 @@ def list_remote_directories(): response = {} response["meta"] = {} - response["meta"]["remotes"] = [json.loads(d.json()) for d in CONFIG.REMOTE_FILESYSTEMS] + response["meta"]["remotes"] = [ + json.loads(d.model_dump_json()) for d in CONFIG.REMOTE_FILESYSTEMS + ] if all_directory_structures: oldest_update = min(d["last_updated"] for d in all_directory_structures) response["meta"]["oldest_cache_update"] = oldest_update.isoformat() @@ -139,7 +141,7 @@ def get_remote_directory(remote_id: str): response: dict[str, Any] = {} response["meta"] = {} - response["meta"]["remote"] = json.loads(d.json()) + response["meta"]["remote"] = json.loads(d.model_dump_json()) response["data"] = directory_structure return jsonify(response), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/users.py b/pydatalab/src/pydatalab/routes/v0_1/users.py index b59aca49d..f790a12bf 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/users.py +++ b/pydatalab/src/pydatalab/routes/v0_1/users.py @@ -6,11 +6,12 @@ from bson import ObjectId from flask import Blueprint, jsonify, request from flask_login import current_user +from pydantic import ValidationError from werkzeug.exceptions import BadRequest, Forbidden, Unauthorized from pydatalab.config import CONFIG from pydatalab.logger import LOGGER -from pydatalab.models.people import AccountStatus, DisplayName, EmailStr, Person +from pydatalab.models.people import Person from pydatalab.mongo import ( USERS_FTS_FIELDS, build_search_pipeline, @@ -49,22 +50,22 @@ def save_user(user_id): update = {} - try: - if display_name: - update["display_name"] = DisplayName(display_name) + if display_name: + update["display_name"] = display_name - except ValueError: - raise BadRequest(f"Invalid display name {display_name!r} was passed") + if contact_email or contact_email in (None, ""): + if contact_email in ("", None): + update["contact_email"] = None + else: + update["contact_email"] = contact_email - try: - if contact_email or contact_email in (None, ""): - if contact_email in ("", None): - update["contact_email"] = None - else: - update["contact_email"] = EmailStr(contact_email) + if account_status: + update["account_status"] = account_status - except ValueError: - raise BadRequest(f"Invalid email address {contact_email!r} was passed") + try: + _ = Person(**update) + except ValidationError as e: + raise BadRequest(f"Invalid user data: {e.errors()}") from e if "contact_email" in update or "display_name" in update: existing = ( @@ -128,12 +129,6 @@ def save_user(user_id): trigger_email_verification = False LOGGER.critical("Unable to send verification email on this deployment: %s", e) - try: - if account_status: - update["account_status"] = AccountStatus(account_status) - except ValueError: - raise BadRequest(f"Invalid account status {account_status!r} was passed") - if not update: return jsonify({"status": "success", "message": "No update to perform."}), 200 From cd324229dafa314e02bd84f3c232475bc23cd31e Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 20:34:24 +0100 Subject: [PATCH 09/40] More model updates More updates for models Fixes for models Simplify item versioning model Remove unecessary by_alias=True --- pydatalab/schemas/cell.json | 46 +++++++++++----- pydatalab/schemas/equipment.json | 50 ++++++++++++----- pydatalab/schemas/sample.json | 46 +++++++++++----- pydatalab/schemas/startingmaterial.json | 54 +++++++++++++------ pydatalab/src/pydatalab/models/blocks.py | 6 ++- pydatalab/src/pydatalab/models/equipment.py | 2 +- pydatalab/src/pydatalab/models/export_task.py | 5 +- pydatalab/src/pydatalab/models/files.py | 4 +- .../pydatalab/models/starting_materials.py | 6 +-- pydatalab/src/pydatalab/models/tasks.py | 12 ++--- pydatalab/src/pydatalab/models/traits.py | 4 +- pydatalab/src/pydatalab/models/versions.py | 52 +++++------------- pydatalab/src/pydatalab/mongo.py | 2 +- pydatalab/src/pydatalab/routes/v0_1/items.py | 7 +-- pydatalab/src/pydatalab/versioning.py | 2 +- 15 files changed, 179 insertions(+), 119 deletions(-) diff --git a/pydatalab/schemas/cell.json b/pydatalab/schemas/cell.json index 9922552af..1a7c02b03 100644 --- a/pydatalab/schemas/cell.json +++ b/pydatalab/schemas/cell.json @@ -84,14 +84,15 @@ "additionalProperties": true, "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "collection_id": { + "immutable_id": { "description": "The immutable ID of the collection.", - "title": "Collection Id", + "format": "objectid", + "title": "Immutable Id", "type": "string" } }, "required": [ - "collection_id" + "immutable_id" ], "title": "CollectionReference", "type": "object" @@ -239,6 +240,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", "title": "B64 Encoded Image" }, @@ -254,6 +256,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "A JSON-encoded string containing the Bokeh plot data, if any.", "title": "Bokeh Plot Data" }, @@ -272,6 +275,20 @@ "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", "title": "Computed" }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, "metadata": { "anyOf": [ { @@ -290,9 +307,7 @@ }, "required": [ "blocktype", - "block_id", - "b64_encoded_image", - "bokeh_plot_data" + "block_id" ], "title": "DataBlockResponse", "type": "object" @@ -609,10 +624,18 @@ "title": "Source" }, "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The timestamp for the original file upload.", - "format": "datetime", - "title": "Time Added", - "type": "string" + "title": "Time Added" }, "metadata": { "anyOf": [ @@ -678,15 +701,14 @@ "type": "null" } ], + "default": null, "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ "name", "extension", - "time_added", - "is_live", - "checksums" + "is_live" ], "title": "File", "type": "object" diff --git a/pydatalab/schemas/equipment.json b/pydatalab/schemas/equipment.json index 25a0fe91f..ab16caa0d 100644 --- a/pydatalab/schemas/equipment.json +++ b/pydatalab/schemas/equipment.json @@ -14,14 +14,15 @@ "additionalProperties": true, "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "collection_id": { + "immutable_id": { "description": "The immutable ID of the collection.", - "title": "Collection Id", + "format": "objectid", + "title": "Immutable Id", "type": "string" } }, "required": [ - "collection_id" + "immutable_id" ], "title": "CollectionReference", "type": "object" @@ -169,6 +170,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", "title": "B64 Encoded Image" }, @@ -184,6 +186,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "A JSON-encoded string containing the Bokeh plot data, if any.", "title": "Bokeh Plot Data" }, @@ -202,6 +205,20 @@ "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", "title": "Computed" }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, "metadata": { "anyOf": [ { @@ -220,9 +237,7 @@ }, "required": [ "blocktype", - "block_id", - "b64_encoded_image", - "bokeh_plot_data" + "block_id" ], "title": "DataBlockResponse", "type": "object" @@ -470,10 +485,18 @@ "title": "Source" }, "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The timestamp for the original file upload.", - "format": "datetime", - "title": "Time Added", - "type": "string" + "title": "Time Added" }, "metadata": { "anyOf": [ @@ -539,15 +562,14 @@ "type": "null" } ], + "default": null, "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ "name", "extension", - "time_added", - "is_live", - "checksums" + "is_live" ], "title": "File", "type": "object" @@ -1306,13 +1328,13 @@ "type": "null" } ], + "default": null, "description": "Contact information for equipment (e.g., email address or phone number).", "title": "Contact" } }, "required": [ - "item_id", - "contact" + "item_id" ], "title": "Equipment", "type": "object" diff --git a/pydatalab/schemas/sample.json b/pydatalab/schemas/sample.json index 727e23f89..1ec3de769 100644 --- a/pydatalab/schemas/sample.json +++ b/pydatalab/schemas/sample.json @@ -14,14 +14,15 @@ "additionalProperties": true, "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "collection_id": { + "immutable_id": { "description": "The immutable ID of the collection.", - "title": "Collection Id", + "format": "objectid", + "title": "Immutable Id", "type": "string" } }, "required": [ - "collection_id" + "immutable_id" ], "title": "CollectionReference", "type": "object" @@ -211,6 +212,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", "title": "B64 Encoded Image" }, @@ -226,6 +228,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "A JSON-encoded string containing the Bokeh plot data, if any.", "title": "Bokeh Plot Data" }, @@ -244,6 +247,20 @@ "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", "title": "Computed" }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, "metadata": { "anyOf": [ { @@ -262,9 +279,7 @@ }, "required": [ "blocktype", - "block_id", - "b64_encoded_image", - "bokeh_plot_data" + "block_id" ], "title": "DataBlockResponse", "type": "object" @@ -581,10 +596,18 @@ "title": "Source" }, "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The timestamp for the original file upload.", - "format": "datetime", - "title": "Time Added", - "type": "string" + "title": "Time Added" }, "metadata": { "anyOf": [ @@ -650,15 +673,14 @@ "type": "null" } ], + "default": null, "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ "name", "extension", - "time_added", - "is_live", - "checksums" + "is_live" ], "title": "File", "type": "object" diff --git a/pydatalab/schemas/startingmaterial.json b/pydatalab/schemas/startingmaterial.json index 2a779a084..f408f1af1 100644 --- a/pydatalab/schemas/startingmaterial.json +++ b/pydatalab/schemas/startingmaterial.json @@ -14,14 +14,15 @@ "additionalProperties": true, "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "collection_id": { + "immutable_id": { "description": "The immutable ID of the collection.", - "title": "Collection Id", + "format": "objectid", + "title": "Immutable Id", "type": "string" } }, "required": [ - "collection_id" + "immutable_id" ], "title": "CollectionReference", "type": "object" @@ -211,6 +212,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", "title": "B64 Encoded Image" }, @@ -226,6 +228,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "A JSON-encoded string containing the Bokeh plot data, if any.", "title": "Bokeh Plot Data" }, @@ -244,6 +247,20 @@ "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", "title": "Computed" }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, "metadata": { "anyOf": [ { @@ -262,9 +279,7 @@ }, "required": [ "blocktype", - "block_id", - "b64_encoded_image", - "bokeh_plot_data" + "block_id" ], "title": "DataBlockResponse", "type": "object" @@ -581,10 +596,18 @@ "title": "Source" }, "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The timestamp for the original file upload.", - "format": "datetime", - "title": "Time Added", - "type": "string" + "title": "Time Added" }, "metadata": { "anyOf": [ @@ -650,15 +673,14 @@ "type": "null" } ], + "default": null, "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ "name", "extension", - "time_added", - "is_live", - "checksums" + "is_live" ], "title": "File", "type": "object" @@ -1523,6 +1545,7 @@ "type": "null" } ], + "default": null, "description": "The name of the substance in the container.", "title": "Name" }, @@ -1567,6 +1590,7 @@ "type": "null" } ], + "default": null, "description": "The chemical purity of this container with regards to the defined substance.", "title": "Chemical Purity" }, @@ -1618,6 +1642,7 @@ "type": "null" } ], + "default": null, "description": "Supplier or manufacturer of the chemical.", "title": "Supplier" }, @@ -1649,10 +1674,7 @@ } }, "required": [ - "item_id", - "name", - "chemical_purity", - "supplier" + "item_id" ], "title": "StartingMaterial", "type": "object" diff --git a/pydatalab/src/pydatalab/models/blocks.py b/pydatalab/src/pydatalab/models/blocks.py index a89af0ffc..f1e02a4d4 100644 --- a/pydatalab/src/pydatalab/models/blocks.py +++ b/pydatalab/src/pydatalab/models/blocks.py @@ -44,12 +44,12 @@ class DataBlockResponse(BaseModel): """Any warnings that occurred during block processing.""" b64_encoded_image: dict[str, str] | None = Field( - datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, datalab_exclude_from_db=True, datalab_exclude_from_load=True ) """Any base64-encoded image data associated with the block, keyed by `file_id`.""" bokeh_plot_data: dict | None = Field( - datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, datalab_exclude_from_db=True, datalab_exclude_from_load=True ) """A JSON-encoded string containing the Bokeh plot data, if any.""" @@ -59,6 +59,8 @@ class DataBlockResponse(BaseModel): Examples could include peak positions, and widths, but not the full spectrum. """ + processed: dict | None = Field(default=None, datalab_exclude_from_load=True) + metadata: dict | None = Field(default=None, datalab_exclude_from_load=True) """Any structured metadata associated with the block, for example, experimental acquisition parameters.""" diff --git a/pydatalab/src/pydatalab/models/equipment.py b/pydatalab/src/pydatalab/models/equipment.py index b86e2e897..7164ccb2f 100644 --- a/pydatalab/src/pydatalab/models/equipment.py +++ b/pydatalab/src/pydatalab/models/equipment.py @@ -26,7 +26,7 @@ class Equipment(Item): location: str | None = None """Place where the equipment is located""" - contact: str | None + contact: str | None = None """Contact information for equipment (e.g., email address or phone number).""" status: EquipmentStatus = Field(default=EquipmentStatus.WORKING) diff --git a/pydatalab/src/pydatalab/models/export_task.py b/pydatalab/src/pydatalab/models/export_task.py index d2085ffde..e2728888d 100644 --- a/pydatalab/src/pydatalab/models/export_task.py +++ b/pydatalab/src/pydatalab/models/export_task.py @@ -1,7 +1,7 @@ from datetime import datetime, timezone from enum import Enum -from pydantic import Field +from pydantic import ConfigDict, Field from pydatalab.models.utils import BaseModel, PyObjectId @@ -48,5 +48,4 @@ class ExportTask(BaseModel): error_message: str | None = None """Error message if status is ERROR""" - class Config: - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) diff --git a/pydatalab/src/pydatalab/models/files.py b/pydatalab/src/pydatalab/models/files.py index edf092fa9..b289c5ccd 100644 --- a/pydatalab/src/pydatalab/models/files.py +++ b/pydatalab/src/pydatalab/models/files.py @@ -53,7 +53,7 @@ class File(Entry, HasOwner, HasRevisionControl): source: str | None = None """The source of the file, e.g. 'remote' or 'uploaded'.""" - time_added: IsoformatDateTime + time_added: IsoformatDateTime | None = None """The timestamp for the original file upload.""" metadata: dict[Any, Any] | None = None @@ -70,7 +70,7 @@ class File(Entry, HasOwner, HasRevisionControl): is_live: bool """Whether or not the file should be watched for future updates.""" - checksums: FileChecksums | None + checksums: FileChecksums | None = None """Content checksums (MD5 and SHA-256) of the file.""" diff --git a/pydatalab/src/pydatalab/models/starting_materials.py b/pydatalab/src/pydatalab/models/starting_materials.py index 5167a1bb5..1852a6d0f 100644 --- a/pydatalab/src/pydatalab/models/starting_materials.py +++ b/pydatalab/src/pydatalab/models/starting_materials.py @@ -26,13 +26,13 @@ class StartingMaterial(Item, HasSynthesisInfo, HasSubstanceInfo): date_opened: IsoformatDateTime | None = Field(None, alias="Date opened") """The date the item was opened""" - chemical_purity: str | None = Field(alias="Chemical purity") + chemical_purity: str | None = Field(None, alias="Chemical purity") """The chemical purity of this container with regards to the defined substance.""" full_percent: str | None = Field(None, alias="Full %") """The amount of the defined substance remaining in the container, expressed as a percentage.""" - name: str | None = Field(alias="Container Name") + name: str | None = Field(None, alias="Container Name") """The name of the substance in the container.""" size: str | None = Field(None, alias="Container Size") @@ -41,7 +41,7 @@ class StartingMaterial(Item, HasSynthesisInfo, HasSubstanceInfo): size_unit: str | None = Field(None, alias="Unit") """Units for the 'size' field.""" - supplier: str | None = Field(alias="Supplier") + supplier: str | None = Field(None, alias="Supplier") """Supplier or manufacturer of the chemical.""" location: str | None = Field(None, alias="Location") diff --git a/pydatalab/src/pydatalab/models/tasks.py b/pydatalab/src/pydatalab/models/tasks.py index 14728df26..ab21d39c8 100644 --- a/pydatalab/src/pydatalab/models/tasks.py +++ b/pydatalab/src/pydatalab/models/tasks.py @@ -2,7 +2,7 @@ from enum import Enum from typing import Literal -from pydantic import Field, validator +from pydantic import ConfigDict, Field, field_validator from pydatalab.models.utils import BaseModel, PyObjectId @@ -90,9 +90,10 @@ class Task(BaseModel): spec: ExportTaskSpec | BlockProcessingTaskSpec """Task-specific data""" - @validator("spec", pre=True, always=True) - def validate_spec_type(cls, v, values): - task_type = values.get("type") + @field_validator("spec", mode="before") + @classmethod + def validate_spec_type(cls, v, info): + task_type = info.data.get("type") if task_type == TaskType.EXPORT: if not isinstance(v, ExportTaskSpec): @@ -103,5 +104,4 @@ def validate_spec_type(cls, v, values): return v - class Config: - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) diff --git a/pydatalab/src/pydatalab/models/traits.py b/pydatalab/src/pydatalab/models/traits.py index e369a92c6..5fd3a0be4 100644 --- a/pydatalab/src/pydatalab/models/traits.py +++ b/pydatalab/src/pydatalab/models/traits.py @@ -72,9 +72,9 @@ class HasBlocks(BaseModel): class CollectionReference(BaseModel): """A reference to a collection, used for inlining collection info within other models.""" - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", populate_by_name=True) - collection_id: str + immutable_id: PyObjectId = Field(alias="_id") """The immutable ID of the collection.""" diff --git a/pydatalab/src/pydatalab/models/versions.py b/pydatalab/src/pydatalab/models/versions.py index 318561fe8..a01311424 100644 --- a/pydatalab/src/pydatalab/models/versions.py +++ b/pydatalab/src/pydatalab/models/versions.py @@ -3,7 +3,7 @@ from datetime import datetime from enum import Enum -from pydantic import Field, validator +from pydantic import ConfigDict, Field, model_validator from pydatalab.models.utils import BaseModel, PyObjectId, Refcode @@ -57,17 +57,16 @@ class ItemVersion(BaseModel): user_agent: str | None = None """User agent string of the client that triggered this version. Will only be stored if it matches a known value from the datalab ecosystem.""" - @validator("restored_from_version") - def validate_restored_from_version(cls, v, values): + @model_validator(mode="after") + def validate_restored_from_version(self): """Ensure restored_from_version is only present when action='restored'.""" - action = values.get("action") - if action == VersionAction.RESTORED and v is None: + if self.action == VersionAction.RESTORED and self.restored_from_version is None: raise ValueError("restored_from_version must be provided when action='restored'") - if action != VersionAction.RESTORED and v is not None: + if self.action != VersionAction.RESTORED and self.restored_from_version is not None: raise ValueError( - f"restored_from_version should only be present when action='restored', got action='{action}'" + f"restored_from_version should only be present when action='restored', got action='{self.action}'" ) - return v + return self class VersionCounter(BaseModel): @@ -83,50 +82,25 @@ class VersionCounter(BaseModel): counter: int = Field(1, ge=1) """Current version counter value (1-indexed, matches version numbers)""" - class Config: - extra = "ignore" # Allow MongoDB's _id field and other internal fields + model_config = ConfigDict(extra="ignore") class RestoreVersionRequest(BaseModel): """Request body for restoring a version.""" - version_id: str + version_id: PyObjectId """ObjectId string of the version to restore to""" - @validator("version_id") - def validate_version_id_format(cls, v): - """Validate that version_id is a valid ObjectId string.""" - try: - from bson import ObjectId - - ObjectId(v) - except Exception as e: - raise ValueError(f"version_id must be a valid ObjectId string: {e}") - return v - - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") class CompareVersionsQuery(BaseModel): """Query parameters for comparing two versions.""" - v1: str + v1: PyObjectId """ObjectId string of the first version""" - v2: str + v2: PyObjectId """ObjectId string of the second version""" - @validator("v1", "v2") - def validate_version_ids(cls, v): - """Validate that version IDs are valid ObjectId strings.""" - try: - from bson import ObjectId - - ObjectId(v) - except Exception as e: - raise ValueError(f"Version ID must be a valid ObjectId string: {e}") - return v - - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") diff --git a/pydatalab/src/pydatalab/mongo.py b/pydatalab/src/pydatalab/mongo.py index a8b7334af..09215c3e6 100644 --- a/pydatalab/src/pydatalab/mongo.py +++ b/pydatalab/src/pydatalab/mongo.py @@ -167,7 +167,7 @@ def insert_pydantic_model_fork_safe(model: BaseModel, collection: str) -> str: """Inserts a Pydantic model into chosen collection, returning the inserted ID.""" return ( get_database()[collection] - .insert_one(model.model_dump(by_alias=True, exclude_none=True)) + .insert_one(model.model_dump(by_alias=False, exclude_none=True)) .inserted_id ) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 9a58a237b..5b5a55cf6 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -731,9 +731,7 @@ def _create_sample( # the `Entry` model. try: result = flask_mongo.db.items.insert_one( - data_model.model_dump( - exclude={"creators", "collections", "groups"}, exclude_none=True, by_alias=True - ) + data_model.model_dump(exclude={"creators", "collections", "groups"}, exclude_none=True) ) except DuplicateKeyError as error: raise Conflict(f"Duplicate key error: {str(error)}.") @@ -1591,7 +1589,7 @@ def restore_version(refcode): # Insert validated data flask_mongo.db.item_versions.insert_one( - validated_restored_version.model_dump(by_alias=True, exclude_none=True) + validated_restored_version.model_dump(exclude_none=True) ) return jsonify( @@ -1830,7 +1828,6 @@ def save_item(): item = ITEM_MODELS[item_type](**item).model_dump( exclude_none=True, exclude_unset=True, - by_alias=True, exclude={"collections", "creators", "immutable_id"}, ) diff --git a/pydatalab/src/pydatalab/versioning.py b/pydatalab/src/pydatalab/versioning.py index 351ddf032..69816ae79 100644 --- a/pydatalab/src/pydatalab/versioning.py +++ b/pydatalab/src/pydatalab/versioning.py @@ -220,7 +220,7 @@ def save_version_snapshot( next_version_number = get_next_version_number(refcode) # Insert validated data (convert to dict and exclude None values) - version_doc = validated_version.dict(by_alias=True, exclude_none=True) + version_doc = validated_version.model_dump(exclude_none=True) version_doc["version"] = next_version_number flask_mongo.db.item_versions.insert_one(version_doc) return ( From 4069ae9744018ea2d29ed84003b702cffc0664c6 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 20:57:45 +0100 Subject: [PATCH 10/40] Minimal test updates --- pydatalab/tests/server/conftest.py | 22 +++---- pydatalab/tests/server/test_item_versions.py | 48 ++------------- pydatalab/tests/server/test_samples.py | 22 +++---- pydatalab/tests/test_models.py | 61 +++++++++++++++----- 4 files changed, 72 insertions(+), 81 deletions(-) diff --git a/pydatalab/tests/server/conftest.py b/pydatalab/tests/server/conftest.py index 5be100adc..92917d578 100644 --- a/pydatalab/tests/server/conftest.py +++ b/pydatalab/tests/server/conftest.py @@ -515,7 +515,7 @@ def fixture_insert_complicated_sample_constituents(user_id): creator_ids=[user_id], refcode=generate_unique_refcode(), ) - flask_mongo.db.items.insert_one(sm.dict(exclude_unset=False)) + flask_mongo.db.items.insert_one(sm.model_dump(exclude_unset=False)) items.append(sm) yield items @@ -528,7 +528,7 @@ def fixture_insert_complicated_sample_constituents(user_id): def example_items(user_id, admin_user_id): """Create a collection of samples with mixed ownership between the user and admin.""" return [ - d.dict(exclude_unset=False) + d.model_dump(exclude_unset=False) for d in [ Sample( **{ @@ -615,23 +615,23 @@ def example_items(user_id, admin_user_id): @pytest.fixture(scope="module", name="default_sample_dict") -def fixture_default_sample_dict(default_sample): - return default_sample.dict(exclude_unset=True) +def fixture_default_sample_model_dump(default_sample): + return default_sample.model_dump(exclude_unset=True) @pytest.fixture(scope="module", name="default_cell_dict") -def fixture_default_cell_dict(default_cell): - return default_cell.dict(exclude_unset=True) +def fixture_default_cell_model_dump(default_cell): + return default_cell.model_dump(exclude_unset=True) @pytest.fixture(scope="module", name="default_starting_material_dict") -def fixture_default_starting_material_dict(default_starting_material): - return default_starting_material.dict(exclude_unset=True) +def fixture_default_starting_material_model_dump(default_starting_material): + return default_starting_material.model_dump(exclude_unset=True) @pytest.fixture(scope="module", name="default_equipment_dict") -def fixture_default_equipment_dict(default_equipment): - return default_equipment.dict(exclude_unset=True) +def fixture_default_equipment_model_dump(default_equipment): + return default_equipment.model_dump(exclude_unset=True) def _insert_and_cleanup_item_from_model(model): @@ -640,7 +640,7 @@ def _insert_and_cleanup_item_from_model(model): refcode = generate_unique_refcode() model.refcode = refcode - flask_mongo.db.items.insert_one(model.dict(exclude_unset=False)) + flask_mongo.db.items.insert_one(model.model_dump(exclude_unset=False)) yield model flask_mongo.db.items.delete_one({"refcode": model.refcode}) diff --git a/pydatalab/tests/server/test_item_versions.py b/pydatalab/tests/server/test_item_versions.py index d99af7d1f..dad4b1ea6 100644 --- a/pydatalab/tests/server/test_item_versions.py +++ b/pydatalab/tests/server/test_item_versions.py @@ -108,7 +108,6 @@ def test_save_version_nonexistent_item(self, client): response = client.post("/items/nonexistent/save-version/") assert response.status_code == 404 - assert response.json["status"] == "error" class TestListVersions: @@ -251,8 +250,6 @@ def test_get_version_nonexistent(self, client, sample_with_version): response = client.get(f"/items/{refcode}/versions/{fake_id}/") assert response.status_code == 404 - assert response.json["status"] == "error" - assert "not found" in response.json["message"].lower() class TestCompareVersions: @@ -301,24 +298,12 @@ def test_compare_versions_missing_parameters(self, client, sample_with_version): # Missing v2 - request.args.get() returns "" for missing params, which fails ObjectId validation response = client.get(f"/items/{refcode}/compare-versions/?v1=some_id") assert response.status_code == 400 - assert response.json["message"] == "Invalid query parameters" - assert "errors" in response.json - errors = response.json["errors"] - # Should have error for v2 (empty string is invalid ObjectId) - v2_errors = [e for e in errors if "v2" in str(e["loc"])] - assert len(v2_errors) == 1 - assert "valid objectid" in v2_errors[0]["msg"].lower() + assert "Invalid query parameters" in response.json["message"] # Missing v1 - same behavior response = client.get(f"/items/{refcode}/compare-versions/?v2=some_id") assert response.status_code == 400 - assert response.json["message"] == "Invalid query parameters" - assert "errors" in response.json - errors = response.json["errors"] - # Should have error for v1 (empty string is invalid ObjectId) - v1_errors = [e for e in errors if "v1" in str(e["loc"])] - assert len(v1_errors) == 1 - assert "valid objectid" in v1_errors[0]["msg"].lower() + assert "Invalid query parameters" in response.json["message"] def test_compare_versions_invalid_id(self, client, sample_with_version): """Test comparing versions with invalid ID format.""" @@ -326,15 +311,7 @@ def test_compare_versions_invalid_id(self, client, sample_with_version): response = client.get(f"/items/{refcode}/compare-versions/?v1=invalid&v2=invalid") assert response.status_code == 400 - assert response.json["message"] == "Invalid query parameters" - # Check Pydantic's structured error response - assert "errors" in response.json - errors = response.json["errors"] - # Should have errors for both v1 and v2 - assert len(errors) == 2 - for error in errors: - assert error["loc"][0] in ["v1", "v2"] - assert "valid ObjectId" in error["msg"] + assert "Invalid query parameters" in response.json["message"] def test_compare_versions_detects_changes(self, client, sample_with_version): """Test that compare_versions properly detects changes using DeepDiff.""" @@ -502,13 +479,7 @@ def test_restore_version_missing_version_id(self, client, sample_with_version): response = client.post(f"/items/{refcode}/restore-version/", json={}) assert response.status_code == 400 - assert response.json["message"] == "Invalid request body" - # Check Pydantic's structured error response - assert "errors" in response.json - errors = response.json["errors"] - assert len(errors) == 1 - assert errors[0]["loc"] == ["version_id"] - assert "required" in errors[0]["msg"].lower() + assert "Invalid request body" in response.json["message"] def test_restore_version_invalid_id(self, client, sample_with_version): """Test restoring with invalid version ID.""" @@ -516,13 +487,7 @@ def test_restore_version_invalid_id(self, client, sample_with_version): response = client.post(f"/items/{refcode}/restore-version/", json={"version_id": "invalid"}) assert response.status_code == 400 - assert response.json["message"] == "Invalid request body" - # Check Pydantic's structured error response - assert "errors" in response.json - errors = response.json["errors"] - assert len(errors) == 1 - assert errors[0]["loc"] == ["version_id"] - assert "valid ObjectId" in errors[0]["msg"] + assert "Invalid request body" in response.json["message"] def test_restore_version_nonexistent(self, client, sample_with_version): """Test restoring non-existent version.""" @@ -531,7 +496,6 @@ def test_restore_version_nonexistent(self, client, sample_with_version): response = client.post(f"/items/{refcode}/restore-version/", json={"version_id": fake_id}) assert response.status_code == 404 - assert "not found" in response.json["message"].lower() def test_restore_version_increments_version_number(self, client, sample_with_version): """Test that restore increments the item version number.""" @@ -594,7 +558,6 @@ def test_delete_version_nonexistent(self, client, sample_with_version): response = client.delete(f"/items/{refcode}/versions/{fake_id}/") assert response.status_code == 404 - assert response.json["status"] == "error" def test_delete_version_invalid_id(self, client, sample_with_version): """Test deleting with invalid ID format.""" @@ -974,7 +937,6 @@ def test_restore_requires_write_permission(self, another_client, sample_with_ver # Should fail because another_client doesn't have access to this item assert response.status_code == 404 - assert "not found or insufficient permissions" in response.json["message"].lower() class TestEdgeCases: diff --git a/pydatalab/tests/server/test_samples.py b/pydatalab/tests/server/test_samples.py index 6e55a35e5..e9d3f34a7 100644 --- a/pydatalab/tests/server/test_samples.py +++ b/pydatalab/tests/server/test_samples.py @@ -624,8 +624,8 @@ def test_create_cell(client, default_cell): assert cell["electrolyte"][1]["item"]["chemform"] == "NaCl" assert ( - cell["positive_electrode"][0]["item"]["name"] - == default_cell.positive_electrode[0].item.name + cell["positive_electrode"][0]["item"]["item_id"] + == default_cell.positive_electrode[0].item.item_id ) # The anode has a "real" entry in the db, so it should be looked up properly # with dereferenced chemical formula and name, even if it was not provided at link time @@ -924,7 +924,7 @@ def test_remove_items_from_collection_success( response = client.post("/new-sample/", json=sample_dict) assert response.status_code == 201 - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_remove" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -996,7 +996,7 @@ def test_remove_items_from_collection_not_found(client): @pytest.mark.dependency() def test_remove_items_from_collection_no_items_provided(client, default_collection): """Test removing with no item IDs provided.""" - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_empty_items" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1012,7 +1012,7 @@ def test_remove_items_from_collection_no_items_provided(client, default_collecti @pytest.mark.dependency() def test_remove_items_from_collection_no_matching_items(client, default_collection): """Test removing items that don't exist.""" - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_no_match" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1041,7 +1041,7 @@ def test_remove_items_from_collection_partial_success( response = client.post("/new-sample/", json=sample_dict) assert response.status_code == 201 - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_partial" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1100,7 +1100,7 @@ def test_copy_sample_and_add_to_collection(client, default_sample_dict, default_ assert response.status_code == 201 assert response.json["status"] == "success" - collection_dict = default_collection.dict().copy() + collection_dict = default_collection.model_dump().copy() collection_dict["collection_id"] = "test_copy_collection" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1132,12 +1132,12 @@ def test_copy_sample_and_add_to_collection(client, default_sample_dict, default_ def test_copy_sample_from_collection_to_different_collection( client, default_sample_dict, default_collection ): - collection1_dict = default_collection.dict().copy() + collection1_dict = default_collection.model_dump().copy() collection1_dict["collection_id"] = "collection_1" response = client.put("/collections", json={"data": collection1_dict}) assert response.status_code == 201 - collection2_dict = default_collection.dict().copy() + collection2_dict = default_collection.model_dump().copy() collection2_dict["collection_id"] = "collection_2" response = client.put("/collections", json={"data": collection2_dict}) assert response.status_code == 201 @@ -1177,7 +1177,7 @@ def test_copy_sample_from_collection_to_different_collection( @pytest.mark.dependency(depends=["test_copy_sample_from_collection_to_different_collection"]) def test_copy_sample_without_copying_collections(client, default_sample_dict, default_collection): - collection_dict = default_collection.dict().copy() + collection_dict = default_collection.model_dump().copy() collection_dict["collection_id"] = "test_no_auto_copy_collection" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1217,7 +1217,7 @@ def test_collections_permissions(client, admin_client, default_sample_dict, defa assert response.json["status"] == "success" refcode = response.json["sample_list_entry"]["refcode"] - admin_collection = default_collection.dict().copy() + admin_collection = default_collection.model_dump().copy() admin_collection["collection_id"] = "admin_only_collection" response = admin_client.put("/collections", json={"data": admin_collection}) diff --git a/pydatalab/tests/test_models.py b/pydatalab/tests/test_models.py index c2ba5959b..c92dbb7ce 100644 --- a/pydatalab/tests/test_models.py +++ b/pydatalab/tests/test_models.py @@ -81,7 +81,7 @@ def test_sample_with_inlined_reference(): @pytest.mark.parametrize("model", ITEM_MODELS.values()) def test_generate_schemas(model): """Test that all item model schemas can be generated.""" - assert model.schema() + assert model.model_json_schema() def test_attribute_docstrings_in_schema(): @@ -187,6 +187,9 @@ def test_file(): def test_custom_and_inherited_items(): class TestItem(Item): type: str = "items_custom" + new_field: str + + TestItem.model_rebuild() item = TestItem( type="items_custom", @@ -195,9 +198,10 @@ class TestItem(Item): creators=None, date="2020-01-01 00:00", item_id="1234", + new_field="This is a new field", ) - item_dict = item.dict() + item_dict = item.model_dump() assert item_dict["type"] == "items_custom" assert item_dict["creator_ids"][0] == ObjectId("0123456789ab0123456789ab") assert item_dict["creator_ids"][1] == ObjectId("1023456789ab0123456789ab") @@ -207,8 +211,8 @@ class TestItem(Item): item_json = json.loads(item.json()) assert item_json["type"] == "items_custom" - assert item_json["creator_ids"][0] == str(ObjectId("0123456789ab0123456789ab")) - assert item_json["creator_ids"][1] == str(ObjectId("1023456789ab0123456789ab")) + assert item_json["creator_ids"][0] == "0123456789ab0123456789ab" + assert item_json["creator_ids"][1] == "1023456789ab0123456789ab" assert ( item_json["date"] == datetime.datetime.fromisoformat("2020-01-01 00:00") @@ -224,7 +228,7 @@ class TestItem(Item): item_id="1234", ) - sample_dict = sample.dict() + sample_dict = sample.model_dump() assert sample_dict["type"] == "samples" assert sample_dict["creator_ids"][0] == ObjectId("0123456789ab0123456789ab") assert sample_dict["creator_ids"][1] == ObjectId("1023456789ab0123456789ab") @@ -261,13 +265,17 @@ class TestItem(Item): "MP2018_TEST_COMMERCIAL", "MP2018_TEST_COMMERCIAL_4.5V_hold", "AAAAAA", - 111111111, + "111111111", ], ) def test_good_ids(id): """Test good human-readable IDs for validity.""" - assert HumanReadableIdentifier(id) + class TestModel(pydantic.BaseModel): + test_id: HumanReadableIdentifier + + model = TestModel(test_id=id) + assert model.test_id == id @pytest.mark.parametrize( @@ -277,6 +285,7 @@ def test_good_ids(id): "mp 1 2 3 4 5 6", "lithium & sodium", "me388-123456789-123456789-really-long-descriptive-identifier-that-should-be-the-name-but-is-otherwise-valid", + 111111111, 1111111111111111111111111111111111111111111111111, "_AAAA", "AAA_", @@ -288,8 +297,11 @@ def test_good_ids(id): def test_bad_ids(id): """Test bad human-readable IDs for invalidity.""" + class TestModel(pydantic.BaseModel): + test_id: HumanReadableIdentifier + with pytest.raises(pydantic.ValidationError): - HumanReadableIdentifier(id) + TestModel(test_id=id) def test_cell_with_inlined_reference(): @@ -312,7 +324,7 @@ def test_cell_with_inlined_reference(): assert cell assert len(cell.relationships) == 1 - cell = Cell(**json.loads(cell.json())) + cell = Cell(**json.loads(cell.model_dump_json())) assert cell assert len(cell.relationships) == 1 @@ -432,7 +444,7 @@ def test_cell_relationship_deduplication(): assert parthood[0].item_id == "test_cathode" # Re-validating an already-clean cell must not grow the relationships list. - cell = Cell(**json.loads(cell.json())) + cell = Cell(**json.loads(cell.model_dump_json())) parthood = [r for r in cell.relationships if r.relation == RelationshipType.PARTHOOD] assert len(parthood) == 1 assert parthood[0].refcode == "grey:ABCDEF" @@ -527,7 +539,7 @@ def test_sample_synthesis_relationship_deduplication(): assert parents[0].item_id == "sm_1" # Re-validating an already-clean sample must not grow the relationships list. - sample = Sample(**json.loads(sample.json())) + sample = Sample(**json.loads(sample.model_dump_json())) parents = [r for r in sample.relationships if r.relation == RelationshipType.PARENT] assert len(parents) == 1 assert parents[0].refcode == "grey:ABCDEF" @@ -590,8 +602,11 @@ def test_good_refcodes(refcode): def test_bad_refcodes(refcode): """Test bad refcodes for invalidity.""" + class TestModel(pydantic.BaseModel): + test_refcode: Refcode + with pytest.raises(pydantic.ValidationError): - Refcode(refcode) + TestModel(test_refcode=refcode) @pytest.mark.parametrize( @@ -606,7 +621,10 @@ def test_bad_refcodes(refcode): def test_good_display_name(display_name): """Test good display name for validity.""" - assert DisplayName(display_name) + class TestModel(pydantic.BaseModel): + name: DisplayName + + assert TestModel(name=display_name) @pytest.mark.parametrize( @@ -620,8 +638,11 @@ def test_good_display_name(display_name): def test_bad_display_name(display_name): """Test bad display_name for invalidity.""" + class TestModel(pydantic.BaseModel): + name: DisplayName + with pytest.raises(ValueError): - DisplayName(display_name) + TestModel(name=display_name) @pytest.mark.parametrize( @@ -631,7 +652,11 @@ def test_bad_display_name(display_name): ], ) def test_good_email(contact_email): - assert EmailStr(contact_email) + + class TestModel(pydantic.BaseModel): + email: EmailStr + + assert TestModel(email=contact_email) @pytest.mark.parametrize( @@ -644,5 +669,9 @@ def test_good_email(contact_email): ], ) def test_bad_email(contact_email): + + class TestModel(pydantic.BaseModel): + email: EmailStr + with pytest.raises(ValueError): - assert EmailStr(contact_email) + TestModel(email=contact_email) From 4e816c2fe3ee59159d9e2657612f2c8077f79331 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 22:30:36 +0100 Subject: [PATCH 11/40] Patch optional vs default value for plugin data blocks --- pydatalab/src/pydatalab/models/utils.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/pydatalab/src/pydatalab/models/utils.py b/pydatalab/src/pydatalab/models/utils.py index 96df03119..696ec620c 100644 --- a/pydatalab/src/pydatalab/models/utils.py +++ b/pydatalab/src/pydatalab/models/utils.py @@ -4,7 +4,7 @@ from collections.abc import Callable from enum import Enum from functools import partial -from typing import Annotated, Any, TypeAlias +from typing import Annotated, Any, TypeAlias, get_args import pint from bson import ObjectId @@ -29,6 +29,25 @@ class BaseModel(PydanticBaseModel): model_config = ConfigDict(use_attribute_docstrings=True) + @classmethod + def __pydantic_init_subclass__(cls, **kwargs): + """Restore pydantic v1's implicit `None` default for optional fields. + + In pydantic v2 a field annotated ``T | None`` with no assigned default is + *required*, whereas in v1 it implicitly defaulted to ``None``. This back-fills + a ``None`` default for any required field that allows ``None``, so that models + (including those in plugins not yet migrated to v2) keep their v1 semantics. + Fields that already declare a default are untouched. + """ + super().__pydantic_init_subclass__(**kwargs) + changed = False + for field in cls.model_fields.values(): + if field.is_required() and type(None) in get_args(field.annotation): + field.default = None + changed = True + if changed: + cls.model_rebuild(force=True) + class ItemType(str, Enum): """An enumeration of the types of items known by this implementation, should be made dynamic in the future.""" From 1be1ee81588aa5599fdd6591648d37bdf454ccad Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 22:31:50 +0100 Subject: [PATCH 12/40] Simplify error handling of all item routes --- pydatalab/src/pydatalab/routes/v0_1/items.py | 206 +++++-------------- 1 file changed, 46 insertions(+), 160 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 5b5a55cf6..d78f09637 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -11,7 +11,7 @@ from flask_login import current_user from pydantic import ValidationError from pymongo.errors import DuplicateKeyError -from werkzeug.exceptions import BadRequest, Conflict, NotFound +from werkzeug.exceptions import BadRequest, Conflict, InternalServerError, NotFound from pydatalab.apps import BLOCK_TYPES from pydatalab.config import CONFIG @@ -525,7 +525,7 @@ def search_items(): types = types.split(",") if not query: - return jsonify({"status": "error", "message": "No query provided."}), 400 + raise BadRequest("No query provided.") permissions = get_default_permissions(user_only=False) pipeline = build_search_pipeline(query, get_items_fts_fields(), permissions) @@ -798,12 +798,9 @@ def create_samples(): sample_jsons = request_json["new_sample_datas"] if len(sample_jsons) > CONFIG.MAX_BATCH_CREATE_SIZE: - return jsonify( - { - "status": "error", - "message": f"Batch size limit exceeded. Maximum allowed: {CONFIG.MAX_BATCH_CREATE_SIZE}, requested: {len(sample_jsons)}", - } - ), 400 + raise BadRequest( + f"Batch size limit exceeded. Maximum allowed: {CONFIG.MAX_BATCH_CREATE_SIZE}, requested: {len(sample_jsons)}" + ) copy_from_item_ids = request_json.get("copy_from_item_ids") generate_ids_automatically = request_json.get("generate_ids_automatically") @@ -890,13 +887,7 @@ def _process_item_permissions( ] if not groups_requested and not creators_requested: - return ( - { - "status": "error", - "message": "No valid creator or group IDs found in the request.", - }, - 400, - ) + raise BadRequest("No valid creator or group IDs found in the request.") # Validate all creator IDs are present in the database if creator_ids: @@ -904,13 +895,7 @@ def _process_item_permissions( d for d in flask_mongo.db.users.find({"_id": {"$in": creator_ids}}, {"_id": 1}) ] if len(found_creator_ids) != len(creator_ids): - return ( - { - "status": "error", - "message": "One or more creator IDs not found in the database.", - }, - 400, - ) + raise BadRequest("One or more creator IDs not found in the database.") if group_ids: # Validate all group IDs are present in the database @@ -918,13 +903,7 @@ def _process_item_permissions( d for d in flask_mongo.db.groups.find({"_id": {"$in": group_ids}}, {"_id": 1}) ] if len(found_group_ids) != len(group_ids): - return ( - { - "status": "error", - "message": "One or more group IDs not found in the database.", - }, - 400, - ) + raise BadRequest("One or more group IDs not found in the database.") if append_mode: if creators_requested: @@ -986,12 +965,8 @@ def _process_item_permissions( ) if result.modified_count != 1: - return ( - { - "status": "error", - "message": "Failed to update permissions: you cannot remove yourself or the base owner as a creator.", - }, - 400, + raise BadRequest( + "Failed to update permissions: you cannot remove yourself or the base owner as a creator." ) # Record the new ownership in the item's history as a version @@ -1045,12 +1020,7 @@ def issue_physical_token(refcode: str): ) if not current_item: - return jsonify( - { - "status": "error", - "message": f"No valid item found with the given {refcode=}.", - } - ), 404 + raise NotFound token = secrets.token_urlsafe(16) access_document = AccessToken( @@ -1104,15 +1074,7 @@ def delete_sample(): ) if result.deleted_count != 1: - return ( - jsonify( - { - "status": "error", - "message": f"Failed to delete item with {item_id=}.", - } - ), - 400, - ) + raise BadRequest(f"Failed to delete item with {item_id=}.") flask_mongo.db.api_keys.delete_many({"refcode": item["refcode"], "type": "access_token"}) @@ -1180,15 +1142,7 @@ def get_item_data( doc = None if not doc: - return ( - jsonify( - { - "status": "error", - "message": f"No matching items for {match=} with current authorization.", - } - ), - 404, - ) + raise NotFound # See LAST_MODIFIED_PROJECTION: same backfill, applied outside an aggregation if not doc.get("last_modified") and isinstance(doc.get("_id"), ObjectId): @@ -1306,9 +1260,7 @@ def list_versions(refcode): # Check if user has access to the item (read access) has_access, _ = check_version_access(refcode, user_only=False) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1360,9 +1312,7 @@ def get_version(refcode, version_id): # Check if user has access to the item (read access) has_access, _ = check_version_access(refcode, user_only=False) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1370,7 +1320,7 @@ def get_version(refcode, version_id): try: version_object_id = ObjectId(version_id) except (InvalidId, TypeError): - return jsonify({"status": "error", "message": f"Invalid version_id: {version_id}"}), 400 + raise BadRequest(f"Invalid version_id: {version_id}") version = list( flask_mongo.db.item_versions.aggregate( @@ -1397,7 +1347,7 @@ def get_version(refcode, version_id): version = version[0] if not version: - return jsonify({"status": "error", "message": "Version not found"}), 404 + raise NotFound return jsonify({"status": "success", "version": version}), 200 @@ -1411,9 +1361,7 @@ def compare_versions(refcode): # Check if user has access to the item (read access) has_access, _ = check_version_access(refcode, user_only=False) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1424,20 +1372,12 @@ def compare_versions(refcode): v1=request.args.get("v1", ""), v2=request.args.get("v2", "") ) except ValidationError as exc: - return jsonify( - {"status": "error", "message": "Invalid query parameters", "errors": exc.errors()} - ), 400 + raise BadRequest(f"Invalid query parameters: {exc}") - try: - v1_object_id = ObjectId(query_params.v1) - v2_object_id = ObjectId(query_params.v2) - except (InvalidId, TypeError) as e: - return jsonify({"status": "error", "message": f"Invalid version ID format: {str(e)}"}), 400 - - v1 = flask_mongo.db.item_versions.find_one({"_id": v1_object_id, "refcode": refcode}) - v2 = flask_mongo.db.item_versions.find_one({"_id": v2_object_id, "refcode": refcode}) + v1 = flask_mongo.db.item_versions.find_one({"_id": query_params.v1, "refcode": refcode}) + v2 = flask_mongo.db.item_versions.find_one({"_id": query_params.v2, "refcode": refcode}) if not v1 or not v2: - return jsonify({"status": "error", "message": "One or both versions not found"}), 404 + return NotFound("One or both versions not found") # Use DeepDiff for proper nested structure comparison # This handles nested dicts, lists, type changes, and provides detailed change information @@ -1482,29 +1422,23 @@ def restore_version(refcode): try: restore_request = RestoreVersionRequest(**request.get_json()) except ValidationError as exc: - return jsonify( - {"status": "error", "message": "Invalid request body", "errors": exc.errors()} - ), 400 + raise BadRequest(f"Invalid request body: {exc}") try: version_object_id = ObjectId(restore_request.version_id) except (InvalidId, TypeError): - return jsonify( - {"status": "error", "message": f"Invalid version_id: {restore_request.version_id}"} - ), 400 + raise BadRequest(f"Invalid version_id: {restore_request.version_id}") # Check permissions - user must have write access current_item = flask_mongo.db.items.find_one( {"refcode": refcode, **get_default_permissions(user_only=True)} ) if not current_item: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound version = flask_mongo.db.item_versions.find_one({"_id": version_object_id, "refcode": refcode}) if not version: - return jsonify({"status": "error", "message": "Version not found"}), 404 + raise NotFound restored_data = version["data"].copy() @@ -1513,12 +1447,9 @@ def restore_version(refcode): # Ensure type consistency if restored_data.get("type") != current_item.get("type"): - return jsonify( - { - "status": "error", - "message": f"Cannot restore version with different type. Current: {current_item.get('type')}, Version: {restored_data.get('type')}", - } - ), 400 + raise BadRequest( + f"Cannot restore version with different type. Current: {current_item.get('type')}, Version: {restored_data.get('type')}" + ) # Atomically get the next version number (used for both version in item_versions and item.version) next_version_number = get_next_version_number(refcode) @@ -1530,19 +1461,15 @@ def restore_version(refcode): # Validate restored data against the item model item_type = current_item["type"] if item_type not in ITEM_MODELS: - return jsonify({"status": "error", "message": f"Invalid item type: {item_type}"}), 400 + raise BadRequest(f"Invalid item type: {item_type}") try: # Validate using the appropriate model ITEM_MODELS[item_type](**restored_data) except ValidationError as exc: - return jsonify( - { - "status": "error", - "message": f"Restored data failed validation: {str(exc)}", - "output": str(exc), - } - ), 400 + raise BadRequest( + f"Restored data failed validation against schema for type {item_type}: {exc}" + ) # Perform the restore first flask_mongo.db.items.update_one({"refcode": refcode}, {"$set": restored_data}) @@ -1579,13 +1506,7 @@ def restore_version(refcode): refcode, str(exc), ) - return jsonify( - { - "status": "error", - "message": f"Restored version data validation failed: {str(exc)}", - "output": str(exc), - } - ), 400 + raise BadRequest(f"Restored version data validation failed: {exc}") # Insert validated data flask_mongo.db.item_versions.insert_one( @@ -1610,9 +1531,7 @@ def delete_version(refcode, version_id): # Check if user has write access to the item (write access required) has_access, _ = check_version_access(refcode, user_only=True) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1620,13 +1539,13 @@ def delete_version(refcode, version_id): try: version_object_id = ObjectId(version_id) except (InvalidId, TypeError): - return jsonify({"status": "error", "message": f"Invalid version_id: {version_id}"}), 400 + raise BadRequest(f"Invalid version_id: {version_id}") result = flask_mongo.db.item_versions.delete_one({"_id": version_object_id, "refcode": refcode}) if result.deleted_count == 1: return jsonify({"status": "success"}), 200 else: - return jsonify({"status": "error", "message": "Version not found"}), 404 + raise NotFound @ITEMS.route("/items//save-version/", methods=["POST"]) @@ -1691,23 +1610,13 @@ def save_item(): ) if not item: - return ( - jsonify( - status="error", - message=f"Unable to find item with appropriate permissions and {item_id=}.", - ), - 404, - ) + raise NotFound # Store refcode for version saving after successful update refcode = item.get("refcode") if not refcode: - return ( - jsonify( - status="error", - message=f"Item {item_id} does not have a refcode.", - ), - 400, + raise InternalServerError( + f"Item {item_id} does not have a refcode; please report this issue." ) user_only = item["type"] not in ("starting_materials", "equipment") @@ -1717,13 +1626,7 @@ def save_item(): ) if not item: - return ( - jsonify( - status="error", - message=f"Unable to find item with appropriate permissions and {item_id=}.", - ), - 404, - ) + raise NotFound stored_blocks = item.get("blocks_obj", {}) for block_id, block_data in updated_data.get("blocks_obj", {}).items(): @@ -1831,14 +1734,9 @@ def save_item(): exclude={"collections", "creators", "immutable_id"}, ) - except ValidationError as exc: - return ( - jsonify( - status="error", - message=f"Unable to update item {item_id=} ({item_type=}) with new data {updated_data}", - output=str(exc), - ), - 400, + except ValidationError: + raise BadRequest( + f"Unable to update item {item_id=} ({item_type=}) with new data {updated_data}" ) if preserve_relationships and original_relationships is not None: @@ -1863,14 +1761,7 @@ def save_item(): ) if result.matched_count != 1: - return ( - jsonify( - status="error", - message=f"{item_id} item update failed. no subdocument matched", - output=result.raw_result, - ), - 400, - ) + raise BadRequest(f"{item_id} item update failed. no subdocument matched") # Now save a version AFTER successful item update. # Only increment item.version and bump last_modified when content actually changed @@ -1927,12 +1818,7 @@ def get_access_token_info(refcode: str): ) if not current_item: - return jsonify( - { - "status": "error", - "message": f"No valid item found with the given {refcode=}.", - } - ), 404 + raise NotFound existing_token = flask_mongo.db.api_keys.find_one( {"refcode": refcode, "active": True, "type": "access_token"}, From b8bf705161762ccfd80ce51635b12775248a90d4 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 23:02:52 +0100 Subject: [PATCH 13/40] Fix exports for pydantic 2 --- pydatalab/src/pydatalab/export.py | 6 +++--- pydatalab/src/pydatalab/routes/v0_1/export.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pydatalab/src/pydatalab/export.py b/pydatalab/src/pydatalab/export.py index 8835ba341..1a83a32b5 100644 --- a/pydatalab/src/pydatalab/export.py +++ b/pydatalab/src/pydatalab/export.py @@ -112,7 +112,7 @@ def write_eln_file( item_folder = root_folder / item[primary_key] - item_metadata = ITEM_MODELS[item.get("type")](**item).json(indent=2) + item_metadata = ITEM_MODELS[item.get("type")](**item).model_dump_json(indent=2) zipf.writestr(str(item_folder / "metadata.json"), item_metadata) for file in item.get("files", []): @@ -378,7 +378,7 @@ def create_eln_file( item_data = list(cursor)[0] ItemModel = ITEM_MODELS[item_data["type"]] - item_data = ItemModel(**item_data).dict() + item_data = ItemModel(**item_data).model_dump() except IndexError: item_data = None @@ -420,7 +420,7 @@ def create_eln_file( for ind, item in enumerate(all_items): ItemModel = ITEM_MODELS[item["type"]] - _all_items.append(ItemModel(**item).dict()) + _all_items.append(ItemModel(**item).model_dump()) all_items = _all_items diff --git a/pydatalab/src/pydatalab/routes/v0_1/export.py b/pydatalab/src/pydatalab/routes/v0_1/export.py index 14f4194d7..fe51ccb0e 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/export.py +++ b/pydatalab/src/pydatalab/routes/v0_1/export.py @@ -134,7 +134,7 @@ def _do_export( def add_stage(message: str, level: str = "info"): stage = TaskStage(timestamp=datetime.now(tz=timezone.utc), message=message, level=level) flask_mongo.db.tasks.update_one( - {"task_id": task_id}, {"$push": {"spec.stages": stage.dict()}} + {"task_id": task_id}, {"$push": {"spec.stages": stage.model_dump()}} ) try: @@ -221,7 +221,7 @@ def start_collection_export(collection_id: str): ), ) - flask_mongo.db.tasks.insert_one(export_task.dict(exclude_none=False)) + flask_mongo.db.tasks.insert_one(export_task.model_dump(exclude_none=False)) task_scheduler.add_job( func=_generate_export_in_background, @@ -390,7 +390,7 @@ def start_item_export(item_id: str): ), ) - flask_mongo.db.tasks.insert_one(export_task.dict(exclude_none=False)) + flask_mongo.db.tasks.insert_one(export_task.model_dump(exclude_none=False)) task_scheduler.add_job( func=_generate_export_in_background, From a5ac1b897d72a964d245ac17ceee8d53b9c83d71 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 23:22:56 +0100 Subject: [PATCH 14/40] Fix info endpoint for pydantic 2 --- pydatalab/src/pydatalab/routes/v0_1/info.py | 37 +++++++++------------ 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/info.py b/pydatalab/src/pydatalab/routes/v0_1/info.py index 40e4fc6a5..4bf7bd6e1 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/info.py +++ b/pydatalab/src/pydatalab/routes/v0_1/info.py @@ -5,6 +5,7 @@ from datetime import timedelta as td from datetime import timezone as tz from functools import lru_cache +from typing import Any, Generic, TypeVar from flask import Blueprint, jsonify, request from pydantic import ( @@ -13,7 +14,6 @@ ConfigDict, Field, field_validator, - model_validator, ) from pydatalab import __version__ @@ -47,14 +47,19 @@ class Links(BaseModel): model_config = ConfigDict(extra="allow") -class Data(BaseModel): +AttributesT = TypeVar("AttributesT") + + +class Data(BaseModel, Generic[AttributesT]): id: str type: str - attributes: Attributes + attributes: AttributesT + """The attributes payload, serialized as whatever concrete type the envelope is + parametrised with (e.g. ``Data[Info]``) so subclass fields are not stripped.""" -class JSONAPIResponse(BaseModel): - data: Data | list[Data] +class JSONAPIResponse(BaseModel, Generic[AttributesT]): + data: Data[AttributesT] | list[Data[AttributesT]] meta: Meta links: Links | None = None @@ -70,7 +75,7 @@ class Info(Attributes, Meta): homepage: AnyUrl | None = None source_repository: AnyUrl | None = None identifier_prefix: str - features: FeatureFlags = FEATURE_FLAGS + features: FeatureFlags | None = None max_upload_bytes: int @field_validator("maintainer", mode="before") @@ -80,18 +85,6 @@ def strip_maintainer_fields(cls, v): return MetaPerson(contact_email=v.contact_email, display_name=v.display_name) return v - @model_validator(mode="after") - def ensure_features_serialization(self): - """Ensure features are properly serialized for frontend consumption.""" - if hasattr(self.features, "model_dump"): - features_dict = self.features.model_dump() - else: - features_dict = self.features - - if not isinstance(self.features, FeatureFlags): - self.features = FeatureFlags(**features_dict) - return self - @lru_cache(maxsize=1) def _get_deployment_metadata_once() -> dict: @@ -118,7 +111,7 @@ def get_info(): """ - response_data = JSONAPIResponse( + response_data = JSONAPIResponse[Info]( data=Data(id="/", type="info", attributes=Info(**_get_deployment_metadata_once())), meta=Meta(query=request.query_string.decode() if request.query_string else ""), links=Links(self=request.url), @@ -150,7 +143,7 @@ def list_block_types(): """Returns a list of all blocks implemented in this server.""" return jsonify( json.loads( - JSONAPIResponse( + JSONAPIResponse[dict[str, Any]]( data=[ Data( id=block_type, @@ -179,7 +172,7 @@ def list_supported_types(): return jsonify( json.loads( - JSONAPIResponse( + JSONAPIResponse[dict[str, Any]]( data=[ Data( id=item_type, @@ -208,7 +201,7 @@ def get_schema_type(item_type): return jsonify( json.loads( - JSONAPIResponse( + JSONAPIResponse[dict[str, Any]]( data=Data( id=item_type, type="item_type", From bbf684d4ae6db7e9490ce46df4dd736b0ad4bd1e Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 23:31:59 +0100 Subject: [PATCH 15/40] Unpick circular dependencies between export/items routes by moving lookups to mongo module --- pydatalab/src/pydatalab/export.py | 4 +- pydatalab/src/pydatalab/mongo.py | 40 +++++++++++++++++ pydatalab/src/pydatalab/routes/v0_1/items.py | 46 ++++---------------- 3 files changed, 50 insertions(+), 40 deletions(-) diff --git a/pydatalab/src/pydatalab/export.py b/pydatalab/src/pydatalab/export.py index 1a83a32b5..a06551765 100644 --- a/pydatalab/src/pydatalab/export.py +++ b/pydatalab/src/pydatalab/export.py @@ -12,10 +12,10 @@ from pydatalab.config import CONFIG from pydatalab.logger import LOGGER from pydatalab.models import ITEM_MODELS -from pydatalab.mongo import flask_mongo -from pydatalab.routes.v0_1.items import ( +from pydatalab.mongo import ( creators_lookup, files_lookup, + flask_mongo, groups_lookup, ) diff --git a/pydatalab/src/pydatalab/mongo.py b/pydatalab/src/pydatalab/mongo.py index 09215c3e6..509bb2d2c 100644 --- a/pydatalab/src/pydatalab/mongo.py +++ b/pydatalab/src/pydatalab/mongo.py @@ -25,6 +25,9 @@ "GROUPS_FTS_FIELDS", "generate_heuristic_regex_search", "build_search_pipeline", + "creators_lookup", + "groups_lookup", + "files_lookup", ) flask_mongo = PyMongo() @@ -35,6 +38,43 @@ """ +def creators_lookup() -> dict: + return { + "from": "users", + "let": {"creator_ids": "$creator_ids"}, + "pipeline": [ + {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$creator_ids", []]}]}}}, + {"$addFields": {"__order": {"$indexOfArray": ["$$creator_ids", "$_id"]}}}, + {"$sort": {"__order": 1}}, + {"$project": {"_id": 1, "display_name": 1, "gravatar_hash": 1}}, + ], + "as": "creators", + } + + +def groups_lookup() -> dict: + return { + "from": "groups", + "let": {"group_ids": "$group_ids"}, + "pipeline": [ + {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$group_ids", []]}]}}}, + {"$addFields": {"__order": {"$indexOfArray": ["$$group_ids", "$_id"]}}}, + {"$sort": {"__order": 1}}, + {"$project": {"_id": 1, "display_name": 1, "group_id": 1}}, + ], + "as": "groups", + } + + +def files_lookup() -> dict: + return { + "from": "files", + "localField": "file_ObjectIds", + "foreignField": "_id", + "as": "files", + } + + @lru_cache(maxsize=1) def get_items_fts_fields() -> set[str]: """Get all string fields from item models for full-text search.""" diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index d78f09637..a28512de6 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -25,7 +25,14 @@ RestoreVersionRequest, VersionAction, ) -from pydatalab.mongo import build_search_pipeline, flask_mongo, get_items_fts_fields +from pydatalab.mongo import ( + build_search_pipeline, + creators_lookup, + files_lookup, + flask_mongo, + get_items_fts_fields, + groups_lookup, +) from pydatalab.permissions import ( PUBLIC_USER_ID, AccessToken, @@ -304,34 +311,6 @@ def get_samples_summary(match: dict | None = None, project: dict | None = None) ) -def creators_lookup() -> dict: - return { - "from": "users", - "let": {"creator_ids": "$creator_ids"}, - "pipeline": [ - {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$creator_ids", []]}]}}}, - {"$addFields": {"__order": {"$indexOfArray": ["$$creator_ids", "$_id"]}}}, - {"$sort": {"__order": 1}}, - {"$project": {"_id": 1, "display_name": 1, "gravatar_hash": 1}}, - ], - "as": "creators", - } - - -def groups_lookup() -> dict: - return { - "from": "groups", - "let": {"group_ids": "$group_ids"}, - "pipeline": [ - {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$group_ids", []]}]}}}, - {"$addFields": {"__order": {"$indexOfArray": ["$$group_ids", "$_id"]}}}, - {"$sort": {"__order": 1}}, - {"$project": {"_id": 1, "display_name": 1, "group_id": 1}}, - ], - "as": "groups", - } - - def entry_reference_lookup(item_doc: dict) -> dict: """Looks up any field that contains an entry reference and resolves it to the item data.""" @@ -411,15 +390,6 @@ def entry_reference_lookup(item_doc: dict) -> dict: return item_doc -def files_lookup() -> dict: - return { - "from": "files", - "localField": "file_ObjectIds", - "foreignField": "_id", - "as": "files", - } - - def collections_lookup() -> dict: """Looks inside the relationships of the item, searches for IDs in the collections table and then projects only the collection ID and name for the response. From f1caab1428b61389f10c7578dbaea50d7403d886 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 23:53:20 +0100 Subject: [PATCH 16/40] Move `datalab_*` fields in schema to `json_schema_extra` and allow carve out for plugins --- pydatalab/src/pydatalab/apps/__init__.py | 11 ++++++++++- pydatalab/src/pydatalab/apps/chat/blocks.py | 4 +++- pydatalab/src/pydatalab/blocks/base.py | 4 ++-- pydatalab/src/pydatalab/models/blocks.py | 18 +++++++++++++----- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/pydatalab/src/pydatalab/apps/__init__.py b/pydatalab/src/pydatalab/apps/__init__.py index 9f49a0950..f48a6427e 100644 --- a/pydatalab/src/pydatalab/apps/__init__.py +++ b/pydatalab/src/pydatalab/apps/__init__.py @@ -3,8 +3,11 @@ """ +import warnings from typing import TYPE_CHECKING +from pydantic.warnings import PydanticDeprecatedSince20 + if TYPE_CHECKING: # This import is required to prevent circular imports for application-specific blocks from pydatalab.blocks.base import DataBlock # noqa @@ -116,7 +119,13 @@ def load_block_plugins(): block_plugins: dict[str, type[DataBlock]] = {} for entry_point in entry_points(group="pydatalab.apps.plugins"): - block = entry_point.load() + # Plugins may not yet be migrated to pydantic v2, so tolerate the v2 + # deprecation warnings their models emit at import time (e.g. extra `Field` + # kwargs like `datalab_exclude_*`) rather than letting them error out plugin + # loading. datalab's own deprecations are still surfaced as errors. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", PydanticDeprecatedSince20) + block = entry_point.load() if not issubclass(block, DataBlock): raise ValueError(f"Plugin {block} must be a subclass of DataBlock") diff --git a/pydatalab/src/pydatalab/apps/chat/blocks.py b/pydatalab/src/pydatalab/apps/chat/blocks.py index b95294faa..78b53b1b2 100644 --- a/pydatalab/src/pydatalab/apps/chat/blocks.py +++ b/pydatalab/src/pydatalab/apps/chat/blocks.py @@ -19,7 +19,9 @@ class ChatBlockResponse(DataBlockResponse): prompt: str | None = None model: str available_models: dict[str, ModelCard] | None = Field( - None, validate_default=True, datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, + validate_default=True, + json_schema_extra={"datalab_exclude_from_db": True, "datalab_exclude_from_load": True}, ) token_count: int | None = None temperature: float diff --git a/pydatalab/src/pydatalab/blocks/base.py b/pydatalab/src/pydatalab/blocks/base.py index 36ea3aac7..b149f7ad2 100644 --- a/pydatalab/src/pydatalab/blocks/base.py +++ b/pydatalab/src/pydatalab/blocks/base.py @@ -195,7 +195,7 @@ def to_db(self) -> dict: ) exclude_fields: set[str] = { f - for (f, s) in self.block_db_model.schema()["properties"].items() + for (f, s) in self.block_db_model.model_json_schema()["properties"].items() if s.get("datalab_exclude_from_db") } return self.block_db_model(**self.data).model_dump( @@ -350,7 +350,7 @@ def update_from_web(self, data: dict, stored_data: dict | None = None): ) exclude_fields: set[str] = { f - for (f, s) in self.block_db_model.schema()["properties"].items() + for (f, s) in self.block_db_model.model_json_schema()["properties"].items() if s.get("datalab_exclude_from_load") } [data.pop(f, None) for f in exclude_fields] diff --git a/pydatalab/src/pydatalab/models/blocks.py b/pydatalab/src/pydatalab/models/blocks.py index f1e02a4d4..e6d9244e3 100644 --- a/pydatalab/src/pydatalab/models/blocks.py +++ b/pydatalab/src/pydatalab/models/blocks.py @@ -44,23 +44,31 @@ class DataBlockResponse(BaseModel): """Any warnings that occurred during block processing.""" b64_encoded_image: dict[str, str] | None = Field( - None, datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, + json_schema_extra={"datalab_exclude_from_db": True, "datalab_exclude_from_load": True}, ) """Any base64-encoded image data associated with the block, keyed by `file_id`.""" bokeh_plot_data: dict | None = Field( - None, datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, + json_schema_extra={"datalab_exclude_from_db": True, "datalab_exclude_from_load": True}, ) """A JSON-encoded string containing the Bokeh plot data, if any.""" - computed: dict | None = Field(default=None, datalab_exclude_from_load=True) + computed: dict | None = Field( + default=None, json_schema_extra={"datalab_exclude_from_load": True} + ) """Any processed or computed data associated with the block, small enough to store and filter directly in the database, i.e., strings or a few hundred numbers not exceeding 16KB in size. Examples could include peak positions, and widths, but not the full spectrum. """ - processed: dict | None = Field(default=None, datalab_exclude_from_load=True) + processed: dict | None = Field( + default=None, json_schema_extra={"datalab_exclude_from_load": True} + ) - metadata: dict | None = Field(default=None, datalab_exclude_from_load=True) + metadata: dict | None = Field( + default=None, json_schema_extra={"datalab_exclude_from_load": True} + ) """Any structured metadata associated with the block, for example, experimental acquisition parameters.""" From 5f4ca8ebda5d82066f81d1bacd7b62d2a1e8c1cf Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 00:16:59 +0100 Subject: [PATCH 17/40] Final wave of model_dump/dict changes --- pydatalab/src/pydatalab/apps/nmr/blocks.py | 6 ++-- pydatalab/src/pydatalab/routes/v0_1/admin.py | 8 ++++-- pydatalab/src/pydatalab/routes/v0_1/blocks.py | 4 +-- pydatalab/src/pydatalab/routes/v0_1/groups.py | 5 +++- pydatalab/src/pydatalab/routes/v0_1/users.py | 2 +- pydatalab/tests/server/test_async_blocks.py | 26 ++++++++--------- pydatalab/tests/server/test_export.py | 28 +++++++++---------- pydatalab/tests/server/test_graph.py | 2 +- pydatalab/tests/server/test_item_graph.py | 13 +++++---- pydatalab/tests/server/test_item_versions.py | 12 ++++---- pydatalab/tests/server/test_samples.py | 21 ++++++++------ pydatalab/tests/server/test_search.py | 2 +- .../tests/server/test_starting_materials.py | 2 +- pydatalab/tests/test_models.py | 6 ++-- 14 files changed, 73 insertions(+), 64 deletions(-) diff --git a/pydatalab/src/pydatalab/apps/nmr/blocks.py b/pydatalab/src/pydatalab/apps/nmr/blocks.py index 9ba68ed9d..de8680bf4 100644 --- a/pydatalab/src/pydatalab/apps/nmr/blocks.py +++ b/pydatalab/src/pydatalab/apps/nmr/blocks.py @@ -130,7 +130,7 @@ def read_bruker_nmr_data( metadata["pulse_program_name"] = a_dic["acqus"]["PULPROG"] metadata["title"] = topspin_title - self.data["metadata"] = NMRMetadata(**metadata).dict() + self.data["metadata"] = NMRMetadata(**metadata).model_dump() return serialized_df, metadata @@ -209,7 +209,7 @@ def read_jcamp_nmr_data( pass serialized_df = df.to_dict() if (df is not None) else None - self.data["metadata"] = NMRMetadata(**metadata).dict() + self.data["metadata"] = NMRMetadata(**metadata).model_dump() return serialized_df, metadata @@ -293,7 +293,7 @@ def read_jeol_nmr_data( metadata["nscans"] = nscans serialized_df = df.to_dict() if (df is not None) else None - self.data["metadata"] = NMRMetadata(**metadata).dict() + self.data["metadata"] = NMRMetadata(**metadata).model_dump() return serialized_df, metadata diff --git a/pydatalab/src/pydatalab/routes/v0_1/admin.py b/pydatalab/src/pydatalab/routes/v0_1/admin.py index 55e886fae..eea5a6478 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/admin.py +++ b/pydatalab/src/pydatalab/routes/v0_1/admin.py @@ -125,7 +125,7 @@ def get_users(): elif not isinstance(user["managers"], list): user["managers"] = [] - return jsonify({"status": "success", "data": [Person(**d).dict() for d in users_list]}) + return jsonify({"status": "success", "data": [Person(**d).model_dump() for d in users_list]}) @ADMIN.route("/roles/", methods=["PATCH"]) @@ -357,7 +357,9 @@ def get_groups(): [{"$match": {}}, {"$lookup": members_lookup}, {"$lookup": managers_lookup}] ) - return jsonify({"status": "success", "data": [Group(**d).dict() for d in group_docs]}), 200 + return jsonify( + {"status": "success", "data": [Group(**d).model_dump() for d in group_docs]} + ), 200 @ADMIN.route("/groups", methods=["PUT"]) @@ -392,7 +394,7 @@ def create_group(): try: group_immutable_id = flask_mongo.db.groups.insert_one( - group.dict(exclude_unset=True) + group.model_dump(exclude_unset=True) ).inserted_id except pymongo.errors.DuplicateKeyError: return jsonify( diff --git a/pydatalab/src/pydatalab/routes/v0_1/blocks.py b/pydatalab/src/pydatalab/routes/v0_1/blocks.py index 6c00d1b2e..8322f1161 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/blocks.py +++ b/pydatalab/src/pydatalab/routes/v0_1/blocks.py @@ -54,7 +54,7 @@ def add_stage(message: str, level: str = "info", traceback: str | None = None): timestamp=datetime.now(tz=timezone.utc), message=message, level=level, detail=traceback ) flask_mongo.db.tasks.update_one( - {"task_id": task_id}, {"$push": {"spec.stages": stage.dict()}} + {"task_id": task_id}, {"$push": {"spec.stages": stage.model_dump()}} ) with app_ctx, req_ctx: @@ -381,7 +381,7 @@ def update_block(): ), ) - flask_mongo.db.tasks.insert_one(block_task.dict()) + flask_mongo.db.tasks.insert_one(block_task.model_dump()) task_scheduler.add_job( func=_process_block_async, diff --git a/pydatalab/src/pydatalab/routes/v0_1/groups.py b/pydatalab/src/pydatalab/routes/v0_1/groups.py index e1a54d5ef..d5b4e5648 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/groups.py +++ b/pydatalab/src/pydatalab/routes/v0_1/groups.py @@ -42,5 +42,8 @@ def search_groups(): cursor = flask_mongo.db.groups.aggregate(pipeline) return jsonify( - {"status": "success", "data": list(json.loads(Group(**d).json()) for d in cursor)} + { + "status": "success", + "data": list(json.loads(Group(**d).model_dump_json()) for d in cursor), + } ), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/users.py b/pydatalab/src/pydatalab/routes/v0_1/users.py index f790a12bf..ab9e737f0 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/users.py +++ b/pydatalab/src/pydatalab/routes/v0_1/users.py @@ -218,5 +218,5 @@ def search_users(): cursor = flask_mongo.db.users.aggregate(pipeline) return jsonify( - {"status": "success", "users": [json.loads(Person(**d).json()) for d in cursor]} + {"status": "success", "users": [json.loads(Person(**d).model_dump_json()) for d in cursor]} ), 200 diff --git a/pydatalab/tests/server/test_async_blocks.py b/pydatalab/tests/server/test_async_blocks.py index 6e85ff4a2..62c59859c 100644 --- a/pydatalab/tests/server/test_async_blocks.py +++ b/pydatalab/tests/server/test_async_blocks.py @@ -147,7 +147,7 @@ def test_status_pending(self, client, user_id, database): block_id="test_block", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -182,7 +182,7 @@ def test_status_processing_with_stages(self, client, user_id, database): stages=stages, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -226,7 +226,7 @@ def test_status_ready_with_gridfs_data(self, client, user_id, database, app): block_id=block_id, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) # Write block data to GridFS from pydatalab.mongo import get_database @@ -276,7 +276,7 @@ def test_status_ready_no_gridfs_data(self, client, user_id, database): block_id=block_id, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -304,7 +304,7 @@ def test_status_error(self, client, user_id, database): block_id="test_block", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -334,7 +334,7 @@ def test_status_ignores_export_tasks(self, client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 404 @@ -369,7 +369,7 @@ def test_status_ready_respects_permissions(self, another_client, user_id, databa block_id=block_id, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) from pydatalab.mongo import get_database @@ -428,7 +428,7 @@ def test_successful_processing(self, app, user_id, database): status=TaskStatus.PENDING, spec=BlockProcessingTaskSpec(item_id=item_id, block_id=block_id), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) block_data = { "blocktype": "comment", @@ -470,7 +470,7 @@ def test_processing_error_handling(self, app, user_id, database): status=TaskStatus.PENDING, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) # Pass an invalid block type to trigger an error block_data = { @@ -509,7 +509,7 @@ def test_marks_timed_out_tasks_as_error(self, app, user_id, database): status=TaskStatus.PROCESSING, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - task_dict = task.dict() + task_dict = task.model_dump() task_dict["created_at"] = old_time database.tasks.insert_one(task_dict) @@ -534,7 +534,7 @@ def test_does_not_timeout_recent_tasks(self, app, user_id, database): status=TaskStatus.PROCESSING, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _cleanup_stale_tasks() @@ -558,7 +558,7 @@ def test_purges_old_completed_tasks_and_gridfs(self, app, user_id, database): completed_at=old_time, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - task_dict = task.dict() + task_dict = task.model_dump() task_dict["created_at"] = old_time database.tasks.insert_one(task_dict) @@ -586,7 +586,7 @@ def test_does_not_purge_recent_completed_tasks(self, app, user_id, database): completed_at=datetime.now(tz=timezone.utc), spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _cleanup_stale_tasks() diff --git a/pydatalab/tests/server/test_export.py b/pydatalab/tests/server/test_export.py index 9ef6ef757..7e8d4c7d3 100644 --- a/pydatalab/tests/server/test_export.py +++ b/pydatalab/tests/server/test_export.py @@ -78,7 +78,7 @@ def test_get_export_status_pending(client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/status") assert response.status_code == 200 @@ -107,7 +107,7 @@ def test_get_export_status_ready(client, user_id, database, tmp_path): file_path=str(file_path), ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/status") assert response.status_code == 200 @@ -136,7 +136,7 @@ def test_get_export_status_error(client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/status") assert response.status_code == 200 @@ -180,7 +180,7 @@ def test_download_export_success(client, user_id, database, tmp_path, monkeypatc file_path=str(file_path), ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/download") assert response.status_code == 200 @@ -203,7 +203,7 @@ def test_not_users_export_download(client, user_id, another_client, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = another_client.get(f"/exports/{task_id}/download") assert response.status_code == 404 @@ -223,7 +223,7 @@ def test_download_export_not_ready(client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/download") assert response.status_code == 400 @@ -248,7 +248,7 @@ def test_download_export_file_missing(client, user_id, database): file_path="/nonexistent/path.eln.zip", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/download") assert response.status_code == 404 @@ -275,7 +275,7 @@ def test_do_export_success(database, sample_collection, insert_default_sample, u export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _do_export(task_id, collection_id=collection_id, export_type="collection") @@ -299,7 +299,7 @@ def test_do_export_success(database, sample_collection, insert_default_sample, u export_type="item", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _do_export(task_id, item_id=item_id, export_type="item") @@ -326,7 +326,7 @@ def test_do_export_error_handling(database, user_id): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _do_export(task_id, collection_id="nonexistent_collection", export_type="collection") @@ -367,7 +367,7 @@ def test_cleanup_old_exports(database, user_id, tmp_path, monkeypatch): file_path=str(old_file), ), ) - database.tasks.insert_one(old_task.dict()) + database.tasks.insert_one(old_task.model_dump()) # A recent export: created within the window, must be retained. recent_task_id = "recent-export-to-keep" @@ -385,7 +385,7 @@ def test_cleanup_old_exports(database, user_id, tmp_path, monkeypatch): file_path=str(recent_file), ), ) - database.tasks.insert_one(recent_task.dict()) + database.tasks.insert_one(recent_task.model_dump()) try: _cleanup_old_exports() @@ -422,7 +422,7 @@ def test_cleanup_old_exports_missing_file(database, user_id): file_path="/nonexistent/already-deleted.eln.zip", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) try: _cleanup_old_exports() @@ -447,7 +447,7 @@ def test_do_export_status_transitions(database, sample_collection, user_id): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) status_during_export = [] diff --git a/pydatalab/tests/server/test_graph.py b/pydatalab/tests/server/test_graph.py index 3e10a9dc9..9c1f30e3d 100644 --- a/pydatalab/tests/server/test_graph.py +++ b/pydatalab/tests/server/test_graph.py @@ -61,7 +61,7 @@ def test_simple_graph(admin_client): ) new_samples = [ - json.loads(d.json()) + json.loads(d.model_dump_json()) for d in [parent, child_1, child_2, child_3, child_4, missing_child, cell] ] diff --git a/pydatalab/tests/server/test_item_graph.py b/pydatalab/tests/server/test_item_graph.py index 08ffa7db4..f04169aa6 100644 --- a/pydatalab/tests/server/test_item_graph.py +++ b/pydatalab/tests/server/test_item_graph.py @@ -15,7 +15,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(material.json())}, + json={"new_sample_data": json.loads(material.model_dump_json())}, ) assert creation.status_code == 201 @@ -39,7 +39,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(parent.json())}, + json={"new_sample_data": json.loads(parent.model_dump_json())}, ) assert creation.status_code == 201 @@ -68,7 +68,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(child.json())}, + json={"new_sample_data": json.loads(child.model_dump_json())}, ) grandchild = Sample( @@ -80,7 +80,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(grandchild.json())}, + json={"new_sample_data": json.loads(grandchild.model_dump_json())}, ) great_grandchild = Sample( @@ -92,7 +92,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(great_grandchild.json())}, + json={"new_sample_data": json.loads(great_grandchild.model_dump_json())}, ) graph = client.get("/item-graph").json @@ -114,7 +114,8 @@ def test_single_starting_material(admin_client, client): ) creation = admin_client.post( - "/new-sample/", json={"new_sample_data": json.loads(admin_great_great_grandchild.json())} + "/new-sample/", + json={"new_sample_data": json.loads(admin_great_great_grandchild.model_dump_json())}, ) graph = admin_client.get("/item-graph/great-grandchild?sudo=1").json diff --git a/pydatalab/tests/server/test_item_versions.py b/pydatalab/tests/server/test_item_versions.py index dad4b1ea6..b1a32680f 100644 --- a/pydatalab/tests/server/test_item_versions.py +++ b/pydatalab/tests/server/test_item_versions.py @@ -44,7 +44,7 @@ def sample_with_version(client, user_id): "synthesis_description": "Initial synthesis", } ) - flask_mongo.db.items.insert_one(sample.dict(exclude_unset=False)) + flask_mongo.db.items.insert_one(sample.model_dump(exclude_unset=False)) yield sample @@ -578,7 +578,7 @@ def test_save_item_creates_version(self, client, sample_with_version): refcode_short = sample_with_version.refcode.split(":")[1] # Modify and save the item using save-item endpoint - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "Updated via save-item" response = client.post("/save-item/", json={"item_id": item_id, "data": item_data}) @@ -602,7 +602,7 @@ def test_save_item_increments_version(self, client, sample_with_version): refcode = sample_with_version.refcode - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "First real change" client.post("/save-item/", json={"item_id": sample_with_version.item_id, "data": item_data}) @@ -670,7 +670,7 @@ def test_save_item_endpoint_creates_manual_save_action(self, client, sample_with refcode_short = sample_with_version.refcode.split(":")[1] # Save via save-item endpoint (user clicking save button) - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "Updated via save-item" client.post("/save-item/", json={"item_id": sample_with_version.item_id, "data": item_data}) @@ -838,7 +838,7 @@ def test_save_item_identical_does_not_create_version(self, client, sample_with_v refcode = sample_with_version.refcode.split(":")[1] full_refcode = sample_with_version.refcode - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "A new description" client.post("/save-item/", json={"item_id": item_id, "data": item_data}) @@ -858,7 +858,7 @@ def test_save_item_with_change_creates_new_version(self, client, sample_with_ver refcode = sample_with_version.refcode.split(":")[1] full_refcode = sample_with_version.refcode - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "First real change" client.post("/save-item/", json={"item_id": item_id, "data": item_data}) diff --git a/pydatalab/tests/server/test_samples.py b/pydatalab/tests/server/test_samples.py index e9d3f34a7..4d7babe74 100644 --- a/pydatalab/tests/server/test_samples.py +++ b/pydatalab/tests/server/test_samples.py @@ -360,7 +360,7 @@ def test_item_regex_search( def test_new_sample_with_relationships( client, complicated_sample, insert_complicated_sample_constituents ): - complicated_sample_json = json.loads(complicated_sample.json()) + complicated_sample_json = json.loads(complicated_sample.model_dump_json()) response = client.post("/new-sample/", json=complicated_sample_json) # Test that 201: Created is emitted assert response.status_code == 201, response.json @@ -424,7 +424,7 @@ def test_new_sample_with_relationships( description="This is a new relationship", ) ) - derived_sample_json = json.loads(derived_sample.json()) + derived_sample_json = json.loads(derived_sample.model_dump_json()) response = client.post("/new-sample/", json=derived_sample_json) # Test that 201: Created is emitted @@ -528,7 +528,7 @@ def test_copy_from_sample(client, complicated_sample): """ complicated_sample.item_id = "new_complicated_sample" - complicated_sample_json = json.loads(complicated_sample.json()) + complicated_sample_json = json.loads(complicated_sample.model_dump_json()) response = client.post("/new-sample/", json=complicated_sample_json) # Test that 201: Created is emitted @@ -558,12 +558,13 @@ def test_copy_from_sample(client, complicated_sample): @pytest.mark.dependency(depends=["test_copy_from_sample"]) def test_create_multiple_samples(client, complicated_sample): - samples = [complicated_sample, complicated_sample.copy()] + samples = [complicated_sample, complicated_sample.model_copy()] samples[0].item_id = "another_new_complicated_sample" samples[1].item_id = "additional_new_complicated_sample" response = client.post( - "/new-samples/", json={"new_sample_datas": [json.loads(s.json()) for s in samples]} + "/new-samples/", + json={"new_sample_datas": [json.loads(s.model_dump_json()) for s in samples]}, ) assert response.status_code == 207, response.json assert response.json["nsuccess"] == 2, response.json @@ -577,7 +578,7 @@ def test_create_multiple_samples(client, complicated_sample): response = client.post( "/new-samples/", json={ - "new_sample_datas": [json.loads(s.json()) for s in samples], + "new_sample_datas": [json.loads(s.model_dump_json()) for s in samples], "copy_from_item_ids": [ "another_new_complicated_sample", "additional_new_complicated_sample", @@ -599,7 +600,7 @@ def test_create_multiple_samples(client, complicated_sample): @pytest.mark.dependency(depends=["test_create_multiple_samples"]) def test_create_cell(client, default_cell): - response = client.post("/new-sample/", json=json.loads(default_cell.json())) + response = client.post("/new-sample/", json=json.loads(default_cell.model_dump_json())) assert response.status_code == 201, response.json assert response.json["status"] == "success" @@ -672,7 +673,9 @@ def test_create_collections(client, default_collection, database): assert response.status_code == 200 # Create an empty collection - response = client.put("/collections", json={"data": json.loads(default_collection.json())}) + response = client.put( + "/collections", json={"data": json.loads(default_collection.model_dump_json())} + ) assert response.status_code == 201, response.json assert response.json["status"] == "success" assert response.json["data"]["collection_id"] == "test_collection" @@ -691,7 +694,7 @@ def test_create_collections(client, default_collection, database): new_collection = copy.deepcopy(default_collection) new_collection.collection_id = "test_collection_2" - data = json.loads(new_collection.json()) + data = json.loads(new_collection.model_dump_json()) data.update( { "starting_members": [ diff --git a/pydatalab/tests/server/test_search.py b/pydatalab/tests/server/test_search.py index 40e4771dd..d9b0cabe9 100644 --- a/pydatalab/tests/server/test_search.py +++ b/pydatalab/tests/server/test_search.py @@ -52,7 +52,7 @@ def test_collection_regex_search(query, should_find, real_mongo_client, client, if real_mongo_client is None: pytest.skip("Skipping search tests, not connected to real MongoDB") - test_collection = default_collection.dict() + test_collection = default_collection.model_dump() clean_query = query.replace("=", "_").replace("%23", "hash_").replace("'", "") test_collection["collection_id"] = f"test_coll_{clean_query}" test_collection["title"] = "test collection" diff --git a/pydatalab/tests/server/test_starting_materials.py b/pydatalab/tests/server/test_starting_materials.py index 3ee28080a..080559e5d 100644 --- a/pydatalab/tests/server/test_starting_materials.py +++ b/pydatalab/tests/server/test_starting_materials.py @@ -133,7 +133,7 @@ def test_starting_materials_list_includes_collections( client, default_starting_material_dict, default_collection ): """GET /starting-materials/ should include a collections array with collection_id for each item.""" - collection_data = json.loads(default_collection.json()) + collection_data = json.loads(default_collection.model_dump_json()) collection_data["starting_members"] = [{"item_id": default_starting_material_dict["item_id"]}] response = client.put("/collections", json={"data": collection_data}) assert response.status_code == 201, response.json diff --git a/pydatalab/tests/test_models.py b/pydatalab/tests/test_models.py index c92dbb7ce..625ae5d08 100644 --- a/pydatalab/tests/test_models.py +++ b/pydatalab/tests/test_models.py @@ -52,7 +52,7 @@ def test_sample_with_inlined_reference(): assert parents[0].item_id == a.item_id assert parents[0].refcode == a.refcode - b_both = Sample(**json.loads(b_both.json())) + b_both = Sample(**json.loads(b_both.model_dump_json())) parents = [r for r in b_both.relationships if r.relation == RelationshipType.PARENT] assert len(parents) == 1 @@ -209,7 +209,7 @@ class TestItem(Item): tzinfo=datetime.timezone.utc ) - item_json = json.loads(item.json()) + item_json = json.loads(item.model_dump_json()) assert item_json["type"] == "items_custom" assert item_json["creator_ids"][0] == "0123456789ab0123456789ab" assert item_json["creator_ids"][1] == "1023456789ab0123456789ab" @@ -239,7 +239,7 @@ class TestItem(Item): "2020-01-01 00:00" ).replace(tzinfo=datetime.timezone.utc) - sample_json = json.loads(sample.json()) + sample_json = json.loads(sample.model_dump_json()) assert sample_json["type"] == "samples" assert sample_json["creator_ids"][0] == str(ObjectId("0123456789ab0123456789ab")) assert sample_json["creator_ids"][1] == str(ObjectId("1023456789ab0123456789ab")) From e48f55bc014459586832bff49e4a988c19b8a067 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 00:19:53 +0100 Subject: [PATCH 18/40] Minor tweaks suggested by copilot --- pydatalab/src/pydatalab/models/people.py | 4 ---- pydatalab/src/pydatalab/routes/v0_1/info.py | 6 +++--- pydatalab/src/pydatalab/routes/v0_1/items.py | 8 ++++++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pydatalab/src/pydatalab/models/people.py b/pydatalab/src/pydatalab/models/people.py index ec3485ade..5a3897419 100644 --- a/pydatalab/src/pydatalab/models/people.py +++ b/pydatalab/src/pydatalab/models/people.py @@ -190,10 +190,6 @@ def add_missing_type(cls, v): v = "people" return v - @classmethod - def set_default_type(cls, _): - return "people" - @staticmethod def new_user_from_identity( identity: Identity, diff --git a/pydatalab/src/pydatalab/routes/v0_1/info.py b/pydatalab/src/pydatalab/routes/v0_1/info.py index 4bf7bd6e1..dd74ebb90 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/info.py +++ b/pydatalab/src/pydatalab/routes/v0_1/info.py @@ -160,7 +160,7 @@ def list_block_types(): ) for block_type, block in BLOCK_TYPES.items() ], - meta=Meta(query=request.query_string), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), ).model_dump_json() ) ) @@ -185,7 +185,7 @@ def list_supported_types(): ) for item_type, schema in ITEM_SCHEMAS.items() ], - meta=Meta(query=request.query_string), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), ).model_dump_json() ) ) @@ -211,7 +211,7 @@ def get_schema_type(item_type): "schema": ITEM_SCHEMAS[item_type], }, ), - meta=Meta(query=request.query_string), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), ).model_dump_json() ) ) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index a28512de6..2e2823ec9 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -710,9 +710,13 @@ def _create_sample( raise BadRequest(f"Failed to add new item {new_sample['item_id']!r} to database.") # Save initial version snapshot after successful item creation + refcode = data_model.refcode + if not isinstance(refcode, str): + raise InternalServerError(f"Invalid refcode generated for new item: {refcode!r}") try: version_resp, version_status = save_version_snapshot( - data_model.refcode, action=VersionAction.CREATED + refcode, + action=VersionAction.CREATED, # type: ignore ) if version_status != 200: LOGGER.error( @@ -1347,7 +1351,7 @@ def compare_versions(refcode): v1 = flask_mongo.db.item_versions.find_one({"_id": query_params.v1, "refcode": refcode}) v2 = flask_mongo.db.item_versions.find_one({"_id": query_params.v2, "refcode": refcode}) if not v1 or not v2: - return NotFound("One or both versions not found") + raise NotFound("One or both versions not found") # Use DeepDiff for proper nested structure comparison # This handles nested dicts, lists, type changes, and provides detailed change information From 4b1ddd40bf881cf70c34371098723b940ddc566a Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 00:28:35 +0100 Subject: [PATCH 19/40] Fix docker build by reordering tasks imports --- pydatalab/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index aeedcd452..e526501e2 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -9,8 +9,6 @@ import tomlkit from invoke import Collection, task -from pydatalab.models.utils import UserRole - ns = Collection() dev = Collection("dev") admin = Collection("admin") @@ -274,13 +272,15 @@ def create_mongo_indices(_): @task -def change_user_role(_, display_name: str, role: "UserRole"): +def change_user_role(_, display_name: str, role: str): """This task takes a user's name and gives them the desired role.""" from bson import ObjectId from pydatalab.models.utils import UserRole from pydatalab.mongo import _get_active_mongo_client + role = UserRole(role.upper()) + try: role = getattr(UserRole, role.upper()) except AttributeError: From 57b55055610eb73cd0b18f1a0f1693ab682bd209 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 10:05:22 +0100 Subject: [PATCH 20/40] Update plugin config validator for pydantic 2 --- pydatalab/tasks.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index e526501e2..63f7844a0 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -64,17 +64,16 @@ class PluginConfigModel(BaseModel): dependencies: list[str] = [] tool: ToolSection = ToolSection() - @model_validator(mode="before") - @classmethod - def _sources_must_match_dependencies(cls, values): - deps = {d.split("[")[0].strip() for d in values.get("dependencies", [])} - sources = values.get("tool", ToolSection()).uv.sources + @model_validator(mode="after") + def _sources_must_match_dependencies(self): + deps = {d.split("[")[0].strip() for d in self.dependencies} + sources = self.tool.uv.sources orphans = sorted(set(sources) - deps) if orphans: raise ValueError( f"[tool.uv.sources] entries have no matching dependency: {orphans}" ) - return values + return self return PluginConfigModel From c75fdae4e606ab00a1ec7392d4747006c9bc566d Mon Sep 17 00:00:00 2001 From: BenSmithGreyGroup Date: Thu, 25 Jun 2026 12:33:14 +0100 Subject: [PATCH 21/40] Added test to check if None fields are preserved with saving items and versions (pydantic 1 to 2 change) --- pydatalab/tests/server/test_item_versions.py | 129 +++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/pydatalab/tests/server/test_item_versions.py b/pydatalab/tests/server/test_item_versions.py index b1a32680f..8a3eb20b3 100644 --- a/pydatalab/tests/server/test_item_versions.py +++ b/pydatalab/tests/server/test_item_versions.py @@ -1513,3 +1513,132 @@ def test_unchanged_permissions_mint_nothing(self, client, api_sample, user_id): versions = client.get(f"/items/{api_sample['refcode']}/versions/").json["versions"] assert len(versions) == before + + +class TestNoneFieldPreservation: + """Tests that None-valued fields are preserved through item creation, versioning, and restore. + + Covers two bugs introduced during the pydantic v1->v2 migration: + + Bug 1 (items.py): model_dump(exclude_none=True) on item creation strips None fields from + MongoDB entirely, so newly created items are missing keys like 'description', 'chemform', etc. + + Bug 2 (versioning.py): model_dump(exclude_none=True) on ItemVersion now recursively strips + None values from inside the 'data' dict (pydantic v2 changed behaviour vs v1), so version + snapshots also lose those keys. When restoring, $set cannot clear a key that is absent + from the restore payload. + """ + + def test_item_creation_preserves_none_fields_in_db(self, client): + """Creating an item with no description should store description=None in MongoDB, + not omit the key entirely.""" + from pydatalab.mongo import flask_mongo + + item_id = "none_field_test_create" + response = client.post( + "/new-sample/", + json={"name": "None field test", "item_id": item_id, "type": "samples"}, + ) + assert response.status_code == 201, response.json + + refcode = response.json["sample_list_entry"]["refcode"] + item = flask_mongo.db.items.find_one({"refcode": refcode}) + + # 'description' is a defined field on Sample — it must be present as a key + # even when the user did not supply a value. + assert "description" in item, ( + "'description' key missing from newly-created item; " + "model_dump(exclude_none=True) is incorrectly stripping None fields on creation" + ) + + # Cleanup + flask_mongo.db.items.delete_one({"refcode": refcode}) + flask_mongo.db.item_versions.delete_many({"refcode": refcode}) + flask_mongo.db.version_counters.delete_one({"refcode": refcode}) + + def test_version_snapshot_preserves_none_fields(self, client): + """Version snapshot stored in item_versions must retain None-valued fields from the + item document; pydantic v2 model_dump(exclude_none=True) must not strip them from + inside the nested 'data' dict.""" + from pydatalab.mongo import flask_mongo + + item_id = "none_field_test_version" + response = client.post( + "/new-sample/", + json={"name": "None field version test", "item_id": item_id, "type": "samples"}, + ) + assert response.status_code == 201, response.json + refcode = response.json["sample_list_entry"]["refcode"] + refcode_short = refcode.split(":")[-1] + + # The initial version is created on item creation; retrieve it. + list_resp = client.get(f"/items/{refcode_short}/versions/") + assert len(list_resp.json["versions"]) == 1 + version_id = list_resp.json["versions"][0]["_id"] + + version_resp = client.get(f"/items/{refcode_short}/versions/{version_id}/") + snapshot_data = version_resp.json["version"]["data"] + + assert "description" in snapshot_data, ( + "'description' key missing from version snapshot 'data'; " + "model_dump(exclude_none=True) on ItemVersion is recursively stripping None " + "values from the nested 'data' dict (pydantic v2 regression)" + ) + + # Cleanup + flask_mongo.db.items.delete_one({"refcode": refcode}) + flask_mongo.db.item_versions.delete_many({"refcode": refcode}) + flask_mongo.db.version_counters.delete_one({"refcode": refcode}) + + def test_restore_to_version_with_none_field_clears_populated_field(self, client): + """Restoring to a version where a field was None must clear that field in the current + item, even if the field has since been populated. + + Failure mode: the version snapshot is missing the key entirely (due to Bug 1/2), so + $set has nothing to write and the field retains its populated value after restore. + """ + from pydatalab.mongo import flask_mongo + + item_id = "none_field_test_restore" + create_resp = client.post( + "/new-sample/", + json={"name": "Restore none test", "item_id": item_id, "type": "samples"}, + ) + assert create_resp.status_code == 201, create_resp.json + refcode = create_resp.json["sample_list_entry"]["refcode"] + refcode_short = refcode.split(":")[-1] + + # Confirm the initial version has no description set (None / absent). + list_resp = client.get(f"/items/{refcode_short}/versions/") + v1_id = list_resp.json["versions"][0]["_id"] + + # Now populate description and save — this creates version 2. + get_resp = client.get(f"/get-item-data/{item_id}") + item_data = get_resp.json["item_data"] + item_data["description"] = "A description added later" + save_resp = client.post("/save-item/", json={"item_id": item_id, "data": item_data}) + assert save_resp.status_code == 200, save_resp.json + + # Verify description is now set in the DB. + current = flask_mongo.db.items.find_one({"refcode": refcode}) + assert current.get("description") == "A description added later" + + # Restore to version 1 (description was None/absent). + restore_resp = client.post( + f"/items/{refcode_short}/restore-version/", json={"version_id": v1_id} + ) + assert restore_resp.status_code == 200, restore_resp.json + + # After restore, description must be None (or absent), not the later value. + restored = flask_mongo.db.items.find_one({"refcode": refcode}) + assert restored.get("description") is None, ( + f"description should be None after restoring to the version where it was unset, " + f"but got {restored.get('description')!r}. " + "This happens because the version snapshot is missing the 'description' key, " + "so $set does not overwrite the populated value." + ) + + # Cleanup + flask_mongo.db.items.delete_one({"refcode": refcode}) + flask_mongo.db.item_versions.delete_many({"refcode": refcode}) + flask_mongo.db.version_counters.delete_one({"refcode": refcode}) From 388bf0b5a1464685cdd938620ca36fb8311a6803 Mon Sep 17 00:00:00 2001 From: BenSmithGreyGroup Date: Thu, 25 Jun 2026 12:36:57 +0100 Subject: [PATCH 22/40] Fix None field stripping in item creation and version snapshots (pydantic v2). --- pydatalab/src/pydatalab/routes/v0_1/items.py | 11 ++++++----- pydatalab/src/pydatalab/versioning.py | 6 +++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 2e2823ec9..1eccc9355 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -701,7 +701,7 @@ def _create_sample( # the `Entry` model. try: result = flask_mongo.db.items.insert_one( - data_model.model_dump(exclude={"creators", "collections", "groups"}, exclude_none=True) + data_model.model_dump(exclude={"creators", "collections", "groups"}) ) except DuplicateKeyError as error: raise Conflict(f"Duplicate key error: {str(error)}.") @@ -1482,10 +1482,11 @@ def restore_version(refcode): ) raise BadRequest(f"Restored version data validation failed: {exc}") - # Insert validated data - flask_mongo.db.item_versions.insert_one( - validated_restored_version.model_dump(exclude_none=True) - ) + # Insert validated data, restoring original restored_data into 'data' so None-valued + # fields are not stripped by model_dump(exclude_none=True) recursing into the dict. + restored_version_doc = validated_restored_version.model_dump(exclude_none=True) + restored_version_doc["data"] = restored_data + flask_mongo.db.item_versions.insert_one(restored_version_doc) return jsonify( { diff --git a/pydatalab/src/pydatalab/versioning.py b/pydatalab/src/pydatalab/versioning.py index 69816ae79..7d7916db6 100644 --- a/pydatalab/src/pydatalab/versioning.py +++ b/pydatalab/src/pydatalab/versioning.py @@ -219,10 +219,14 @@ def save_version_snapshot( # permanent gap in the item's version history. next_version_number = get_next_version_number(refcode) - # Insert validated data (convert to dict and exclude None values) + # Insert validated data, but restore the original item dict into 'data' so that + # None-valued fields are not stripped by model_dump(exclude_none=True) recursing into it. version_doc = validated_version.model_dump(exclude_none=True) + version_doc["data"] = item version_doc["version"] = next_version_number + flask_mongo.db.item_versions.insert_one(version_doc) + return ( {"status": "success", "message": "Version saved.", "version": next_version_number}, 200, From 78b440b72fa3648391af99f221020a34d2a73008 Mon Sep 17 00:00:00 2001 From: BenSmithGreyGroup Date: Thu, 25 Jun 2026 15:44:58 +0100 Subject: [PATCH 23/40] Add failing test. Checking if saving None overwrites a previously saved value. --- pydatalab/tests/server/test_samples.py | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pydatalab/tests/server/test_samples.py b/pydatalab/tests/server/test_samples.py index 4d7babe74..15b3920e7 100644 --- a/pydatalab/tests/server/test_samples.py +++ b/pydatalab/tests/server/test_samples.py @@ -1293,6 +1293,40 @@ def test_save_item_with_malformed_constituent_returns_400(client, default_sample assert response.json["status"] == "error" +def test_clearing_field_via_save_item_persists_to_db(client): + """Setting a field then clearing it (setting to None) via save-item must + actually clear it in MongoDB. Fails if exclude_none=True strips the None + from the $set payload, leaving the old value untouched.""" + item_id = "clear_field_test" + response = client.post( + "/new-sample/", + json={"name": "Clear field test", "item_id": item_id, "type": "samples"}, + ) + assert response.status_code == 201, response.json + + # Set molar_mass and save. + item_data = client.get(f"/get-item-data/{item_id}").json["item_data"] + item_data["molar_mass"] = 123.45 + assert ( + client.post("/save-item/", json={"item_id": item_id, "data": item_data}).status_code == 200 + ) + + assert client.get(f"/get-item-data/{item_id}").json["item_data"]["molar_mass"] == 123.45 + + # Now clear molar_mass and save again. + item_data = client.get(f"/get-item-data/{item_id}").json["item_data"] + item_data["molar_mass"] = None + assert ( + client.post("/save-item/", json={"item_id": item_id, "data": item_data}).status_code == 200 + ) + + result = client.get(f"/get-item-data/{item_id}").json["item_data"].get("molar_mass") + assert result is None, ( + f"molar_mass should be None after being cleared but got {result!r}; " + "exclude_none=True in save_item is stripping the None from the $set payload" + ) + + def test_get_item_with_malformed_stored_constituent_returns_500(client, database, user_id): """If an item with a malformed constituent ended up in the database (e.g. from an older write path), GET should surface it as a server-side From 953c784e1f41d68e968de4aa25e622e2b411c065 Mon Sep 17 00:00:00 2001 From: BenSmithGreyGroup Date: Thu, 25 Jun 2026 15:45:15 +0100 Subject: [PATCH 24/40] Remove exclude None from the save_item route --- pydatalab/src/pydatalab/routes/v0_1/items.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 1eccc9355..93bcca50f 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -1704,7 +1704,6 @@ def save_item(): try: item = entry_reference_lookup(item) item = ITEM_MODELS[item_type](**item).model_dump( - exclude_none=True, exclude_unset=True, exclude={"collections", "creators", "immutable_id"}, ) From 7391c01b687eff4a217b458379550d260896f8d4 Mon Sep 17 00:00:00 2001 From: Julien Bouquiaux Date: Fri, 12 Jun 2026 09:50:52 +0200 Subject: [PATCH 25/40] Fix status dropdown broken by Pydantic v2 schema migration Pydantic v2's model_json_schema() uses "$defs" instead of "definitions" (Pydantic v1). Update the four item-information components to use the correct key so possibleItemStatuses is no longer undefined. --- webapp/src/components/CellInformation.vue | 2 +- webapp/src/components/EquipmentInformation.vue | 2 +- webapp/src/components/SampleInformation.vue | 2 +- webapp/src/components/StartingMaterialInformation.vue | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/webapp/src/components/CellInformation.vue b/webapp/src/components/CellInformation.vue index 1d1416675..8c0b34a5b 100644 --- a/webapp/src/components/CellInformation.vue +++ b/webapp/src/components/CellInformation.vue @@ -179,7 +179,7 @@ export default { return this.$store.state.schemas[this.item?.type]; }, possibleItemStatuses() { - return this.schema?.attributes?.schema?.definitions?.CellStatus?.enum; + return this.schema?.attributes?.schema?.["$defs"]?.CellStatus?.enum; }, }, }; diff --git a/webapp/src/components/EquipmentInformation.vue b/webapp/src/components/EquipmentInformation.vue index 66401d1a6..7668b3201 100644 --- a/webapp/src/components/EquipmentInformation.vue +++ b/webapp/src/components/EquipmentInformation.vue @@ -135,7 +135,7 @@ export default { return this.$store.state.schemas[this.item?.type]; }, possibleItemStatuses() { - return this.schema?.attributes?.schema?.definitions?.EquipmentStatus?.enum; + return this.schema?.attributes?.schema?.["$defs"]?.EquipmentStatus?.enum; }, uniqueLocations() { return [ diff --git a/webapp/src/components/SampleInformation.vue b/webapp/src/components/SampleInformation.vue index d47c8b8f1..29f2fcded 100644 --- a/webapp/src/components/SampleInformation.vue +++ b/webapp/src/components/SampleInformation.vue @@ -126,7 +126,7 @@ export default { return this.$store.state.schemas[this.item?.type]; }, possibleItemStatuses() { - return this.schema?.attributes?.schema?.definitions?.ItemStatus?.enum; + return this.schema?.attributes?.schema?.["$defs"]?.ItemStatus?.enum; }, }, }; diff --git a/webapp/src/components/StartingMaterialInformation.vue b/webapp/src/components/StartingMaterialInformation.vue index 26a78494f..7bb2bdc3f 100644 --- a/webapp/src/components/StartingMaterialInformation.vue +++ b/webapp/src/components/StartingMaterialInformation.vue @@ -184,7 +184,7 @@ export default { return this.$store.state.schemas[this.item?.type]; }, possibleItemStatuses() { - return this.schema?.attributes?.schema?.definitions?.StartingMaterialsStatus?.enum; + return this.schema?.attributes?.schema?.["$defs"]?.StartingMaterialsStatus?.enum; }, Barcode: createComputedSetterForItemField("barcode"), uniqueSuppliers() { From 66b99db2ae97b5c27085f032c5de445560faf117 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 4 Aug 2026 22:54:53 +0100 Subject: [PATCH 26/40] Fix enum value sanitisation in change-user-role --- pydatalab/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index 63f7844a0..f40f8a3f7 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -278,7 +278,7 @@ def change_user_role(_, display_name: str, role: str): from pydatalab.models.utils import UserRole from pydatalab.mongo import _get_active_mongo_client - role = UserRole(role.upper()) + role = UserRole(role.lower()) try: role = getattr(UserRole, role.upper()) From ea2653dc286697467814bc062783dd7018030ab8 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 4 Aug 2026 23:26:44 +0100 Subject: [PATCH 27/40] Use `model_dump(mode=json)` to prevent repeated deserialization --- pydatalab/src/pydatalab/routes/v0_1/auth.py | 3 +- .../src/pydatalab/routes/v0_1/collections.py | 7 +- pydatalab/src/pydatalab/routes/v0_1/groups.py | 4 +- pydatalab/src/pydatalab/routes/v0_1/info.py | 99 +++++++++---------- .../src/pydatalab/routes/v0_1/remotes.py | 7 +- pydatalab/src/pydatalab/routes/v0_1/users.py | 3 +- 6 files changed, 53 insertions(+), 70 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/auth.py b/pydatalab/src/pydatalab/routes/v0_1/auth.py index f14efe6ad..764a6e664 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/auth.py +++ b/pydatalab/src/pydatalab/routes/v0_1/auth.py @@ -5,7 +5,6 @@ """ import datetime -import json import os import re import secrets @@ -1057,7 +1056,7 @@ def redirect_to_ui(blueprint, token): # pylint: disable=unused-argument def get_authenticated_user_info(): """Returns metadata associated with the currently authenticated user.""" if current_user.is_authenticated: - current_user_response = json.loads(current_user.person.model_dump_json()) + current_user_response = current_user.person.model_dump(mode="json") current_user_response["role"] = current_user.role.value return jsonify(current_user_response), 200 else: diff --git a/pydatalab/src/pydatalab/routes/v0_1/collections.py b/pydatalab/src/pydatalab/routes/v0_1/collections.py index fd3f6dc6f..a0b3c4f6e 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/collections.py +++ b/pydatalab/src/pydatalab/routes/v0_1/collections.py @@ -1,5 +1,4 @@ import datetime -import json from bson import ObjectId from flask import Blueprint, jsonify, request @@ -87,7 +86,7 @@ def get_collection(collection_id): { "status": "success", "collection_id": collection_id, - "data": json.loads(collection.model_dump_json(exclude_unset=True)), + "data": collection.model_dump(mode="json", exclude_unset=True), "child_items": list(samples), } ) @@ -218,7 +217,7 @@ def create_collection(): response = { "status": "success", - "data": json.loads(data_model.model_dump_json()), + "data": data_model.model_dump(mode="json"), } if errors: @@ -534,7 +533,7 @@ def search_collections(): ) cursor = [ - json.loads(Collection(**doc).model_dump_json(exclude_unset=True)) + Collection(**doc).model_dump(mode="json", exclude_unset=True) for doc in flask_mongo.db.collections.aggregate(pipeline) ] diff --git a/pydatalab/src/pydatalab/routes/v0_1/groups.py b/pydatalab/src/pydatalab/routes/v0_1/groups.py index d5b4e5648..779166452 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/groups.py +++ b/pydatalab/src/pydatalab/routes/v0_1/groups.py @@ -1,5 +1,3 @@ -import json - from flask import Blueprint, jsonify, request from pydatalab.models.people import Group @@ -44,6 +42,6 @@ def search_groups(): return jsonify( { "status": "success", - "data": list(json.loads(Group(**d).model_dump_json()) for d in cursor), + "data": [Group(**d).model_dump(mode="json") for d in cursor], } ), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/info.py b/pydatalab/src/pydatalab/routes/v0_1/info.py index dd74ebb90..b2b45aafb 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/info.py +++ b/pydatalab/src/pydatalab/routes/v0_1/info.py @@ -1,6 +1,5 @@ """This submodule defines introspective info endpoints of the API.""" -import json from datetime import datetime from datetime import timedelta as td from datetime import timezone as tz @@ -118,7 +117,7 @@ def get_info(): ) return ( - jsonify(json.loads(response_data.model_dump_json())), + jsonify(response_data.model_dump(mode="json")), 200, ) @@ -142,27 +141,23 @@ def get_stats(): def list_block_types(): """Returns a list of all blocks implemented in this server.""" return jsonify( - json.loads( - JSONAPIResponse[dict[str, Any]]( - data=[ - Data( - id=block_type, - type="block_type", - attributes={ - "name": getattr(block, "name", ""), - "description": getattr(block, "description", ""), - "version": getattr(block, "version", __version__), - "accepted_file_extensions": getattr( - block, "accepted_file_extensions", [] - ), - "multi_file": getattr(block, "multi_file", False), - }, - ) - for block_type, block in BLOCK_TYPES.items() - ], - meta=Meta(query=request.query_string.decode() if request.query_string else ""), - ).model_dump_json() - ) + JSONAPIResponse[dict[str, Any]]( + data=[ + Data( + id=block_type, + type="block_type", + attributes={ + "name": getattr(block, "name", ""), + "description": getattr(block, "description", ""), + "version": getattr(block, "version", __version__), + "accepted_file_extensions": getattr(block, "accepted_file_extensions", []), + "multi_file": getattr(block, "multi_file", False), + }, + ) + for block_type, block in BLOCK_TYPES.items() + ], + meta=Meta(query=request.query_string.decode() if request.query_string else ""), + ).model_dump(mode="json") ) @@ -171,23 +166,21 @@ def list_supported_types(): """Returns a list of supported schemas.""" return jsonify( - json.loads( - JSONAPIResponse[dict[str, Any]]( - data=[ - Data( - id=item_type, - type="item_type", - attributes={ - "version": __version__, - "api_version": __api_version__, - "schema": schema, - }, - ) - for item_type, schema in ITEM_SCHEMAS.items() - ], - meta=Meta(query=request.query_string.decode() if request.query_string else ""), - ).model_dump_json() - ) + JSONAPIResponse[dict[str, Any]]( + data=[ + Data( + id=item_type, + type="item_type", + attributes={ + "version": __version__, + "api_version": __api_version__, + "schema": schema, + }, + ) + for item_type, schema in ITEM_SCHEMAS.items() + ], + meta=Meta(query=request.query_string.decode() if request.query_string else ""), + ).model_dump(mode="json") ) @@ -200,20 +193,18 @@ def get_schema_type(item_type): ), 404 return jsonify( - json.loads( - JSONAPIResponse[dict[str, Any]]( - data=Data( - id=item_type, - type="item_type", - attributes={ - "version": __version__, - "api_version": __api_version__, - "schema": ITEM_SCHEMAS[item_type], - }, - ), - meta=Meta(query=request.query_string.decode() if request.query_string else ""), - ).model_dump_json() - ) + JSONAPIResponse[dict[str, Any]]( + data=Data( + id=item_type, + type="item_type", + attributes={ + "version": __version__, + "api_version": __api_version__, + "schema": ITEM_SCHEMAS[item_type], + }, + ), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), + ).model_dump(mode="json") ) diff --git a/pydatalab/src/pydatalab/routes/v0_1/remotes.py b/pydatalab/src/pydatalab/routes/v0_1/remotes.py index 4e69952a2..dab3fa00f 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/remotes.py +++ b/pydatalab/src/pydatalab/routes/v0_1/remotes.py @@ -1,4 +1,3 @@ -import json from typing import Any from flask import Blueprint, jsonify, request @@ -76,9 +75,7 @@ def list_remote_directories(): response = {} response["meta"] = {} - response["meta"]["remotes"] = [ - json.loads(d.model_dump_json()) for d in CONFIG.REMOTE_FILESYSTEMS - ] + response["meta"]["remotes"] = [d.model_dump(mode="json") for d in CONFIG.REMOTE_FILESYSTEMS] if all_directory_structures: oldest_update = min(d["last_updated"] for d in all_directory_structures) response["meta"]["oldest_cache_update"] = oldest_update.isoformat() @@ -141,7 +138,7 @@ def get_remote_directory(remote_id: str): response: dict[str, Any] = {} response["meta"] = {} - response["meta"]["remote"] = json.loads(d.model_dump_json()) + response["meta"]["remote"] = d.model_dump(mode="json") response["data"] = directory_structure return jsonify(response), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/users.py b/pydatalab/src/pydatalab/routes/v0_1/users.py index ab9e737f0..15cc3ca48 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/users.py +++ b/pydatalab/src/pydatalab/routes/v0_1/users.py @@ -1,4 +1,3 @@ -import json from datetime import datetime from datetime import timedelta as td from datetime import timezone as tz @@ -218,5 +217,5 @@ def search_users(): cursor = flask_mongo.db.users.aggregate(pipeline) return jsonify( - {"status": "success", "users": [json.loads(Person(**d).model_dump_json()) for d in cursor]} + {"status": "success", "users": [Person(**d).model_dump(mode="json") for d in cursor]} ), 200 From 6acc3faa4df811eb480a54b0b2b8f20b68cf68c7 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Wed, 12 Aug 2026 16:56:06 +0100 Subject: [PATCH 28/40] Catch final remaining dict() --- pydatalab/src/pydatalab/routes/v0_1/items.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 93bcca50f..1879bf2c8 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -1008,7 +1008,7 @@ def issue_physical_token(refcode: str): ) try: - result = flask_mongo.db.api_keys.insert_one(access_document.dict()) + result = flask_mongo.db.api_keys.insert_one(access_document.model_dump()) if not result.inserted_id: return jsonify( {"status": "error", "message": "Unknown error generating token for item."} From 1e350f222b7b53691e6fd5a225cc9aadaab4780a Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Thu, 20 Aug 2026 11:23:31 +0100 Subject: [PATCH 29/40] Final regeneration of schemas --- pydatalab/schemas/plugin_config.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pydatalab/schemas/plugin_config.json b/pydatalab/schemas/plugin_config.json index 71db64db6..57975833d 100644 --- a/pydatalab/schemas/plugin_config.json +++ b/pydatalab/schemas/plugin_config.json @@ -1,7 +1,6 @@ { "$defs": { "ToolSection": { - "additionalProperties": false, "properties": { "uv": { "$ref": "#/$defs/UvSection", @@ -14,7 +13,6 @@ "type": "object" }, "UvSection": { - "additionalProperties": false, "properties": { "sources": { "additionalProperties": { @@ -29,7 +27,6 @@ "type": "object" }, "UvSource": { - "additionalProperties": false, "description": "A single entry under `[tool.uv.sources]` in plugins.toml.", "properties": { "git": { @@ -109,7 +106,6 @@ "type": "object" } }, - "additionalProperties": false, "description": "The schema for the top-level plugins.toml file.", "properties": { "dependencies": { From c075b7ab845dd1ed699de240e5b2e84009d9b2f4 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Thu, 20 Aug 2026 11:28:11 +0100 Subject: [PATCH 30/40] Move HasFiles trait into files model module --- pydatalab/src/pydatalab/models/files.py | 15 +++++++++------ pydatalab/src/pydatalab/models/items.py | 6 +----- pydatalab/src/pydatalab/models/traits.py | 13 +------------ 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/pydatalab/src/pydatalab/models/files.py b/pydatalab/src/pydatalab/models/files.py index b289c5ccd..7e965a537 100644 --- a/pydatalab/src/pydatalab/models/files.py +++ b/pydatalab/src/pydatalab/models/files.py @@ -2,10 +2,9 @@ from pydantic import Field -from pydatalab.models import traits from pydatalab.models.entries import Entry from pydatalab.models.traits import HasOwner, HasRevisionControl -from pydatalab.models.utils import BaseModel, IsoformatDateTime +from pydatalab.models.utils import BaseModel, IsoformatDateTime, PyObjectId class FileChecksums(BaseModel): @@ -74,7 +73,11 @@ class File(Entry, HasOwner, HasRevisionControl): """Content checksums (MD5 and SHA-256) of the file.""" -# `traits.HasFiles` cannot import `File` at module scope (this module needs the -# traits at class-definition time), so inject it and resolve the forward ref here. -traits.File = File # type: ignore[misc] -traits.HasFiles.model_rebuild() +class HasFiles(BaseModel): + """Trait mixin for models that can have files attached to them.""" + + files: list["File"] | None = None + """Any files attached to this item.""" + + file_ObjectIds: list[PyObjectId] = Field(default_factory=list) + """Links to object IDs of files stored within the database.""" diff --git a/pydatalab/src/pydatalab/models/items.py b/pydatalab/src/pydatalab/models/items.py index 326946f84..a405ce086 100644 --- a/pydatalab/src/pydatalab/models/items.py +++ b/pydatalab/src/pydatalab/models/items.py @@ -3,13 +3,9 @@ from pydantic import field_validator from pydatalab.models.entries import Entry - -# Importing `files` here resolves the deferred `File` annotation on `HasFiles` -# before any `Item` subclass is built. -from pydatalab.models.files import File # noqa: F401 +from pydatalab.models.files import HasFiles from pydatalab.models.traits import ( HasBlocks, - HasFiles, HasOwner, HasRevisionControl, IsCollectable, diff --git a/pydatalab/src/pydatalab/models/traits.py b/pydatalab/src/pydatalab/models/traits.py index 5fd3a0be4..774ec4348 100644 --- a/pydatalab/src/pydatalab/models/traits.py +++ b/pydatalab/src/pydatalab/models/traits.py @@ -7,13 +7,12 @@ from pydatalab.models.utils import BaseModel, Constituent, InlineSubstance, PyObjectId if TYPE_CHECKING: - from pydatalab.models.files import File + pass __all__ = ( "HasOwner", "HasRevisionControl", "HasBlocks", - "HasFiles", "IsCollectable", "HasSynthesisInfo", "HasSubstanceInfo", @@ -49,16 +48,6 @@ class HasRevisionControl(BaseModel): """The version number used by the version control system for tracking snapshots.""" -class HasFiles(BaseModel): - """Trait mixin for models that can have files attached to them.""" - - files: list["File"] | None = None - """Any files attached to this item.""" - - file_ObjectIds: list[PyObjectId] = Field(default_factory=list) - """Links to object IDs of files stored within the database.""" - - class HasBlocks(BaseModel): """Trait mixin for models that can have data blocks attached to them.""" From c211fedea54648af05b79e7c6b379f0ecf0edaa8 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sat, 22 Aug 2026 15:56:50 +0100 Subject: [PATCH 31/40] Update cell docstrings --- pydatalab/schemas/cell.json | 4 ++++ pydatalab/src/pydatalab/models/cells.py | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/pydatalab/schemas/cell.json b/pydatalab/schemas/cell.json index 1a7c02b03..dc15cedd8 100644 --- a/pydatalab/schemas/cell.json +++ b/pydatalab/schemas/cell.json @@ -1522,6 +1522,7 @@ "title": "Characteristic Molar Mass" }, "positive_electrode": { + "description": "The components of the positive electrode (cathode) of the cell.", "items": { "$ref": "#/$defs/CellComponent" }, @@ -1529,6 +1530,7 @@ "type": "array" }, "negative_electrode": { + "description": "The components of the negative electrode (anode) of the cell.", "items": { "$ref": "#/$defs/CellComponent" }, @@ -1536,6 +1538,7 @@ "type": "array" }, "electrolyte": { + "description": "The components of the electrolyte of the cell.", "items": { "$ref": "#/$defs/CellComponent" }, @@ -1557,6 +1560,7 @@ }, "active_ion_charge": { "default": 1, + "description": "The charge of the active ion species.", "title": "Active Ion Charge", "type": "number" } diff --git a/pydatalab/src/pydatalab/models/cells.py b/pydatalab/src/pydatalab/models/cells.py index 54461ce52..930717a58 100644 --- a/pydatalab/src/pydatalab/models/cells.py +++ b/pydatalab/src/pydatalab/models/cells.py @@ -55,11 +55,20 @@ class Cell(Item): """The molar mass of the active material, in g/mol. Will be inferred from the chemical formula, or can be supplied if it cannot be supplied""" positive_electrode: list[CellComponent] = Field(default_factory=list) + """The components of the positive electrode (cathode) of the cell.""" + negative_electrode: list[CellComponent] = Field(default_factory=list) + """The components of the negative electrode (anode) of the cell.""" + electrolyte: list[CellComponent] = Field(default_factory=list) + """The components of the electrolyte of the cell.""" + active_ion: str | None = None """The active ion species.""" + active_ion_charge: float = 1 + """The charge of the active ion species.""" + status: CellStatus = Field(default=CellStatus.ACTIVE) """The status of the cells, indicating its current state.""" From 390edf61742d6ed59b8135d760e5d26aa17a5658 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sat, 22 Aug 2026 15:57:21 +0100 Subject: [PATCH 32/40] Chatblock fix: make sure to set item data properly --- pydatalab/src/pydatalab/apps/chat/blocks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pydatalab/src/pydatalab/apps/chat/blocks.py b/pydatalab/src/pydatalab/apps/chat/blocks.py index 78b53b1b2..61e73ae10 100644 --- a/pydatalab/src/pydatalab/apps/chat/blocks.py +++ b/pydatalab/src/pydatalab/apps/chat/blocks.py @@ -207,8 +207,8 @@ def _prepare_item_json_for_chat(self, item_id: str, item_data: dict | None = Non item_model.blocks_obj = { k: block for k, block in item_model.blocks_obj.items() if block.blocktype != "chat" } - item_info = item_model.model_dump(exclude_none=True, exclude_unset=True) - item_info["type"] = item_model.type + item_data = item_model.model_dump(exclude_none=True, exclude_unset=True) + item_data["type"] = item_model.type # strip irrelevant or large fields item_filenames = { From 239a785d4ad48064948b268c7bad0c4c27a7fe1c Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sat, 22 Aug 2026 16:00:05 +0100 Subject: [PATCH 33/40] Fix plugin config schema regression --- pydatalab/schemas/plugin_config.json | 4 ++++ pydatalab/tasks.py | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pydatalab/schemas/plugin_config.json b/pydatalab/schemas/plugin_config.json index 57975833d..71db64db6 100644 --- a/pydatalab/schemas/plugin_config.json +++ b/pydatalab/schemas/plugin_config.json @@ -1,6 +1,7 @@ { "$defs": { "ToolSection": { + "additionalProperties": false, "properties": { "uv": { "$ref": "#/$defs/UvSection", @@ -13,6 +14,7 @@ "type": "object" }, "UvSection": { + "additionalProperties": false, "properties": { "sources": { "additionalProperties": { @@ -27,6 +29,7 @@ "type": "object" }, "UvSource": { + "additionalProperties": false, "description": "A single entry under `[tool.uv.sources]` in plugins.toml.", "properties": { "git": { @@ -106,6 +109,7 @@ "type": "object" } }, + "additionalProperties": false, "description": "The schema for the top-level plugins.toml file.", "properties": { "dependencies": { diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index f40f8a3f7..1c41cb5f5 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -29,11 +29,13 @@ def load_plugin_schema(): need them. A JSON Schema generated from the returned model is emitted to `pydatalab/schemas/plugin_config.json` by `invoke dev.generate-schemas`. """ - from pydantic import BaseModel, model_validator + from pydantic import BaseModel, ConfigDict, model_validator class UvSource(BaseModel): """A single entry under `[tool.uv.sources]` in plugins.toml.""" + model_config = ConfigDict(extra="forbid") + git: str | None = None rev: str | None = None branch: str | None = None @@ -53,14 +55,20 @@ def _exactly_one_source(cls, values): return values class UvSection(BaseModel): + model_config = ConfigDict(extra="forbid") + sources: dict[str, UvSource] = {} class ToolSection(BaseModel): + model_config = ConfigDict(extra="forbid") + uv: UvSection = UvSection() class PluginConfigModel(BaseModel): """The schema for the top-level plugins.toml file.""" + model_config = ConfigDict(extra="forbid") + dependencies: list[str] = [] tool: ToolSection = ToolSection() From 5b49756861299c8f81671f98077c1781b0bb66ec Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sat, 22 Aug 2026 16:02:52 +0100 Subject: [PATCH 34/40] Use post-pydantic validator in config --- pydatalab/src/pydatalab/config.py | 35 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/pydatalab/src/pydatalab/config.py b/pydatalab/src/pydatalab/config.py index 793d371d6..57fa57508 100644 --- a/pydatalab/src/pydatalab/config.py +++ b/pydatalab/src/pydatalab/config.py @@ -282,17 +282,19 @@ class ServerConfig(BaseSettings): description="The desired backup configuration.", ) - @model_validator(mode="before") - @classmethod - def validate_cache_ages(cls, values): - min_age = values.get("REMOTE_CACHE_MIN_AGE") - max_age = values.get("REMOTE_CACHE_MAX_AGE") + @model_validator(mode="after") + def validate_cache_ages(self): + """Check the cache ages after parsing, so that the defaults are covered even + when only one of the two values is set explicitly. + """ + min_age = self.REMOTE_CACHE_MIN_AGE + max_age = self.REMOTE_CACHE_MAX_AGE - if min_age is not None and max_age is not None and min_age > max_age: + if min_age > max_age: raise RuntimeError( f"The maximum cache age must be greater than the minimum cache age: min {min_age=}, max {max_age=}" ) - return values + return self @field_validator("SECRET_KEY", mode="before") @classmethod @@ -349,13 +351,18 @@ def validate_identifier_prefix(cls, v, info): ) return v - @model_validator(mode="before") - @classmethod - def deactivate_backup_strategies_during_testing(cls, values): - if values.get("TESTING"): - for name in values.get("BACKUP_STRATEGIES", {}): - values["BACKUP_STRATEGIES"][name].active = False - return values + @model_validator(mode="after") + def deactivate_backup_strategies_during_testing(self): + """Disable every backup strategy when running in testing mode. + + This runs after parsing so that the default strategies are covered (defaults + never appear in the raw input) and so that strategies loaded from a config + file are `BackupStrategy` instances rather than plain dicts. + """ + if self.TESTING and self.BACKUP_STRATEGIES: + for strategy in self.BACKUP_STRATEGIES.values(): + strategy.active = False + return self @field_validator("LOG_FILE", mode="before") @classmethod From 00b6b24a2878f5f6d4037b46f0d4baad93ccdad8 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 23 Aug 2026 10:20:40 +0100 Subject: [PATCH 35/40] Fix double-coercion in admin task --- pydatalab/tasks.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index 1c41cb5f5..8dc57b2b8 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -286,11 +286,9 @@ def change_user_role(_, display_name: str, role: str): from pydatalab.models.utils import UserRole from pydatalab.mongo import _get_active_mongo_client - role = UserRole(role.lower()) - try: - role = getattr(UserRole, role.upper()) - except AttributeError: + role = UserRole(role.lower()) + except ValueError: raise SystemExit(f"Invalid role: {role!r}. Must be one of {UserRole.__members__}") from None matches = list( From 2d7f24604f477a8d30f4cbd94e083181099ad7e7 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 23 Aug 2026 10:21:34 +0100 Subject: [PATCH 36/40] Remove global containing item_fts_fields --- pydatalab/src/pydatalab/mongo.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pydatalab/src/pydatalab/mongo.py b/pydatalab/src/pydatalab/mongo.py index 509bb2d2c..5b988abe7 100644 --- a/pydatalab/src/pydatalab/mongo.py +++ b/pydatalab/src/pydatalab/mongo.py @@ -19,7 +19,7 @@ "insert_pydantic_model_fork_safe", "gravatar_hash_for", "run_startup_migrations", - "ITEMS_FTS_FIELDS", + "get_items_fts_fields", "USERS_FTS_FIELDS", "COLLECTIONS_FTS_FIELDS", "GROUPS_FTS_FIELDS", @@ -106,8 +106,6 @@ def get_items_fts_fields() -> set[str]: return fields -ITEMS_FTS_FIELDS: set[str] = set() - USERS_FTS_FIELDS: set[str] = {"identities.name", "display_name", "contact_email"} """Fields to search for users.""" @@ -293,12 +291,9 @@ def create_default_indices( """ - global ITEMS_FTS_FIELDS - - if not ITEMS_FTS_FIELDS: - ITEMS_FTS_FIELDS = get_items_fts_fields() + items_fts_fields = get_items_fts_fields() - if not ITEMS_FTS_FIELDS: + if not items_fts_fields: raise ValueError("Cannot create text indices: no fields available for full-text search") if client is None: @@ -325,7 +320,7 @@ def create_fts(): ret += create_or_recreate_text_index( db.items, - ITEMS_FTS_FIELDS, + items_fts_fields, weights={"refcode": 3, "item_id": 3, "name": 3, "chemform": 3}, ) From 4056ee24861f8a02a2939e17d406b06aed847ea6 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 23 Aug 2026 10:21:59 +0100 Subject: [PATCH 37/40] Revert to allow collection_id to be None --- pydatalab/src/pydatalab/models/collections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydatalab/src/pydatalab/models/collections.py b/pydatalab/src/pydatalab/models/collections.py index 139798eba..9423ef330 100644 --- a/pydatalab/src/pydatalab/models/collections.py +++ b/pydatalab/src/pydatalab/models/collections.py @@ -17,7 +17,7 @@ class Collection(Entry, HasOwner, HasBlocks): type: Literal["collections"] = "collections" - collection_id: HumanReadableIdentifier + collection_id: HumanReadableIdentifier | None = None """A short human-readable/usable name for the collection.""" title: str | None = None From 623edac88cdafb79f6211b94e874de311086adbc Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 23 Aug 2026 10:54:19 +0100 Subject: [PATCH 38/40] Ditch now-extraneous collection relationship block --- pydatalab/src/pydatalab/routes/v0_1/items.py | 42 +------------------- 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 1879bf2c8..c2e4d8e6d 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -1658,42 +1658,6 @@ def save_item(): ] updated_data["collections"] = inaccessible - existing_item = flask_mongo.db.items.find_one({"item_id": item_id}) - if existing_item: - existing_relationships = existing_item.get("relationships", []) - non_collection_relationships = [ - rel for rel in existing_relationships if rel.get("type") != "collections" - ] - - collection_relationships = [] - for coll in updated_data.get("collections", []): - immutable_id = coll.get("immutable_id") - collection_id = coll.get("collection_id") - - if immutable_id: - if isinstance(immutable_id, str): - from bson import ObjectId - - immutable_id = ObjectId(immutable_id) - elif collection_id: - collection_doc = flask_mongo.db.collections.find_one( - {"collection_id": collection_id} - ) - if collection_doc: - immutable_id = collection_doc["_id"] - - if immutable_id: - collection_relationships.append( - { - "relation": None, - "immutable_id": immutable_id, - "type": "collections", - "description": "Is a member of", - } - ) - - updated_data["relationships"] = non_collection_relationships + collection_relationships - item_type = item["type"] preserve_relationships = "collections" not in updated_data @@ -1708,10 +1672,8 @@ def save_item(): exclude={"collections", "creators", "immutable_id"}, ) - except ValidationError: - raise BadRequest( - f"Unable to update item {item_id=} ({item_type=}) with new data {updated_data}" - ) + except ValidationError as exc: + raise BadRequest(f"Unable to update item {item_id=} ({item_type=}): {exc}") if preserve_relationships and original_relationships is not None: item["relationships"] = original_relationships From b472313bbcc6877e5568350f62af9774f26bdc56 Mon Sep 17 00:00:00 2001 From: Guido Petretto Date: Tue, 21 Jul 2026 15:34:20 +0200 Subject: [PATCH 39/40] separate blocks from items --- pydatalab/src/pydatalab/blocks/store.py | 391 +++++++++ pydatalab/src/pydatalab/export.py | 5 + pydatalab/src/pydatalab/models/__init__.py | 5 +- pydatalab/src/pydatalab/models/blocks.py | 43 + pydatalab/src/pydatalab/models/collections.py | 3 +- pydatalab/src/pydatalab/models/items.py | 2 +- pydatalab/src/pydatalab/models/traits.py | 12 - pydatalab/src/pydatalab/models/versions.py | 48 + pydatalab/src/pydatalab/mongo.py | 16 + pydatalab/src/pydatalab/routes/v0_1/blocks.py | 79 +- pydatalab/src/pydatalab/routes/v0_1/graphs.py | 37 +- pydatalab/src/pydatalab/routes/v0_1/items.py | 310 +++++-- pydatalab/src/pydatalab/versioning.py | 62 +- .../tests/server/test_block_separation.py | 822 ++++++++++++++++++ pydatalab/tests/server/test_blocks.py | 7 +- webapp/src/server_fetch_utils.js | 3 +- webapp/src/store/index.js | 15 +- 17 files changed, 1754 insertions(+), 106 deletions(-) create mode 100644 pydatalab/src/pydatalab/blocks/store.py create mode 100644 pydatalab/tests/server/test_block_separation.py diff --git a/pydatalab/src/pydatalab/blocks/store.py b/pydatalab/src/pydatalab/blocks/store.py new file mode 100644 index 000000000..f894b8709 --- /dev/null +++ b/pydatalab/src/pydatalab/blocks/store.py @@ -0,0 +1,391 @@ +"""Central helpers for the `blocks` and `block_versions` collections. + +This module is the single chokepoint for reading and writing separated block +documents, and for the branching logic between the two coexisting storage +forms of an item's `blocks_obj` entry: + +- **Embedded (legacy)**: the full block payload stored inline in the item + document. Legacy blocks are frozen in this form — they are read, written, + versioned, and restored exactly as before blocks were separated, and are + never converted. +- **Reference (new)**: a ``{"immutable_id": ObjectId}`` pointer to a document + in the `blocks` collection, which is the source of truth for the payload. + Only blocks created after the separation are in this form. Item version + snapshots store version-pinned references ``{"immutable_id", "version"}`` + that point at `block_versions` entries. + +Security invariant: the parent item is the sole authority for reading block +content. Every function here that returns block (or block-version) payloads +must only be called downstream of an already-authorized item read — no route +may authorize block content by the block document's own owner fields, and no +direct `blocks`/`block_versions` listing or fetch-by-id endpoint may exist. +""" + +import datetime +from typing import TYPE_CHECKING, Any + +from bson import ObjectId +from flask_login import current_user +from pymongo import ReturnDocument +from pymongo.errors import DuplicateKeyError +from werkzeug.exceptions import BadRequest, NotFound + +from pydatalab.logger import LOGGER +from pydatalab.models.blocks import Block +from pydatalab.models.versions import BlockVersion, VersionAction +from pydatalab.mongo import flask_mongo + +if TYPE_CHECKING: + from pydatalab.blocks.base import DataBlock + +__all__ = ( + "is_block_reference", + "authorize_and_get_form", + "load_blocks_obj", + "resolve_snapshot_blocks_obj", + "create_block_document", + "update_block_document", + "delete_block_document", + "snapshot_block_version", + "restore_block_version", +) + +_REFERENCE_KEYS = {"immutable_id", "version"} + + +def _now_isoformat() -> str: + return datetime.datetime.now(tz=datetime.timezone.utc).isoformat() + + +def _current_user_id() -> ObjectId | None: + if current_user and current_user.is_authenticated: + return current_user.person.immutable_id + return None + + +def is_block_reference(entry: Any) -> bool: + """Whether a stored `blocks_obj` entry is a reference to a `blocks` document + (``{"immutable_id": ...}``) rather than a legacy embedded block. + """ + return ( + isinstance(entry, dict) and "immutable_id" in entry and set(entry.keys()) <= _REFERENCE_KEYS + ) + + +def authorize_and_get_form(item_id: str | None, block_id: str) -> dict | None: + """The single permissioned query at the head of every block write. + + Checks in one query that the current user may write blocks on the item and + returns the stored `blocks_obj` entry for `block_id` (embedded payload or + reference). + + Read-level access is sufficient, matching the permissions applied to block + writes on embedded blocks before blocks were separated out (#1951). + + Returns: + The stored `blocks_obj` entry, or `None` if the item has no entry for + this `block_id`. + """ + from pydatalab.permissions import get_default_permissions + + if not item_id: + raise BadRequest("`item_id` must be provided when saving a block.") + + item = flask_mongo.db.items.find_one( + {"item_id": item_id, **get_default_permissions(user_only=False)}, + {f"blocks_obj.{block_id}": 1}, + ) + if not item: + raise NotFound(f"Could not find accessible item {item_id!r} to save block to.") + + return (item.get("blocks_obj") or {}).get(block_id) + + +def load_blocks_obj(parent_doc: dict) -> dict[str, Any]: + """Resolve a live parent document's `blocks_obj` into full payloads keyed by + `block_id`, i.e., the API shape used before blocks were separated. + + Embedded entries pass through verbatim; referenced entries are resolved from + the `blocks` collection in a single batched query. Must only be called on a + document the caller is already authorized to read. + """ + blocks_obj = parent_doc.get("blocks_obj") or {} + referenced: dict[str, ObjectId] = { + block_id: entry["immutable_id"] + for block_id, entry in blocks_obj.items() + if is_block_reference(entry) + } + + resolved_docs: dict[ObjectId, dict] = {} + if referenced: + resolved_docs = { + doc["_id"]: doc + for doc in flask_mongo.db.blocks.find( + {"_id": {"$in": list(referenced.values())}}, {"data": 1} + ) + } + + loaded: dict[str, Any] = {} + for block_id, entry in blocks_obj.items(): + if block_id not in referenced: + loaded[block_id] = entry + continue + doc = resolved_docs.get(referenced[block_id]) + if doc is None: + LOGGER.error( + "Dangling block reference: blocks_obj[%r] points at missing blocks doc %s", + block_id, + referenced[block_id], + ) + continue + loaded[block_id] = doc.get("data", {}) + + return loaded + + +def resolve_snapshot_blocks_obj(snapshot_data: dict) -> dict[str, Any]: + """Resolve an `item_versions` snapshot's `blocks_obj` into full payloads keyed + by `block_id`. + + Embedded entries pass through verbatim; version-pinned references + ``{"immutable_id", "version"}`` are resolved from `block_versions` in a single + batched query. + Must only be called downstream of an authorized item(-version) read. + """ + blocks_obj = snapshot_data.get("blocks_obj") or {} + + referenced: dict[str, tuple[ObjectId, int]] = { + block_id: (entry["immutable_id"], entry["version"]) + for block_id, entry in blocks_obj.items() + if is_block_reference(entry) and entry.get("version") is not None + } + + resolved_versions: dict[tuple[ObjectId, int], dict] = {} + if referenced: + resolved_versions = { + (doc["block_immutable_id"], doc["version"]): doc + for doc in flask_mongo.db.block_versions.find( + { + "$or": [ + {"block_immutable_id": immutable_id, "version": version} + for immutable_id, version in referenced.values() + ] + }, + {"block_immutable_id": 1, "version": 1, "data": 1}, + ) + } + + resolved: dict[str, Any] = {} + for block_id, entry in blocks_obj.items(): + if not is_block_reference(entry): + resolved[block_id] = entry + continue + doc = resolved_versions.get(referenced[block_id]) if block_id in referenced else None + if doc is None: + LOGGER.error( + "Dangling block reference in version snapshot: blocks_obj[%r] -> %s", + block_id, + entry, + ) + continue + resolved[block_id] = doc.get("data", {}) + + return resolved + + +def create_block_document(block: "DataBlock") -> ObjectId: + """Insert a new `blocks` document for a newly created block and return its + immutable ID, to be used as reference. + + The new document starts with no committed version (`version` 0); its first + `block_versions` entry is cut by the next item version snapshot. Ownership + fields are populated with the creating user only. Access to the block + depends on the item. + """ + creator_id = _current_user_id() + block_doc = Block( + block_id=block.block_id, + blocktype=block.blocktype, + data=block.to_db(), + creator_ids=[creator_id] if creator_id else [], + last_modified=_now_isoformat(), + ) + + result = flask_mongo.db.blocks.insert_one( + block_doc.model_dump(exclude={"immutable_id", "creators", "groups"}) + ) + if not result.acknowledged: + raise RuntimeError(f"Failed to insert new block document for block {block.block_id!r}.") + + return result.inserted_id + + +def update_block_document(immutable_id: ObjectId, data: dict) -> None: + """Overwrite the live payload of a referenced block (`data` sub-document). + + Writes the live document only: no `block_versions` entry is created and the + `version` counter is not bumped. + """ + result = flask_mongo.db.blocks.update_one( + {"_id": immutable_id}, + {"$set": {"data": data, "last_modified": _now_isoformat()}}, + ) + if result.matched_count != 1: + raise BadRequest(f"Failed to save block: no block document found for {immutable_id}.") + + +def delete_block_document(immutable_id: ObjectId) -> None: + """Remove a referenced block's live document. + + Its `block_versions` history is retained so that item versions + pinning it remain restorable. Mirrors `item_versions`. + """ + flask_mongo.db.blocks.delete_one({"_id": immutable_id}) + + +def snapshot_block_version( + immutable_id: ObjectId, + action: VersionAction = VersionAction.MANUAL_SAVE, + user_id: ObjectId | None = None, +) -> int | None: + """Cut a `block_versions` entry for a referenced block if its live payload + has changed since its last committed version, and return the block's + now-current committed version number. + + If no live `blocks` document exists (e.g. the block was deleted between the + caller's item read and now — deletion retains history), falls back to + re-pinning the latest committed version. Returns `None` only when there is + no live document *and* no committed history, i.e. there is no content + anywhere to snapshot. + """ + from pydatalab import __version__ + + block_doc = flask_mongo.db.blocks.find_one({"_id": immutable_id}) + if not block_doc: + last_version = flask_mongo.db.block_versions.find_one( + {"block_immutable_id": immutable_id}, + {"version": 1, "_id": 0}, + sort=[("version", -1)], + ) + return last_version["version"] if last_version else None + + data = block_doc.get("data", {}) + + last_version = flask_mongo.db.block_versions.find_one( + {"block_immutable_id": immutable_id}, sort=[("version", -1)] + ) + if last_version and last_version.get("data") == data: + return last_version["version"] + + updated_doc = flask_mongo.db.blocks.find_one_and_update( + {"_id": immutable_id}, + {"$inc": {"version": 1}}, + return_document=ReturnDocument.AFTER, + ) + if updated_doc is None: + return None + next_version = updated_doc["version"] + + version_entry = BlockVersion( + block_immutable_id=immutable_id, + block_id=block_doc["block_id"], + version=next_version, + timestamp=datetime.datetime.now(tz=datetime.timezone.utc), + action=action, + user_id=user_id if user_id is not None else _current_user_id(), + datalab_version=__version__, + data=data, + ) + + # Restore the raw payload into 'data' so that None-valued fields are not + # stripped by model_dump(exclude_none=True), mirroring item snapshots. + version_doc = version_entry.model_dump(exclude_none=True) + version_doc["data"] = data + try: + flask_mongo.db.block_versions.insert_one(version_doc) + except DuplicateKeyError: + # A concurrent snapshot minted this version number first; both captured + # the same live payload, so re-pinning the number is safe. + LOGGER.warning( + "Concurrent block version snapshot for block %s (version %d)", + immutable_id, + next_version, + ) + + return next_version + + +def restore_block_version( + immutable_id: ObjectId, + version: int, + user_id: ObjectId | None = None, +) -> int | None: + """Append-only restore of a version-pinned block reference. + + Writes the pinned `block_versions` payload back as the new current state of + the live `blocks` document and creates a new RESTORED `block_versions` entry. + Returns the new committed version number, or `None` (restoring nothing) if no + matching `block_versions` entry exists. + """ + from pydatalab import __version__ + + pinned = flask_mongo.db.block_versions.find_one( + {"block_immutable_id": immutable_id, "version": version} + ) + if not pinned: + LOGGER.error( + "Cannot restore block %s to version %s: no such block_versions entry", + immutable_id, + version, + ) + return None + + latest = flask_mongo.db.block_versions.find_one( + {"block_immutable_id": immutable_id}, + {"version": 1, "_id": 0}, + sort=[("version", -1)], + ) + next_version = (latest["version"] if latest else 0) + 1 + + data = pinned["data"] + if user_id is None: + user_id = _current_user_id() + + # Build the full Block model so that a recreated document contains all + # the fields. + block_doc = Block( + block_id=pinned["block_id"], + blocktype=data.get("blocktype", "unknown"), + data=data, + version=next_version, + last_modified=_now_isoformat(), + creator_ids=[user_id] if user_id else [], + ) + envelope = block_doc.model_dump(exclude={"immutable_id", "creators", "groups"}) + set_fields = { + field: envelope.pop(field) + for field in ("data", "version", "last_modified", "block_id", "blocktype", "type") + } + + flask_mongo.db.blocks.update_one( + {"_id": immutable_id}, + {"$set": set_fields, "$setOnInsert": envelope}, + upsert=True, + ) + + version_entry = BlockVersion( + block_immutable_id=immutable_id, + block_id=pinned["block_id"], + version=next_version, + timestamp=datetime.datetime.now(tz=datetime.timezone.utc), + action=VersionAction.RESTORED, + restored_from_version=pinned["_id"], + user_id=user_id, + datalab_version=__version__, + data=data, + ) + version_doc = version_entry.model_dump(exclude_none=True) + version_doc["data"] = data + flask_mongo.db.block_versions.insert_one(version_doc) + + return next_version diff --git a/pydatalab/src/pydatalab/export.py b/pydatalab/src/pydatalab/export.py index a06551765..c4a35b5b6 100644 --- a/pydatalab/src/pydatalab/export.py +++ b/pydatalab/src/pydatalab/export.py @@ -9,6 +9,7 @@ from pathlib import Path from pydatalab import __version__ +from pydatalab.blocks.store import load_blocks_obj from pydatalab.config import CONFIG from pydatalab.logger import LOGGER from pydatalab.models import ITEM_MODELS @@ -377,6 +378,8 @@ def create_eln_file( try: item_data = list(cursor)[0] + if item_data.get("blocks_obj"): + item_data["blocks_obj"] = load_blocks_obj(item_data) ItemModel = ITEM_MODELS[item_data["type"]] item_data = ItemModel(**item_data).model_dump() @@ -419,6 +422,8 @@ def create_eln_file( _all_items = [] for ind, item in enumerate(all_items): + if item.get("blocks_obj"): + item["blocks_obj"] = load_blocks_obj(item) ItemModel = ITEM_MODELS[item["type"]] _all_items.append(ItemModel(**item).model_dump()) diff --git a/pydatalab/src/pydatalab/models/__init__.py b/pydatalab/src/pydatalab/models/__init__.py index ea76f4514..fca0924a6 100644 --- a/pydatalab/src/pydatalab/models/__init__.py +++ b/pydatalab/src/pydatalab/models/__init__.py @@ -1,5 +1,6 @@ import functools +from pydatalab.models.blocks import Block from pydatalab.models.cells import Cell from pydatalab.models.collections import Collection from pydatalab.models.equipment import Equipment @@ -8,7 +9,7 @@ from pydatalab.models.people import Person from pydatalab.models.samples import Sample from pydatalab.models.starting_materials import StartingMaterial -from pydatalab.models.versions import ItemVersion +from pydatalab.models.versions import BlockVersion, ItemVersion @functools.lru_cache(maxsize=1) @@ -31,6 +32,8 @@ def generate_schemas() -> dict[str, dict]: ITEM_SCHEMAS = generate_schemas() __all__ = ( + "Block", + "BlockVersion", "File", "Sample", "StartingMaterial", diff --git a/pydatalab/src/pydatalab/models/blocks.py b/pydatalab/src/pydatalab/models/blocks.py index e6d9244e3..6fe280659 100644 --- a/pydatalab/src/pydatalab/models/blocks.py +++ b/pydatalab/src/pydatalab/models/blocks.py @@ -1,5 +1,9 @@ +from typing import Any, Literal + from pydantic import ConfigDict, Field +from pydatalab.models.entries import Entry +from pydatalab.models.traits import HasOwner, HasRevisionControl from pydatalab.models.utils import BaseModel, PyObjectId @@ -72,3 +76,42 @@ class DataBlockResponse(BaseModel): ) """Any structured metadata associated with the block, for example, experimental acquisition parameters.""" + + +# Here to avoid circular import +class HasBlocks(BaseModel): + """Trait mixin for models that can have data blocks attached to them.""" + + blocks_obj: dict[str, DataBlockResponse] = Field({}) + """A mapping from block ID to block data.""" + + display_order: list[str] = Field([]) + """The order in which to display block data in the UI.""" + + +class Block(Entry, HasOwner, HasRevisionControl): + """A model for a data block stored as its own document in the `blocks` collection. + + This is the persistence envelope around a block's payload (the output of + `DataBlock.to_db()`, stored verbatim under `data`); the payload itself is + described by `DataBlockResponse` and its per-block-type subclasses. + """ + + type: Literal["blocks"] = "blocks" + + block_id: str + """The runtime-generated shorthand ID for the block, used as the key in the + parent item's `blocks_obj`/`display_order` and in the DOM.""" + + blocktype: str + """A short string key specifying the type (technique) of the block.""" + + data: dict[str, Any] = Field(default_factory=dict) + """The block payload, exactly as produced by `DataBlock.to_db()`.""" + + version: int = 0 + """The latest committed version number of this block in `block_versions`. + + A newly created block starts at 0, i.e., with no committed version; the first + version is cut when an item version snapshot is next saved. Block creation + does not create a versioned entry. Only when item is saved.""" diff --git a/pydatalab/src/pydatalab/models/collections.py b/pydatalab/src/pydatalab/models/collections.py index 9423ef330..578c1c209 100644 --- a/pydatalab/src/pydatalab/models/collections.py +++ b/pydatalab/src/pydatalab/models/collections.py @@ -4,8 +4,9 @@ model_validator, ) +from pydatalab.models.blocks import HasBlocks from pydatalab.models.entries import Entry -from pydatalab.models.traits import HasBlocks, HasOwner +from pydatalab.models.traits import HasOwner from pydatalab.models.utils import HumanReadableIdentifier diff --git a/pydatalab/src/pydatalab/models/items.py b/pydatalab/src/pydatalab/models/items.py index a405ce086..812b834b8 100644 --- a/pydatalab/src/pydatalab/models/items.py +++ b/pydatalab/src/pydatalab/models/items.py @@ -2,10 +2,10 @@ from pydantic import field_validator +from pydatalab.models.blocks import HasBlocks from pydatalab.models.entries import Entry from pydatalab.models.files import HasFiles from pydatalab.models.traits import ( - HasBlocks, HasOwner, HasRevisionControl, IsCollectable, diff --git a/pydatalab/src/pydatalab/models/traits.py b/pydatalab/src/pydatalab/models/traits.py index 774ec4348..ce5c57f1d 100644 --- a/pydatalab/src/pydatalab/models/traits.py +++ b/pydatalab/src/pydatalab/models/traits.py @@ -2,7 +2,6 @@ from pydantic import AliasChoices, ConfigDict, Field, field_validator, model_validator -from pydatalab.models.blocks import DataBlockResponse from pydatalab.models.people import Group, Person from pydatalab.models.utils import BaseModel, Constituent, InlineSubstance, PyObjectId @@ -12,7 +11,6 @@ __all__ = ( "HasOwner", "HasRevisionControl", - "HasBlocks", "IsCollectable", "HasSynthesisInfo", "HasSubstanceInfo", @@ -48,16 +46,6 @@ class HasRevisionControl(BaseModel): """The version number used by the version control system for tracking snapshots.""" -class HasBlocks(BaseModel): - """Trait mixin for models that can have data blocks attached to them.""" - - blocks_obj: dict[str, DataBlockResponse] = Field({}) - """A mapping from block ID to block data.""" - - display_order: list[str] = Field([]) - """The order in which to display block data in the UI.""" - - class CollectionReference(BaseModel): """A reference to a collection, used for inlining collection info within other models.""" diff --git a/pydatalab/src/pydatalab/models/versions.py b/pydatalab/src/pydatalab/models/versions.py index a01311424..ecec3e90f 100644 --- a/pydatalab/src/pydatalab/models/versions.py +++ b/pydatalab/src/pydatalab/models/versions.py @@ -69,6 +69,54 @@ def validate_restored_from_version(self): return self +class BlockVersion(BaseModel): + """A snapshot of a data block's payload at a specific point in time. + + This model represents a version entry in the `block_versions` collection. + Entries are created whenever an item version snapshot is cut and the + block's payload has changed since its last committed version (or by a + version restore). + """ + + block_immutable_id: PyObjectId + """The immutable ID (`_id`) of the `blocks` document this version belongs to""" + + block_id: str + """The shorthand block ID, denormalised for lookups by block ID""" + + version: int = Field(ge=1) + """Sequential version number (1-indexed), scoped to the block""" + + timestamp: datetime + """When this version was created (ISO format with timezone)""" + + action: VersionAction + """The action that triggered this version, matching the item snapshot that cut it""" + + user_id: PyObjectId | None = None + """User's ObjectId for efficient querying and indexing""" + + datalab_version: str + """Version of datalab-server that created this snapshot""" + + data: dict + """Complete snapshot of the block payload at this version""" + + restored_from_version: PyObjectId | None = None + """ObjectId of the `block_versions` entry that was restored from (only present if action='restored')""" + + @model_validator(mode="after") + def validate_restored_from_version(self): + """Ensure restored_from_version is only present when action='restored'.""" + if self.action == VersionAction.RESTORED and self.restored_from_version is None: + raise ValueError("restored_from_version must be provided when action='restored'") + if self.action != VersionAction.RESTORED and self.restored_from_version is not None: + raise ValueError( + f"restored_from_version should only be present when action='restored', got action='{self.action}'" + ) + return self + + class VersionCounter(BaseModel): """Atomic counter for tracking version numbers per item. diff --git a/pydatalab/src/pydatalab/mongo.py b/pydatalab/src/pydatalab/mongo.py index 5b988abe7..733bfd1e1 100644 --- a/pydatalab/src/pydatalab/mongo.py +++ b/pydatalab/src/pydatalab/mongo.py @@ -435,6 +435,22 @@ def create_group_fts(): "refcode", unique=True, name="unique refcode counter", background=background ) + # Block storage indexes. `block_id` is only effectively unique (random, + # historically unique within a single item), so is not a unique index. + ret += db.blocks.create_index("block_id", name="block ID", background=background) + ret += db.blocks.create_index("blocktype", name="block type", background=background) + ret += db.blocks.create_index("creator_ids", name="block creators", background=background) + ret += db.blocks.create_index("group_ids", name="block groups", background=background) + ret += db.block_versions.create_index( + [("block_immutable_id", pymongo.ASCENDING), ("version", pymongo.DESCENDING)], + unique=True, + name="block immutable ID and version", + background=background, + ) + ret += db.block_versions.create_index( + "block_id", name="block version block ID", background=background + ) + return ret diff --git a/pydatalab/src/pydatalab/routes/v0_1/blocks.py b/pydatalab/src/pydatalab/routes/v0_1/blocks.py index 8322f1161..395afb3e7 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/blocks.py +++ b/pydatalab/src/pydatalab/routes/v0_1/blocks.py @@ -10,6 +10,7 @@ from werkzeug.exceptions import BadRequest, NotFound, NotImplemented from pydatalab.apps import BLOCK_TYPES +from pydatalab.blocks import store from pydatalab.blocks.base import DataBlock from pydatalab.logger import LOGGER from pydatalab.login import get_by_id @@ -77,9 +78,8 @@ def add_stage(message: str, level: str = "info", traceback: str | None = None): {"item_id": block_data["item_id"], **get_default_permissions(user_only=True)}, projection={f"blocks_obj.{block_data['block_id']}": 1}, ) - stored_block_data = ( - (stored_item or {}).get("blocks_obj", {}).get(block_data["block_id"]) - ) + stored_blocks = store.load_blocks_obj(stored_item or {}) + stored_block_data = stored_blocks.get(block_data["block_id"]) block = BLOCK_TYPES[block_type].from_web(block_data, stored_data=stored_block_data) @@ -256,15 +256,34 @@ def add_data_block(): else: display_order_update = block.block_id + # Check the item exists and the user has write access before creating the + # block document, to avoid orphaning it below. + if not flask_mongo.db.items.find_one( + {"item_id": item_id, **get_default_permissions(user_only=True)}, {"_id": 1} + ): + return ( + jsonify( + status="error", + message=f"Update failed. {item_id=} is probably incorrect.", + ), + 400, + ) + + # New blocks are born as separate documents in the `blocks` collection; + # the item only stores a reference to them. No version is committed until + # the next item version snapshot. + block_immutable_id = store.create_block_document(block) + result = flask_mongo.db.items.update_one( {"item_id": item_id, **get_default_permissions(user_only=True)}, { "$push": {"display_order": display_order_update}, - "$set": {f"blocks_obj.{block.block_id}": block.to_db()}, + "$set": {f"blocks_obj.{block.block_id}": {"immutable_id": block_immutable_id}}, }, ) if result.modified_count < 1: + store.delete_block_document(block_immutable_id) return ( jsonify( status="error", @@ -292,6 +311,11 @@ def _save_block_to_db(block: DataBlock): """Save data for a single block within an item to the database, overwriting previous data saved there. + The write is authorized against the parent item and then branches on the + block's stored form: a referenced block updates its `blocks` document, + while a legacy embedded block is written back into the item's `blocks_obj` + as before (a block never changes form). + Parameters: block: The instance of DataBlock to save. @@ -299,18 +323,27 @@ def _save_block_to_db(block: DataBlock): updated_block = block.to_db() update = {"$set": {f"blocks_obj.{block.block_id}": updated_block}} - match = { - "item_id": block.data["item_id"], - f"blocks_obj.{block.block_id}": {"$exists": True}, - **get_default_permissions(user_only=False), - } - result = flask_mongo.db.items.update_one(match, update) - - if result.matched_count != 1: + stored_entry = store.authorize_and_get_form(block.data.get("item_id"), block.block_id) + if stored_entry is None: raise BadRequest( - f"Failed to save block, likely because item_id ({block.data.get('item_id')}), and/or block_id ({block.block_id}) wasn't found" + f"Failed to save block, likely because block_id ({block.block_id}) wasn't found on item ({block.data.get('item_id')})" ) + if store.is_block_reference(stored_entry): + store.update_block_document(stored_entry["immutable_id"], updated_block) + else: + match = { + "item_id": block.data["item_id"], + f"blocks_obj.{block.block_id}": {"$exists": True}, + **get_default_permissions(user_only=False), + } + result = flask_mongo.db.items.update_one(match, update) + + if result.matched_count != 1: + raise BadRequest( + f"Failed to save block, likely because item_id ({block.data.get('item_id')}) and/or block_id ({block.block_id}) wasn't found" + ) + @BLOCKS.route("/update-block/", methods=["POST"]) @BLOCKS.route("/blocks/", methods=["POST"]) @@ -342,8 +375,10 @@ def update_block(): if not item: raise NotFound(f"Item with item_id {item_id} not found or not accessible") + # `blocks_obj` holds a reference for separated blocks, so the stored payload + # has to be resolved before it can restore server-authoritative fields. block = BLOCK_TYPES[block_type].from_web( - block_data, stored_data=item.get("blocks_obj", {}).get(block_data["block_id"]) + block_data, stored_data=store.load_blocks_obj(item).get(block_data["block_id"]) ) from pydatalab.config import CONFIG @@ -430,7 +465,9 @@ def delete_block(): item_id = request_json["item_id"] block_id = request_json["block_id"] - result = flask_mongo.db.items.update_one( + # Atomically remove the block from the item and read back its pre-deletion + # state. + doc_before = flask_mongo.db.items.find_one_and_update( {"item_id": item_id, **get_default_permissions(user_only=True)}, { "$pull": { @@ -439,9 +476,15 @@ def delete_block(): }, "$unset": {f"blocks_obj.{block_id}": ""}, }, + projection={f"blocks_obj.{block_id}": 1, "display_order": 1}, ) - if result.modified_count < 1: + stored_entry = (doc_before.get("blocks_obj") or {}).get(block_id) if doc_before else None + + block_was_present = stored_entry is not None or ( + doc_before is not None and block_id in (doc_before.get("display_order") or []) + ) + if not block_was_present: return ( jsonify( { @@ -451,6 +494,10 @@ def delete_block(): ), 400, ) + + if store.is_block_reference(stored_entry): + store.delete_block_document(stored_entry["immutable_id"]) + return ( jsonify({"status": "success"}), 200, diff --git a/pydatalab/src/pydatalab/routes/v0_1/graphs.py b/pydatalab/src/pydatalab/routes/v0_1/graphs.py index 72fb68317..674fc1184 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/graphs.py +++ b/pydatalab/src/pydatalab/routes/v0_1/graphs.py @@ -1,5 +1,6 @@ from flask import Blueprint, jsonify, request +from pydatalab.blocks.store import is_block_reference from pydatalab.mongo import flask_mongo from pydatalab.permissions import active_users_or_get_only, get_default_permissions @@ -47,12 +48,19 @@ def get_graph_cy_format( } else: query = {} - all_documents = flask_mongo.db.items.find( - {**query, **get_default_permissions(user_only=False)}, - projection={"item_id": 1, "name": 1, "type": 1, "relationships": 1, "blocks_obj": 1}, + all_documents = list( + flask_mongo.db.items.find( + {**query, **get_default_permissions(user_only=False)}, + projection={ + "item_id": 1, + "name": 1, + "type": 1, + "relationships": 1, + "blocks_obj": 1, + }, + ) ) node_ids: set[str] = {document["item_id"] for document in all_documents} - all_documents.rewind() else: main_item = flask_mongo.db.items.find_one( @@ -138,6 +146,22 @@ def add_related_items(current_item_id: str, current_depth: int): nodes = [] edges = [] + # Resolve the blocktype of referenced blocks_obj entries in one batched query. + referenced_block_ids = { + entry["immutable_id"] + for document in all_documents + for entry in (document.get("blocks_obj") or {}).values() + if is_block_reference(entry) + } + referenced_blocktypes: dict = {} + if referenced_block_ids: + referenced_blocktypes = { + doc["_id"]: doc.get("blocktype", "unknown") + for doc in flask_mongo.db.blocks.find( + {"_id": {"$in": list(referenced_block_ids)}}, {"blocktype": 1} + ) + } + # Collect the elements that have already been added to the graph, to avoid duplication drawn_elements = set() node_collections: set[str] = set() @@ -230,7 +254,10 @@ def add_related_items(current_item_id: str, current_depth: int): for block_id, block_data in (document.get("blocks_obj") or {}).items(): if block_id not in drawn_elements: drawn_elements.add(block_id) - blocktype = block_data.get("blocktype", "unknown") + if is_block_reference(block_data): + blocktype = referenced_blocktypes.get(block_data["immutable_id"], "unknown") + else: + blocktype = block_data.get("blocktype", "unknown") nodes.append( { "data": { diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index c2e4d8e6d..b9479c2a9 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -14,6 +14,7 @@ from werkzeug.exceptions import BadRequest, Conflict, InternalServerError, NotFound from pydatalab.apps import BLOCK_TYPES +from pydatalab.blocks import store as block_store from pydatalab.config import CONFIG from pydatalab.logger import LOGGER from pydatalab.models import ITEM_MODELS, ItemVersion @@ -57,6 +58,91 @@ # creation time embedded in their ObjectId. LAST_MODIFIED_PROJECTION = {"$ifNull": ["$last_modified", {"$toDate": "$_id"}]} +_BLOCKS_OBJ_AS_ARRAY = {"$objectToArray": {"$ifNull": ["$blocks_obj", {}]}} +"""Aggregation expression turning `blocks_obj` into an array of +`{"k": , "v": }` documents.""" + +_BLOCKS_OBJ_ENTRY_IS_REFERENCE = {"$ne": [{"$type": "$$b.v.immutable_id"}, "missing"]} +"""Aggregation condition telling a `{"immutable_id": ...}` block reference apart +from a legacy embedded block payload (bound to a `blocks_obj` entry as `$$b`).""" + + +def blocks_preview_stages() -> list[dict]: + """Aggregation stages that resolve the previews (block_id/blocktype/title) of an item's + referenced blocks from the `blocks` collection into a temporary + `_referenced_blocks` field, for use by `blocks_preview_projection()`. + + Must be placed before the `$project` stage that consumes the result. Safe to + use on any permission-filtered items query: previews carry no payload beyond + the block ID, type and title. + """ + return [ + { + "$set": { + "_referenced_block_ids": { + "$map": { + "input": { + "$filter": { + "input": _BLOCKS_OBJ_AS_ARRAY, + "as": "b", + "cond": _BLOCKS_OBJ_ENTRY_IS_REFERENCE, + } + }, + "as": "b", + "in": "$$b.v.immutable_id", + } + } + } + }, + { + "$lookup": { + "from": "blocks", + "let": {"refs": "$_referenced_block_ids"}, + "pipeline": [ + {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$refs", []]}]}}}, + { + "$project": { + "_id": 0, + "block_id": 1, + "blocktype": 1, + "title": "$data.title", + } + }, + ], + "as": "_referenced_blocks", + } + }, + ] + + +def blocks_preview_projection() -> dict: + """Projection expression for the `blocks` preview array (block_id/blocktype/title + per block), combining legacy embedded `blocks_obj` entries with the referenced + blocks resolved by `blocks_preview_stages()`. + """ + return { + "$concatArrays": [ + { + "$map": { + "input": { + "$filter": { + "input": _BLOCKS_OBJ_AS_ARRAY, + "as": "b", + "cond": {"$not": _BLOCKS_OBJ_ENTRY_IS_REFERENCE}, + } + }, + "as": "b", + "in": { + "block_id": "$$b.k", + "blocktype": "$$b.v.blocktype", + "title": "$$b.v.title", + }, + } + }, + {"$ifNull": ["$_referenced_blocks", []]}, + ] + } + @ITEMS.before_request @active_users_or_get_only @@ -106,20 +192,12 @@ def get_starting_materials(): } }, {"$lookup": collections_lookup()}, + *blocks_preview_stages(), { "$project": { "_id": 0, "item_id": 1, - "blocks": { - "$map": { - "input": {"$objectToArray": {"$ifNull": ["$blocks_obj", {}]}}, - "as": "b", - "in": { - "blocktype": "$$b.v.blocktype", - "title": "$$b.v.title", - }, - } - }, + "blocks": blocks_preview_projection(), "collections": { "collection_id": 1, }, @@ -172,17 +250,7 @@ def get_items_summary(match: dict | None = None, project: dict | None = None) -> _project = { "_id": 0, - "blocks": { - "$map": { - "input": {"$objectToArray": {"$ifNull": ["$blocks_obj", {}]}}, - "as": "b", - "in": { - "block_id": "$$b.k", - "blocktype": "$$b.v.blocktype", - "title": "$$b.v.title", - }, - } - }, + "blocks": blocks_preview_projection(), "creators": { "display_name": 1, "gravatar_hash": 1, @@ -227,6 +295,7 @@ def get_items_summary(match: dict | None = None, project: dict | None = None) -> {"$lookup": creators_lookup()}, {"$lookup": groups_lookup()}, {"$lookup": collections_lookup()}, + *blocks_preview_stages(), {"$project": _project}, {"$sort": {"date": -1}}, ] @@ -250,16 +319,7 @@ def get_samples_summary(match: dict | None = None, project: dict | None = None) _project = { "_id": 0, - "blocks": { - "$map": { - "input": {"$objectToArray": {"$ifNull": ["$blocks_obj", {}]}}, - "as": "b", - "in": { - "blocktype": "$$b.v.blocktype", - "title": "$$b.v.title", - }, - } - }, + "blocks": blocks_preview_projection(), "creators": { "display_name": 1, "gravatar_hash": 1, @@ -304,6 +364,7 @@ def get_samples_summary(match: dict | None = None, project: dict | None = None) {"$lookup": creators_lookup()}, {"$lookup": groups_lookup()}, {"$lookup": collections_lookup()}, + *blocks_preview_stages(), {"$project": _project}, {"$sort": {"date": -1}}, ] @@ -1029,7 +1090,7 @@ def delete_sample(): item = flask_mongo.db.items.find_one( {"item_id": item_id, **get_default_permissions(user_only=True, deleting=True)}, - {"refcode": 1}, + {"refcode": 1, "blocks_obj": 1}, ) if not item: @@ -1050,6 +1111,16 @@ def delete_sample(): if result.deleted_count != 1: raise BadRequest(f"Failed to delete item with {item_id=}.") + # Deleting an item removes the live documents of its referenced blocks, but + # their `block_versions` history is retained. + referenced_block_ids = [ + entry["immutable_id"] + for entry in (item.get("blocks_obj") or {}).values() + if block_store.is_block_reference(entry) + ] + if referenced_block_ids: + flask_mongo.db.blocks.delete_many({"_id": {"$in": referenced_block_ids}}) + flask_mongo.db.api_keys.delete_many({"refcode": item["refcode"], "type": "access_token"}) return jsonify({"status": "success"}), 200 @@ -1131,6 +1202,11 @@ def get_item_data( else: raise BadRequest(f"Item {item_id=} has no type field in document.") + # Resolve any referenced blocks into full payloads so the response keeps + # the legacy `blocks_obj` shape (the item read above was already permission-filtered). + if doc.get("blocks_obj"): + doc["blocks_obj"] = block_store.load_blocks_obj(doc) + try: doc = entry_reference_lookup(doc) doc = ItemModel(**doc) @@ -1323,6 +1399,12 @@ def get_version(refcode, version_id): if not version: raise NotFound + # Resolve any version-pinned block references in the snapshot into full + # payloads so the response keeps the pre-separation `blocks_obj` shape + # (access was already checked against the parent item). + if version.get("data", {}).get("blocks_obj"): + version["data"]["blocks_obj"] = block_store.resolve_snapshot_blocks_obj(version["data"]) + return jsonify({"status": "success", "version": version}), 200 @@ -1353,6 +1435,11 @@ def compare_versions(refcode): if not v1 or not v2: raise NotFound("One or both versions not found") + # Resolve any block references so the diff shows block content changes. + for v in (v1, v2): + if v.get("data", {}).get("blocks_obj"): + v["data"]["blocks_obj"] = block_store.resolve_snapshot_blocks_obj(v["data"]) + # Use DeepDiff for proper nested structure comparison # This handles nested dicts, lists, type changes, and provides detailed change information deep_diff = DeepDiff( @@ -1438,26 +1525,96 @@ def restore_version(refcode): raise BadRequest(f"Invalid item type: {item_type}") try: - # Validate using the appropriate model - ITEM_MODELS[item_type](**restored_data) + # Validate using the appropriate model. Version-pinned block references + # do not validate as block payloads, so validate a *resolved* copy while + # keeping each entry's original form for the writes below. + validation_data = restored_data + if any( + block_store.is_block_reference(entry) + for entry in (restored_data.get("blocks_obj") or {}).values() + ): + validation_data = { + **restored_data, + "blocks_obj": block_store.resolve_snapshot_blocks_obj(restored_data), + } + ITEM_MODELS[item_type](**validation_data) except ValidationError as exc: raise BadRequest( f"Restored data failed validation against schema for type {item_type}: {exc}" ) - # Perform the restore first - flask_mongo.db.items.update_one({"refcode": refcode}, {"$set": restored_data}) - # Extract user information for hybrid storage approach user_id = None if current_user.is_authenticated: user_id = current_user.person.immutable_id + # Apply each snapshotted blocks_obj entry in its own form: embedded payloads + # are written back inline as before, while the references get + # the payload from the `block_versions` and becomes the new current `blocks` + # state, plus a new RESTORED `block_versions` entry. + restored_block_pins: dict[str, dict] = {} + if "blocks_obj" in restored_data: + live_blocks_obj = {} + for block_id, entry in (restored_data.get("blocks_obj") or {}).items(): + if not block_store.is_block_reference(entry): + live_blocks_obj[block_id] = entry + continue + # Snapshot references should always be version-pinned. + # An unpinned one is not restorable. + pinned_version = entry.get("version") + new_block_version = ( + block_store.restore_block_version( + entry["immutable_id"], pinned_version, user_id=user_id + ) + if pinned_version is not None + else None + ) + if new_block_version is None: + # No block_versions entry to restore from (should not happen): + # drop the entry rather than leaving a dangling reference. + LOGGER.error( + "Dropping unrestorable block reference %s (%s) while restoring %s", + block_id, + entry, + refcode, + ) + if isinstance(restored_data.get("display_order"), list): + restored_data["display_order"] = [ + b for b in restored_data["display_order"] if b != block_id + ] + continue + live_blocks_obj[block_id] = {"immutable_id": entry["immutable_id"]} + restored_block_pins[block_id] = { + "immutable_id": entry["immutable_id"], + "version": new_block_version, + } + restored_data["blocks_obj"] = live_blocks_obj + + # Referenced blocks on the current item that are absent from the restored + # snapshot disappear from the item (as the legacy blocks), so remove + # their live documents too. + for block_id, entry in (current_item.get("blocks_obj") or {}).items(): + if block_store.is_block_reference(entry) and block_id not in live_blocks_obj: + block_store.delete_block_document(entry["immutable_id"]) + + # Perform the restore first + flask_mongo.db.items.update_one({"refcode": refcode}, {"$set": restored_data}) + # Get the software version from pydatalab import __version__ software_version = __version__ + # The RESTORED snapshot pins its referenced blocks_obj entries to the + # new RESTORED block version numbers, so that this snapshot can be + # restored later. + restored_snapshot_data = restored_data + if restored_block_pins: + restored_snapshot_data = { + **restored_data, + "blocks_obj": {**restored_data["blocks_obj"], **restored_block_pins}, + } + # Save the RESTORED state as a new version snapshot (after restore) restored_version_entry = { "refcode": refcode, @@ -1468,7 +1625,7 @@ def restore_version(refcode): "restored_from_version": version_object_id, "user_id": user_id, # ObjectId for efficient querying "datalab_version": software_version, - "data": restored_data, # Store the complete snapshot of the restored state + "data": restored_snapshot_data, # Store the complete snapshot of the restored state } # Validate with Pydantic before inserting @@ -1482,10 +1639,10 @@ def restore_version(refcode): ) raise BadRequest(f"Restored version data validation failed: {exc}") - # Insert validated data, restoring original restored_data into 'data' so None-valued + # Insert validated data, restoring the original snapshot dict into 'data' so None-valued # fields are not stripped by model_dump(exclude_none=True) recursing into the dict. restored_version_doc = validated_restored_version.model_dump(exclude_none=True) - restored_version_doc["data"] = restored_data + restored_version_doc["data"] = restored_snapshot_data flask_mongo.db.item_versions.insert_one(restored_version_doc) return jsonify( @@ -1603,15 +1760,55 @@ def save_item(): if not item: raise NotFound - stored_blocks = item.get("blocks_obj", {}) - for block_id, block_data in updated_data.get("blocks_obj", {}).items(): - blocktype = block_data["blocktype"] - - block = BLOCK_TYPES.get(blocktype, BLOCK_TYPES["notsupported"]).from_web( - block_data, stored_data=stored_blocks.get(block_id) - ) - - updated_data["blocks_obj"][block_id] = block.to_db() + # Reconcile the incoming blocks against their stored form (the request + # always carries full payloads, so the form is only knowable from the stored + # item): referenced blocks are written to the `blocks` collection, legacy + # embedded blocks stay inline in the item, and unknown block IDs are born + # referenced. A block never changes form. The block-document writes are + # deferred until the item payload has passed validation below. + stored_blocks_obj = item.get("blocks_obj") or {} + block_reference_map: dict[str, dict] = {} + pending_block_creations: dict[str, object] = {} + pending_block_updates: list[tuple[ObjectId, dict]] = [] + pending_block_deletions: list[ObjectId] = [] + + if "blocks_obj" in updated_data: + incoming_blocks = updated_data["blocks_obj"] or {} + # Server-authoritative fields (`metadata`, `computed`, ...) never come back + # from the web, so they have to be restored from the stored payload; for a + # referenced block that payload lives in the `blocks` collection, not in + # the item's `blocks_obj` entry. + stored_payloads = block_store.load_blocks_obj(item) if stored_blocks_obj else {} + for block_id, block_data in incoming_blocks.items(): + blocktype = block_data["blocktype"] + block = BLOCK_TYPES.get(blocktype, BLOCK_TYPES["notsupported"]).from_web( + block_data, stored_data=stored_payloads.get(block_id) + ) + incoming_blocks[block_id] = block.to_db() + + stored_entry = stored_blocks_obj.get(block_id) + if stored_entry is None: + pending_block_creations[block_id] = block + elif block_store.is_block_reference(stored_entry): + pending_block_updates.append( + (stored_entry["immutable_id"], incoming_blocks[block_id]) + ) + block_reference_map[block_id] = {"immutable_id": stored_entry["immutable_id"]} + # else: legacy embedded block — the payload is kept inline as before + + # A stored referenced block missing from the payload is treated as a + # deletion, avoiding orphaned `blocks` documents. + for block_id, stored_entry in stored_blocks_obj.items(): + if block_id not in incoming_blocks and block_store.is_block_reference(stored_entry): + pending_block_deletions.append(stored_entry["immutable_id"]) + elif any(block_store.is_block_reference(entry) for entry in stored_blocks_obj.values()): + # The client did not send blocks_obj, so the stored blocks are untouched, + # but referenced entries must be resolved into full payloads for the item + # validation below and swapped back before the final write. + for block_id, stored_entry in stored_blocks_obj.items(): + if block_store.is_block_reference(stored_entry): + block_reference_map[block_id] = stored_entry + item["blocks_obj"] = block_store.load_blocks_obj(item) if "collections" in updated_data: requested_collections = updated_data["collections"] @@ -1690,6 +1887,21 @@ def save_item(): if isinstance(existing_last_modified, datetime.datetime): existing_last_modified = existing_last_modified.isoformat() + # Now that the item payload has passed validation, apply the deferred block + # document writes and swap `{"immutable_id": ...}` references back into the + # document in place of the full payloads of referenced blocks. + for block_id, block in pending_block_creations.items(): + new_block_immutable_id = block_store.create_block_document(block) # type: ignore[arg-type] + block_reference_map[block_id] = {"immutable_id": new_block_immutable_id} + for block_immutable_id, block_payload in pending_block_updates: + block_store.update_block_document(block_immutable_id, block_payload) + for block_immutable_id in pending_block_deletions: + block_store.delete_block_document(block_immutable_id) + + if block_reference_map: + item.setdefault("blocks_obj", {}) + item["blocks_obj"].update(block_reference_map) + # Update the item FIRST (transaction safety: item update before version save) result = flask_mongo.db.items.update_one( {"item_id": item_id, **get_default_permissions(user_only=True)}, diff --git a/pydatalab/src/pydatalab/versioning.py b/pydatalab/src/pydatalab/versioning.py index 7d7916db6..26394b882 100644 --- a/pydatalab/src/pydatalab/versioning.py +++ b/pydatalab/src/pydatalab/versioning.py @@ -138,6 +138,7 @@ def save_version_snapshot( Tuple of (response_dict, status_code) """ from pydatalab import __version__ + from pydatalab.blocks.store import is_block_reference, snapshot_block_version from pydatalab.config import CONFIG if len(refcode.split(":")) != 2: @@ -152,20 +153,6 @@ def save_version_snapshot( if not item: raise NotFound(f"Item {refcode} not found.") - # Skip creating a new version if content is identical to the last snapshot. - # Excludes fields that change mechanically on every save and don't reflect real edits. - last_version = flask_mongo.db.item_versions.find_one( - {"refcode": refcode}, sort=[("version", -1)] - ) - if last_version: - current_data = {k: v for k, v in item.items() if k not in MECHANICAL_FIELDS} - last_data = {k: v for k, v in last_version["data"].items() if k not in MECHANICAL_FIELDS} - if current_data == last_data: - LOGGER.debug( - "No changes detected for %s, skipping version save: %s", refcode, current_data - ) - return {"status": "success", "message": "No changes detected, version not saved."}, 200 - # Extract user information for hybrid storage approach user_id = None if current_user.is_authenticated: @@ -185,6 +172,53 @@ def save_version_snapshot( action = VersionAction.MANUAL_SAVE if action is None else action + # Cut `block_versions` entries for any *referenced* blocks whose payload has + # changed since their last committed version, and pin each referenced + # `blocks_obj` entry to its now-current version number in this snapshot. + # Embedded blocks are captured verbatim in the item snapshot. + # This must happen before the de-duplication guard below so that a + # block content change is seen as an item change. + for block_id, entry in list((item.get("blocks_obj") or {}).items()): + if not is_block_reference(entry): + continue + block_version = snapshot_block_version( + entry["immutable_id"], action=action, user_id=user_id + ) + if block_version is None: + # There is no live block document and no committed history for + # this reference (e.g. a dangling reference left by a crash), so + # there is no content anywhere to snapshot: record the block as + # absent rather than storing an unresolvable reference. Only this + # in-memory snapshot is altered — the live item is not modified. + LOGGER.warning( + "Dropping dangling block reference %r (%s) from version snapshot of %s", + block_id, + entry["immutable_id"], + refcode, + ) + del item["blocks_obj"][block_id] + if isinstance(item.get("display_order"), list): + item["display_order"] = [b for b in item["display_order"] if b != block_id] + continue + item["blocks_obj"][block_id] = { + "immutable_id": entry["immutable_id"], + "version": block_version, + } + + # Skip creating a new version if content is identical to the last snapshot. + # Excludes fields that change mechanically on every save and don't reflect real edits. + last_version = flask_mongo.db.item_versions.find_one( + {"refcode": refcode}, sort=[("version", -1)] + ) + if last_version: + current_data = {k: v for k, v in item.items() if k not in MECHANICAL_FIELDS} + last_data = {k: v for k, v in last_version["data"].items() if k not in MECHANICAL_FIELDS} + if current_data == last_data: + LOGGER.debug( + "No changes detected for %s, skipping version save: %s", refcode, current_data + ) + return {"status": "success", "message": "No changes detected, version not saved."}, 200 + version_entry = { "refcode": refcode, "version": 1, diff --git a/pydatalab/tests/server/test_block_separation.py b/pydatalab/tests/server/test_block_separation.py new file mode 100644 index 000000000..a600c89d7 --- /dev/null +++ b/pydatalab/tests/server/test_block_separation.py @@ -0,0 +1,822 @@ +"""Core verification tests for the separation of blocks into their own +`blocks`/`block_versions` collections. + +These cover the two coexisting storage forms of an item's `blocks_obj` entry: +legacy *embedded* payloads (frozen in that form for life) and *referenced* +blocks (`{"immutable_id": ObjectId}` pointers, used for all newly created +blocks), plus the versioning/restore/reconciliation flows across them. +""" + +import pytest +from bson import ObjectId + + +@pytest.fixture() +def sample_with_block(admin_client, default_sample_dict, random_string): + """Create a fresh sample with one (referenced) comment block attached, + returning `(sample_id, refcode, block_id)`.""" + sample_id = f"test_block_sep_{random_string[:10]}" + sample_data = default_sample_dict.copy() + sample_data["item_id"] = sample_id + + response = admin_client.post("/new-sample/", json=sample_data) + assert response.status_code == 201, response.json + + response = admin_client.post( + "/add-data-block/", + json={"block_type": "comment", "item_id": sample_id, "index": 0}, + ) + assert response.status_code == 200, response.json + block_id = response.json["new_block_obj"]["block_id"] + + response = admin_client.get(f"/get-item-data/{sample_id}") + assert response.status_code == 200 + refcode = response.json["item_data"]["refcode"] + + yield sample_id, refcode, block_id + + admin_client.post("/delete-sample/", json={"item_id": sample_id}) + + +def _get_item_data(client, sample_id): + response = client.get(f"/get-item-data/{sample_id}") + assert response.status_code == 200, response.json + return response.json["item_data"] + + +def _save_item(client, sample_id, item_data): + response = client.post("/save-item/", json={"item_id": sample_id, "data": item_data}) + assert response.status_code == 200, response.json + return response + + +def _insert_legacy_embedded_block(database, sample_id, block_id, comment="legacy comment"): + """Simulate a pre-separation item by embedding a full block payload directly + in the item document.""" + payload = { + "blocktype": "comment", + "block_id": block_id, + "item_id": sample_id, + "title": "Comment", + "freeform_comment": comment, + } + result = database.items.update_one( + {"item_id": sample_id}, + {"$set": {f"blocks_obj.{block_id}": payload}, "$push": {"display_order": block_id}}, + ) + assert result.modified_count == 1 + return payload + + +def test_new_block_is_referenced_and_reassembled(sample_with_block, admin_client, database): + """A newly created block lands in `blocks` with a reference in the item, has + no committed version yet, and API reads reassemble the full payload.""" + sample_id, _, block_id = sample_with_block + + stored_entry = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"][ + block_id + ] + assert set(stored_entry) == {"immutable_id"} + + block_doc = database.blocks.find_one({"_id": stored_entry["immutable_id"]}) + assert block_doc is not None + assert block_doc["block_id"] == block_id + assert block_doc["blocktype"] == "comment" + assert block_doc["type"] == "blocks" + assert block_doc["version"] == 0 + assert block_doc["data"]["blocktype"] == "comment" + assert block_doc["data"]["block_id"] == block_id + + # No block version is committed at creation + assert database.block_versions.count_documents({"block_id": block_id}) == 0 + + # The API response keeps the pre-separation shape + item_data = _get_item_data(admin_client, sample_id) + assert item_data["blocks_obj"][block_id]["blocktype"] == "comment" + assert item_data["blocks_obj"][block_id]["block_id"] == block_id + assert item_data["display_order"] == [block_id] + + +def test_summary_and_graph_resolve_referenced_blocks(sample_with_block, admin_client): + """Summary block previews and the item graph report the blocktype of + referenced blocks.""" + sample_id, _, block_id = sample_with_block + + response = admin_client.get("/samples/") + assert response.status_code == 200 + entries = [s for s in response.json["samples"] if s["item_id"] == sample_id] + assert len(entries) == 1 + assert entries[0]["nblocks"] == 1 + assert {b["blocktype"] for b in entries[0]["blocks"]} == {"comment"} + + response = admin_client.get(f"/item-graph/{sample_id}") + assert response.status_code == 200 + block_nodes = [n for n in response.json["nodes"] if n["data"]["id"] == block_id] + assert len(block_nodes) == 1 + assert block_nodes[0]["data"]["name"] == "comment" + + +def test_update_block_and_versioning_flow(sample_with_block, admin_client, database): + """`update-block` only writes the live `blocks` doc; versions are cut by item + snapshots (any entry point), de-duplicated when the payload is unchanged.""" + sample_id, refcode, block_id = sample_with_block + + immutable_id = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"][ + block_id + ]["immutable_id"] + + # Update the block: live doc is written, but no version is committed + item_data = _get_item_data(admin_client, sample_id) + block_data = item_data["blocks_obj"][block_id] + block_data["freeform_comment"] = "first comment" + response = admin_client.post("/update-block/", json={"block_data": block_data}) + assert response.status_code == 200, response.json + + block_doc = database.blocks.find_one({"_id": immutable_id}) + assert block_doc["data"]["freeform_comment"] == "first comment" + assert block_doc["version"] == 0 + assert database.block_versions.count_documents({"block_immutable_id": immutable_id}) == 0 + + # Saving the item cuts the first block version and pins it in the item snapshot + item_data = _get_item_data(admin_client, sample_id) + _save_item(admin_client, sample_id, item_data) + + assert database.blocks.find_one({"_id": immutable_id})["version"] == 1 + versions = list(database.block_versions.find({"block_immutable_id": immutable_id})) + assert len(versions) == 1 + assert versions[0]["version"] == 1 + assert versions[0]["data"]["freeform_comment"] == "first comment" + + item_version = database.item_versions.find_one({"refcode": refcode}, sort=[("version", -1)]) + assert item_version["data"]["blocks_obj"][block_id] == { + "immutable_id": immutable_id, + "version": 1, + } + + # An unchanged save does not mint a new block or item version + n_item_versions = database.item_versions.count_documents({"refcode": refcode}) + item_data = _get_item_data(admin_client, sample_id) + _save_item(admin_client, sample_id, item_data) + assert database.blocks.find_one({"_id": immutable_id})["version"] == 1 + assert database.block_versions.count_documents({"block_immutable_id": immutable_id}) == 1 + assert database.item_versions.count_documents({"refcode": refcode}) == n_item_versions + + # A block content change alone produces a new item version (the changed pin + # defeats the de-dup guard) and a new block version + item_data = _get_item_data(admin_client, sample_id) + item_data["blocks_obj"][block_id]["freeform_comment"] = "second comment" + _save_item(admin_client, sample_id, item_data) + + assert database.blocks.find_one({"_id": immutable_id})["version"] == 2 + assert database.block_versions.count_documents({"block_immutable_id": immutable_id}) == 2 + assert database.item_versions.count_documents({"refcode": refcode}) == n_item_versions + 1 + item_version = database.item_versions.find_one({"refcode": refcode}, sort=[("version", -1)]) + assert item_version["data"]["blocks_obj"][block_id] == { + "immutable_id": immutable_id, + "version": 2, + } + + # The manual /save-version/ endpoint also snapshots drifted block content + item_data = _get_item_data(admin_client, sample_id) + block_data = item_data["blocks_obj"][block_id] + block_data["freeform_comment"] = "drifted comment" + response = admin_client.post("/update-block/", json={"block_data": block_data}) + assert response.status_code == 200 + + response = admin_client.post(f"/items/{refcode}/save-version/") + assert response.status_code == 200, response.json + + assert database.blocks.find_one({"_id": immutable_id})["version"] == 3 + latest_block_version = database.block_versions.find_one( + {"block_immutable_id": immutable_id}, sort=[("version", -1)] + ) + assert latest_block_version["data"]["freeform_comment"] == "drifted comment" + item_version = database.item_versions.find_one({"refcode": refcode}, sort=[("version", -1)]) + assert item_version["data"]["blocks_obj"][block_id]["version"] == 3 + + +def test_version_views_resolve_referenced_payloads(sample_with_block, admin_client): + """Version previews and diffs show full block payloads, not + `{immutable_id, version}` pins.""" + sample_id, refcode, block_id = sample_with_block + + item_data = _get_item_data(admin_client, sample_id) + item_data["blocks_obj"][block_id]["freeform_comment"] = "first comment" + _save_item(admin_client, sample_id, item_data) + + item_data = _get_item_data(admin_client, sample_id) + item_data["blocks_obj"][block_id]["freeform_comment"] = "second comment" + _save_item(admin_client, sample_id, item_data) + + response = admin_client.get(f"/items/{refcode}/versions/") + assert response.status_code == 200 + versions = response.json["versions"] + assert len(versions) >= 3 # created + two saves + v_first = next(v for v in versions if v["version"] == 2) + v_second = next(v for v in versions if v["version"] == 3) + + response = admin_client.get(f"/items/{refcode}/versions/{v_first['_id']}/") + assert response.status_code == 200 + snapshot_block = response.json["version"]["data"]["blocks_obj"][block_id] + assert snapshot_block["blocktype"] == "comment" + assert snapshot_block["freeform_comment"] == "first comment" + + response = admin_client.get( + f"/items/{refcode}/compare-versions/?v1={v_first['_id']}&v2={v_second['_id']}" + ) + assert response.status_code == 200 + diff = response.json["diff"] + changed = diff.get("values_changed", {}) + comment_key = f"root['blocks_obj']['{block_id}']['freeform_comment']" + assert comment_key in changed + assert changed[comment_key]["old_value"] == "first comment" + assert changed[comment_key]["new_value"] == "second comment" + + +def test_restore_referenced_block(sample_with_block, admin_client, database): + """Restore is append-only for referenced blocks: the pinned payload becomes + the new current state plus a RESTORED history entry, and a since-deleted + block is recreated with its original immutable_id.""" + sample_id, refcode, block_id = sample_with_block + + immutable_id = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"][ + block_id + ]["immutable_id"] + + item_data = _get_item_data(admin_client, sample_id) + item_data["blocks_obj"][block_id]["freeform_comment"] = "first comment" + _save_item(admin_client, sample_id, item_data) + + item_data = _get_item_data(admin_client, sample_id) + item_data["blocks_obj"][block_id]["freeform_comment"] = "second comment" + _save_item(admin_client, sample_id, item_data) + + response = admin_client.get(f"/items/{refcode}/versions/") + v_first = next(v for v in response.json["versions"] if v["version"] == 2) + + response = admin_client.post( + f"/items/{refcode}/restore-version/", json={"version_id": v_first["_id"]} + ) + assert response.status_code == 200, response.json + + # The live block payload is back to the pinned content... + block_doc = database.blocks.find_one({"_id": immutable_id}) + assert block_doc["data"]["freeform_comment"] == "first comment" + # ...via a new RESTORED history entry (append-only) + latest_block_version = database.block_versions.find_one( + {"block_immutable_id": immutable_id}, sort=[("version", -1)] + ) + assert latest_block_version["action"] == "restored" + assert latest_block_version["version"] == block_doc["version"] == 3 + # The live item points at the block without a pin; the RESTORED item + # snapshot pins the freshly minted block version + stored_entry = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"][ + block_id + ] + assert stored_entry == {"immutable_id": immutable_id} + restored_item_version = database.item_versions.find_one( + {"refcode": refcode}, sort=[("version", -1)] + ) + assert restored_item_version["action"] == "restored" + assert restored_item_version["data"]["blocks_obj"][block_id] == { + "immutable_id": immutable_id, + "version": 3, + } + + item_data = _get_item_data(admin_client, sample_id) + assert item_data["blocks_obj"][block_id]["freeform_comment"] == "first comment" + + # Delete the block, then restore again: the block document is recreated + # with its original immutable_id from its retained history + response = admin_client.post( + "/delete-block/", json={"item_id": sample_id, "block_id": block_id} + ) + assert response.status_code == 200 + assert database.blocks.find_one({"_id": immutable_id}) is None + assert database.block_versions.count_documents({"block_immutable_id": immutable_id}) == 3 + + response = admin_client.post( + f"/items/{refcode}/restore-version/", json={"version_id": v_first["_id"]} + ) + assert response.status_code == 200, response.json + + block_doc = database.blocks.find_one({"_id": immutable_id}) + assert block_doc is not None + assert block_doc["data"]["freeform_comment"] == "first comment" + item_data = _get_item_data(admin_client, sample_id) + assert item_data["blocks_obj"][block_id]["freeform_comment"] == "first comment" + + +def test_legacy_embedded_block_stays_embedded(sample_with_block, admin_client, database): + """A legacy embedded block is frozen in embedded form across update-block, + save-item, snapshots, and restores — no `blocks` document is ever created for + it — while a referenced sibling in the same item works normally (mixed state).""" + sample_id, refcode, referenced_block_id = sample_with_block + + legacy_block_id = "legacyblock12345" + _insert_legacy_embedded_block(database, sample_id, legacy_block_id) + + # Both forms are reassembled in the API response + item_data = _get_item_data(admin_client, sample_id) + assert item_data["blocks_obj"][legacy_block_id]["freeform_comment"] == "legacy comment" + assert item_data["blocks_obj"][referenced_block_id]["blocktype"] == "comment" + + # update-block on the legacy block stays on the embedded path + block_data = item_data["blocks_obj"][legacy_block_id] + block_data["freeform_comment"] = "legacy edited" + response = admin_client.post("/update-block/", json={"block_data": block_data}) + assert response.status_code == 200, response.json + + stored = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"] + assert stored[legacy_block_id]["freeform_comment"] == "legacy edited" + assert set(stored[referenced_block_id]) == {"immutable_id"} + assert database.blocks.find_one({"block_id": legacy_block_id}) is None + + # save-item keeps each entry in its form; the snapshot embeds the legacy + # payload verbatim and pins the referenced entry + item_data = _get_item_data(admin_client, sample_id) + item_data["blocks_obj"][legacy_block_id]["freeform_comment"] = "legacy saved" + _save_item(admin_client, sample_id, item_data) + + stored = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"] + assert stored[legacy_block_id]["freeform_comment"] == "legacy saved" + assert set(stored[referenced_block_id]) == {"immutable_id"} + assert database.blocks.find_one({"block_id": legacy_block_id}) is None + + item_version = database.item_versions.find_one({"refcode": refcode}, sort=[("version", -1)]) + snapshot_blocks = item_version["data"]["blocks_obj"] + assert snapshot_blocks[legacy_block_id]["freeform_comment"] == "legacy saved" + assert set(snapshot_blocks[referenced_block_id]) == {"immutable_id", "version"} + assert item_version["data"]["display_order"] == [referenced_block_id, legacy_block_id] + saved_version_id = str(item_version["_id"]) + + # Edit again and restore the previous snapshot: the legacy block is restored + # embedded and still has no blocks document (anti-dangling) + item_data = _get_item_data(admin_client, sample_id) + item_data["blocks_obj"][legacy_block_id]["freeform_comment"] = "legacy edited again" + _save_item(admin_client, sample_id, item_data) + + response = admin_client.post( + f"/items/{refcode}/restore-version/", json={"version_id": saved_version_id} + ) + assert response.status_code == 200, response.json + + stored = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"] + assert stored[legacy_block_id]["freeform_comment"] == "legacy saved" + assert set(stored[referenced_block_id]) == {"immutable_id"} + assert database.blocks.find_one({"block_id": legacy_block_id}) is None + + item_data = _get_item_data(admin_client, sample_id) + assert item_data["blocks_obj"][legacy_block_id]["freeform_comment"] == "legacy saved" + + +def test_save_item_reconciliation(sample_with_block, admin_client, database): + """save-item deletes the `blocks` doc of a referenced block omitted from the + payload (retaining its history), and births unknown block IDs as references.""" + sample_id, _, block_id = sample_with_block + + immutable_id = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"][ + block_id + ]["immutable_id"] + + # Commit a version so there is history to retain + item_data = _get_item_data(admin_client, sample_id) + _save_item(admin_client, sample_id, item_data) + assert database.block_versions.count_documents({"block_immutable_id": immutable_id}) == 1 + + # A payload omitting the stored referenced block deletes its live document + item_data = _get_item_data(admin_client, sample_id) + del item_data["blocks_obj"][block_id] + item_data["display_order"] = [] + _save_item(admin_client, sample_id, item_data) + + stored = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"] + assert block_id not in stored + assert database.blocks.find_one({"_id": immutable_id}) is None + assert database.block_versions.count_documents({"block_immutable_id": immutable_id}) == 1 + + # A payload with an unknown block_id births a *referenced* block + new_block_id = "clientmadeblock1" + item_data = _get_item_data(admin_client, sample_id) + item_data.setdefault("blocks_obj", {})[new_block_id] = { + "blocktype": "comment", + "block_id": new_block_id, + "item_id": sample_id, + "freeform_comment": "born via save-item", + } + item_data["display_order"] = [new_block_id] + _save_item(admin_client, sample_id, item_data) + + stored = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"] + assert set(stored[new_block_id]) == {"immutable_id"} + block_doc = database.blocks.find_one({"_id": stored[new_block_id]["immutable_id"]}) + assert block_doc["data"]["freeform_comment"] == "born via save-item" + + item_data = _get_item_data(admin_client, sample_id) + assert item_data["blocks_obj"][new_block_id]["freeform_comment"] == "born via save-item" + + +def test_server_authoritative_fields_survive_saves_of_referenced_blocks( + sample_with_block, admin_client, database +): + """`metadata`/`computed` are server-authoritative: they are stripped from any + web payload and must be restored from the *stored* block. For a referenced + block that stored payload lives in the `blocks` collection, not in the item's + `blocks_obj` entry, so both /save-item/ and /update-block/ have to resolve it + before rebuilding the block. + """ + sample_id, _, block_id = sample_with_block + + immutable_id = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"][ + block_id + ]["immutable_id"] + + # Stand in for fields that only block code or block events can write + database.blocks.update_one( + {"_id": immutable_id}, + {"$set": {"data.metadata": {"instrument": "abc"}, "data.computed": {"peak": 1.23}}}, + ) + + # Saving the item round-trips the block through `from_web`, which must not + # drop the fields the client never received + item_data = _get_item_data(admin_client, sample_id) + item_data["blocks_obj"][block_id]["freeform_comment"] = "edited via save-item" + _save_item(admin_client, sample_id, item_data) + + block_doc = database.blocks.find_one({"_id": immutable_id}) + assert block_doc["data"]["freeform_comment"] == "edited via save-item" + assert block_doc["data"]["metadata"] == {"instrument": "abc"} + assert block_doc["data"]["computed"] == {"peak": 1.23} + + # ...and neither may the dedicated block-update route + block_data = _get_item_data(admin_client, sample_id)["blocks_obj"][block_id] + block_data["freeform_comment"] = "edited via update-block" + response = admin_client.post("/update-block/", json={"block_data": block_data}) + assert response.status_code == 200, response.json + + block_doc = database.blocks.find_one({"_id": immutable_id}) + assert block_doc["data"]["freeform_comment"] == "edited via update-block" + assert block_doc["data"]["metadata"] == {"instrument": "abc"} + assert block_doc["data"]["computed"] == {"peak": 1.23} + + +def test_delete_block_and_item_retain_history(sample_with_block, admin_client, database): + """Deleting a block (or its whole item) removes the live `blocks` document(s) + but retains `block_versions`, mirroring `item_versions`.""" + sample_id, _, block_id = sample_with_block + + immutable_id = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"][ + block_id + ]["immutable_id"] + + # Commit a version, then delete the block + item_data = _get_item_data(admin_client, sample_id) + _save_item(admin_client, sample_id, item_data) + + response = admin_client.post( + "/delete-block/", json={"item_id": sample_id, "block_id": block_id} + ) + assert response.status_code == 200 + assert database.blocks.find_one({"_id": immutable_id}) is None + assert database.block_versions.count_documents({"block_immutable_id": immutable_id}) == 1 + stored = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1, "display_order": 1}) + assert block_id not in stored["blocks_obj"] + assert block_id not in stored["display_order"] + + # Deleting the same (now-absent) block again returns 400, on an accessible + # item — the single-query find_one_and_update preserves this behaviour. + response = admin_client.post( + "/delete-block/", json={"item_id": sample_id, "block_id": block_id} + ) + assert response.status_code == 400 + + # Add another block and delete the whole item + response = admin_client.post( + "/add-data-block/", json={"block_type": "comment", "item_id": sample_id, "index": 0} + ) + assert response.status_code == 200 + second_block_id = response.json["new_block_obj"]["block_id"] + item_data = _get_item_data(admin_client, sample_id) + _save_item(admin_client, sample_id, item_data) + second_immutable_id = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})[ + "blocks_obj" + ][second_block_id]["immutable_id"] + + response = admin_client.post("/delete-sample/", json={"item_id": sample_id}) + assert response.status_code == 200 + assert database.blocks.find_one({"_id": second_immutable_id}) is None + assert database.block_versions.count_documents({"block_immutable_id": second_immutable_id}) == 1 + + +def test_block_created_and_deleted_before_save_leaves_no_residue( + sample_with_block, admin_client, database +): + """A block created and deleted before any item save has no `block_versions` + entries to retain — nothing is left behind.""" + sample_id, _, _ = sample_with_block + + response = admin_client.post( + "/add-data-block/", json={"block_type": "comment", "item_id": sample_id, "index": 0} + ) + assert response.status_code == 200 + transient_block_id = response.json["new_block_obj"]["block_id"] + + transient_immutable_id = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})[ + "blocks_obj" + ][transient_block_id]["immutable_id"] + + response = admin_client.post( + "/delete-block/", json={"item_id": sample_id, "block_id": transient_block_id} + ) + assert response.status_code == 200 + assert database.blocks.find_one({"_id": transient_immutable_id}) is None + assert ( + database.block_versions.count_documents({"block_immutable_id": transient_immutable_id}) == 0 + ) + + +def test_snapshot_pins_from_history_when_live_doc_missing( + sample_with_block, admin_client, database +): + """If a referenced block's live document is missing at snapshot time (e.g. a + deletion race) but its history is retained, the snapshot pins the latest + committed version instead of storing an unpinned reference.""" + sample_id, refcode, block_id = sample_with_block + + # Commit version 1 of the block + item_data = _get_item_data(admin_client, sample_id) + _save_item(admin_client, sample_id, item_data) + + immutable_id = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"][ + block_id + ]["immutable_id"] + assert database.block_versions.count_documents({"block_immutable_id": immutable_id}) == 1 + + # Simulate the race: the live doc disappears while the item still references + # it, and the item changes so the snapshot de-dup guard does not skip + database.blocks.delete_one({"_id": immutable_id}) + database.items.update_one( + {"item_id": sample_id}, {"$set": {"description": "changed under race"}} + ) + + response = admin_client.post(f"/items/{refcode}/save-version/") + assert response.status_code == 200, response.json + + item_version = database.item_versions.find_one({"refcode": refcode}, sort=[("version", -1)]) + assert item_version["data"]["description"] == "changed under race" + assert item_version["data"]["blocks_obj"][block_id] == { + "immutable_id": immutable_id, + "version": 1, + } + # No new block version was minted by the fallback + assert database.block_versions.count_documents({"block_immutable_id": immutable_id}) == 1 + + +def test_snapshot_drops_reference_with_no_content_anywhere( + sample_with_block, admin_client, database +): + """A dangling reference with no live document and no committed history is + recorded as absent in the snapshot (never as an unresolvable reference); + the live item document is left untouched.""" + sample_id, refcode, block_id = sample_with_block + + immutable_id = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"][ + block_id + ]["immutable_id"] + + # The block was never saved (no history); make its reference dangle and + # change the item so the de-dup guard does not skip the snapshot + database.blocks.delete_one({"_id": immutable_id}) + database.items.update_one({"item_id": sample_id}, {"$set": {"description": "now dangling"}}) + + response = admin_client.post(f"/items/{refcode}/save-version/") + assert response.status_code == 200, response.json + + item_version = database.item_versions.find_one({"refcode": refcode}, sort=[("version", -1)]) + assert item_version["data"]["description"] == "now dangling" + assert block_id not in item_version["data"]["blocks_obj"] + assert block_id not in item_version["data"]["display_order"] + + # The live item still carries the dangling reference (snapshotting must not + # mutate the item); the read path just skips it + stored = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1, "display_order": 1}) + assert stored["blocks_obj"][block_id] == {"immutable_id": immutable_id} + assert block_id in stored["display_order"] + item_data = _get_item_data(admin_client, sample_id) + assert block_id not in item_data["blocks_obj"] + + +def test_file_attachment_linkage_with_referenced_block( + sample_with_block, admin_client, database, tmpdir +): + """A file attached to a referenced block keeps the pre-separation linkage: + the item lists the file, the block payload carries its `file_id`, and the + reassembled API response reflects both.""" + sample_id, _, block_id = sample_with_block + + test_file = tmpdir.join("block_attachment.txt") + test_file.write("file contents for block attachment") + with open(str(test_file), "rb") as f: + response = admin_client.post( + "/upload-file/", + buffered=True, + content_type="multipart/form-data", + data={ + "item_id": sample_id, + "file": [(f, "block_attachment.txt")], + "type": "application/octet-stream", + "replace_file": "null", + "relativePath": "null", + }, + ) + assert response.status_code == 201, response.json + file_id = response.json["file_id"] + + # Attach the file to the block and save it + item_data = _get_item_data(admin_client, sample_id) + block_data = item_data["blocks_obj"][block_id] + block_data["file_id"] = file_id + response = admin_client.post("/update-block/", json={"block_data": block_data}) + assert response.status_code == 200, response.json + + # The file document is linked to the item, and the item lists the file + file_doc = database.files.find_one({"_id": ObjectId(file_id)}) + assert file_doc is not None + assert sample_id in file_doc["item_ids"] + item_doc = database.items.find_one({"item_id": sample_id}, {"file_ObjectIds": 1}) + assert ObjectId(file_id) in item_doc["file_ObjectIds"] + + # The block's payload (in the `blocks` collection) carries the file_id + stored_entry = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"][ + block_id + ] + block_doc = database.blocks.find_one({"_id": stored_entry["immutable_id"]}) + assert str(block_doc["data"]["file_id"]) == file_id + + # And the reassembled API response resolves the linkage + item_data = _get_item_data(admin_client, sample_id) + assert item_data["blocks_obj"][block_id]["file_id"] == file_id + assert file_id in [f["immutable_id"] for f in item_data.get("files", [])] + + +def test_no_block_content_for_user_without_item_access( + admin_client, another_client, database, default_sample_dict, random_string +): + """Every path that can return block or block-version content is gated by the + parent item's permissions, and the block document's own owner fields must + not grant access when the item denies it.""" + # The shared fixture sample carries group_ids granting the demo group read + # access (which another_client is in), so build one without any group grant + sample_id = f"test_block_sep_priv_{random_string[:8]}" + sample_data = default_sample_dict.copy() + sample_data["item_id"] = sample_id + sample_data.pop("group_ids", None) + + response = admin_client.post("/new-sample/", json=sample_data) + assert response.status_code == 201, response.json + + response = admin_client.post( + "/add-data-block/", json={"block_type": "comment", "item_id": sample_id, "index": 0} + ) + assert response.status_code == 200, response.json + block_id = response.json["new_block_obj"]["block_id"] + + # Commit a version so version endpoints have content to protect + item_data = _get_item_data(admin_client, sample_id) + refcode = item_data["refcode"] + _save_item(admin_client, sample_id, item_data) + version_id = admin_client.get(f"/items/{refcode}/versions/").json["versions"][0]["_id"] + + def assert_no_access(): + assert another_client.get(f"/get-item-data/{sample_id}").status_code == 404 + assert another_client.get(f"/item-graph/{sample_id}").status_code == 404 + assert another_client.get(f"/items/{refcode}/versions/").status_code == 404 + assert another_client.get(f"/items/{refcode}/versions/{version_id}/").status_code == 404 + assert ( + another_client.post( + f"/items/{refcode}/restore-version/", json={"version_id": version_id} + ).status_code + == 404 + ) + samples = another_client.get("/samples/").json["samples"] + assert sample_id not in [s["item_id"] for s in samples] + + assert_no_access() + + # Even if the *block document* names the other user as creator/group member, + # access is still denied: the parent item is the sole access authority + immutable_id = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"][ + block_id + ]["immutable_id"] + another_user_id = ObjectId(24 * "7") # `another_client`'s user, per conftest + database.blocks.update_one( + {"_id": immutable_id}, + {"$set": {"creator_ids": [another_user_id], "group_ids": [another_user_id]}}, + ) + assert_no_access() + + admin_client.post("/delete-sample/", json={"item_id": sample_id}) + + +def test_item_created_with_client_blocks_obj_stays_embedded( + admin_client, default_sample_dict, database, random_string +): + """An API client can still supply `blocks_obj` at item creation; such blocks + are stored embedded (legacy form) and live on the embedded path.""" + sample_id = f"test_block_sep_create_{random_string[:8]}" + embedded_block_id = "clientcreated123" + sample_data = default_sample_dict.copy() + sample_data["item_id"] = sample_id + sample_data["blocks_obj"] = { + embedded_block_id: { + "blocktype": "comment", + "block_id": embedded_block_id, + "item_id": sample_id, + "freeform_comment": "born embedded at creation", + } + } + sample_data["display_order"] = [embedded_block_id] + + response = admin_client.post("/new-sample/", json=sample_data) + assert response.status_code == 201, response.json + + stored = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"] + assert stored[embedded_block_id]["freeform_comment"] == "born embedded at creation" + assert database.blocks.find_one({"block_id": embedded_block_id}) is None + + item_data = _get_item_data(admin_client, sample_id) + assert item_data["blocks_obj"][embedded_block_id]["blocktype"] == "comment" + + admin_client.post("/delete-sample/", json={"item_id": sample_id}) + + +def test_block_write_permissions( + sample_with_block, + admin_client, + another_client, + database, + default_sample_dict, + random_string, +): + """Block writes are authorized via the parent item. Since #1951 read access + to the item is enough to update its blocks, and that must hold for a + referenced block just as it did for an embedded one; a user with no access + at all is still refused, as is a request without an `item_id`. Deletion + remains creator-only. + """ + sample_id, _, block_id = sample_with_block + + item_data = _get_item_data(admin_client, sample_id) + block_data = item_data["blocks_obj"][block_id] + block_data["freeform_comment"] = "group edit" + + immutable_id = database.items.find_one({"item_id": sample_id}, {"blocks_obj": 1})["blocks_obj"][ + block_id + ]["immutable_id"] + + # The fixture sample grants the demo group (which `another_client` belongs + # to) read access, so this write is permitted and must land on the block + # document rather than being silently dropped. + response = another_client.post("/update-block/", json={"block_data": block_data}) + assert response.status_code == 200, response.json + assert database.blocks.find_one({"_id": immutable_id})["data"]["freeform_comment"] == ( + "group edit" + ) + + # A user with no access to the item at all still cannot write its blocks + private_id = f"test_block_sep_perm_{random_string[:8]}" + private_sample = default_sample_dict.copy() + private_sample["item_id"] = private_id + private_sample.pop("group_ids", None) + assert admin_client.post("/new-sample/", json=private_sample).status_code == 201 + + response = admin_client.post( + "/add-data-block/", json={"block_type": "comment", "item_id": private_id, "index": 0} + ) + assert response.status_code == 200, response.json + private_block_id = response.json["new_block_obj"]["block_id"] + + private_block_data = _get_item_data(admin_client, private_id)["blocks_obj"][private_block_id] + private_block_data["freeform_comment"] = "sneaky edit" + response = another_client.post("/update-block/", json={"block_data": private_block_data}) + assert response.status_code == 404 + + private_immutable_id = database.items.find_one({"item_id": private_id}, {"blocks_obj": 1})[ + "blocks_obj" + ][private_block_id]["immutable_id"] + private_doc = database.blocks.find_one({"_id": private_immutable_id}) + assert private_doc["data"].get("freeform_comment") is None + + admin_client.post("/delete-sample/", json={"item_id": private_id}) + + # A request without an item_id is a 400 + no_item_block_data = {k: v for k, v in block_data.items() if k != "item_id"} + response = admin_client.post("/update-block/", json={"block_data": no_item_block_data}) + assert response.status_code == 400 + + # Deleting is creator-only, so read access does not extend to it + response = another_client.post( + "/delete-block/", json={"item_id": sample_id, "block_id": block_id} + ) + assert response.status_code == 400 + assert database.blocks.find_one({"_id": immutable_id}) is not None diff --git a/pydatalab/tests/server/test_blocks.py b/pydatalab/tests/server/test_blocks.py index 7a16f10e5..b8628f925 100644 --- a/pydatalab/tests/server/test_blocks.py +++ b/pydatalab/tests/server/test_blocks.py @@ -793,8 +793,13 @@ def test_create_sample_with_example_files( assert "blocks_obj" in item_data if block_type == "xrd": + # Newly created blocks are stored as `{"immutable_id": ...}` references to + # the `blocks` collection, so follow the reference to check the stored payload. doc = database.items.find_one({"item_id": sample_id}, projection={"blocks_obj": 1}) - assert doc["blocks_obj"][block_id]["computed"]["peak_data"] is not None + stored_entry = doc["blocks_obj"][block_id] + assert set(stored_entry) == {"immutable_id"} + block_doc = database.blocks.find_one({"_id": stored_entry["immutable_id"]}) + assert block_doc["data"]["computed"]["peak_data"] is not None @pytest.fixture() diff --git a/webapp/src/server_fetch_utils.js b/webapp/src/server_fetch_utils.js index d8865df3c..8a5e0a4c0 100644 --- a/webapp/src/server_fetch_utils.js +++ b/webapp/src/server_fetch_utils.js @@ -1085,11 +1085,10 @@ export function deleteBlock(item_id, block_id) { // eslint-disable-next-line no-unused-vars .then(function (response_json) { // response_json should always just be {status: "success"}, so we don't actually use it - store.commit("removeBlockFromDisplay", { + store.commit("removeBlock", { item_id: item_id, block_id: block_id, }); - // currently, we don't actually delete the block from the store, so it may get re-added to the db on the next save. Fix once new schemas are established }) .catch((error) => { DialogService.error({ diff --git a/webapp/src/store/index.js b/webapp/src/store/index.js index 8dd7cf01e..7fb6c35a8 100644 --- a/webapp/src/store/index.js +++ b/webapp/src/store/index.js @@ -320,14 +320,21 @@ export default createStore({ // item_id, isSaved state.saved_status_collections[payload.collection_id] = payload.isSaved; }, - removeBlockFromDisplay(state, payload) { + removeBlock(state, payload) { // requires the following fields in payload: // item_id, block_id - var display_order = state.all_item_data[payload.item_id].display_order; - const index = display_order.indexOf(payload.block_id); + // Remove the block from the display order, but also drop its data from + // blocks_obj so that the next save-item does not resurrect the deleted + // block in the database, and clear any stale saved status. + const item_data = state.all_item_data[payload.item_id]; + const index = item_data.display_order.indexOf(payload.block_id); if (index > -1) { - display_order.splice(index, 1); + item_data.display_order.splice(index, 1); + } + if (item_data.blocks_obj) { + delete item_data.blocks_obj[payload.block_id]; } + delete state.saved_status_blocks[payload.block_id]; }, addFile(state, payload) { // requires the following fileds in payload: From 2e49fd48749389d2c07cfb5a6667ad6014a0be07 Mon Sep 17 00:00:00 2001 From: Guido Petretto Date: Fri, 24 Jul 2026 23:54:53 +0200 Subject: [PATCH 40/40] address comments --- pydatalab/src/pydatalab/blocks/store.py | 108 ++++++++++-------- pydatalab/src/pydatalab/routes/v0_1/blocks.py | 22 ++-- pydatalab/src/pydatalab/routes/v0_1/graphs.py | 8 +- pydatalab/src/pydatalab/routes/v0_1/items.py | 74 ++++++------ pydatalab/src/pydatalab/versioning.py | 12 +- 5 files changed, 119 insertions(+), 105 deletions(-) diff --git a/pydatalab/src/pydatalab/blocks/store.py b/pydatalab/src/pydatalab/blocks/store.py index f894b8709..7ffc19034 100644 --- a/pydatalab/src/pydatalab/blocks/store.py +++ b/pydatalab/src/pydatalab/blocks/store.py @@ -2,7 +2,7 @@ This module is the single chokepoint for reading and writing separated block documents, and for the branching logic between the two coexisting storage -forms of an item's `blocks_obj` entry: +forms of an item's `blocks_obj` value: - **Embedded (legacy)**: the full block payload stored inline in the item document. Legacy blocks are frozen in this form — they are read, written, @@ -40,7 +40,7 @@ __all__ = ( "is_block_reference", - "authorize_and_get_form", + "authorize_and_get_blocks_data", "load_blocks_obj", "resolve_snapshot_blocks_obj", "create_block_document", @@ -63,27 +63,29 @@ def _current_user_id() -> ObjectId | None: return None -def is_block_reference(entry: Any) -> bool: - """Whether a stored `blocks_obj` entry is a reference to a `blocks` document +def is_block_reference(blocks_obj_value: Any) -> bool: + """Whether a stored `blocks_obj` value is a reference to a `blocks` document (``{"immutable_id": ...}``) rather than a legacy embedded block. """ return ( - isinstance(entry, dict) and "immutable_id" in entry and set(entry.keys()) <= _REFERENCE_KEYS + isinstance(blocks_obj_value, dict) + and "immutable_id" in blocks_obj_value + and set(blocks_obj_value.keys()) <= _REFERENCE_KEYS ) -def authorize_and_get_form(item_id: str | None, block_id: str) -> dict | None: +def authorize_and_get_blocks_data(item_id: str | None, block_id: str) -> dict | None: """The single permissioned query at the head of every block write. Checks in one query that the current user may write blocks on the item and - returns the stored `blocks_obj` entry for `block_id` (embedded payload or + returns the stored `blocks_obj` value for `block_id` (embedded payload or reference). Read-level access is sufficient, matching the permissions applied to block writes on embedded blocks before blocks were separated out (#1951). Returns: - The stored `blocks_obj` entry, or `None` if the item has no entry for + The stored `blocks_obj` value, or `None` if the item has no entry for this `block_id`. """ from pydatalab.permissions import get_default_permissions @@ -101,19 +103,46 @@ def authorize_and_get_form(item_id: str | None, block_id: str) -> dict | None: return (item.get("blocks_obj") or {}).get(block_id) +def _assemble_blocks_obj( + blocks_obj: dict, resolved_payloads: dict[str, Any], source_label: str +) -> dict[str, Any]: + """Reassemble a `blocks_obj` into full payloads keyed by `block_id`. + + Embedded values pass through verbatim; each referenced value is replaced by + its payload from `resolved_payloads` (keyed by `block_id`). + A reference with missing reference is dropped. + """ + assembled: dict[str, Any] = {} + for block_id, blocks_obj_value in blocks_obj.items(): + if not is_block_reference(blocks_obj_value): + assembled[block_id] = blocks_obj_value + continue + payload = resolved_payloads.get(block_id) + if not payload: + LOGGER.error( + "Dangling block reference in %s: blocks_obj[%r] -> %s", + source_label, + block_id, + blocks_obj_value, + ) + continue + assembled[block_id] = payload + return assembled + + def load_blocks_obj(parent_doc: dict) -> dict[str, Any]: """Resolve a live parent document's `blocks_obj` into full payloads keyed by `block_id`, i.e., the API shape used before blocks were separated. - Embedded entries pass through verbatim; referenced entries are resolved from + Embedded values pass through verbatim; referenced values are resolved from the `blocks` collection in a single batched query. Must only be called on a document the caller is already authorized to read. """ blocks_obj = parent_doc.get("blocks_obj") or {} referenced: dict[str, ObjectId] = { - block_id: entry["immutable_id"] - for block_id, entry in blocks_obj.items() - if is_block_reference(entry) + block_id: blocks_obj_value["immutable_id"] + for block_id, blocks_obj_value in blocks_obj.items() + if is_block_reference(blocks_obj_value) } resolved_docs: dict[ObjectId, dict] = {} @@ -125,39 +154,30 @@ def load_blocks_obj(parent_doc: dict) -> dict[str, Any]: ) } - loaded: dict[str, Any] = {} - for block_id, entry in blocks_obj.items(): - if block_id not in referenced: - loaded[block_id] = entry - continue - doc = resolved_docs.get(referenced[block_id]) - if doc is None: - LOGGER.error( - "Dangling block reference: blocks_obj[%r] points at missing blocks doc %s", - block_id, - referenced[block_id], - ) - continue - loaded[block_id] = doc.get("data", {}) - - return loaded + resolved_payloads = { + block_id: resolved_docs[immutable_id]["data"] + for block_id, immutable_id in referenced.items() + if resolved_docs.get(immutable_id) and resolved_docs[immutable_id].get("data") + } + return _assemble_blocks_obj(blocks_obj, resolved_payloads, "live item") def resolve_snapshot_blocks_obj(snapshot_data: dict) -> dict[str, Any]: """Resolve an `item_versions` snapshot's `blocks_obj` into full payloads keyed by `block_id`. - Embedded entries pass through verbatim; version-pinned references + Embedded values pass through verbatim; version-pinned references ``{"immutable_id", "version"}`` are resolved from `block_versions` in a single - batched query. + batched query (an unpinned reference has no committed version to resolve and + is treated as dangling). Must only be called downstream of an authorized item(-version) read. """ blocks_obj = snapshot_data.get("blocks_obj") or {} referenced: dict[str, tuple[ObjectId, int]] = { - block_id: (entry["immutable_id"], entry["version"]) - for block_id, entry in blocks_obj.items() - if is_block_reference(entry) and entry.get("version") is not None + block_id: (blocks_obj_value["immutable_id"], blocks_obj_value["version"]) + for block_id, blocks_obj_value in blocks_obj.items() + if is_block_reference(blocks_obj_value) and blocks_obj_value.get("version") is not None } resolved_versions: dict[tuple[ObjectId, int], dict] = {} @@ -175,22 +195,12 @@ def resolve_snapshot_blocks_obj(snapshot_data: dict) -> dict[str, Any]: ) } - resolved: dict[str, Any] = {} - for block_id, entry in blocks_obj.items(): - if not is_block_reference(entry): - resolved[block_id] = entry - continue - doc = resolved_versions.get(referenced[block_id]) if block_id in referenced else None - if doc is None: - LOGGER.error( - "Dangling block reference in version snapshot: blocks_obj[%r] -> %s", - block_id, - entry, - ) - continue - resolved[block_id] = doc.get("data", {}) - - return resolved + resolved_payloads = { + block_id: resolved_versions[key]["data"] + for block_id, key in referenced.items() + if resolved_versions.get(key) and resolved_versions[key].get("data") + } + return _assemble_blocks_obj(blocks_obj, resolved_payloads, "version snapshot") def create_block_document(block: "DataBlock") -> ObjectId: diff --git a/pydatalab/src/pydatalab/routes/v0_1/blocks.py b/pydatalab/src/pydatalab/routes/v0_1/blocks.py index 395afb3e7..652b59c8e 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/blocks.py +++ b/pydatalab/src/pydatalab/routes/v0_1/blocks.py @@ -7,6 +7,7 @@ import gridfs from flask import Blueprint, jsonify, request from flask_login import current_user, login_user +from pymongo import ReturnDocument from werkzeug.exceptions import BadRequest, NotFound, NotImplemented from pydatalab.apps import BLOCK_TYPES @@ -323,14 +324,16 @@ def _save_block_to_db(block: DataBlock): updated_block = block.to_db() update = {"$set": {f"blocks_obj.{block.block_id}": updated_block}} - stored_entry = store.authorize_and_get_form(block.data.get("item_id"), block.block_id) - if stored_entry is None: + stored_blocks_obj_value = store.authorize_and_get_blocks_data( + block.data.get("item_id"), block.block_id + ) + if stored_blocks_obj_value is None: raise BadRequest( f"Failed to save block, likely because block_id ({block.block_id}) wasn't found on item ({block.data.get('item_id')})" ) - if store.is_block_reference(stored_entry): - store.update_block_document(stored_entry["immutable_id"], updated_block) + if store.is_block_reference(stored_blocks_obj_value): + store.update_block_document(stored_blocks_obj_value["immutable_id"], updated_block) else: match = { "item_id": block.data["item_id"], @@ -477,11 +480,14 @@ def delete_block(): "$unset": {f"blocks_obj.{block_id}": ""}, }, projection={f"blocks_obj.{block_id}": 1, "display_order": 1}, + return_document=ReturnDocument.BEFORE, ) - stored_entry = (doc_before.get("blocks_obj") or {}).get(block_id) if doc_before else None + stored_blocks_obj_value = ( + (doc_before.get("blocks_obj") or {}).get(block_id) if doc_before else None + ) - block_was_present = stored_entry is not None or ( + block_was_present = stored_blocks_obj_value is not None or ( doc_before is not None and block_id in (doc_before.get("display_order") or []) ) if not block_was_present: @@ -495,8 +501,8 @@ def delete_block(): 400, ) - if store.is_block_reference(stored_entry): - store.delete_block_document(stored_entry["immutable_id"]) + if store.is_block_reference(stored_blocks_obj_value): + store.delete_block_document(stored_blocks_obj_value["immutable_id"]) return ( jsonify({"status": "success"}), diff --git a/pydatalab/src/pydatalab/routes/v0_1/graphs.py b/pydatalab/src/pydatalab/routes/v0_1/graphs.py index 674fc1184..081ce14ff 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/graphs.py +++ b/pydatalab/src/pydatalab/routes/v0_1/graphs.py @@ -146,12 +146,12 @@ def add_related_items(current_item_id: str, current_depth: int): nodes = [] edges = [] - # Resolve the blocktype of referenced blocks_obj entries in one batched query. + # Resolve the blocktype of referenced blocks_obj values in one batched query. referenced_block_ids = { - entry["immutable_id"] + blocks_obj_value["immutable_id"] for document in all_documents - for entry in (document.get("blocks_obj") or {}).values() - if is_block_reference(entry) + for blocks_obj_value in (document.get("blocks_obj") or {}).values() + if is_block_reference(blocks_obj_value) } referenced_blocktypes: dict = {} if referenced_block_ids: diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index b9479c2a9..6438dbb3f 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -1114,9 +1114,9 @@ def delete_sample(): # Deleting an item removes the live documents of its referenced blocks, but # their `block_versions` history is retained. referenced_block_ids = [ - entry["immutable_id"] - for entry in (item.get("blocks_obj") or {}).values() - if block_store.is_block_reference(entry) + blocks_obj_value["immutable_id"] + for blocks_obj_value in (item.get("blocks_obj") or {}).values() + if block_store.is_block_reference(blocks_obj_value) ] if referenced_block_ids: flask_mongo.db.blocks.delete_many({"_id": {"$in": referenced_block_ids}}) @@ -1527,16 +1527,11 @@ def restore_version(refcode): try: # Validate using the appropriate model. Version-pinned block references # do not validate as block payloads, so validate a *resolved* copy while - # keeping each entry's original form for the writes below. - validation_data = restored_data - if any( - block_store.is_block_reference(entry) - for entry in (restored_data.get("blocks_obj") or {}).values() - ): - validation_data = { - **restored_data, - "blocks_obj": block_store.resolve_snapshot_blocks_obj(restored_data), - } + # keeping each blocks_obj value's original form for the writes below. + validation_data = { + **restored_data, + "blocks_obj": block_store.resolve_snapshot_blocks_obj(restored_data), + } ITEM_MODELS[item_type](**validation_data) except ValidationError as exc: raise BadRequest( @@ -1548,23 +1543,23 @@ def restore_version(refcode): if current_user.is_authenticated: user_id = current_user.person.immutable_id - # Apply each snapshotted blocks_obj entry in its own form: embedded payloads + # Apply each snapshotted blocks_obj value in its own form: embedded payloads # are written back inline as before, while the references get # the payload from the `block_versions` and becomes the new current `blocks` # state, plus a new RESTORED `block_versions` entry. restored_block_pins: dict[str, dict] = {} if "blocks_obj" in restored_data: live_blocks_obj = {} - for block_id, entry in (restored_data.get("blocks_obj") or {}).items(): - if not block_store.is_block_reference(entry): - live_blocks_obj[block_id] = entry + for block_id, blocks_obj_value in (restored_data.get("blocks_obj") or {}).items(): + if not block_store.is_block_reference(blocks_obj_value): + live_blocks_obj[block_id] = blocks_obj_value continue # Snapshot references should always be version-pinned. # An unpinned one is not restorable. - pinned_version = entry.get("version") + pinned_version = blocks_obj_value.get("version") new_block_version = ( block_store.restore_block_version( - entry["immutable_id"], pinned_version, user_id=user_id + blocks_obj_value["immutable_id"], pinned_version, user_id=user_id ) if pinned_version is not None else None @@ -1575,7 +1570,7 @@ def restore_version(refcode): LOGGER.error( "Dropping unrestorable block reference %s (%s) while restoring %s", block_id, - entry, + blocks_obj_value, refcode, ) if isinstance(restored_data.get("display_order"), list): @@ -1583,9 +1578,9 @@ def restore_version(refcode): b for b in restored_data["display_order"] if b != block_id ] continue - live_blocks_obj[block_id] = {"immutable_id": entry["immutable_id"]} + live_blocks_obj[block_id] = {"immutable_id": blocks_obj_value["immutable_id"]} restored_block_pins[block_id] = { - "immutable_id": entry["immutable_id"], + "immutable_id": blocks_obj_value["immutable_id"], "version": new_block_version, } restored_data["blocks_obj"] = live_blocks_obj @@ -1593,9 +1588,9 @@ def restore_version(refcode): # Referenced blocks on the current item that are absent from the restored # snapshot disappear from the item (as the legacy blocks), so remove # their live documents too. - for block_id, entry in (current_item.get("blocks_obj") or {}).items(): - if block_store.is_block_reference(entry) and block_id not in live_blocks_obj: - block_store.delete_block_document(entry["immutable_id"]) + for block_id, blocks_obj_value in (current_item.get("blocks_obj") or {}).items(): + if block_store.is_block_reference(blocks_obj_value) and block_id not in live_blocks_obj: + block_store.delete_block_document(blocks_obj_value["immutable_id"]) # Perform the restore first flask_mongo.db.items.update_one({"refcode": refcode}, {"$set": restored_data}) @@ -1786,28 +1781,31 @@ def save_item(): ) incoming_blocks[block_id] = block.to_db() - stored_entry = stored_blocks_obj.get(block_id) - if stored_entry is None: + blocks_obj_value = stored_blocks_obj.get(block_id) + if blocks_obj_value is None: pending_block_creations[block_id] = block - elif block_store.is_block_reference(stored_entry): + elif block_store.is_block_reference(blocks_obj_value): pending_block_updates.append( - (stored_entry["immutable_id"], incoming_blocks[block_id]) + (blocks_obj_value["immutable_id"], incoming_blocks[block_id]) ) - block_reference_map[block_id] = {"immutable_id": stored_entry["immutable_id"]} + block_reference_map[block_id] = {"immutable_id": blocks_obj_value["immutable_id"]} # else: legacy embedded block — the payload is kept inline as before # A stored referenced block missing from the payload is treated as a # deletion, avoiding orphaned `blocks` documents. - for block_id, stored_entry in stored_blocks_obj.items(): - if block_id not in incoming_blocks and block_store.is_block_reference(stored_entry): - pending_block_deletions.append(stored_entry["immutable_id"]) - elif any(block_store.is_block_reference(entry) for entry in stored_blocks_obj.values()): + for block_id, blocks_obj_value in stored_blocks_obj.items(): + if block_id not in incoming_blocks and block_store.is_block_reference(blocks_obj_value): + pending_block_deletions.append(blocks_obj_value["immutable_id"]) + elif any( + block_store.is_block_reference(blocks_obj_value) + for blocks_obj_value in stored_blocks_obj.values() + ): # The client did not send blocks_obj, so the stored blocks are untouched, - # but referenced entries must be resolved into full payloads for the item + # but referenced values must be resolved into full payloads for the item # validation below and swapped back before the final write. - for block_id, stored_entry in stored_blocks_obj.items(): - if block_store.is_block_reference(stored_entry): - block_reference_map[block_id] = stored_entry + for block_id, blocks_obj_value in stored_blocks_obj.items(): + if block_store.is_block_reference(blocks_obj_value): + block_reference_map[block_id] = blocks_obj_value item["blocks_obj"] = block_store.load_blocks_obj(item) if "collections" in updated_data: diff --git a/pydatalab/src/pydatalab/versioning.py b/pydatalab/src/pydatalab/versioning.py index 26394b882..937e19fdb 100644 --- a/pydatalab/src/pydatalab/versioning.py +++ b/pydatalab/src/pydatalab/versioning.py @@ -174,15 +174,15 @@ def save_version_snapshot( # Cut `block_versions` entries for any *referenced* blocks whose payload has # changed since their last committed version, and pin each referenced - # `blocks_obj` entry to its now-current version number in this snapshot. + # `blocks_obj` value to its now-current version number in this snapshot. # Embedded blocks are captured verbatim in the item snapshot. # This must happen before the de-duplication guard below so that a # block content change is seen as an item change. - for block_id, entry in list((item.get("blocks_obj") or {}).items()): - if not is_block_reference(entry): + for block_id, blocks_obj_value in list((item.get("blocks_obj") or {}).items()): + if not is_block_reference(blocks_obj_value): continue block_version = snapshot_block_version( - entry["immutable_id"], action=action, user_id=user_id + blocks_obj_value["immutable_id"], action=action, user_id=user_id ) if block_version is None: # There is no live block document and no committed history for @@ -193,7 +193,7 @@ def save_version_snapshot( LOGGER.warning( "Dropping dangling block reference %r (%s) from version snapshot of %s", block_id, - entry["immutable_id"], + blocks_obj_value["immutable_id"], refcode, ) del item["blocks_obj"][block_id] @@ -201,7 +201,7 @@ def save_version_snapshot( item["display_order"] = [b for b in item["display_order"] if b != block_id] continue item["blocks_obj"][block_id] = { - "immutable_id": entry["immutable_id"], + "immutable_id": blocks_obj_value["immutable_id"], "version": block_version, }