From 45e05f293a9d860a8f2b0fabcf68b7f750fe6ac4 Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Sun, 7 Jun 2026 15:40:18 +0100 Subject: [PATCH 01/31] Update pydantic version across various files --- .github/dependabot.yml | 2 - .pre-commit-config.yaml | 2 +- pydatalab/pyproject.toml | 4 +- pydatalab/uv.lock | 123 +++++++++++++++++++++++++++++++-------- 4 files changed, 104 insertions(+), 27 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1bf622ebd..785ee179c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -25,8 +25,6 @@ updates: - minor - patch ignore: - - dependency-name: "pydantic" - versions: [ ">=2" ] - dependency-name: "bokeh" versions: [ ">=3" ] - dependency-name: "langchain" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 184a1b695..74b398eb9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,7 +67,7 @@ repos: hooks: - id: mypy additional_dependencies: - ["types-python-dateutil", "types-requests", "types-paramiko", "pydantic~=1.10"] + ["types-python-dateutil", "types-requests", "types-paramiko", "pydantic", "pydantic-settings"] args: ["--config-file", "pydatalab/pyproject.toml"] - repo: local diff --git a/pydatalab/pyproject.toml b/pydatalab/pyproject.toml index f98455654..81a20ec19 100644 --- a/pydatalab/pyproject.toml +++ b/pydatalab/pyproject.toml @@ -21,7 +21,9 @@ dependencies = [ "bokeh ~= 2.4, < 3.0", "matplotlib ~= 3.8", "periodictable ~= 1.7", - "pydantic[email, dotenv] < 2.0", + "pydantic[email] >= 2.11, < 3.0", + "pydantic-settings ~= 2.10", + "python-dotenv ~= 1.1", "pint ~= 0.24", "pandas[excel] ~= 2.2", "pymongo ~= 4.7", diff --git a/pydatalab/uv.lock b/pydatalab/uv.lock index 8425e07c5..fce0c9203 100644 --- a/pydatalab/uv.lock +++ b/pydatalab/uv.lock @@ -99,6 +99,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, ] +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + [[package]] name = "anthropic" version = "0.60.0" @@ -621,8 +630,10 @@ dependencies = [ { name = "pandas", extra = ["excel"] }, { name = "periodictable" }, { name = "pint" }, - { name = "pydantic", extra = ["dotenv", "email"] }, + { name = "pydantic", extra = ["email"] }, + { name = "pydantic-settings" }, { name = "pymongo" }, + { name = "python-dotenv" }, { name = "tomlkit" }, ] @@ -746,10 +757,12 @@ requires-dist = [ { name = "psutil", marker = "extra == 'apps'", specifier = ">=7.0.0" }, { name = "pyarrow", marker = "extra == 'apps'", specifier = "~=23.0.1" }, { name = "pybaselines", marker = "extra == 'apps'", specifier = "~=1.1" }, - { name = "pydantic", extras = ["dotenv", "email"], specifier = "<2.0" }, + { name = "pydantic", extras = ["email"], specifier = ">=2.11,<3.0" }, + { name = "pydantic-settings", specifier = "~=2.10" }, { name = "pyjwt", marker = "extra == 'server'", specifier = "~=2.9" }, { name = "pymongo", specifier = "~=4.7" }, { name = "python-dateutil", marker = "extra == 'apps'", specifier = "~=2.9" }, + { name = "python-dotenv", specifier = "~=1.1" }, { name = "python-dotenv", marker = "extra == 'server'", specifier = "~=1.0" }, { name = "renishawwire", marker = "extra == 'apps'", specifier = ">=0.1.16" }, { name = "scipy", marker = "extra == 'apps'", specifier = "~=1.13" }, @@ -2326,38 +2339,90 @@ wheels = [ [[package]] name = "pydantic" -version = "1.10.22" +version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, { name = "typing-extensions" }, + { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9a/57/5996c63f0deec09e9e901a2b838247c97c6844999562eac4e435bcb83938/pydantic-1.10.22.tar.gz", hash = "sha256:ee1006cebd43a8e7158fb7190bb8f4e2da9649719bff65d0c287282ec38dec6d", size = 356771, upload-time = "2025-04-24T13:38:43.605Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/92/91eb5c75a1460292e1f2f3e577122574ebb942fbac19ad2369ff00b9eb24/pydantic-1.10.22-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:57889565ccc1e5b7b73343329bbe6198ebc472e3ee874af2fa1865cfe7048228", size = 2852481, upload-time = "2025-04-24T13:36:55.045Z" }, - { url = "https://files.pythonhosted.org/packages/08/f3/dd54b49fc5caaed06f5a0d0a5ec35a81cf722cd6b42455f408dad1ef3f7d/pydantic-1.10.22-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90729e22426de79bc6a3526b4c45ec4400caf0d4f10d7181ba7f12c01bb3897d", size = 2585586, upload-time = "2025-04-24T13:36:58.453Z" }, - { url = "https://files.pythonhosted.org/packages/ec/9b/48d10180cc614ffb66da486e99bc1f8b639fb44edf322864f2fb161e2351/pydantic-1.10.22-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8684d347f351554ec94fdcb507983d3116dc4577fb8799fed63c65869a2d10", size = 3336974, upload-time = "2025-04-24T13:37:00.652Z" }, - { url = "https://files.pythonhosted.org/packages/ff/80/b55ad0029ae8e7b8b5c81ad7c4e800774a52107d26f70c6696857dc733d5/pydantic-1.10.22-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8dad498ceff2d9ef1d2e2bc6608f5b59b8e1ba2031759b22dfb8c16608e1802", size = 3362338, upload-time = "2025-04-24T13:37:02.42Z" }, - { url = "https://files.pythonhosted.org/packages/65/e0/8a5cd2cd29a5632581ba466f5792194b2a568aa052ce9da9ba98b634debf/pydantic-1.10.22-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fac529cc654d4575cf8de191cce354b12ba705f528a0a5c654de6d01f76cd818", size = 3519505, upload-time = "2025-04-24T13:37:04.322Z" }, - { url = "https://files.pythonhosted.org/packages/38/c5/c776d03ec374f22860802b2cee057b41e866be3c80826b53d4c001692db3/pydantic-1.10.22-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4148232aded8dd1dd13cf910a01b32a763c34bd79a0ab4d1ee66164fcb0b7b9d", size = 3485878, upload-time = "2025-04-24T13:37:06.102Z" }, - { url = "https://files.pythonhosted.org/packages/d1/a2/1efd064513a2c1bcb5c2b0e022cdf77d132ef7f7f20d91bb439d759f6a88/pydantic-1.10.22-cp310-cp310-win_amd64.whl", hash = "sha256:ece68105d9e436db45d8650dc375c760cc85a6793ae019c08769052902dca7db", size = 2299673, upload-time = "2025-04-24T13:37:07.969Z" }, - { url = "https://files.pythonhosted.org/packages/42/03/e435ed85a9abda29e3fbdb49c572fe4131a68c6daf3855a01eebda9e1b27/pydantic-1.10.22-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8e530a8da353f791ad89e701c35787418605d35085f4bdda51b416946070e938", size = 2845682, upload-time = "2025-04-24T13:37:10.142Z" }, - { url = "https://files.pythonhosted.org/packages/72/ea/4a625035672f6c06d3f1c7e33aa0af6bf1929991e27017e98b9c2064ae0b/pydantic-1.10.22-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:654322b85642e9439d7de4c83cb4084ddd513df7ff8706005dada43b34544946", size = 2553286, upload-time = "2025-04-24T13:37:11.946Z" }, - { url = "https://files.pythonhosted.org/packages/a4/f0/424ad837746e69e9f061ba9be68c2a97aef7376d1911692904d8efbcd322/pydantic-1.10.22-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8bece75bd1b9fc1c32b57a32831517943b1159ba18b4ba32c0d431d76a120ae", size = 3141232, upload-time = "2025-04-24T13:37:14.394Z" }, - { url = "https://files.pythonhosted.org/packages/14/67/4979c19e8cfd092085a292485e0b42d74e4eeefbb8cd726aa8ba38d06294/pydantic-1.10.22-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eccb58767f13c6963dcf96d02cb8723ebb98b16692030803ac075d2439c07b0f", size = 3214272, upload-time = "2025-04-24T13:37:16.201Z" }, - { url = "https://files.pythonhosted.org/packages/1a/04/32339ce43e97519d19e7759902515c750edbf4832a13063a4ab157f83f42/pydantic-1.10.22-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7778e6200ff8ed5f7052c1516617423d22517ad36cc7a3aedd51428168e3e5e8", size = 3321646, upload-time = "2025-04-24T13:37:19.086Z" }, - { url = "https://files.pythonhosted.org/packages/92/35/dffc1b29cb7198aadab68d75447191e59bdbc1f1d2d51826c9a4460d372f/pydantic-1.10.22-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bffe02767d27c39af9ca7dc7cd479c00dda6346bb62ffc89e306f665108317a2", size = 3244258, upload-time = "2025-04-24T13:37:20.929Z" }, - { url = "https://files.pythonhosted.org/packages/11/c5/c4ce6ebe7f528a879441eabd2c6dd9e2e4c54f320a8c9344ba93b3aa8701/pydantic-1.10.22-cp311-cp311-win_amd64.whl", hash = "sha256:23bc19c55427091b8e589bc08f635ab90005f2dc99518f1233386f46462c550a", size = 2309702, upload-time = "2025-04-24T13:37:23.296Z" }, - { url = "https://files.pythonhosted.org/packages/e9/e0/1ed151a56869be1588ad2d8cda9f8c1d95b16f74f09a7cea879ca9b63a8b/pydantic-1.10.22-py3-none-any.whl", hash = "sha256:343037d608bcbd34df937ac259708bfc83664dadf88afe8516c4f282d7d471a9", size = 166503, upload-time = "2025-04-24T13:38:41.374Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, ] [package.optional-dependencies] -dotenv = [ - { name = "python-dotenv" }, -] email = [ { name = "email-validator" }, ] +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/08/f1ba952f1c8ae5581c70fa9c6da89f247b83e3dd8c09c035d5d7931fc23d/pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4", size = 2113146, upload-time = "2026-05-06T13:37:36.537Z" }, + { url = "https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5", size = 1949769, upload-time = "2026-05-06T13:37:46.365Z" }, + { url = "https://files.pythonhosted.org/packages/64/ba/bfb1d928fd5b49e1258935ff104ae356e9fd89384a55bf9f847e9193ad40/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba", size = 1974958, upload-time = "2026-05-06T13:37:28.611Z" }, + { url = "https://files.pythonhosted.org/packages/4e/74/76223bfb117b64af743c9b6670d1364516f5c0604f96b48f3272f6af6cc6/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b", size = 2042118, upload-time = "2026-05-06T13:36:55.216Z" }, + { url = "https://files.pythonhosted.org/packages/cb/7b/848732968bc8f48f3187542f08358b9d842db564147b256669426ebb1652/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c", size = 2222876, upload-time = "2026-05-06T13:38:25.455Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2f/e90b63ee2e14bd8d3db8f705a6d75d64e6ee1b7c2c8833747ce706e1e0ce/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50", size = 2286703, upload-time = "2026-05-06T13:37:53.304Z" }, + { url = "https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd", size = 2092042, upload-time = "2026-05-06T13:38:46.981Z" }, + { url = "https://files.pythonhosted.org/packages/a9/da/0a422b57bf8504102bf3c4ccea9c41bab5a5cee6a54650acf8faf67f5a24/pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01", size = 2117231, upload-time = "2026-05-06T13:39:23.146Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2a/2ac13c3af305843e23c5078c53d135656b3f05a2fd78cb7bbbb12e97b473/pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d", size = 2168388, upload-time = "2026-05-06T13:40:08.06Z" }, + { url = "https://files.pythonhosted.org/packages/72/04/2beacf7e1607e93eefe4aed1b4709f079b905fb77530179d4f7c71745f22/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4", size = 2184769, upload-time = "2026-05-06T13:38:13.901Z" }, + { url = "https://files.pythonhosted.org/packages/9e/29/d2b9fd9f539133548eaf622c06a4ce176cb46ac59f32d0359c4abc0de047/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f", size = 2319312, upload-time = "2026-05-06T13:39:08.24Z" }, + { url = "https://files.pythonhosted.org/packages/7c/af/0f7a5b85fec6075bea96e3ef9187de38fccced0de92c1e7feda8d5cc7bb9/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39", size = 2361817, upload-time = "2026-05-06T13:38:43.2Z" }, + { url = "https://files.pythonhosted.org/packages/25/a4/73363fec545fd3ec025490bdda2743c56d0dd5b6266b1a53bbe9e4265375/pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d", size = 1987085, upload-time = "2026-05-06T13:39:25.497Z" }, + { url = "https://files.pythonhosted.org/packages/01/aa/62f082da2c91fac1c234bc9ee0066257ce83f0604abd72e4c9d5991f2d84/pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf", size = 2074311, upload-time = "2026-05-06T13:39:59.922Z" }, + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", size = 235551, upload-time = "2026-05-08T13:40:06.542Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -3062,6 +3127,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, ] +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + [[package]] name = "tzdata" version = "2025.2" From faa34641ab1d96144156dc1f72752681bbd2d563 Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Tue, 11 Nov 2025 01:06:15 +0000 Subject: [PATCH 02/31] Refactor server config for pydantic v2 --- pydatalab/src/pydatalab/config.py | 116 ++++++++++++++++++------------ 1 file changed, 71 insertions(+), 45 deletions(-) diff --git a/pydatalab/src/pydatalab/config.py b/pydatalab/src/pydatalab/config.py index 8b72dac4d..793d371d6 100644 --- a/pydatalab/src/pydatalab/config.py +++ b/pydatalab/src/pydatalab/config.py @@ -9,12 +9,13 @@ from pydantic import ( AnyUrl, BaseModel, - BaseSettings, + ConfigDict, Field, ValidationError, - root_validator, - validator, + field_validator, + model_validator, ) +from pydantic_settings import BaseSettings, SettingsConfigDict from pydatalab.models import Person from pydatalab.models.utils import RandomAlphabeticalRefcodeFactory, RefCodeFactory @@ -24,7 +25,7 @@ config_logger = logging.getLogger("pydatalab.config") -def config_file_settings(settings: BaseSettings) -> dict[str, Any]: +def config_file_settings(settings_cls: type[BaseSettings] | None = None) -> dict[str, Any]: """Returns a dictionary of server settings loaded from the default or specified JSON config file location (via the env var `PYDATALAB_CONFIG_FILE`). @@ -34,7 +35,7 @@ def config_file_settings(settings: BaseSettings) -> dict[str, Any]: res = {} if config_file.is_file(): config_logger.debug("Loading from config file at %s", config_file) - config_file_content = config_file.read_text(encoding=settings.__config__.env_file_encoding) + config_file_content = config_file.read_text(encoding="utf-8") try: res = json.loads(config_file_content) @@ -51,20 +52,20 @@ def config_file_settings(settings: BaseSettings) -> dict[str, Any]: class DeploymentMetadata(BaseModel): """A model for specifying metadata about a datalab deployment.""" - maintainer: Person | None + maintainer: Person | None = None issue_tracker: AnyUrl | None = Field("https://github.com/datalab-org/datalab/issues") - homepage: AnyUrl | None + homepage: AnyUrl | None = None source_repository: AnyUrl | None = Field("https://github.com/datalab-org/datalab") - @validator("maintainer") + @field_validator("maintainer") + @classmethod def strip_fields_from_person(cls, v): if not v.contact_email: raise ValueError("Must provide contact email for maintainer.") return Person(contact_email=v.contact_email, display_name=v.display_name) - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class BackupStrategy(BaseModel): @@ -75,7 +76,8 @@ class BackupStrategy(BaseModel): description="Whether this backup strategy is active; i.e., whether it is actually used. All strategies will be disabled in testing scenarios.", ) hostname: str | None = Field( - description="The hostname of the SSH-accessible server on which to store the backup (`None` indicates local backups)." + None, + description="The hostname of the SSH-accessible server on which to store the backup (`None` indicates local backups).", ) location: Path = Field( description="The location under which to store the backups on the host. Each backup will be date-stamped and stored in a subdirectory of this location." @@ -87,7 +89,7 @@ class BackupStrategy(BaseModel): frequency: str | None = Field( None, description="The frequency of the backup, described in the crontab syntax.", - pattern=r"^(?:\*|\d+(?:-\d+)?)(?:\/\d+)?(?:,\d+(?:-\d+)?(?:\/\d+)?)*$", + examples=["5 4 * * *", "5 2 1 1,4,7,10 *"], ) notification_email_address: str | None = Field( None, description="An email address to send backup notifications to." @@ -143,7 +145,7 @@ class ServerConfig(BaseSettings): False, description="Whether to run the server in testing mode, i.e., without user auth." ) - SECRET_KEY: str = Field( + SECRET_KEY: str | None = Field( None, description="The secret key to use for Flask. This value should be changed and/or loaded from an environment variable for production deployments.", ) @@ -180,7 +182,7 @@ class ServerConfig(BaseSettings): REMOTE_FILESYSTEMS: list[RemoteFilesystem] = Field( [], - descripton="A list of dictionaries describing remote filesystems to be accessible from the server.", + description="A list of dictionaries describing remote filesystems to be accessible from the server.", ) REMOTE_CACHE_MAX_AGE: int = Field( @@ -280,18 +282,24 @@ class ServerConfig(BaseSettings): description="The desired backup configuration.", ) - @root_validator + @model_validator(mode="before") + @classmethod def validate_cache_ages(cls, values): - if values.get("REMOTE_CACHE_MIN_AGE") > values.get("REMOTE_CACHE_MAX_AGE"): + min_age = values.get("REMOTE_CACHE_MIN_AGE") + max_age = values.get("REMOTE_CACHE_MAX_AGE") + + if min_age is not None and max_age is not None and min_age > max_age: raise RuntimeError( - f"The maximum cache age must be greater than the minimum cache age: min {values.get('REMOTE_CACHE_MIN_AGE')=}, max {values.get('REMOTE_CACHE_MAX_AGE')=}" + f"The maximum cache age must be greater than the minimum cache age: min {min_age=}, max {max_age=}" ) return values - @validator("SECRET_KEY", pre=True, always=True) - def validate_secret_key(cls, v, values): + @field_validator("SECRET_KEY", mode="before") + @classmethod + def validate_secret_key(cls, v, info): if v is None: - if values.get("TESTING"): + data = info.data if hasattr(info, "data") else {} + if data.get("TESTING"): config_logger.error( "`CONFIG.TESTING` is enabled - generating a deterministic secret key for testing purposes. This MUST be updated for production deployments." ) @@ -301,7 +309,8 @@ def validate_secret_key(cls, v, values): return v - @validator("ROOT_PATH") + @field_validator("ROOT_PATH", mode="before") + @classmethod def validate_root_path(cls, v): if not v.startswith("/"): v = "/" + v @@ -311,20 +320,22 @@ def validate_root_path(cls, v): return v - @validator("IDENTIFIER_PREFIX", pre=True, always=True) - def validate_identifier_prefix(cls, v, values): + @field_validator("IDENTIFIER_PREFIX", mode="before") + @classmethod + def validate_identifier_prefix(cls, v, info): """Make sure that the identifier prefix is set and is valid, raising clear error messages if not. If in testing mode, then set the prefix to 'test' too. The app startup will test for this value and should also warn aggressively that this is unset. """ - if values.get("TESTING") or v is None: + data = info.data if hasattr(info, "data") else {} + if data.get("TESTING") or v is None: return "test" if len(v) > 12: raise RuntimeError( - "Identifier prefix must be less than 12 characters long, received {v=}" + f"Identifier prefix must be less than 12 characters long, received {v=}" ) # test a trial refcode @@ -336,18 +347,18 @@ def validate_identifier_prefix(cls, v, values): raise RuntimeError( f"Invalid identifier prefix: {v}. Validation with refcode `AAAAAA` returned error: {exc}" ) - return v - @root_validator + @model_validator(mode="before") + @classmethod def deactivate_backup_strategies_during_testing(cls, values): if values.get("TESTING"): for name in values.get("BACKUP_STRATEGIES", {}): values["BACKUP_STRATEGIES"][name].active = False - return values - @validator("LOG_FILE") + @field_validator("LOG_FILE", mode="before") + @classmethod def make_missing_log_directory(cls, v): """Make sure that the log directory exists and is writable.""" if v is None: @@ -360,25 +371,40 @@ def make_missing_log_directory(cls, v): raise RuntimeError(f"Unable to create log file at {v}") from exc return v - class Config: - env_prefix = "pydatalab_" - extra = "allow" - env_file = ".env" - env_file_encoding = "utf-8" - validate_assignment = True - - @classmethod - def customise_sources( - cls, + def update(self, values: dict): + """Update the configuration with new values, following Pydantic v1 behavior.""" + for key, value in values.items(): + key_upper = key.upper() + if hasattr(self, key_upper): + setattr(self, key_upper, value) + else: + setattr(self, key_upper, value) + + model_config = SettingsConfigDict( + env_prefix="PYDATALAB_", + extra="allow", + env_file=".env", + env_file_encoding="utf-8", + validate_assignment=True, + case_sensitive=False, + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings, + env_settings, + dotenv_settings, + file_secret_settings, + ): + return ( init_settings, env_settings, + dotenv_settings, + config_file_settings, file_secret_settings, - ): - return (init_settings, env_settings, config_file_settings, file_secret_settings) - - def update(self, mapping): - for key in mapping: - setattr(self, key.upper(), mapping[key]) + ) CONFIG: ServerConfig = ServerConfig() From f1bcce9f80d6b5a4a1125b1e7304bb8e00dd0e14 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 15:51:35 +0100 Subject: [PATCH 03/31] Update traits module to pydantic v2 --- pydatalab/src/pydatalab/models/traits.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pydatalab/src/pydatalab/models/traits.py b/pydatalab/src/pydatalab/models/traits.py index 140f5182c..435b39d34 100644 --- a/pydatalab/src/pydatalab/models/traits.py +++ b/pydatalab/src/pydatalab/models/traits.py @@ -1,11 +1,20 @@ from typing import Any -from pydantic import BaseModel, Field, root_validator, validator +from pydantic import BaseModel, Field, field_validator, model_validator from pydatalab.models.blocks import DataBlockResponse from pydatalab.models.people import Group, Person from pydatalab.models.utils import Constituent, InlineSubstance, PyObjectId +__all__ = ( + "HasOwner", + "HasRevisionControl", + "HasBlocks", + "IsCollectable", + "HasSynthesisInfo", + "HasSubstanceInfo", +) + class HasOwner(BaseModel): creator_ids: list[PyObjectId] = Field([]) @@ -50,7 +59,8 @@ class IsCollectable(BaseModel): collections: list[Collection] = Field([]) """Inlined info for the collections associated with this item.""" - @root_validator + @model_validator(mode="before") + @classmethod def add_missing_collection_relationships(cls, values): from pydatalab.models.relationships import TypedRelationship @@ -99,7 +109,8 @@ class HasSynthesisInfo(BaseModel): synthesis_description: str | None = None """Free-text details of the procedure applied to synthesise the sample""" - @root_validator + @model_validator(mode="before") + @classmethod def add_missing_synthesis_relationships(cls, values): """Add any missing sample synthesis constituents to parent relationships""" from pydatalab.models.relationships import RelationshipType, TypedRelationship @@ -205,7 +216,8 @@ class HasSubstanceInfo(BaseModel): CAS: str | None = Field(alias="Substance CAS") """The CAS Registry Number for the substance described by this entry.""" - @validator("molar_mass") + @field_validator("molar_mass", mode="before") + @classmethod def add_molar_mass(cls, v, values): from periodictable import formula From 126f42b6f74056b73823d8a5db231aa4f9232ea1 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 11 Nov 2025 01:06:39 +0000 Subject: [PATCH 04/31] Refactor model imports to avoid circular deps --- pydatalab/src/pydatalab/models/__init__.py | 29 ++++++++++++++++------ 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/pydatalab/src/pydatalab/models/__init__.py b/pydatalab/src/pydatalab/models/__init__.py index 92db5f475..ea76f4514 100644 --- a/pydatalab/src/pydatalab/models/__init__.py +++ b/pydatalab/src/pydatalab/models/__init__.py @@ -1,20 +1,34 @@ -from pydantic import BaseModel +import functools from pydatalab.models.cells import Cell from pydatalab.models.collections import Collection from pydatalab.models.equipment import Equipment from pydatalab.models.files import File +from pydatalab.models.items import Item from pydatalab.models.people import Person from pydatalab.models.samples import Sample from pydatalab.models.starting_materials import StartingMaterial from pydatalab.models.versions import ItemVersion -ITEM_MODELS: dict[str, type[BaseModel]] = { - "samples": Sample, - "starting_materials": StartingMaterial, - "cells": Cell, - "equipment": Equipment, -} + +@functools.lru_cache(maxsize=1) +def get_item_models() -> dict[str, type[Item]]: + """ + Returns a dictionary of item models keyed by their type. + """ + return { + model.model_json_schema()["properties"]["type"]["default"]: model + for model in Item.__subclasses__() + } + + +@functools.lru_cache(maxsize=1) +def generate_schemas() -> dict[str, dict]: + return {t: model.model_json_schema(by_alias=False) for t, model in get_item_models().items()} + + +ITEM_MODELS: dict[str, type[Item]] = get_item_models() +ITEM_SCHEMAS = generate_schemas() __all__ = ( "File", @@ -26,4 +40,5 @@ "Equipment", "ItemVersion", "ITEM_MODELS", + "ITEM_SCHEMAS", ) From 239f84dab38c37190ae52e0f7a97df1dbdc5b824 Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Tue, 11 Nov 2025 01:07:23 +0000 Subject: [PATCH 05/31] Rework all models and tasks into pydantic v2 amenable form Move back to descriptions in field docstrings and configure this with customised BaseModel More model updates for pydantic v2 --- pydatalab/schemas/cell.json | 2147 ++++++++++------ pydatalab/schemas/equipment.json | 1771 ++++++++----- pydatalab/schemas/plugin_config.json | 172 +- pydatalab/schemas/sample.json | 2104 ++++++++++------ pydatalab/schemas/startingmaterial.json | 2241 +++++++++++------ pydatalab/src/pydatalab/models/blocks.py | 11 +- pydatalab/src/pydatalab/models/cells.py | 79 +- pydatalab/src/pydatalab/models/collections.py | 19 +- pydatalab/src/pydatalab/models/entries.py | 22 +- pydatalab/src/pydatalab/models/equipment.py | 10 +- pydatalab/src/pydatalab/models/export_task.py | 50 +- pydatalab/src/pydatalab/models/files.py | 32 +- pydatalab/src/pydatalab/models/items.py | 23 +- pydatalab/src/pydatalab/models/people.py | 87 +- .../src/pydatalab/models/relationships.py | 43 +- pydatalab/src/pydatalab/models/samples.py | 4 +- .../pydatalab/models/starting_materials.py | 25 +- pydatalab/src/pydatalab/models/tasks.py | 89 +- pydatalab/src/pydatalab/models/traits.py | 117 +- pydatalab/src/pydatalab/models/utils.py | 262 +- pydatalab/src/pydatalab/models/versions.py | 80 +- pydatalab/tasks.py | 8 +- pydatalab/tests/test_models.py | 21 + 23 files changed, 5979 insertions(+), 3438 deletions(-) diff --git a/pydatalab/schemas/cell.json b/pydatalab/schemas/cell.json index 94dbbb862..771d40fb0 100644 --- a/pydatalab/schemas/cell.json +++ b/pydatalab/schemas/cell.json @@ -1,674 +1,706 @@ { - "title": "Cell", - "description": "A model for representing electrochemical cells.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "cells", - "const": "cells", - "pattern": "^cells$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Name", + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" + ], + "title": "AccountStatus", "type": "string" }, - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "default": "active", - "allOf": [ - { - "$ref": "#/definitions/CellStatus" + "CellComponent": { + "properties": { + "item": { + "anyOf": [ + { + "$ref": "#/$defs/EntryReference" + }, + { + "$ref": "#/$defs/InlineSubstance" + } + ], + "description": "A reference to item (sample or starting material) entry for the constituent substance.", + "title": "Item" + }, + "quantity": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the constituent material used to create the sample.", + "title": "Quantity" + }, + "unit": { + "default": "g", + "description": "The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).", + "title": "Unit", + "type": "string" } - ] - }, - "cell_format": { - "$ref": "#/definitions/CellFormat" - }, - "cell_format_description": { - "title": "Cell Format Description", - "type": "string" + }, + "required": [ + "item" + ], + "title": "CellComponent", + "type": "object" }, - "cell_preparation_description": { - "title": "Cell Preparation Description", + "CellFormat": { + "enum": [ + "coin", + "pouch", + "in situ (XRD)", + "in situ (NMR)", + "in situ (SQUID)", + "in situ (optical)", + "swagelok", + "cylindrical", + "other" + ], + "title": "CellFormat", "type": "string" }, - "characteristic_mass": { - "title": "Characteristic Mass", - "type": "number" - }, - "characteristic_chemical_formula": { - "title": "Characteristic Chemical Formula", + "CellStatus": { + "description": "An enumeration of the status of cells", + "enum": [ + "active", + "planned", + "disposed", + "cycled", + "shorted", + "dismantled", + "other" + ], + "title": "CellStatus", "type": "string" }, - "characteristic_molar_mass": { - "title": "Characteristic Molar Mass", - "type": "number" - }, - "positive_electrode": { - "title": "Positive Electrode", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/CellComponent" - } - }, - "negative_electrode": { - "title": "Negative Electrode", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/CellComponent" - } - }, - "electrolyte": { - "title": "Electrolyte", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/CellComponent" - } + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", + "properties": { + "collection_id": { + "description": "The immutable ID of the collection.", + "title": "Collection Id", + "type": "string" + } + }, + "required": [ + "collection_id" + ], + "title": "CollectionReference", + "type": "object" }, - "active_ion_charge": { - "title": "Active Ion Charge", - "default": 1, - "type": "number" - } - }, - "required": [ - "item_id" - ], - "definitions": { "DataBlockResponse": { - "title": "DataBlockResponse", + "additionalProperties": true, "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", "properties": { "blocktype": { + "description": "The type of the block.", "title": "Blocktype", "type": "string" }, "block_id": { + "description": "A shorthand random ID for the block.", "title": "Block Id", "type": "string" }, "item_id": { - "title": "Item Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" }, "collection_id": { - "title": "Collection Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" }, "title": { - "title": "Title", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" }, "freeform_comment": { - "title": "Freeform Comment", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" }, "file_id": { - "title": "File Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" }, "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" }, "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" }, "warnings": { - "title": "Warnings", - "type": "array", - "items": { - "type": "string" - } - }, - "b64_encoded_image": { - "title": "B64 Encoded Image", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" + }, + "b64_encoded_image": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" }, "bokeh_plot_data": { - "title": "Bokeh Plot Data", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object" + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" }, "computed": { - "title": "Computed", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" }, "metadata": { - "title": "Metadata", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" } }, "required": [ "blocktype", - "block_id" - ] - }, - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" - ], - "type": "string" - }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", - "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" + "block_id", + "b64_encoded_image", + "bokeh_plot_data" ], - "type": "string" + "title": "DataBlockResponse", + "type": "object" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "EntryReference": { + "additionalProperties": true, + "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", "properties": { - "description": { - "title": "Description", + "type": { + "title": "Type", "type": "string" }, - "relation": { - "$ref": "#/definitions/RelationshipType" - }, - "type": { - "$ref": "#/definitions/KnownType" + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" }, "immutable_id": { - "title": "Immutable Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Immutable Id" }, "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Item Id" }, "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Refcode" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" } }, "required": [ "type" - ] - }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" ], - "type": "string" + "title": "EntryReference", + "type": "object" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" }, - "identifier": { - "title": "Identifier", - "type": "string" + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" }, - "name": { - "title": "Name", - "type": "string" + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" }, - "display_name": { - "title": "Display Name", - "type": "string" - } - }, - "required": [ - "identity_type", - "identifier", - "name" - ] - }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" - ], - "type": "string" - }, - "Group": { - "title": "Group", - "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", - "properties": { "type": { + "const": "files", + "default": "files", "title": "Type", - "default": "groups", - "const": "groups", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" }, - "members": { - "title": "Members", - "type": "array", + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" + }, + "item_ids": { + "description": "A list of item IDs associated with this file.", "items": { - "type": "object" - } + "type": "string" + }, + "title": "Item Ids", + "type": "array" }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, + "blocks": { + "description": "A list of block IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Blocks", + "type": "array" + }, + "name": { + "description": "The filename on disk.", + "title": "Name", "type": "string" }, - "description": { - "title": "Description", + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", "type": "string" }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object" - } - ] - } - } - } - }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", - "enum": [ - "active", - "unverified", - "deactivated" - ], - "type": "string" - }, - "Person": { - "title": "Person", - "description": "A model that describes an individual and their digital identities.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "default": "people", - "const": "people", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "identities": { - "title": "Identities", - "type": "array", - "items": { - "$ref": "#/definitions/Identity" - } - }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" - }, - "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" - }, - "gravatar_hash": { - "title": "Gravatar Hash", - "type": "string" - }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } - }, - "role": { - "default": "user", - "allOf": [ + "original_name": { + "anyOf": [ + { + "type": "string" + }, { - "$ref": "#/definitions/UserRole" + "type": "null" } - ] - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" }, - "account_status": { - "default": "unverified", - "allOf": [ + "location": { + "anyOf": [ + { + "type": "string" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] - } - } - }, - "Collection": { - "title": "Collection", - "description": "An Entry is an abstract base class for any model that can be\ndeserialized and stored in the database.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", - "type": "string" + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + "time_added": { + "description": "The timestamp for the original file upload.", + "format": "datetime", + "title": "Time Added", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" }, - "title": { - "title": "Title", - "type": "string" + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" }, - "description": { - "title": "Description", - "type": "string" + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", + "type": "boolean" }, - "num_items": { - "title": "Num Items", - "type": "integer" + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "description": "Content checksums (MD5 and SHA-256) of the file." } - } + }, + "required": [ + "name", + "extension", + "time_added", + "is_live", + "checksums" + ], + "title": "File", + "type": "object" }, "FileChecksums": { - "title": "FileChecksums", "description": "Content checksums for a file.", - "type": "object", "properties": { "md5": { + "description": "The MD5 hex digest of the file contents.", "title": "Md5", "type": "string" }, "sha256": { + "description": "The SHA-256 hex digest of the file contents.", "title": "Sha256", "type": "string" } @@ -676,283 +708,840 @@ "required": [ "md5", "sha256" - ] + ], + "title": "FileChecksums", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", + "Group": { + "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, "type": { + "const": "groups", + "default": "groups", + "description": "The entry type as a string.", "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "size": { - "title": "Size", - "type": "integer" + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" + "group_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, - "item_ids": { - "title": "Item Ids", - "type": "array", + "members": { + "description": "A list of people that belong to this group; stored on the user objects.", "items": { - "type": "string" - } + "additionalProperties": true, + "type": "object" + }, + "title": "Members", + "type": "array" }, - "blocks": { - "title": "Blocks", - "type": "array", - "items": { - "type": "string" - } + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" }, - "name": { - "title": "Name", - "type": "string" + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" + }, + "managers": { + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", + "items": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + } + ] + }, + "title": "Managers", + "type": "array" + } + }, + "title": "Group", + "type": "object" + }, + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", + "properties": { + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." + }, + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", + "type": "string" + }, + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", + "type": "string" + }, + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" + }, + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" + } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "InlineSubstance": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "name" + ], + "title": "InlineSubstance", + "type": "object" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" + ], + "title": "KnownType", + "type": "string" + }, + "Person": { + "description": "A model that describes an individual and their digital identities.", + "properties": { + "type": { + "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "identities": { + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", + "items": { + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" + }, + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" + }, + "contact_email": { + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" + }, + "gravatar_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" + }, + "managers": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" + }, + "role": { + "$ref": "#/$defs/UserRole", + "default": "user", + "description": "The role assigned to this person." + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" + }, + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." + } + }, + "title": "Person", + "type": "object" + }, + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" + }, + "TypedRelationship": { + "additionalProperties": true, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" + }, + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." + }, + "type": { + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" + }, + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" + } + }, + "required": [ + "type" + ], + "title": "TypedRelationship", + "type": "object" + }, + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing electrochemical cells.", + "properties": { + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" }, - "extension": { - "title": "Extension", - "type": "string" + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" }, - "original_name": { - "title": "Original Name", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "cells", + "default": "cells", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", "type": "string" }, - "location": { - "title": "Location", + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "url_path": { - "title": "Url Path", + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", "type": "string" }, - "source": { - "title": "Source", + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { "type": "string" }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" - }, - "metadata": { - "title": "Metadata", - "type": "object" - }, - "representation": { - "title": "Representation" - }, - "source_server_name": { - "title": "Source Server Name", + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "source_path": { - "title": "Source Path", + { + "type": "null" + } + ], + "default": null, + "description": "A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).", + "title": "Date" + }, + "name": { + "anyOf": [ + { "type": "string" }, - "is_live": { - "title": "Is Live", - "type": "boolean" + { + "type": "null" + } + ], + "default": null, + "description": "An optional human-readable/usable name for the entry.", + "title": "Name" + }, + "files": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" }, - "checksums": { - "$ref": "#/definitions/FileChecksums" + { + "type": "null" } + ], + "default": null, + "description": "Any files attached to this sample.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" }, - "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + "title": "File Objectids", + "type": "array" }, - "CellStatus": { - "title": "CellStatus", - "description": "An enumeration of the status of cells", - "enum": [ - "active", - "planned", - "disposed", - "cycled", - "shorted", - "dismantled", - "other" - ], - "type": "string" + "status": { + "$ref": "#/$defs/CellStatus", + "default": "active", + "description": "The status of the cells, indicating its current state." }, - "CellFormat": { - "title": "CellFormat", - "description": "An enumeration.", - "enum": [ - "coin", - "pouch", - "in situ (XRD)", - "in situ (NMR)", - "in situ (SQUID)", - "in situ (optical)", - "swagelok", - "cylindrical", - "other" + "cell_format": { + "anyOf": [ + { + "$ref": "#/$defs/CellFormat" + }, + { + "type": "null" + } ], - "type": "string" + "default": null, + "description": "The form factor of the cell, e.g., coin, pouch, in situ or otherwise." }, - "EntryReference": { - "title": "EntryReference", - "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "immutable_id": { - "title": "Immutable Id", + "cell_format_description": { + "anyOf": [ + { "type": "string" }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + { + "type": "null" + } + ], + "default": null, + "description": "Additional human-readable description of the cell form factor, e.g., 18650, AMPIX, CAMPIX", + "title": "Cell Format Description" + }, + "cell_preparation_description": { + "anyOf": [ + { "type": "string" }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + { + "type": "null" + } + ], + "default": null, + "description": "Description of how the cell was prepared.", + "title": "Cell Preparation Description" + }, + "characteristic_mass": { + "anyOf": [ + { + "type": "number" }, - "chemform": { - "title": "Chemform", - "type": "string" + { + "type": "null" } - }, - "required": [ - "type" - ] + ], + "default": null, + "description": "The characteristic mass of the cell in milligrams. Can be used to normalize capacities.", + "title": "Characteristic Mass" }, - "InlineSubstance": { - "title": "InlineSubstance", - "type": "object", - "properties": { - "name": { - "title": "Name", + "characteristic_chemical_formula": { + "anyOf": [ + { "type": "string" }, - "chemform": { - "title": "Chemform", - "type": "string" + { + "type": "null" } - }, - "required": [ - "name" - ] + ], + "default": null, + "description": "The chemical formula of the active material. Can be used to calculated molar mass in g/mol for normalizing capacities.", + "title": "Characteristic Chemical Formula" }, - "CellComponent": { - "title": "CellComponent", - "description": "A constituent of a sample.", - "type": "object", - "properties": { - "item": { - "title": "Item", - "anyOf": [ - { - "$ref": "#/definitions/EntryReference" - }, - { - "$ref": "#/definitions/InlineSubstance" - } - ] - }, - "quantity": { - "title": "Quantity", - "minimum": 0, + "characteristic_molar_mass": { + "anyOf": [ + { "type": "number" }, - "unit": { - "title": "Unit", - "default": "g", - "type": "string" + { + "type": "null" } + ], + "default": null, + "description": "The molar mass of the active material, in g/mol. Will be inferred from the chemical formula, or can be supplied if it cannot be supplied", + "title": "Characteristic Molar Mass" + }, + "positive_electrode": { + "items": { + "$ref": "#/$defs/CellComponent" }, - "required": [ - "item", - "quantity" - ] + "title": "Positive Electrode", + "type": "array" + }, + "negative_electrode": { + "items": { + "$ref": "#/$defs/CellComponent" + }, + "title": "Negative Electrode", + "type": "array" + }, + "electrolyte": { + "items": { + "$ref": "#/$defs/CellComponent" + }, + "title": "Electrolyte", + "type": "array" + }, + "active_ion": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The active ion species.", + "title": "Active Ion" + }, + "active_ion_charge": { + "default": 1, + "title": "Active Ion Charge", + "type": "number" } - } + }, + "required": [ + "item_id" + ], + "title": "Cell", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/equipment.json b/pydatalab/schemas/equipment.json index d8c82722a..5482268c6 100644 --- a/pydatalab/schemas/equipment.json +++ b/pydatalab/schemas/equipment.json @@ -1,816 +1,1319 @@ { - "title": "Equipment", - "description": "A model for representing an experimental sample.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "equipment", - "const": "equipment", - "pattern": "^equipment$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Name", + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" + ], + "title": "AccountStatus", "type": "string" }, - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "default": "working", - "allOf": [ - { - "$ref": "#/definitions/EquipmentStatus" + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", + "properties": { + "collection_id": { + "description": "The immutable ID of the collection.", + "title": "Collection Id", + "type": "string" } - ] - }, - "serial_numbers": { - "title": "Serial Numbers", - "type": "string" - }, - "manufacturer": { - "title": "Manufacturer", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" + }, + "required": [ + "collection_id" + ], + "title": "CollectionReference", + "type": "object" }, - "contact": { - "title": "Contact", - "type": "string" - } - }, - "required": [ - "item_id" - ], - "definitions": { "DataBlockResponse": { - "title": "DataBlockResponse", + "additionalProperties": true, "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", "properties": { "blocktype": { + "description": "The type of the block.", "title": "Blocktype", "type": "string" }, "block_id": { + "description": "A shorthand random ID for the block.", "title": "Block Id", "type": "string" }, "item_id": { - "title": "Item Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" }, "collection_id": { - "title": "Collection Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" }, "title": { - "title": "Title", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" }, "freeform_comment": { - "title": "Freeform Comment", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" }, "file_id": { - "title": "File Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" }, "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" }, "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" }, "warnings": { - "title": "Warnings", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" }, "b64_encoded_image": { - "title": "B64 Encoded Image", + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" }, "bokeh_plot_data": { - "title": "Bokeh Plot Data", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object" + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" }, "computed": { - "title": "Computed", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" }, "metadata": { - "title": "Metadata", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" } }, "required": [ "blocktype", - "block_id" - ] - }, - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" + "block_id", + "b64_encoded_image", + "bokeh_plot_data" ], - "type": "string" + "title": "DataBlockResponse", + "type": "object" }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "EquipmentStatus": { + "description": "An enumeration of the status of equipments", "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" + "working", + "broken", + "being_fixed", + "defunct", + "not_being_fixed", + "other" ], + "title": "EquipmentStatus", "type": "string" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", "properties": { - "description": { - "title": "Description", - "type": "string" + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" }, - "relation": { - "$ref": "#/definitions/RelationshipType" + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" }, - "type": { - "$ref": "#/definitions/KnownType" + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" }, - "immutable_id": { - "title": "Immutable Id", - "type": "string" + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - } - }, - "required": [ - "type" - ] - }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" - ], - "type": "string" - }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", - "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" }, - "identifier": { - "title": "Identifier", + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "files", + "default": "files", + "title": "Type", "type": "string" }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" + }, + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" + }, + "item_ids": { + "description": "A list of item IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Item Ids", + "type": "array" + }, + "blocks": { + "description": "A list of block IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Blocks", + "type": "array" + }, "name": { + "description": "The filename on disk.", "title": "Name", "type": "string" }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", + "type": "string" }, - "display_name": { - "title": "Display Name", + "original_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" + }, + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" + }, + "time_added": { + "description": "The timestamp for the original file upload.", + "format": "datetime", + "title": "Time Added", "type": "string" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" + }, + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" + }, + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" + }, + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", + "type": "boolean" + }, + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ - "identity_type", - "identifier", - "name" - ] + "name", + "extension", + "time_added", + "is_live", + "checksums" + ], + "title": "File", + "type": "object" }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" + "FileChecksums": { + "description": "Content checksums for a file.", + "properties": { + "md5": { + "description": "The MD5 hex digest of the file contents.", + "title": "Md5", + "type": "string" + }, + "sha256": { + "description": "The SHA-256 hex digest of the file contents.", + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "md5", + "sha256" ], - "type": "string" + "title": "FileChecksums", + "type": "object" }, "Group": { - "title": "Group", "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", "properties": { "type": { - "title": "Type", - "default": "groups", "const": "groups", + "default": "groups", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, "members": { - "title": "Members", - "type": "array", + "description": "A list of people that belong to this group; stored on the user objects.", "items": { + "additionalProperties": true, "type": "object" - } + }, + "title": "Members", + "type": "array" }, "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" }, "description": { - "title": "Description", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" }, "managers": { - "title": "Managers", - "type": "array", + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", "items": { "anyOf": [ { + "format": "objectid", "type": "string" }, { + "additionalProperties": true, "type": "object" } ] - } + }, + "title": "Managers", + "type": "array" } - } - }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", - "enum": [ - "active", - "unverified", - "deactivated" - ], - "type": "string" + }, + "title": "Group", + "type": "object" }, - "Person": { - "title": "Person", - "description": "A model that describes an individual and their digital identities.", - "type": "object", + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", "properties": { - "type": { - "title": "Type", - "default": "people", - "const": "people", - "type": "string" + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "identities": { - "title": "Identities", - "type": "array", - "items": { - "$ref": "#/definitions/Identity" - } - }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", "type": "string" }, - "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" - }, - "gravatar_hash": { - "title": "Gravatar Hash", - "type": "string" + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } - }, - "role": { - "default": "user", - "allOf": [ + "display_name": { + "anyOf": [ { - "$ref": "#/definitions/UserRole" - } - ] - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "account_status": { - "default": "unverified", - "allOf": [ + "type": "string" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" } - } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" + ], + "title": "KnownType", + "type": "string" }, - "Collection": { - "title": "Collection", - "description": "An Entry is an abstract base class for any model that can be\ndeserialized and stored in the database.", - "type": "object", + "Person": { + "description": "A model that describes an individual and their digital identities.", "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, "type": { + "const": "people", + "default": "people", + "description": "The entry type as a string.", "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, "title": "Relationships", - "type": "array", + "type": "array" + }, + "identities": { + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" }, - "title": { - "title": "Title", - "type": "string" + "contact_email": { + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" }, - "description": { - "title": "Description", - "type": "string" + "gravatar_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" }, - "num_items": { - "title": "Num Items", - "type": "integer" - } - } - }, - "FileChecksums": { - "title": "FileChecksums", - "description": "Content checksums for a file.", - "type": "object", - "properties": { - "md5": { - "title": "Md5", - "type": "string" + "managers": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" }, - "sha256": { - "title": "Sha256", - "type": "string" + "role": { + "$ref": "#/$defs/UserRole", + "default": "user", + "description": "The role assigned to this person." + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" + }, + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." } }, - "required": [ - "md5", - "sha256" - ] + "title": "Person", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" + }, + "TypedRelationship": { + "additionalProperties": true, "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." }, "type": { - "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", - "type": "string" + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." }, "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" }, - "size": { - "title": "Size", - "type": "integer" + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" + } + }, + "required": [ + "type" + ], + "title": "TypedRelationship", + "type": "object" + }, + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing an experimental sample.", + "properties": { + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" }, - "item_ids": { - "title": "Item Ids", - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { "items": { - "type": "string" - } + "$ref": "#/$defs/Person" + }, + "type": "array" }, - "blocks": { - "title": "Blocks", - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { "items": { - "type": "string" - } + "$ref": "#/$defs/Group" + }, + "type": "array" }, - "name": { - "title": "Name", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "equipment", + "default": "equipment", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", "type": "string" }, - "extension": { - "title": "Extension", + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "original_name": { - "title": "Original Name", + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", "type": "string" }, - "location": { - "title": "Location", + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { "type": "string" }, - "url_path": { - "title": "Url Path", + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "source": { - "title": "Source", + { + "type": "null" + } + ], + "default": null, + "description": "A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).", + "title": "Date" + }, + "name": { + "anyOf": [ + { "type": "string" }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" - }, - "metadata": { - "title": "Metadata", - "type": "object" - }, - "representation": { - "title": "Representation" + { + "type": "null" + } + ], + "default": null, + "description": "An optional human-readable/usable name for the entry.", + "title": "Name" + }, + "files": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" }, - "source_server_name": { - "title": "Source Server Name", + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this sample.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "status": { + "$ref": "#/$defs/EquipmentStatus", + "default": "working", + "description": "The status of the equipment, indicating its current state." + }, + "serial_numbers": { + "anyOf": [ + { "type": "string" }, - "source_path": { - "title": "Source Path", + { + "type": "null" + } + ], + "default": null, + "description": "A string describing one or more serial numbers for the instrument.", + "title": "Serial Numbers" + }, + "manufacturer": { + "anyOf": [ + { "type": "string" }, - "is_live": { - "title": "Is Live", - "type": "boolean" + { + "type": "null" + } + ], + "default": null, + "description": "The manufacturer of this piece of equipment", + "title": "Manufacturer" + }, + "location": { + "anyOf": [ + { + "type": "string" }, - "checksums": { - "$ref": "#/definitions/FileChecksums" + { + "type": "null" } - }, - "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + ], + "default": null, + "description": "Place where the equipment is located", + "title": "Location" }, - "EquipmentStatus": { - "title": "EquipmentStatus", - "description": "An enumeration of the status of equipments", - "enum": [ - "working", - "broken", - "being_fixed", - "defunct", - "not_being_fixed", - "other" + "contact": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "type": "string" + "description": "Contact information for equipment (e.g., email address or phone number).", + "title": "Contact" } - } + }, + "required": [ + "item_id", + "contact" + ], + "title": "Equipment", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/plugin_config.json b/pydatalab/schemas/plugin_config.json index ae64bcb3c..57975833d 100644 --- a/pydatalab/schemas/plugin_config.json +++ b/pydatalab/schemas/plugin_config.json @@ -1,96 +1,130 @@ { - "title": "PluginConfigModel", - "description": "The schema for the top-level plugins.toml file.", - "type": "object", - "properties": { - "dependencies": { - "title": "Dependencies", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "tool": { - "title": "Tool", - "default": { + "$defs": { + "ToolSection": { + "properties": { "uv": { - "sources": {} + "$ref": "#/$defs/UvSection", + "default": { + "sources": {} + } } }, - "allOf": [ - { - "$ref": "#/definitions/ToolSection" + "title": "ToolSection", + "type": "object" + }, + "UvSection": { + "properties": { + "sources": { + "additionalProperties": { + "$ref": "#/$defs/UvSource" + }, + "default": {}, + "title": "Sources", + "type": "object" } - ] - } - }, - "additionalProperties": false, - "definitions": { + }, + "title": "UvSection", + "type": "object" + }, "UvSource": { - "title": "UvSource", "description": "A single entry under `[tool.uv.sources]` in plugins.toml.", - "type": "object", "properties": { "git": { - "title": "Git", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Git" }, "rev": { - "title": "Rev", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Rev" }, "branch": { - "title": "Branch", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Branch" }, "tag": { - "title": "Tag", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tag" }, "path": { - "title": "Path", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" }, "editable": { - "title": "Editable", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Editable" } }, - "additionalProperties": false - }, - "UvSection": { - "title": "UvSection", - "type": "object", - "properties": { - "sources": { - "title": "Sources", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/UvSource" - } - } + "title": "UvSource", + "type": "object" + } + }, + "description": "The schema for the top-level plugins.toml file.", + "properties": { + "dependencies": { + "default": [], + "items": { + "type": "string" }, - "additionalProperties": false + "title": "Dependencies", + "type": "array" }, - "ToolSection": { - "title": "ToolSection", - "type": "object", - "properties": { + "tool": { + "$ref": "#/$defs/ToolSection", + "default": { "uv": { - "title": "Uv", - "default": { - "sources": {} - }, - "allOf": [ - { - "$ref": "#/definitions/UvSection" - } - ] + "sources": {} } - }, - "additionalProperties": false + } } - } + }, + "title": "PluginConfigModel", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/sample.json b/pydatalab/schemas/sample.json index 92488c652..57b0fb8ce 100644 --- a/pydatalab/schemas/sample.json +++ b/pydatalab/schemas/sample.json @@ -1,766 +1,678 @@ { - "title": "Sample", - "description": "A model for representing an experimental sample.", - "type": "object", - "properties": { - "chemform": { - "title": "Chemform", - "example": [ - "Na3P", - "Na3P", - "LiNiO2@C", - "Na3+xP", - "LiNi1/3Co0.1Mn0.1O2" - ], - "type": "string" - }, - "smiles": { - "title": "Smiles", - "aliases": [ - "SMILES", - "smiles_representation" - ], - "type": "string" - }, - "inchi": { - "title": "Inchi", - "type": "string" - }, - "inchi_key": { - "title": "Inchi Key", - "type": "string" - }, - "GHS_codes": { - "title": "Ghs H-Codes", - "examples": [ - "H224", - "H303, H316, H319" + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" ], + "title": "AccountStatus", "type": "string" }, - "molar_mass": { - "title": "Molecular Weight", - "type": "number" - }, - "CAS": { - "title": "Substance Cas", - "type": "string" - }, - "synthesis_constituents": { - "title": "Synthesis Constituents", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Constituent" - } - }, - "synthesis_description": { - "title": "Synthesis Description", - "type": "string" - }, - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "samples", - "const": "samples", - "pattern": "^samples$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Name", - "type": "string" - }, - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "default": "active", - "allOf": [ - { - "$ref": "#/definitions/ItemStatus" - } - ] - } - }, - "required": [ - "item_id" - ], - "definitions": { - "EntryReference": { - "title": "EntryReference", - "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "immutable_id": { - "title": "Immutable Id", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "chemform": { - "title": "Chemform", - "type": "string" - } - }, - "required": [ - "type" - ] - }, - "InlineSubstance": { - "title": "InlineSubstance", - "type": "object", + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "name": { - "title": "Name", - "type": "string" - }, - "chemform": { - "title": "Chemform", + "collection_id": { + "description": "The immutable ID of the collection.", + "title": "Collection Id", "type": "string" } }, "required": [ - "name" - ] + "collection_id" + ], + "title": "CollectionReference", + "type": "object" }, "Constituent": { - "title": "Constituent", "description": "A constituent of a sample.", - "type": "object", "properties": { "item": { - "title": "Item", "anyOf": [ { - "$ref": "#/definitions/EntryReference" + "$ref": "#/$defs/EntryReference" }, { - "$ref": "#/definitions/InlineSubstance" + "$ref": "#/$defs/InlineSubstance" } - ] + ], + "description": "A reference to item (sample or starting material) entry for the constituent substance.", + "title": "Item" }, "quantity": { - "title": "Quantity", - "minimum": 0, - "type": "number" + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the constituent material used to create the sample.", + "title": "Quantity" }, "unit": { - "title": "Unit", "default": "g", + "description": "The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).", + "title": "Unit", "type": "string" } }, "required": [ - "item", - "quantity" - ] + "item" + ], + "title": "Constituent", + "type": "object" }, "DataBlockResponse": { - "title": "DataBlockResponse", + "additionalProperties": true, "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", "properties": { "blocktype": { + "description": "The type of the block.", "title": "Blocktype", "type": "string" }, "block_id": { + "description": "A shorthand random ID for the block.", "title": "Block Id", "type": "string" }, "item_id": { - "title": "Item Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" }, "collection_id": { - "title": "Collection Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" }, "title": { - "title": "Title", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" }, "freeform_comment": { - "title": "Freeform Comment", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" }, "file_id": { - "title": "File Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" }, "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" }, "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" }, "warnings": { - "title": "Warnings", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" }, "b64_encoded_image": { - "title": "B64 Encoded Image", + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" }, "bokeh_plot_data": { - "title": "Bokeh Plot Data", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object" + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" }, "computed": { - "title": "Computed", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" }, "metadata": { - "title": "Metadata", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" } }, "required": [ "blocktype", - "block_id" - ] - }, - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" - ], - "type": "string" - }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", - "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" + "block_id", + "b64_encoded_image", + "bokeh_plot_data" ], - "type": "string" + "title": "DataBlockResponse", + "type": "object" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "EntryReference": { + "additionalProperties": true, + "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", "properties": { - "description": { - "title": "Description", + "type": { + "title": "Type", "type": "string" }, - "relation": { - "$ref": "#/definitions/RelationshipType" - }, - "type": { - "$ref": "#/definitions/KnownType" + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" }, "immutable_id": { - "title": "Immutable Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Immutable Id" }, "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Item Id" }, "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Refcode" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" } }, "required": [ "type" - ] - }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" ], - "type": "string" + "title": "EntryReference", + "type": "object" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" }, - "identifier": { - "title": "Identifier", - "type": "string" + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" }, - "name": { - "title": "Name", - "type": "string" + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" }, - "display_name": { - "title": "Display Name", - "type": "string" - } - }, - "required": [ - "identity_type", - "identifier", - "name" - ] - }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" - ], - "type": "string" - }, - "Group": { - "title": "Group", - "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", - "properties": { "type": { + "const": "files", + "default": "files", "title": "Type", - "default": "groups", - "const": "groups", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" }, - "members": { - "title": "Members", - "type": "array", + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" + }, + "item_ids": { + "description": "A list of item IDs associated with this file.", "items": { - "type": "object" - } + "type": "string" + }, + "title": "Item Ids", + "type": "array" }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, + "blocks": { + "description": "A list of block IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Blocks", + "type": "array" + }, + "name": { + "description": "The filename on disk.", + "title": "Name", "type": "string" }, - "description": { - "title": "Description", + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", "type": "string" }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object" - } - ] - } - } - } - }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", - "enum": [ - "active", - "unverified", - "deactivated" - ], - "type": "string" - }, - "Person": { - "title": "Person", - "description": "A model that describes an individual and their digital identities.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "default": "people", - "const": "people", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "identities": { - "title": "Identities", - "type": "array", - "items": { - "$ref": "#/definitions/Identity" - } - }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" - }, - "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" - }, - "gravatar_hash": { - "title": "Gravatar Hash", - "type": "string" - }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } - }, - "role": { - "default": "user", - "allOf": [ + "original_name": { + "anyOf": [ + { + "type": "string" + }, { - "$ref": "#/definitions/UserRole" + "type": "null" } - ] + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "account_status": { - "default": "unverified", - "allOf": [ + "location": { + "anyOf": [ + { + "type": "string" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] - } - } - }, - "Collection": { - "title": "Collection", - "description": "An Entry is an abstract base class for any model that can be\ndeserialized and stored in the database.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", - "type": "string" + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + "time_added": { + "description": "The timestamp for the original file upload.", + "format": "datetime", + "title": "Time Added", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" }, - "title": { - "title": "Title", - "type": "string" + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" }, - "description": { - "title": "Description", - "type": "string" + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", + "type": "boolean" }, - "num_items": { - "title": "Num Items", - "type": "integer" + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "description": "Content checksums (MD5 and SHA-256) of the file." } - } + }, + "required": [ + "name", + "extension", + "time_added", + "is_live", + "checksums" + ], + "title": "File", + "type": "object" }, "FileChecksums": { - "title": "FileChecksums", "description": "Content checksums for a file.", - "type": "object", "properties": { "md5": { + "description": "The MD5 hex digest of the file contents.", "title": "Md5", "type": "string" }, "sha256": { + "description": "The SHA-256 hex digest of the file contents.", "title": "Sha256", "type": "string" } @@ -768,167 +680,214 @@ "required": [ "md5", "sha256" - ] + ], + "title": "FileChecksums", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", + "Group": { + "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, "type": { + "const": "groups", + "default": "groups", + "description": "The entry type as a string.", "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "size": { - "title": "Size", - "type": "integer" - }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "item_ids": { - "title": "Item Ids", - "type": "array", - "items": { - "type": "string" - } + "group_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, - "blocks": { - "title": "Blocks", - "type": "array", + "members": { + "description": "A list of people that belong to this group; stored on the user objects.", "items": { - "type": "string" - } - }, - "name": { - "title": "Name", - "type": "string" + "additionalProperties": true, + "type": "object" + }, + "title": "Members", + "type": "array" }, - "extension": { - "title": "Extension", - "type": "string" + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" }, - "original_name": { - "title": "Original Name", - "type": "string" + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" }, - "location": { - "title": "Location", - "type": "string" + "managers": { + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", + "items": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + } + ] + }, + "title": "Managers", + "type": "array" + } + }, + "title": "Group", + "type": "object" + }, + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", + "properties": { + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." }, - "url_path": { - "title": "Url Path", + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", "type": "string" }, - "source": { - "title": "Source", + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", "type": "string" }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" - }, - "metadata": { - "title": "Metadata", - "type": "object" - }, - "representation": { - "title": "Representation" - }, - "source_server_name": { - "title": "Source Server Name", - "type": "string" + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" }, - "source_path": { - "title": "Source Path", + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" + } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "InlineSubstance": { + "properties": { + "name": { + "title": "Name", "type": "string" }, - "is_live": { - "title": "Is Live", - "type": "boolean" - }, - "checksums": { - "$ref": "#/definitions/FileChecksums" + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" } }, "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + "name" + ], + "title": "InlineSubstance", + "type": "object" }, "ItemStatus": { - "title": "ItemStatus", "description": "An enumeration of the status of items", "enum": [ "active", @@ -938,7 +897,644 @@ "failed", "other" ], + "title": "ItemStatus", + "type": "string" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" + ], + "title": "KnownType", + "type": "string" + }, + "Person": { + "description": "A model that describes an individual and their digital identities.", + "properties": { + "type": { + "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "identities": { + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", + "items": { + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" + }, + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" + }, + "contact_email": { + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" + }, + "gravatar_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" + }, + "managers": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" + }, + "role": { + "$ref": "#/$defs/UserRole", + "default": "user", + "description": "The role assigned to this person." + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" + }, + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." + } + }, + "title": "Person", + "type": "object" + }, + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" + }, + "TypedRelationship": { + "additionalProperties": true, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" + }, + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." + }, + "type": { + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" + }, + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" + } + }, + "required": [ + "type" + ], + "title": "TypedRelationship", + "type": "object" + }, + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing an experimental sample.", + "properties": { + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A string representation of the chemical formula or composition associated with this sample.\n\nThe representation is relatively free-form; clients are expected parse and interpret HTML markup for subscripts\nand accept unicode characters for greek letters.", + "examples": [ + "Na3P", + "Na3P", + "LiNiO2@C", + "Na3+xP", + "LiNi1/3Co0.1Mn0.1O2" + ], + "title": "Chemform" + }, + "smiles": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A SMILES string representation of the chemical structure associated with this sample.", + "title": "Smiles" + }, + "inchi": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An International Chemical Identifier (InChI) string representation of chemicals/molecules associated with this sample.", + "title": "Inchi" + }, + "inchi_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A unique key derived from the InChI.", + "title": "Inchi Key" + }, + "GHS_codes": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A string describing any GHS hazard codes associated with this item. See https://pubchem.ncbi.nlm.nih.gov/ghs/ for code definitions.", + "examples": [ + "H224", + "H303, H316, H319" + ], + "title": "Ghs Codes" + }, + "molar_mass": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Mass per formula unit, in g/mol.", + "title": "Molar Mass" + }, + "CAS": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The CAS Registry Number for the substance described by this entry.", + "title": "Cas" + }, + "synthesis_constituents": { + "default": [], + "description": "A list of references to constituent materials giving the amount and relevant inlined details of consituent items.", + "items": { + "$ref": "#/$defs/Constituent" + }, + "title": "Synthesis Constituents", + "type": "array" + }, + "synthesis_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Free-text details of the procedure applied to synthesise the sample", + "title": "Synthesis Description" + }, + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "samples", + "default": "samples", + "title": "Type", "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).", + "title": "Date" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional human-readable/usable name for the entry.", + "title": "Name" + }, + "files": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this sample.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "status": { + "$ref": "#/$defs/ItemStatus", + "default": "active", + "description": "The status of the sample, indicating its current state." } - } + }, + "required": [ + "item_id" + ], + "title": "Sample", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/startingmaterial.json b/pydatalab/schemas/startingmaterial.json index b51586d6b..40ea9e4c1 100644 --- a/pydatalab/schemas/startingmaterial.json +++ b/pydatalab/schemas/startingmaterial.json @@ -1,803 +1,678 @@ { - "title": "StartingMaterial", - "description": "A model for representing an experimental sample, based on the connection\nwith cheminventory.net, which mixes container-level and substance-level\ninformation.", - "type": "object", - "properties": { - "chemform": { - "title": "Chemform", - "example": [ - "Na3P", - "Na3P", - "LiNiO2@C", - "Na3+xP", - "LiNi1/3Co0.1Mn0.1O2" - ], - "type": "string" - }, - "smiles": { - "title": "Smiles", - "aliases": [ - "SMILES", - "smiles_representation" - ], - "type": "string" - }, - "inchi": { - "title": "Inchi", - "type": "string" - }, - "inchi_key": { - "title": "Inchi Key", - "type": "string" - }, - "GHS_codes": { - "title": "Ghs H-Codes", - "examples": [ - "H224", - "H303, H316, H319" + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" ], + "title": "AccountStatus", "type": "string" }, - "molar_mass": { - "title": "Molecular Weight", - "type": "number" - }, - "CAS": { - "title": "Substance Cas", - "type": "string" - }, - "synthesis_constituents": { - "title": "Synthesis Constituents", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Constituent" - } - }, - "synthesis_description": { - "title": "Synthesis Description", - "type": "string" - }, - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "starting_materials", - "const": "starting_materials", - "pattern": "^starting_materials$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date Acquired", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Container Name", - "type": "string" - }, - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "default": "available", - "allOf": [ - { - "$ref": "#/definitions/StartingMaterialsStatus" - } - ] - }, - "barcode": { - "title": "Barcode", - "type": "string" - }, - "date_opened": { - "title": "Date Opened", - "type": "string", - "format": "date-time" - }, - "chemical_purity": { - "title": "Chemical Purity", - "type": "string" - }, - "full_percent": { - "title": "Full %", - "type": "string" - }, - "size": { - "title": "Container Size", - "type": "string" - }, - "size_unit": { - "title": "Unit", - "type": "string" - }, - "supplier": { - "title": "Supplier", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" - }, - "comment": { - "title": "Comments", - "type": "string" - } - }, - "required": [ - "item_id" - ], - "definitions": { - "EntryReference": { - "title": "EntryReference", - "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "immutable_id": { - "title": "Immutable Id", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "chemform": { - "title": "Chemform", - "type": "string" - } - }, - "required": [ - "type" - ] - }, - "InlineSubstance": { - "title": "InlineSubstance", - "type": "object", + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "name": { - "title": "Name", - "type": "string" - }, - "chemform": { - "title": "Chemform", + "collection_id": { + "description": "The immutable ID of the collection.", + "title": "Collection Id", "type": "string" } }, "required": [ - "name" - ] + "collection_id" + ], + "title": "CollectionReference", + "type": "object" }, "Constituent": { - "title": "Constituent", "description": "A constituent of a sample.", - "type": "object", "properties": { "item": { - "title": "Item", "anyOf": [ { - "$ref": "#/definitions/EntryReference" + "$ref": "#/$defs/EntryReference" }, { - "$ref": "#/definitions/InlineSubstance" + "$ref": "#/$defs/InlineSubstance" } - ] + ], + "description": "A reference to item (sample or starting material) entry for the constituent substance.", + "title": "Item" }, "quantity": { - "title": "Quantity", - "minimum": 0, - "type": "number" + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the constituent material used to create the sample.", + "title": "Quantity" }, "unit": { - "title": "Unit", "default": "g", + "description": "The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).", + "title": "Unit", "type": "string" } }, "required": [ - "item", - "quantity" - ] + "item" + ], + "title": "Constituent", + "type": "object" }, "DataBlockResponse": { - "title": "DataBlockResponse", + "additionalProperties": true, "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", "properties": { "blocktype": { + "description": "The type of the block.", "title": "Blocktype", "type": "string" }, "block_id": { + "description": "A shorthand random ID for the block.", "title": "Block Id", "type": "string" }, "item_id": { - "title": "Item Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" }, "collection_id": { - "title": "Collection Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" }, "title": { - "title": "Title", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" }, "freeform_comment": { - "title": "Freeform Comment", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" }, "file_id": { - "title": "File Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" }, "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" }, "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" }, "warnings": { - "title": "Warnings", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" }, "b64_encoded_image": { - "title": "B64 Encoded Image", + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" }, "bokeh_plot_data": { - "title": "Bokeh Plot Data", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object" + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" }, "computed": { - "title": "Computed", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" }, "metadata": { - "title": "Metadata", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" } }, "required": [ "blocktype", - "block_id" - ] - }, - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" - ], - "type": "string" - }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", - "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" + "block_id", + "b64_encoded_image", + "bokeh_plot_data" ], - "type": "string" + "title": "DataBlockResponse", + "type": "object" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "EntryReference": { + "additionalProperties": true, + "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", "properties": { - "description": { - "title": "Description", + "type": { + "title": "Type", "type": "string" }, - "relation": { - "$ref": "#/definitions/RelationshipType" - }, - "type": { - "$ref": "#/definitions/KnownType" + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" }, "immutable_id": { - "title": "Immutable Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Immutable Id" }, "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Item Id" }, "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Refcode" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" } }, "required": [ "type" - ] - }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" ], - "type": "string" + "title": "EntryReference", + "type": "object" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" }, - "identifier": { - "title": "Identifier", - "type": "string" + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" }, - "name": { - "title": "Name", - "type": "string" + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" }, - "display_name": { - "title": "Display Name", - "type": "string" - } - }, - "required": [ - "identity_type", - "identifier", - "name" - ] - }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" - ], - "type": "string" - }, - "Group": { - "title": "Group", - "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", - "properties": { "type": { + "const": "files", + "default": "files", "title": "Type", - "default": "groups", - "const": "groups", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "members": { - "title": "Members", - "type": "array", - "items": { - "type": "object" - } + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object" - } - ] - } - } - } - }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", - "enum": [ - "active", - "unverified", - "deactivated" - ], - "type": "string" - }, - "Person": { - "title": "Person", - "description": "A model that describes an individual and their digital identities.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "default": "people", - "const": "people", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" }, - "relationships": { - "title": "Relationships", - "type": "array", + "item_ids": { + "description": "A list of item IDs associated with this file.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "type": "string" + }, + "title": "Item Ids", + "type": "array" }, - "identities": { - "title": "Identities", - "type": "array", + "blocks": { + "description": "A list of block IDs associated with this file.", "items": { - "$ref": "#/definitions/Identity" - } + "type": "string" + }, + "title": "Blocks", + "type": "array" }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, + "name": { + "description": "The filename on disk.", + "title": "Name", "type": "string" }, - "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" - }, - "gravatar_hash": { - "title": "Gravatar Hash", + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", "type": "string" }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } - }, - "role": { - "default": "user", - "allOf": [ + "original_name": { + "anyOf": [ + { + "type": "string" + }, { - "$ref": "#/definitions/UserRole" + "type": "null" } - ] - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" }, - "account_status": { - "default": "unverified", - "allOf": [ + "location": { + "anyOf": [ + { + "type": "string" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] - } - } - }, - "Collection": { - "title": "Collection", - "description": "An Entry is an abstract base class for any model that can be\ndeserialized and stored in the database.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", - "type": "string" + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + "time_added": { + "description": "The timestamp for the original file upload.", + "format": "datetime", + "title": "Time Added", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" }, - "title": { - "title": "Title", - "type": "string" + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" }, - "description": { - "title": "Description", - "type": "string" + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", + "type": "boolean" }, - "num_items": { - "title": "Num Items", - "type": "integer" + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "description": "Content checksums (MD5 and SHA-256) of the file." } - } + }, + "required": [ + "name", + "extension", + "time_added", + "is_live", + "checksums" + ], + "title": "File", + "type": "object" }, "FileChecksums": { - "title": "FileChecksums", "description": "Content checksums for a file.", - "type": "object", "properties": { "md5": { + "description": "The MD5 hex digest of the file contents.", "title": "Md5", "type": "string" }, "sha256": { + "description": "The SHA-256 hex digest of the file contents.", "title": "Sha256", "type": "string" } @@ -805,167 +680,383 @@ "required": [ "md5", "sha256" - ] + ], + "title": "FileChecksums", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", + "Group": { + "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, "type": { + "const": "groups", + "default": "groups", + "description": "The entry type as a string.", "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "size": { - "title": "Size", - "type": "integer" + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" + "group_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, - "item_ids": { - "title": "Item Ids", - "type": "array", + "members": { + "description": "A list of people that belong to this group; stored on the user objects.", "items": { - "type": "string" - } - }, - "blocks": { - "title": "Blocks", - "type": "array", + "additionalProperties": true, + "type": "object" + }, + "title": "Members", + "type": "array" + }, + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" + }, + "managers": { + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", "items": { - "type": "string" - } + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + } + ] + }, + "title": "Managers", + "type": "array" + } + }, + "title": "Group", + "type": "object" + }, + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", + "properties": { + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." + }, + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", + "type": "string" }, "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", "title": "Name", "type": "string" }, - "extension": { - "title": "Extension", - "type": "string" + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" }, - "original_name": { - "title": "Original Name", + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" + } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "InlineSubstance": { + "properties": { + "name": { + "title": "Name", "type": "string" }, - "location": { - "title": "Location", + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "name" + ], + "title": "InlineSubstance", + "type": "object" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" + ], + "title": "KnownType", + "type": "string" + }, + "Person": { + "description": "A model that describes an individual and their digital identities.", + "properties": { + "type": { + "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, - "url_path": { - "title": "Url Path", - "type": "string" + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" }, - "source": { - "title": "Source", - "type": "string" + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "metadata": { - "title": "Metadata", - "type": "object" + "identities": { + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", + "items": { + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" }, - "representation": { - "title": "Representation" + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" }, - "source_server_name": { - "title": "Source Server Name", - "type": "string" + "contact_email": { + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" }, - "source_path": { - "title": "Source Path", - "type": "string" + "gravatar_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" }, - "is_live": { - "title": "Is Live", - "type": "boolean" + "managers": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" }, - "checksums": { - "$ref": "#/definitions/FileChecksums" + "role": { + "$ref": "#/$defs/UserRole", + "default": "user", + "description": "The role assigned to this person." + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" + }, + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." } }, - "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + "title": "Person", + "type": "object" + }, + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" }, "StartingMaterialsStatus": { - "title": "StartingMaterialsStatus", "description": "An enumeration of the status of starting materials", "enum": [ "ordered", @@ -976,7 +1067,593 @@ "exhausted", "other" ], + "title": "StartingMaterialsStatus", "type": "string" - } - } + }, + "TypedRelationship": { + "additionalProperties": true, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" + }, + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." + }, + "type": { + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" + }, + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" + } + }, + "required": [ + "type" + ], + "title": "TypedRelationship", + "type": "object" + }, + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing an experimental sample, based on the connection\nwith cheminventory.net, which mixes container-level and substance-level\ninformation.", + "properties": { + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A string representation of the chemical formula or composition associated with this sample.\n\nThe representation is relatively free-form; clients are expected parse and interpret HTML markup for subscripts\nand accept unicode characters for greek letters.", + "examples": [ + "Na3P", + "Na3P", + "LiNiO2@C", + "Na3+xP", + "LiNi1/3Co0.1Mn0.1O2" + ], + "title": "Chemform" + }, + "smiles": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A SMILES string representation of the chemical structure associated with this sample.", + "title": "Smiles" + }, + "inchi": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An International Chemical Identifier (InChI) string representation of chemicals/molecules associated with this sample.", + "title": "Inchi" + }, + "inchi_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A unique key derived from the InChI.", + "title": "Inchi Key" + }, + "GHS_codes": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A string describing any GHS hazard codes associated with this item. See https://pubchem.ncbi.nlm.nih.gov/ghs/ for code definitions.", + "examples": [ + "H224", + "H303, H316, H319" + ], + "title": "Ghs Codes" + }, + "molar_mass": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Mass per formula unit, in g/mol.", + "title": "Molar Mass" + }, + "CAS": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The CAS Registry Number for the substance described by this entry.", + "title": "Cas" + }, + "synthesis_constituents": { + "default": [], + "description": "A list of references to constituent materials giving the amount and relevant inlined details of consituent items.", + "items": { + "$ref": "#/$defs/Constituent" + }, + "title": "Synthesis Constituents", + "type": "array" + }, + "synthesis_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Free-text details of the procedure applied to synthesise the sample", + "title": "Synthesis Description" + }, + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "starting_materials", + "default": "starting_materials", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The date the item was acquired", + "title": "Date" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The name of the substance in the container.", + "title": "Name" + }, + "files": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this sample.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "status": { + "$ref": "#/$defs/StartingMaterialsStatus", + "default": "available", + "description": "The status of the starting materials, indicating its current state." + }, + "barcode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A unique barcode provided by an external source, e.g., cheminventory.", + "title": "Barcode" + }, + "date_opened": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The date the item was opened", + "title": "Date Opened" + }, + "chemical_purity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The chemical purity of this container with regards to the defined substance.", + "title": "Chemical Purity" + }, + "full_percent": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the defined substance remaining in the container, expressed as a percentage.", + "title": "Full Percent" + }, + "size": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The total size of the container, in units of `size_unit`.", + "title": "Size" + }, + "size_unit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Units for the 'size' field.", + "title": "Size Unit" + }, + "supplier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Supplier or manufacturer of the chemical.", + "title": "Supplier" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The place where the container is located.", + "title": "Location" + }, + "comment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional comments or notes about the container.", + "title": "Comment" + } + }, + "required": [ + "item_id", + "name", + "chemical_purity", + "supplier" + ], + "title": "StartingMaterial", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/src/pydatalab/models/blocks.py b/pydatalab/src/pydatalab/models/blocks.py index 9e33aec65..a89af0ffc 100644 --- a/pydatalab/src/pydatalab/models/blocks.py +++ b/pydatalab/src/pydatalab/models/blocks.py @@ -1,6 +1,6 @@ -from pydantic import BaseModel, Field +from pydantic import ConfigDict, Field -from pydatalab.models.utils import JSON_ENCODERS, PyObjectId +from pydatalab.models.utils import BaseModel, PyObjectId class DataBlockResponse(BaseModel): @@ -11,6 +11,8 @@ class DataBlockResponse(BaseModel): where possible. """ + model_config = ConfigDict(validate_by_name=True, extra="allow") + blocktype: str """The type of the block.""" @@ -60,8 +62,3 @@ class DataBlockResponse(BaseModel): metadata: dict | None = Field(default=None, datalab_exclude_from_load=True) """Any structured metadata associated with the block, for example, experimental acquisition parameters.""" - - class Config: - allow_population_by_field_name = True - json_encoders = JSON_ENCODERS - extra = "allow" diff --git a/pydatalab/src/pydatalab/models/cells.py b/pydatalab/src/pydatalab/models/cells.py index 8474ad056..e8690db9a 100644 --- a/pydatalab/src/pydatalab/models/cells.py +++ b/pydatalab/src/pydatalab/models/cells.py @@ -1,13 +1,16 @@ from enum import Enum +from typing import Literal -from pydantic import Field, root_validator, validator +from pydantic import ( + Field, + field_validator, + model_validator, +) from pydatalab.models.entries import EntryReference from pydatalab.models.items import Item from pydatalab.models.utils import CellStatus, Constituent -# from pydatalab.logger import LOGGER - class CellComponent(Constituent): ... @@ -27,72 +30,66 @@ class CellFormat(str, Enum): class Cell(Item): """A model for representing electrochemical cells.""" - type: str = Field("cells", const="cells", pattern="^cells$") + type: Literal["cells"] = "cells" - cell_format: CellFormat | None + cell_format: CellFormat | None = None """The form factor of the cell, e.g., coin, pouch, in situ or otherwise.""" - cell_format_description: str | None + cell_format_description: str | None = None """Additional human-readable description of the cell form factor, e.g., 18650, AMPIX, CAMPIX""" - cell_preparation_description: str | None + cell_preparation_description: str | None = None + """Description of how the cell was prepared.""" - characteristic_mass: float | None + characteristic_mass: float | None = None """The characteristic mass of the cell in milligrams. Can be used to normalize capacities.""" - characteristic_chemical_formula: str | None + characteristic_chemical_formula: str | None = None """The chemical formula of the active material. Can be used to calculated molar mass in g/mol for normalizing capacities.""" - characteristic_molar_mass: float | None + characteristic_molar_mass: float | None = None """The molar mass of the active material, in g/mol. Will be inferred from the chemical formula, or can be supplied if it cannot be supplied""" - positive_electrode: list[CellComponent] = [] - - negative_electrode: list[CellComponent] = [] - - electrolyte: list[CellComponent] = [] - + positive_electrode: list[CellComponent] = Field(default_factory=list) + negative_electrode: list[CellComponent] = Field(default_factory=list) + electrolyte: list[CellComponent] = Field(default_factory=list) + active_ion: str | None = None + """The active ion species.""" active_ion_charge: float = 1 - status: CellStatus = Field(default=CellStatus.ACTIVE) """The status of the cells, indicating its current state.""" - @validator("characteristic_molar_mass", always=True, pre=True) - def set_molar_mass(cls, v, values): + @field_validator("characteristic_molar_mass", mode="before") + @classmethod + def set_molar_mass(cls, v, info): from periodictable import formula - if not v: - chemical_formula = values.get("characteristic_chemical_formula") - + if not v and hasattr(info, "data") and info.data: + chemical_formula = info.data.get("characteristic_chemical_formula") if chemical_formula: try: return formula(chemical_formula).mass except Exception: return None - return v - @root_validator - def add_missing_electrode_relationships(cls, values): + @model_validator(mode="after") + def add_missing_electrode_relationships(self): """Add any missing cell component constituents to parent relationships""" from pydatalab.models.relationships import RelationshipType, TypedRelationship - existing_parthood_relationships = {} - if values.get("relationships") is not None: - # Index by refcode *and* item_id so a stored relationship carrying an - # item_id still matches a refcode-enriched constituent (and vice-versa). - existing_parthood_relationships = { - identifier: relationship - for relationship in values["relationships"] - if relationship.relation == RelationshipType.PARTHOOD - for identifier in (relationship.refcode, relationship.item_id) - if identifier - } - else: - values["relationships"] = [] + # Index by refcode *and* item_id so a stored relationship carrying an + # item_id still matches a refcode-enriched constituent (and vice-versa). + existing_parthood_relationships = { + identifier: relationship + for relationship in self.relationships + if relationship.relation == RelationshipType.PARTHOOD + for identifier in (relationship.refcode, relationship.item_id) + if identifier + } for component in ("positive_electrode", "negative_electrode", "electrolyte"): - for constituent in values.get(component, []): + for constituent in getattr(self, component): if not isinstance(constituent.item, EntryReference): continue @@ -110,7 +107,7 @@ def add_missing_electrode_relationships(cls, values): type=constituent.item.type, description="Is a constituent of", ) - values["relationships"].append(relationship) + self.relationships.append(relationship) else: # Back-fill any identifier missing from the stored relationship relationship.refcode = relationship.refcode or refcode @@ -123,4 +120,4 @@ def add_missing_electrode_relationships(cls, values): if identifier: existing_parthood_relationships[identifier] = relationship - return values + return self diff --git a/pydatalab/src/pydatalab/models/collections.py b/pydatalab/src/pydatalab/models/collections.py index 4ce6b7b5b..be47e0631 100644 --- a/pydatalab/src/pydatalab/models/collections.py +++ b/pydatalab/src/pydatalab/models/collections.py @@ -1,4 +1,8 @@ -from pydantic import Field, root_validator +from typing import Literal + +from pydantic import ( + model_validator, +) from pydatalab.models.entries import Entry from pydatalab.models.traits import HasBlocks, HasOwner @@ -6,21 +10,22 @@ class Collection(Entry, HasOwner, HasBlocks): - type: str = Field("collections", const="collections", pattern="^collections$") + type: Literal["collections"] = "collections" - collection_id: HumanReadableIdentifier = Field(None) + collection_id: HumanReadableIdentifier """A short human-readable/usable name for the collection.""" - title: str | None + title: str | None = None """A descriptive title for the collection.""" - description: str | None + description: str | None = None """A description of the collection, either in plain-text or a markup language.""" - num_items: int | None = Field(None) + num_items: int | None = None """Inlined number of items associated with this collection.""" - @root_validator + @model_validator(mode="before") + @classmethod def check_ids(cls, values): if not any(values.get(k) is not None for k in ("collection_id", "immutable_id")): raise ValueError("Collection must have at least collection_id or immutable_id") diff --git a/pydatalab/src/pydatalab/models/entries.py b/pydatalab/src/pydatalab/models/entries.py index 694a312ad..a5b61f9de 100644 --- a/pydatalab/src/pydatalab/models/entries.py +++ b/pydatalab/src/pydatalab/models/entries.py @@ -1,10 +1,10 @@ import abc -from pydantic import BaseModel, Field, root_validator +from pydantic import ConfigDict, Field, model_validator from pydatalab.models.relationships import TypedRelationship from pydatalab.models.utils import ( - JSON_ENCODERS, + BaseModel, EntryReference, IsoformatDateTime, PyObjectId, @@ -20,21 +20,22 @@ class Entry(BaseModel, abc.ABC): type: str """The resource type of the entry.""" - immutable_id: PyObjectId = Field( + immutable_id: PyObjectId | None = Field( None, title="Immutable ID", alias="_id", - format="uuid", + json_schema_extra={"format": "uuid"}, ) """The immutable database ID of the entry.""" last_modified: IsoformatDateTime | None = None """The timestamp at which the entry was last modified.""" - relationships: list[TypedRelationship] | None = None + relationships: list[TypedRelationship] = Field(default_factory=list) """A list of related entries and their types.""" - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def check_id_names(cls, values): """Slightly upsetting hack: this case *should* be covered by the pydantic setting for populating fields by alias names. @@ -42,6 +43,10 @@ def check_id_names(cls, values): if "_id" in values: values["immutable_id"] = values.pop("_id") + # Coerce a stored/explicit null into an empty list so the field is always a list. + if values.get("relationships") is None: + values["relationships"] = [] + return values def to_reference(self, additional_fields: list[str] | None = None) -> "EntryReference": @@ -63,7 +68,4 @@ def to_reference(self, additional_fields: list[str] | None = None) -> "EntryRefe return EntryReference(**data) - class Config: - allow_population_by_field_name = True - json_encoders = JSON_ENCODERS - extra = "ignore" + model_config = ConfigDict(populate_by_name=True, extra="ignore") diff --git a/pydatalab/src/pydatalab/models/equipment.py b/pydatalab/src/pydatalab/models/equipment.py index 1d60a1f5c..820961905 100644 --- a/pydatalab/src/pydatalab/models/equipment.py +++ b/pydatalab/src/pydatalab/models/equipment.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import Field from pydatalab.models.items import Item @@ -9,15 +11,15 @@ class Equipment(Item): """A model for representing an experimental sample.""" - type: str = Field("equipment", const="equipment", pattern="^equipment$") + type: Literal["equipment"] = "equipment" - serial_numbers: str | None + serial_numbers: str | None = None """A string describing one or more serial numbers for the instrument.""" - manufacturer: str | None + manufacturer: str | None = None """The manufacturer of this piece of equipment""" - location: str | None + location: str | None = None """Place where the equipment is located""" contact: str | None diff --git a/pydatalab/src/pydatalab/models/export_task.py b/pydatalab/src/pydatalab/models/export_task.py index 171567038..d2085ffde 100644 --- a/pydatalab/src/pydatalab/models/export_task.py +++ b/pydatalab/src/pydatalab/models/export_task.py @@ -1,9 +1,9 @@ from datetime import datetime, timezone from enum import Enum -from pydantic import BaseModel, Field +from pydantic import Field -from pydatalab.models.utils import PyObjectId +from pydatalab.models.utils import BaseModel, PyObjectId class ExportStatus(str, Enum): @@ -18,23 +18,35 @@ class ExportStatus(str, Enum): class ExportTask(BaseModel): """Model for an export task.""" - task_id: str = Field(..., description="Unique identifier for the export task") - collection_id: str | None = Field(None, description="ID of the collection being exported") - item_id: str | None = Field(None, description="ID of the item being exported") - export_type: str = Field( - default="collection", description="Type of export: 'collection' or 'item' or 'graph'" - ) - status: ExportStatus = Field( - default=ExportStatus.PENDING, description="Current status of the task" - ) - creator_id: PyObjectId = Field(..., description="ID of the user who created the export") - created_at: datetime = Field( - default_factory=lambda: datetime.now(tz=timezone.utc), - description="When the task was created", - ) - completed_at: datetime | None = Field(None, description="When the task was completed") - file_path: str | None = Field(None, description="Path to the generated .eln file") - error_message: str | None = Field(None, description="Error message if status is ERROR") + task_id: str + """Unique identifier for the export task""" + + collection_id: str | None = None + """ID of the collection being exported""" + + item_id: str | None = None + """ID of the item being exported""" + + export_type: str = "collection" + """Type of export: 'collection' or 'item' or 'graph'""" + + status: ExportStatus = ExportStatus.PENDING + """Current status of the task""" + + creator_id: PyObjectId + """ID of the user who created the export""" + + created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc)) + """When the task was created""" + + completed_at: datetime | None = None + """When the task was completed""" + + file_path: str | None = None + """Path to the generated .eln file""" + + error_message: str | None = None + """Error message if status is ERROR""" class Config: use_enum_values = True diff --git a/pydatalab/src/pydatalab/models/files.py b/pydatalab/src/pydatalab/models/files.py index 1cdc744af..ca17eadd5 100644 --- a/pydatalab/src/pydatalab/models/files.py +++ b/pydatalab/src/pydatalab/models/files.py @@ -1,10 +1,10 @@ -from typing import Any +from typing import Any, Literal -from pydantic import BaseModel, Field +from pydantic import Field from pydatalab.models.entries import Entry from pydatalab.models.traits import HasOwner, HasRevisionControl -from pydatalab.models.utils import IsoformatDateTime +from pydatalab.models.utils import BaseModel, IsoformatDateTime class FileChecksums(BaseModel): @@ -20,18 +20,18 @@ class FileChecksums(BaseModel): class File(Entry, HasOwner, HasRevisionControl): """A model for representing a file that has been tracked or uploaded to datalab.""" - type: str = Field("files", const="files", pattern="^files$") + type: Literal["files"] = "files" - size: int | None + size: int | None = None """The size of the file on disk in bytes.""" - last_modified_remote: IsoformatDateTime | None + last_modified_remote: IsoformatDateTime | None = None """The last date/time at which the remote file was modified.""" - item_ids: list[str] + item_ids: list[str] = Field(default_factory=list) """A list of item IDs associated with this file.""" - blocks: list[str] + blocks: list[str] = Field(default_factory=list) """A list of block IDs associated with this file.""" name: str @@ -40,30 +40,30 @@ class File(Entry, HasOwner, HasRevisionControl): extension: str """The file extension that the file was uploaded with.""" - original_name: str | None + original_name: str | None = None """The raw filename as uploaded.""" - location: str | None + location: str | None = None """The location of the file on disk.""" - url_path: str | None + url_path: str | None = None """The path to a remote file.""" - source: str | None + source: str | None = None """The source of the file, e.g. 'remote' or 'uploaded'.""" time_added: IsoformatDateTime """The timestamp for the original file upload.""" - metadata: dict[Any, Any] | None + metadata: dict[Any, Any] | None = None """Any additional metadata.""" - representation: Any | None + representation: Any | None = None - source_server_name: str | None + source_server_name: str | None = None """The server name at which the file is stored.""" - source_path: str | None + source_path: str | None = None """The path to the file on the remote resource.""" is_live: bool diff --git a/pydatalab/src/pydatalab/models/items.py b/pydatalab/src/pydatalab/models/items.py index f3eb1799b..d1295d180 100644 --- a/pydatalab/src/pydatalab/models/items.py +++ b/pydatalab/src/pydatalab/models/items.py @@ -1,6 +1,6 @@ import abc -from pydantic import Field, validator +from pydantic import Field, field_validator from pydatalab.models.entries import Entry from pydatalab.models.files import File @@ -21,33 +21,32 @@ class Item(Entry, HasOwner, HasRevisionControl, IsCollectable, HasBlocks, abc.ABC): """The generic model for data types that will be exposed with their own named endpoints.""" - refcode: Refcode = None # type: ignore - """A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) - and a locally unique string, ideally created with some consistent scheme. - """ + refcode: Refcode | None = None + """A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.""" item_id: HumanReadableIdentifier """A locally unique, human-readable identifier for the entry. This ID is mutable.""" - description: str | None + description: str | None = None """A description of the item, either in plain-text or a markup language.""" - date: IsoformatDateTime | None + date: IsoformatDateTime | None = None """A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).""" - name: str | None + name: str | None = None """An optional human-readable/usable name for the entry.""" - files: list[File] | None + files: list[File] | None = None """Any files attached to this sample.""" - file_ObjectIds: list[PyObjectId] = Field([]) + file_ObjectIds: list[PyObjectId] = Field(default_factory=list) """Links to object IDs of files stored within the database.""" - status: str | None + status: str | None = None """The status of the item, with allowed values defined by the specific item class.""" - @validator("refcode", pre=True, always=True) + @field_validator("refcode", mode="before") + @classmethod def refcode_validator(cls, v): """Generate a refcode if not provided.""" diff --git a/pydatalab/src/pydatalab/models/people.py b/pydatalab/src/pydatalab/models/people.py index a86a2d4a2..ec3485ade 100644 --- a/pydatalab/src/pydatalab/models/people.py +++ b/pydatalab/src/pydatalab/models/people.py @@ -1,14 +1,17 @@ from __future__ import annotations from enum import Enum +from typing import Annotated, Literal -import bson -import bson.errors -from pydantic import BaseModel, ConstrainedStr, Field, parse_obj_as, validator from pydantic import EmailStr as PydanticEmailStr +from pydantic import ( + Field, + StringConstraints, + field_validator, +) from pydatalab.models.entries import Entry -from pydatalab.models.utils import HumanReadableIdentifier, PyObjectId, UserRole +from pydatalab.models.utils import BaseModel, HumanReadableIdentifier, PyObjectId, UserRole class IdentityType(str, Enum): @@ -36,29 +39,32 @@ class Identity(BaseModel): name: str """The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.""" - verified: bool = Field(False) + verified: bool = False """Whether the identity has been verified (by some means, e.g., OAuth2 or email)""" - display_name: str | None + display_name: str | None = None """The user's display name associated with the identity, also to be exposed in free text searches.""" - @validator("name", pre=True, always=True) - def add_missing_name(cls, v, values): + @field_validator("name", mode="before") + @classmethod + def add_missing_name(cls, v, info): """If the identity is created without a free-text 'name', then for certain providers, populate this field so that it can appear in the free text index, e.g., an ORCID, or an institutional username from an email address. """ - if v is None: - if values["identity_type"] == IdentityType.ORCID: - return values["identifier"] - if values["identity_type"] == IdentityType.EMAIL: - return values["identifier"].split("@")[0] - + if v is None and hasattr(info, "data") and info.data: + data = info.data + if data.get("identity_type") == IdentityType.ORCID: + return data.get("identifier") + if data.get("identity_type") == IdentityType.EMAIL: + identifier = data.get("identifier", "") + return identifier.split("@")[0] if "@" in identifier else identifier return v - @validator("verified", pre=True, always=True) + @field_validator("verified", mode="before") + @classmethod def add_missing_verification(cls, v): """Fills in missing value for `verified` if not given.""" if not v: @@ -66,18 +72,11 @@ def add_missing_verification(cls, v): return v -class DisplayName(ConstrainedStr): - """A constrained string less than 150 characters long but with - non-empty content, intended to be entered by the user. - - """ - - max_length = 150 - min_length = 1 - strip_whitespace = True - - def __new__(cls, value): - return parse_obj_as(cls, value) +DisplayName = Annotated[ + str, + StringConstraints(min_length=1, max_length=150, strip_whitespace=True), +] +"""A constrained string less than 150 characters long but with non-empty content, intended to be entered by the user.""" class EmailStr(PydanticEmailStr): @@ -112,38 +111,40 @@ class Group(Entry): """ - type: str = Field("groups", const=True) + type: Literal["groups"] = "groups" """The entry type as a string.""" - group_id: HumanReadableIdentifier | None = Field(None) + group_id: HumanReadableIdentifier | None = None """A short, locally-unique ID for the group.""" - members: list[dict] = Field(None) - """A list of people that belong to this group.""" + members: list[dict] = Field(default_factory=list) + """A list of people that belong to this group; stored on the user objects.""" - display_name: DisplayName | None = Field(None) + display_name: DisplayName | None = None """The chosen display name for the group""" - description: str | None = Field(None) + description: str | None = None """A description of the group""" managers: list[PyObjectId | dict] = Field(default_factory=list) - """A list of user IDs that can manage this group.""" + """A list of user IDs that can manage this group; stored in db as list of IDs.""" - @validator("members", pre=True, always=True) + @field_validator("members", mode="before") + @classmethod def cast_members_to_people(cls, v): """Casts members to list of people if not None.""" if v is not None: - return [Person(**member).dict(exclude_unset=True) for member in v] + return [Person(**member).model_dump(exclude_unset=True) for member in v] return v - @validator("managers", pre=True, always=True) + @field_validator("managers", mode="before") + @classmethod def cast_managers_to_people(cls, v): """Casts managers to list of people if not None.""" if v and isinstance(v[0], dict): return [ - Person(**member).dict(exclude_unset=True) + Person(**member).model_dump(exclude_unset=True) for member in v if isinstance(member, dict) ] @@ -154,7 +155,7 @@ def cast_managers_to_people(cls, v): class Person(Entry): """A model that describes an individual and their digital identities.""" - type: str = Field("people", const=True) + type: Literal["people"] = "people" """The entry type as a string.""" identities: list[Identity] = Field(default_factory=list) @@ -181,14 +182,15 @@ class Person(Entry): account_status: AccountStatus = Field(AccountStatus.UNVERIFIED) """The status of the user's account.""" - @validator("type", pre=True, always=True) + @field_validator("type", mode="before") + @classmethod def add_missing_type(cls, v): """Fill in missing `type` field if not provided.""" if v is None: v = "people" return v - @validator("type", pre=True) + @classmethod def set_default_type(cls, _): return "people" @@ -214,8 +216,6 @@ def new_user_from_identity( A `Person` object with only the provided identity. """ - user_id = bson.ObjectId() - display_name = None if use_display_name: display_name = identity.display_name @@ -227,7 +227,6 @@ def new_user_from_identity( from pydatalab.mongo import gravatar_hash_for return Person( - immutable_id=user_id, identities=[identity], display_name=display_name, contact_email=contact_email, diff --git a/pydatalab/src/pydatalab/models/relationships.py b/pydatalab/src/pydatalab/models/relationships.py index e844c2498..e4c53ec33 100644 --- a/pydatalab/src/pydatalab/models/relationships.py +++ b/pydatalab/src/pydatalab/models/relationships.py @@ -1,8 +1,13 @@ from enum import Enum -from pydantic import BaseModel, root_validator, validator +from pydantic import ( + ConfigDict, + field_validator, + model_validator, +) from pydatalab.models.utils import ( + BaseModel, HumanReadableIdentifier, KnownType, PyObjectId, @@ -29,46 +34,52 @@ class entryC SIBLING = "sibling" PARTHOOD = "is_part_of" OTHER = "other" + COLLECTIONS = "collections" class TypedRelationship(BaseModel): - description: str | None + description: str | None = None """A description of the relationship.""" - relation: RelationshipType | None + relation: RelationshipType | None = None """The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided.""" type: KnownType """The type of the related resource.""" - immutable_id: PyObjectId | None + immutable_id: PyObjectId | None = None """The immutable ID of the entry that is related to this entry.""" - item_id: HumanReadableIdentifier | None + item_id: HumanReadableIdentifier | None = None """The ID of the entry that is related to this entry.""" - refcode: Refcode | None + refcode: Refcode | None = None """The refcode of the entry that is related to this entry.""" - @validator("relation") - def check_for_description(cls, v, values): - if v == RelationshipType.OTHER and values.get("description") is None: - raise ValueError( - f"A description must be provided if the relationship type is {RelationshipType.OTHER.value!r}." - ) - + @field_validator("relation") + @classmethod + def check_for_description(cls, v, info): + if v == RelationshipType.OTHER: + data = info.data if hasattr(info, "data") and info.data else {} + if data.get("description") is None: + raise ValueError( + f"A description must be provided if the relationship type is {RelationshipType.OTHER.value!r}." + ) return v - @root_validator + @model_validator(mode="before") + @classmethod def check_id_fields(cls, values): """Check that at least one of the possible identifier fields is provided.""" id_fields = ("immutable_id", "item_id", "refcode") - if all(values[f] is None for f in id_fields): + if all(values.get(f) is None for f in id_fields): raise ValueError(f"Must provide at least one of {id_fields!r}") if values.get("refcode") and values.get("item_id"): pass elif values.get("immutable_id") and (values.get("refcode") or values.get("item_id")): - raise ValueError("Must provide only one of {id_fields!r}") + raise ValueError(f"Must provide only one of {id_fields!r}") return values + + model_config = ConfigDict(extra="allow") diff --git a/pydatalab/src/pydatalab/models/samples.py b/pydatalab/src/pydatalab/models/samples.py index 25a2eda93..1664e7874 100644 --- a/pydatalab/src/pydatalab/models/samples.py +++ b/pydatalab/src/pydatalab/models/samples.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import Field from pydatalab.models.items import Item @@ -8,7 +10,7 @@ class Sample(Item, HasSynthesisInfo, HasSubstanceInfo): """A model for representing an experimental sample.""" - type: str = Field("samples", const="samples", pattern="^samples$") + type: Literal["samples"] = "samples" status: SampleStatus = Field(default=SampleStatus.ACTIVE) """The status of the sample, indicating its current state.""" diff --git a/pydatalab/src/pydatalab/models/starting_materials.py b/pydatalab/src/pydatalab/models/starting_materials.py index c44ec4408..845bd6e5a 100644 --- a/pydatalab/src/pydatalab/models/starting_materials.py +++ b/pydatalab/src/pydatalab/models/starting_materials.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import Field from pydatalab.models.items import Item @@ -9,46 +11,41 @@ class StartingMaterial(Item, HasSynthesisInfo, HasSubstanceInfo): """A model for representing an experimental sample, based on the connection with cheminventory.net, which mixes container-level and substance-level information. - """ - type: str = Field( - "starting_materials", const="starting_materials", pattern="^starting_materials$" - ) + type: Literal["starting_materials"] = "starting_materials" - barcode: str | None = Field( - alias="Barcode", - ) + barcode: str | None = Field(None, alias="Barcode") """A unique barcode provided by an external source, e.g., cheminventory.""" - date: IsoformatDateTime | None = Field(alias="Date Acquired") + date: IsoformatDateTime | None = Field(None, alias="Date Acquired") """The date the item was acquired""" - date_opened: IsoformatDateTime | None = Field(alias="Date opened") + date_opened: IsoformatDateTime | None = Field(None, alias="Date opened") """The date the item was opened""" chemical_purity: str | None = Field(alias="Chemical purity") """The chemical purity of this container with regards to the defined substance.""" - full_percent: str | None = Field(alias="Full %") + full_percent: str | None = Field(None, alias="Full %") """The amount of the defined substance remaining in the container, expressed as a percentage.""" name: str | None = Field(alias="Container Name") """The name of the substance in the container.""" - size: str | None = Field(alias="Container Size") + size: str | None = Field(None, alias="Container Size") """The total size of the container, in units of `size_unit`.""" - size_unit: str | None = Field(alias="Unit") + size_unit: str | None = Field(None, alias="Unit") """Units for the 'size' field.""" supplier: str | None = Field(alias="Supplier") """Supplier or manufacturer of the chemical.""" - location: str | None = Field(alias="Location") + location: str | None = Field(None, alias="Location") """The place where the container is located.""" - comment: str | None = Field(alias="Comments") + comment: str | None = Field(None, alias="Comments") """Any additional comments or notes about the container.""" status: StartingMaterialsStatus = Field(default=StartingMaterialsStatus.AVAILABLE) diff --git a/pydatalab/src/pydatalab/models/tasks.py b/pydatalab/src/pydatalab/models/tasks.py index fa64714c3..14728df26 100644 --- a/pydatalab/src/pydatalab/models/tasks.py +++ b/pydatalab/src/pydatalab/models/tasks.py @@ -2,9 +2,9 @@ from enum import Enum from typing import Literal -from pydantic import BaseModel, Field, validator +from pydantic import Field, validator -from pydatalab.models.utils import PyObjectId +from pydatalab.models.utils import BaseModel, PyObjectId class TaskStatus(str, Enum): @@ -20,12 +20,17 @@ class TaskType(str, Enum): class TaskStage(BaseModel): - timestamp: datetime = Field(..., description="When this stage occurred") - message: str = Field(..., description="Description of this processing stage") - level: Literal["info", "warning", "error"] = Field( - default="info", description="Severity level of this stage" - ) - detail: str | None = Field(None, description="Optional detailed information about this stage") + timestamp: datetime + """When this stage occurred""" + + message: str + """Description of this processing stage""" + + level: Literal["info", "warning", "error"] = "info" + """Severity level of this stage""" + + detail: str | None = None + """Optional detailed information about this stage""" class TaskSpec(BaseModel): @@ -33,35 +38,57 @@ class TaskSpec(BaseModel): class ExportTaskSpec(TaskSpec): - collection_id: str | None = Field(None, description="Collection ID being exported") - item_id: str | None = Field(None, description="Item ID being exported") - export_type: str = Field(..., description="Type of export: collection/item/graph") - file_path: str | None = Field(None, description="Path to generated .eln file") - stages: list[TaskStage] = Field( - default_factory=list, description="Timestamped processing stages" - ) + collection_id: str | None = None + """Collection ID being exported""" + + item_id: str | None = None + """Item ID being exported""" + + export_type: str + """Type of export: collection/item/graph""" + + file_path: str | None = None + """Path to generated .eln file""" + + stages: list[TaskStage] = Field(default_factory=list) + """Timestamped processing stages""" class BlockProcessingTaskSpec(TaskSpec): - item_id: str = Field(..., description="Item ID containing the block") - block_id: str = Field(..., description="Block ID being processed") - stages: list[TaskStage] = Field( - default_factory=list, description="Timestamped processing stages" - ) + item_id: str + """Item ID containing the block""" + + block_id: str + """Block ID being processed""" + + stages: list[TaskStage] = Field(default_factory=list) + """Timestamped processing stages""" class Task(BaseModel): - task_id: str = Field(..., description="Unique identifier for the task") - type: TaskType = Field(..., description="Type of task") - status: TaskStatus = Field(default=TaskStatus.PENDING, description="Current status") - creator_id: PyObjectId = Field(..., description="ID of the user who created the task") - created_at: datetime = Field( - default_factory=lambda: datetime.now(tz=timezone.utc), - description="When the task was created", - ) - completed_at: datetime | None = Field(None, description="When completed") - error_message: str | None = Field(None, description="Error message if status is ERROR") - spec: ExportTaskSpec | BlockProcessingTaskSpec = Field(..., description="Task-specific data") + task_id: str + """Unique identifier for the task""" + + type: TaskType + """Type of task""" + + status: TaskStatus = TaskStatus.PENDING + """Current status""" + + creator_id: PyObjectId + """ID of the user who created the task""" + + created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc)) + """When the task was created""" + + completed_at: datetime | None = None + """When completed""" + + error_message: str | None = None + """Error message if status is ERROR""" + + spec: ExportTaskSpec | BlockProcessingTaskSpec + """Task-specific data""" @validator("spec", pre=True, always=True) def validate_spec_type(cls, v, values): diff --git a/pydatalab/src/pydatalab/models/traits.py b/pydatalab/src/pydatalab/models/traits.py index 435b39d34..8cc0f9deb 100644 --- a/pydatalab/src/pydatalab/models/traits.py +++ b/pydatalab/src/pydatalab/models/traits.py @@ -1,10 +1,10 @@ from typing import Any -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import AliasChoices, ConfigDict, Field, field_validator, model_validator from pydatalab.models.blocks import DataBlockResponse from pydatalab.models.people import Group, Person -from pydatalab.models.utils import Constituent, InlineSubstance, PyObjectId +from pydatalab.models.utils import BaseModel, Constituent, InlineSubstance, PyObjectId __all__ = ( "HasOwner", @@ -49,34 +49,36 @@ class HasBlocks(BaseModel): """The order in which to display block data in the UI.""" +class CollectionReference(BaseModel): + """A reference to a collection, used for inlining collection info within other models.""" + + model_config = ConfigDict(extra="allow") + + collection_id: str + """The immutable ID of the collection.""" + + class IsCollectable(BaseModel): """Trait mixin for models that can be added to collections. """ - from pydatalab.models.collections import Collection - - collections: list[Collection] = Field([]) + collections: list[CollectionReference] = Field(default_factory=list) """Inlined info for the collections associated with this item.""" - @model_validator(mode="before") - @classmethod - def add_missing_collection_relationships(cls, values): + @model_validator(mode="after") + def add_missing_collection_relationships(self): from pydatalab.models.relationships import TypedRelationship - if values.get("collections") is not None: - new_ids = {coll.immutable_id for coll in values["collections"]} - existing_collection_relationship_ids = set() - if values.get("relationships") is not None: - existing_collection_relationship_ids = { - relationship.immutable_id - for relationship in values["relationships"] - if relationship.type == "collections" - } - else: - values["relationships"] = [] - - for collection in values.get("collections", []): + if self.collections is not None: + new_ids = {coll.immutable_id for coll in self.collections} + existing_collection_relationship_ids = { + relationship.immutable_id + for relationship in self.relationships + if relationship.type == "collections" + } + + for collection in self.collections: if collection.immutable_id not in existing_collection_relationship_ids: relationship = TypedRelationship( relation=None, @@ -84,20 +86,18 @@ def add_missing_collection_relationships(cls, values): type="collections", description="Is a member of", ) - values["relationships"].append(relationship) + self.relationships.append(relationship) - values["relationships"] = [ + self.relationships = [ d - for d in values.get("relationships", []) + for d in self.relationships if d.type != "collections" or d.immutable_id in new_ids ] - if len([d for d in values.get("relationships", []) if d.type == "collections"]) != len( - values.get("collections", []) - ): + if len([d for d in self.relationships if d.type == "collections"]) != len(self.collections): raise RuntimeError("Relationships and collections mismatch") - return values + return self class HasSynthesisInfo(BaseModel): @@ -109,29 +109,24 @@ class HasSynthesisInfo(BaseModel): synthesis_description: str | None = None """Free-text details of the procedure applied to synthesise the sample""" - @model_validator(mode="before") - @classmethod - def add_missing_synthesis_relationships(cls, values): + @model_validator(mode="after") + def add_missing_synthesis_relationships(self): """Add any missing sample synthesis constituents to parent relationships""" from pydatalab.models.relationships import RelationshipType, TypedRelationship constituents_set = set() - if values.get("synthesis_constituents") is not None: - existing_parent_relationships = {} - if values.get("relationships") is not None: - # Index by refcode *and* item_id so a stored relationship carrying an - # item_id still matches a refcode-enriched constituent (and vice-versa). - existing_parent_relationships = { - identifier: relationship - for relationship in values["relationships"] - if relationship.relation == RelationshipType.PARENT - for identifier in (relationship.refcode, relationship.item_id) - if identifier - } - else: - values["relationships"] = [] - - for constituent in values.get("synthesis_constituents", []): + if self.synthesis_constituents is not None: + # Index by refcode *and* item_id so a stored relationship carrying an + # item_id still matches a refcode-enriched constituent (and vice-versa). + existing_parent_relationships = { + identifier: relationship + for relationship in self.relationships + if relationship.relation == RelationshipType.PARENT + for identifier in (relationship.refcode, relationship.item_id) + if identifier + } + + for constituent in self.synthesis_constituents: # If this is an inline relationship, just skip it if isinstance(constituent.item, InlineSubstance): continue @@ -150,7 +145,7 @@ def add_missing_synthesis_relationships(cls, values): type=constituent.item.type, description="Is a constituent of", ) - values["relationships"].append(relationship) + self.relationships.append(relationship) else: # Back-fill any identifier missing from the stored relationship relationship.refcode = relationship.refcode or refcode @@ -168,9 +163,9 @@ def add_missing_synthesis_relationships(cls, values): # Finally, filter out any parent relationships with item that were removed # from the synthesis constituents - values["relationships"] = [ + self.relationships = [ rel - for rel in values["relationships"] + for rel in self.relationships if not ( rel.refcode not in constituents_set and rel.item_id not in constituents_set @@ -179,14 +174,15 @@ def add_missing_synthesis_relationships(cls, values): ) ] - return values + return self class HasSubstanceInfo(BaseModel): """Trait mixin for models that have substance information.""" chemform: str | None = Field( - example=["Na3P", "Na3P", "LiNiO2@C", "Na3+xP", "LiNi1/3Co0.1Mn0.1O2"], + None, + examples=["Na3P", "Na3P", "LiNiO2@C", "Na3+xP", "LiNi1/3Co0.1Mn0.1O2"], ) """A string representation of the chemical formula or composition associated with this sample. @@ -195,7 +191,9 @@ class HasSubstanceInfo(BaseModel): """ - smiles: str | None = Field(None, aliases=["SMILES", "smiles_representation"]) + smiles: str | None = Field( + None, validation_alias=AliasChoices("smiles_representation", "SMILES") + ) """A SMILES string representation of the chemical structure associated with this sample.""" inchi: str | None = Field(None) @@ -205,25 +203,28 @@ class HasSubstanceInfo(BaseModel): """A unique key derived from the InChI.""" GHS_codes: str | None = Field( + None, alias="GHS H-codes", examples=["H224", "H303, H316, H319"], ) """A string describing any GHS hazard codes associated with this item. See https://pubchem.ncbi.nlm.nih.gov/ghs/ for code definitions.""" - molar_mass: float | None = Field(alias="Molecular Weight") + molar_mass: float | None = Field(None, alias="Molecular Weight", validate_default=True) """Mass per formula unit, in g/mol.""" - CAS: str | None = Field(alias="Substance CAS") + CAS: str | None = Field(None, alias="Substance CAS") """The CAS Registry Number for the substance described by this entry.""" @field_validator("molar_mass", mode="before") @classmethod - def add_molar_mass(cls, v, values): + def add_molar_mass(cls, v, info): + """Fill in the molar mass if not already set and a chemical formula is provided.""" from periodictable import formula - if v is None and values.get("chemform"): + chemform = info.data.get("chemform") + if v is None and chemform: try: - return formula(values.get("chemform")).mass + return formula(chemform).mass except Exception: return None diff --git a/pydatalab/src/pydatalab/models/utils.py b/pydatalab/src/pydatalab/models/utils.py index 58230b184..96df03119 100644 --- a/pydatalab/src/pydatalab/models/utils.py +++ b/pydatalab/src/pydatalab/models/utils.py @@ -4,18 +4,30 @@ from collections.abc import Callable from enum import Enum from functools import partial -from typing import TypeAlias +from typing import Annotated, Any, TypeAlias import pint -from bson.objectid import ObjectId +from bson import ObjectId from pydantic import ( - BaseModel, - ConstrainedStr, + BaseModel as PydanticBaseModel, +) +from pydantic import ( + ConfigDict, Field, - parse_obj_as, - root_validator, - validator, + StringConstraints, + field_validator, + model_validator, ) +from pydantic_core import core_schema + + +class BaseModel(PydanticBaseModel): + """The base model for all pydatalab models, configured to pull field + descriptions from attribute docstrings into the generated JSON schema. + + """ + + model_config = ConfigDict(use_attribute_docstrings=True) class ItemType(str, Enum): @@ -92,43 +104,30 @@ class KnownType(str, Enum): """ -class HumanReadableIdentifier(ConstrainedStr): - """Used to constrain human-readable and URL-safe identifiers for items.""" - - min_length = 1 - max_length = 40 - strip_whitespace = True - to_lower = False - strict = False - regex = IDENTIFIER_REGEX - - def __init__(self, value): - self.value = parse_obj_as(type(self), value) - - def __str__(self): - return self.value - - def __repr__(self): - return self.value - - def __bool__(self): - return bool(self.value) - - -class Refcode(HumanReadableIdentifier): - regex = r"^[a-z]{2,10}:" + IDENTIFIER_REGEX[1:] - """A regex to match refcodes that have a lower-case prefix between 2-10 chars, followed by a colon, - and then the normal rules for an ID (url-safe etc.). - - """ - - @property - def prefix(self): - return self.value.split(":")[0] - - @property - def identifier(self): - return self.value.split(":")[1] +HumanReadableIdentifier = Annotated[ + str, + StringConstraints( + min_length=1, + max_length=40, + strip_whitespace=True, + to_lower=False, + strict=False, + pattern=IDENTIFIER_REGEX, + ), +] +"""Used to constrain human-readable and URL-safe identifiers for items.""" + + +Refcode = Annotated[ + str, + StringConstraints( + min_length=1, + max_length=40, + strip_whitespace=True, + pattern=r"^[a-z]{2,10}:" + IDENTIFIER_REGEX[1:], + ), +] +"""A regex to match refcodes that have a lower-case prefix between 2-10 chars, followed by a colon, and then the normal rules for an ID (url-safe etc.).""" class UserRole(str, Enum): @@ -149,8 +148,12 @@ def __init__(self, dimensions: str): self._dimensions = dimensions @classmethod - def __get_validators__(self): - yield self.validate + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls.validate, + core_schema.str_schema(), + serialization=core_schema.plain_serializer_function_ser_schema(str, when_used="json"), + ) @classmethod def validate(self, v): @@ -159,77 +162,120 @@ def validate(self, v): raise ValueError("Value {v} must have dimensions of mass, not {v.dimensions}") return q - @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="string") - Mass: TypeAlias = PintType("[mass]") # type: ignore # noqa Volume: TypeAlias = PintType("[volume]") # type: ignore # noqa class PyObjectId(ObjectId): - """A wrapper class for a BSON ObjectId that can be used as a Pydantic field type. - - Modified from "Getting started iwth MongoDB and FastAPI": - https://www.mongodb.com/developer/languages/python/python-quickstart-fastapi/. - - """ + @classmethod + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls.validate, + core_schema.union_schema( + [ + core_schema.str_schema(), + core_schema.is_instance_schema(ObjectId), + core_schema.is_instance_schema(cls), + core_schema.dict_schema(), + core_schema.none_schema(), + ] + ), + serialization=core_schema.plain_serializer_function_ser_schema( + lambda x: str(x) if x else None, when_used="json" + ), + ) @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_json_schema__(cls, core_schema, handler): + return { + "type": "string", + "format": "objectid", + } @classmethod def validate(cls, v): - if isinstance(v, dict) and "$oid" in v: - v = v["$oid"] + if v is None: + return None + if isinstance(v, cls): + return v + if isinstance(v, ObjectId): + return cls(v) - if not ObjectId.is_valid(v): - raise ValueError("Invalid ObjectId") + if isinstance(v, dict): + if "$oid" in v: + return cls(ObjectId(v["$oid"])) + elif "_id" in v and isinstance(v["_id"], (str, ObjectId)): + return cls(ObjectId(v["_id"])) + elif len(v) == 1: + first_val = next(iter(v.values())) + if isinstance(first_val, str) and ObjectId.is_valid(first_val): + return cls(ObjectId(first_val)) + raise ValueError(f"Cannot convert dict to ObjectId: {v}") - return ObjectId(v) + if isinstance(v, str): + if not ObjectId.is_valid(v): + raise ValueError("Invalid ObjectId string") + return cls(ObjectId(v)) - @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="string") + raise ValueError(f"Cannot convert {type(v)} to ObjectId: {v}") class IsoformatDateTime(datetime.datetime): """A datetime container that is more flexible than the pydantic default.""" @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls.validate, + core_schema.union_schema( + [ + core_schema.str_schema(), + core_schema.is_instance_schema(datetime.datetime), + ] + ), + serialization=core_schema.plain_serializer_function_ser_schema( + lambda x: x.isoformat() if x else None, when_used="json" + ), + ) + + @classmethod + def __get_pydantic_json_schema__(cls, core_schema, handler): + return { + "type": "string", + "format": "datetime", + } @classmethod def validate(cls, v) -> datetime.datetime | None: """Cast isoformat strings to datetimes and enforce UTC if tzinfo is missing.""" + if v is None: + return None + + if isinstance(v, datetime.datetime): + if v.tzinfo is None: + v = v.replace(tzinfo=datetime.timezone.utc) + return v + if isinstance(v, str): - if v in ["0", " "]: + if v in ["0", " ", ""]: return None v = datetime.datetime.fromisoformat(v) + if v.tzinfo is None: + v = v.replace(tzinfo=datetime.timezone.utc) + return v - if v.tzinfo is None: - v = v.replace(tzinfo=datetime.timezone.utc) - - return v - - -JSON_ENCODERS = { - pint.Quantity: str, - ObjectId: str, -} + raise ValueError(f"Invalid datetime value: {v}") class RefCodeFactory: refcode_generator: Callable @classmethod - def generate(self): + def generate(cls): from pydatalab.config import CONFIG - return f"{CONFIG.IDENTIFIER_PREFIX}:{self.refcode_generator()}" + return f"{CONFIG.IDENTIFIER_PREFIX}:{cls.refcode_generator()}" def random_uppercase(length: int = 6): @@ -257,7 +303,7 @@ def generate_unique_refcode(): class InlineSubstance(BaseModel): name: str - chemform: str | None + chemform: str | None = None class EntryReference(BaseModel): @@ -269,15 +315,19 @@ class EntryReference(BaseModel): """ type: str - name: str | None - immutable_id: PyObjectId | None - item_id: HumanReadableIdentifier | None - refcode: Refcode | None - chemform: str | None + name: str | None = None + immutable_id: PyObjectId | None = None + item_id: HumanReadableIdentifier | None = None + refcode: Refcode | None = None + chemform: str | None = None - @root_validator + @model_validator(mode="before") + @classmethod def check_id_fields(cls, values): """Check that at least one of the possible identifier fields is provided.""" + if not isinstance(values, dict): + return values + id_fields = ("immutable_id", "item_id", "refcode") if all(values.get(f) is None for f in id_fields): @@ -285,8 +335,7 @@ def check_id_fields(cls, values): return values - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class Constituent(BaseModel): @@ -295,23 +344,22 @@ class Constituent(BaseModel): item: EntryReference | InlineSubstance """A reference to item (sample or starting material) entry for the constituent substance.""" - quantity: float | None = Field(..., ge=0) + quantity: float | None = Field(default=None, ge=0) """The amount of the constituent material used to create the sample.""" - unit: str = Field("g") - """The unit symbol for the value provided in `quantity`, default is mass - in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol). - """ + unit: str = "g" + """The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).""" - @validator("item") + @field_validator("item") + @classmethod def check_itemhood(cls, v): """Check that the reference within the constituent is to an item type.""" - if "type" in (v.value for v in ItemType): - raise ValueError(f"`type` must be one of {ItemType!r}") - + if hasattr(v, "type") and v.type not in [item_type.value for item_type in ItemType]: + raise ValueError(f"`type` must be one of {[t.value for t in ItemType]!r}") return v - @validator("item", pre=True, always=True) + @field_validator("item", mode="before") + @classmethod def coerce_reference(cls, v): if isinstance(v, dict): refcode = v.pop("refcode", None) @@ -327,4 +375,18 @@ def coerce_reference(cls, v): if not name: raise ValueError("Inline substance must have a name!") return InlineSubstance(name=name, chemform=chemform) + elif hasattr(v, "model_dump"): + item_id = getattr(v, "item_id", None) + refcode = getattr(v, "refcode", None) + item_type = getattr(v, "type", None) + name = getattr(v, "name", None) + chemform = getattr(v, "chemform", None) + + if item_id or refcode: + return EntryReference( + item_id=item_id, refcode=refcode, type=item_type, name=name, chemform=chemform + ) + else: + return InlineSubstance(name=name or str(v), chemform=chemform) + return v diff --git a/pydatalab/src/pydatalab/models/versions.py b/pydatalab/src/pydatalab/models/versions.py index f62914df5..6efce3a2c 100644 --- a/pydatalab/src/pydatalab/models/versions.py +++ b/pydatalab/src/pydatalab/models/versions.py @@ -3,9 +3,9 @@ from datetime import datetime from enum import Enum -from pydantic import BaseModel, Field, validator +from pydantic import Field, validator -from pydatalab.models.utils import PyObjectId, Refcode +from pydatalab.models.utils import BaseModel, PyObjectId, Refcode class VersionAction(str, Enum): @@ -26,34 +26,35 @@ class ItemVersion(BaseModel): view history and restore previous states. """ - refcode: Refcode = Field(..., description="The refcode of the item this version belongs to") - version: int = Field(..., ge=1, description="Sequential version number (1-indexed)") - timestamp: datetime = Field( - ..., description="When this version was created (ISO format with timezone)" - ) - action: VersionAction = Field( - ..., - description="The action that triggered this version: 'created' (item creation), " - "'manual_save' (user save), 'auto_save' (system save), or 'restored' (version restore)", - ) - user_id: PyObjectId | None = Field( - None, description="User's ObjectId for efficient querying and indexing" - ) - creator: dict | None = Field( - None, description="Inlined information about the user who created this version (e.g., name)" - ) - datalab_version: str = Field( - ..., description="Version of datalab-server that created this snapshot" - ) - data: dict = Field(..., description="Complete snapshot of the item data at this version") - restored_from_version: PyObjectId | None = Field( - None, - description="ObjectId of the version that was restored from (only present if action='restored')", - ) - user_agent: str | None = Field( - None, - description="User agent string of the client that triggered this version. Will only be stored if it matches a known value from the datalab ecosystem.", - ) + refcode: Refcode + """The refcode of the item this version belongs to""" + + version: int = Field(ge=1) + """Sequential version number (1-indexed)""" + + timestamp: datetime + """When this version was created (ISO format with timezone)""" + + action: VersionAction + """The action that triggered this version: 'created' (item creation), 'manual_save' (user save), 'auto_save' (system save), or 'restored' (version restore)""" + + user_id: PyObjectId | None = None + """User's ObjectId for efficient querying and indexing""" + + creator: dict | None = None + """Inlined information about the user who created this version (e.g., name)""" + + datalab_version: str + """Version of datalab-server that created this snapshot""" + + data: dict + """Complete snapshot of the item data at this version""" + + restored_from_version: PyObjectId | None = None + """ObjectId of the version that was restored from (only present if action='restored')""" + + user_agent: str | None = None + """User agent string of the client that triggered this version. Will only be stored if it matches a known value from the datalab ecosystem.""" @validator("restored_from_version") def validate_restored_from_version(cls, v, values): @@ -75,10 +76,11 @@ class VersionCounter(BaseModel): It ensures atomic increment of version numbers to prevent race conditions. """ - refcode: Refcode = Field(..., description="The refcode this counter belongs to") - counter: int = Field( - 1, ge=1, description="Current version counter value (1-indexed, matches version numbers)" - ) + refcode: Refcode + """The refcode this counter belongs to""" + + counter: int = Field(1, ge=1) + """Current version counter value (1-indexed, matches version numbers)""" class Config: extra = "ignore" # Allow MongoDB's _id field and other internal fields @@ -87,7 +89,8 @@ class Config: class RestoreVersionRequest(BaseModel): """Request body for restoring a version.""" - version_id: str = Field(..., description="ObjectId string of the version to restore to") + version_id: str + """ObjectId string of the version to restore to""" @validator("version_id") def validate_version_id_format(cls, v): @@ -107,8 +110,11 @@ class Config: class CompareVersionsQuery(BaseModel): """Query parameters for comparing two versions.""" - v1: str = Field(..., description="ObjectId string of the first version") - v2: str = Field(..., description="ObjectId string of the second version") + v1: str + """ObjectId string of the first version""" + + v2: str + """ObjectId string of the second version""" @validator("v1", "v2") def validate_version_ids(cls, v): diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index 6ba1387be..be3868b45 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -33,7 +33,7 @@ def load_plugin_schema(): need them. A JSON Schema generated from the returned model is emitted to `pydatalab/schemas/plugin_config.json` by `invoke dev.generate-schemas`. """ - from pydantic import BaseModel, root_validator + from pydantic import BaseModel, model_validator class UvSource(BaseModel): """A single entry under `[tool.uv.sources]` in plugins.toml.""" @@ -48,7 +48,8 @@ class UvSource(BaseModel): class Config: extra = "forbid" - @root_validator + @model_validator(mode="before") + @classmethod def _exactly_one_source(cls, values): has_git = values.get("git") is not None has_path = values.get("path") is not None @@ -79,7 +80,8 @@ class PluginConfigModel(BaseModel): class Config: extra = "forbid" - @root_validator + @model_validator(mode="before") + @classmethod def _sources_must_match_dependencies(cls, values): deps = {d.split("[")[0].strip() for d in values.get("dependencies", [])} sources = values.get("tool", ToolSection()).uv.sources diff --git a/pydatalab/tests/test_models.py b/pydatalab/tests/test_models.py index 1e59bfbce..c2ba5959b 100644 --- a/pydatalab/tests/test_models.py +++ b/pydatalab/tests/test_models.py @@ -84,6 +84,27 @@ def test_generate_schemas(model): assert model.schema() +def test_attribute_docstrings_in_schema(): + """Field docstrings should be pulled into the JSON schema as descriptions. + + Relies on ``use_attribute_docstrings`` being set on the shared ``BaseModel`` + in ``pydatalab.models.utils``; covers fields defined across several different + trait mixins to confirm the config propagates through inheritance. + """ + properties = Sample.model_json_schema(by_alias=False)["properties"] + + expected = { + "creators": "Inlined info for the people associated with this item.", # HasOwner + "blocks_obj": "A mapping from block ID to block data.", # HasBlocks + "revision": "The revision number of the entry.", # HasRevisionControl + "collections": "Inlined info for the collections associated with this item.", # IsCollectable + "name": "An optional human-readable/usable name for the entry.", # Item + } + + for field, description in expected.items(): + assert properties[field]["description"] == description + + def test_relationship_with_custom_type(): """Test that a relationship with a custom type can be created.""" relationship = TypedRelationship( From 66d24ead1dd75bbe355c74d69da1ec1ad6a393fb Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Tue, 11 Nov 2025 01:08:22 +0000 Subject: [PATCH 06/31] Load FTS fields for MongoDB from model schemas directly Rework ITEMS_FTS_FIELDS lookup --- pydatalab/src/pydatalab/mongo.py | 60 ++++++++++++++------ pydatalab/src/pydatalab/routes/v0_1/items.py | 4 +- pydatalab/tests/server/test_items.py | 4 +- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/pydatalab/src/pydatalab/mongo.py b/pydatalab/src/pydatalab/mongo.py index b69b51451..a8b7334af 100644 --- a/pydatalab/src/pydatalab/mongo.py +++ b/pydatalab/src/pydatalab/mongo.py @@ -10,7 +10,6 @@ from pymongo.errors import ConnectionFailure from pydatalab.logger import LOGGER -from pydatalab.models import ITEM_MODELS __all__ = ( "flask_mongo", @@ -34,20 +33,40 @@ """One-liner that pulls all non-semantic string fields out of all item models implemented for this server. """ -ITEMS_FTS_FIELDS: set[str] = set().union( - *( - { - f - for f, p in model.schema(by_alias=False)["properties"].items() - if ( - p.get("type") == "string" - and p.get("format") not in ("date-time", "uuid") - and f != "type" - ) - } - for model in ITEM_MODELS.values() - ) -) + + +@lru_cache(maxsize=1) +def get_items_fts_fields() -> set[str]: + """Get all string fields from item models for full-text search.""" + from pydatalab.models import ITEM_MODELS + + fields = set() + + for model_name, model in ITEM_MODELS.items(): + schema = model.model_json_schema(by_alias=False) + + model_fields = set() + for f, p in schema.get("properties", {}).items(): + if f == "type": + continue + + if p.get("type") == "string" and p.get("format") not in ("date-time", "uuid"): + model_fields.add(f) + elif "anyOf" in p: + for option in p["anyOf"]: + if option.get("type") == "string" and option.get("format") not in ( + "date-time", + "uuid", + ): + model_fields.add(f) + break + + fields.update(model_fields) + + return fields + + +ITEMS_FTS_FIELDS: set[str] = set() USERS_FTS_FIELDS: set[str] = {"identities.name", "display_name", "contact_email"} """Fields to search for users.""" @@ -148,7 +167,7 @@ def insert_pydantic_model_fork_safe(model: BaseModel, collection: str) -> str: """Inserts a Pydantic model into chosen collection, returning the inserted ID.""" return ( get_database()[collection] - .insert_one(model.dict(by_alias=True, exclude_none=True)) + .insert_one(model.model_dump(by_alias=True, exclude_none=True)) .inserted_id ) @@ -167,7 +186,6 @@ def _get_active_mongo_client(timeoutMS: int = 1000) -> pymongo.MongoClient: """ from pydatalab.config import CONFIG - from pydatalab.logger import LOGGER try: client = pymongo.MongoClient( @@ -235,6 +253,14 @@ def create_default_indices( """ + global ITEMS_FTS_FIELDS + + if not ITEMS_FTS_FIELDS: + ITEMS_FTS_FIELDS = get_items_fts_fields() + + if not ITEMS_FTS_FIELDS: + raise ValueError("Cannot create text indices: no fields available for full-text search") + if client is None: client = _get_active_mongo_client() db = client.get_database() diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 48b662cc6..1c4db0314 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -25,7 +25,7 @@ RestoreVersionRequest, VersionAction, ) -from pydatalab.mongo import ITEMS_FTS_FIELDS, build_search_pipeline, flask_mongo +from pydatalab.mongo import build_search_pipeline, flask_mongo, get_items_fts_fields from pydatalab.permissions import ( PUBLIC_USER_ID, access_token_or_active_users, @@ -526,7 +526,7 @@ def search_items(): return jsonify({"status": "error", "message": "No query provided."}), 400 permissions = get_default_permissions(user_only=False) - pipeline = build_search_pipeline(query, ITEMS_FTS_FIELDS, permissions) + pipeline = build_search_pipeline(query, get_items_fts_fields(), permissions) if types is not None: if pipeline and "$match" in pipeline[0]: diff --git a/pydatalab/tests/server/test_items.py b/pydatalab/tests/server/test_items.py index e58bafd2d..9c9ad1598 100644 --- a/pydatalab/tests/server/test_items.py +++ b/pydatalab/tests/server/test_items.py @@ -22,7 +22,7 @@ def test_single_item_endpoints(client, inserted_default_items): def test_fts_fields(): """Test non-exhaustively that certain fields make it into the fts index.""" - from pydatalab.mongo import ITEMS_FTS_FIELDS + from pydatalab.mongo import get_items_fts_fields fields = ("item_id", "name", "description", "refcode", "synthesis_description", "supplier") - assert all(field in ITEMS_FTS_FIELDS for field in fields) + assert all(field in get_items_fts_fields() for field in fields) From dcae303db4f2be4ee066ebec6d109fe3e4e49e67 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 11 Nov 2025 01:09:59 +0000 Subject: [PATCH 07/31] Misc pydantic v2 bumps around code --- pydatalab/src/pydatalab/backups.py | 2 +- pydatalab/src/pydatalab/errors.py | 1 + pydatalab/src/pydatalab/file_utils.py | 10 +++++----- pydatalab/src/pydatalab/main.py | 4 ++-- pydatalab/tasks.py | 23 +++++------------------ 5 files changed, 14 insertions(+), 26 deletions(-) diff --git a/pydatalab/src/pydatalab/backups.py b/pydatalab/src/pydatalab/backups.py index bea81e84d..f058a62a6 100644 --- a/pydatalab/src/pydatalab/backups.py +++ b/pydatalab/src/pydatalab/backups.py @@ -92,7 +92,7 @@ def take_snapshot(snapshot_path: Path, encrypt: bool = False) -> None: LOGGER.debug("Dumping server config.") with tempfile.TemporaryDirectory() as temp_dir: with open(tmp_config := Path(temp_dir) / "config.json", "w") as f: - data = CONFIG.json(indent=2, exclude_unset=True) + data = CONFIG.model_dump_json(indent=2, exclude_unset=True) f.write(data) tar.add( diff --git a/pydatalab/src/pydatalab/errors.py b/pydatalab/src/pydatalab/errors.py index 5868d9124..0996a54ad 100644 --- a/pydatalab/src/pydatalab/errors.py +++ b/pydatalab/src/pydatalab/errors.py @@ -70,6 +70,7 @@ def handle_pydantic_validation_error(exc: ValidationError) -> tuple[Response, in These always come from malformed data, so should not necessarily trigger the Flask debugger. """ + LOGGER.critical("Pydantic validation error: %s", exc, exc_info=True) response = { "title": exc.__class__.__name__, "message": str(exc.args[:]) if exc.args else "", diff --git a/pydatalab/src/pydatalab/file_utils.py b/pydatalab/src/pydatalab/file_utils.py index 33bb8958b..5bda4eabc 100644 --- a/pydatalab/src/pydatalab/file_utils.py +++ b/pydatalab/src/pydatalab/file_utils.py @@ -336,7 +336,7 @@ def get_file_info_by_id(file_id: str | ObjectId, update_if_live: bool = True) -> if update_if_live and file_info.is_live: file_info = _check_and_sync_file(file_info, file_id) - return file_info.dict() + return file_info.model_dump() def update_uploaded_file(file: FileStorage, file_id: ObjectId, size_bytes: int | None = None): @@ -405,7 +405,7 @@ def update_uploaded_file(file: FileStorage, file_id: ObjectId, size_bytes: int | file.save(updated_file_entry.location) - ret = updated_file_entry.dict() + ret = updated_file_entry.model_dump() ret.update({"_id": file_id}) return ret @@ -498,7 +498,7 @@ def save_uploaded_file( f"Cannot store file: insufficient space available on disk (required: {size_bytes // 1024**3} GB). Please contact your datalab administrator." ) file_collection = client.get_database().files - result = file_collection.insert_one(new_file_document.dict(), session=session) + result = file_collection.insert_one(new_file_document.model_dump(), session=session) if not result.acknowledged: raise RuntimeError( f"db operation failed when trying to insert new file. Result: {result}" @@ -534,7 +534,7 @@ def save_uploaded_file( f"db operation failed when trying to insert new file ObjectId into sample: {item_id}" ) - ret = updated_file_entry.dict() + ret = updated_file_entry.model_dump() ret.update({"_id": inserted_id}) return ret @@ -610,7 +610,7 @@ def add_file_from_remote_directory( creator_ids=creator_ids if creator_ids is not None else [], ) - result = file_collection.insert_one(new_file_document.dict()) + result = file_collection.insert_one(new_file_document.model_dump()) if not result.acknowledged: raise OSError(f"db operation failed when trying to insert new file. Result: {result}") diff --git a/pydatalab/src/pydatalab/main.py b/pydatalab/src/pydatalab/main.py index f9d0d7b3d..072087b54 100644 --- a/pydatalab/src/pydatalab/main.py +++ b/pydatalab/src/pydatalab/main.py @@ -43,7 +43,7 @@ def create_app( CONFIG.update(config_override) app.config.from_prefixed_env() - app.config.update(CONFIG.dict()) + app.config.update(CONFIG.model_dump()) # This value will still be overwritten by any dotenv values app.config["MAIL_DEBUG"] = app.config.get("MAIL_DEBUG") or CONFIG.TESTING @@ -51,7 +51,7 @@ def create_app( # percolate datalab mail settings up to the `MAIL_` env vars/app config # for use by Flask Mail if CONFIG.EMAIL_AUTH_SMTP_SETTINGS is not None: - mail_settings = CONFIG.EMAIL_AUTH_SMTP_SETTINGS.dict() + mail_settings = CONFIG.EMAIL_AUTH_SMTP_SETTINGS.model_dump() for key in mail_settings: app.config[key] = mail_settings[key] diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index be3868b45..aeedcd452 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -5,13 +5,11 @@ import shutil import subprocess import time -import typing import tomlkit from invoke import Collection, task -if typing.TYPE_CHECKING: - from pydatalab.models.utils import UserRole +from pydatalab.models.utils import UserRole ns = Collection() dev = Collection("dev") @@ -45,9 +43,6 @@ class UvSource(BaseModel): path: str | None = None editable: bool | None = None - class Config: - extra = "forbid" - @model_validator(mode="before") @classmethod def _exactly_one_source(cls, values): @@ -62,24 +57,15 @@ def _exactly_one_source(cls, values): class UvSection(BaseModel): sources: dict[str, UvSource] = {} - class Config: - extra = "forbid" - class ToolSection(BaseModel): uv: UvSection = UvSection() - class Config: - extra = "forbid" - class PluginConfigModel(BaseModel): """The schema for the top-level plugins.toml file.""" dependencies: list[str] = [] tool: ToolSection = ToolSection() - class Config: - extra = "forbid" - @model_validator(mode="before") @classmethod def _sources_must_match_dependencies(cls, values): @@ -117,7 +103,7 @@ def generate_schemas(_): schemas_path = pathlib.Path(__file__).parent / "schemas" for model in ITEM_MODELS.values(): - schema = model.schema(by_alias=False) + schema = model.model_json_schema(by_alias=False) with open(schemas_path / f"{model.__name__.lower()}.json", "w") as f: json.dump(schema, f, indent=2) @@ -429,11 +415,12 @@ def add_missing_refcodes(_): def _check_id(id=None, base_url=None, api_key=None): - from pydatalab.logger import setup_log - """Checks the given item ID served at the base URL and logs the result.""" + import requests + from pydatalab.logger import setup_log + log = setup_log("check_item_validity") response = requests.get( f"{base_url}/get-item-data/{id}", headers={"DATALAB-API-KEY": api_key}, timeout=30 From 796cac3028e6bd70d5d7be11a444cd8c5a1f8bec Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Tue, 11 Nov 2025 01:12:51 +0000 Subject: [PATCH 08/31] Update various routes for pydantic v2 Fixes for routes Chatblock patch; set default values in block More updates for blocks --- pydatalab/src/pydatalab/apps/chat/blocks.py | 15 +-- pydatalab/src/pydatalab/apps/chat/models.py | 2 +- pydatalab/src/pydatalab/apps/nmr/models.py | 5 +- pydatalab/src/pydatalab/apps/xrd/blocks.py | 2 +- pydatalab/src/pydatalab/blocks/base.py | 6 +- pydatalab/src/pydatalab/routes/v0_1/auth.py | 13 +-- .../src/pydatalab/routes/v0_1/collections.py | 10 +- pydatalab/src/pydatalab/routes/v0_1/graphs.py | 4 +- pydatalab/src/pydatalab/routes/v0_1/info.py | 108 +++++++++--------- pydatalab/src/pydatalab/routes/v0_1/items.py | 72 +++++++++--- .../src/pydatalab/routes/v0_1/remotes.py | 6 +- pydatalab/src/pydatalab/routes/v0_1/users.py | 35 +++--- 12 files changed, 161 insertions(+), 117 deletions(-) diff --git a/pydatalab/src/pydatalab/apps/chat/blocks.py b/pydatalab/src/pydatalab/apps/chat/blocks.py index c2bce7df3..b95294faa 100644 --- a/pydatalab/src/pydatalab/apps/chat/blocks.py +++ b/pydatalab/src/pydatalab/apps/chat/blocks.py @@ -2,7 +2,7 @@ import warnings from langchain_core.messages import AIMessage, HumanMessage, SystemMessage -from pydantic import Field, validator +from pydantic import Field, field_validator from pydatalab.blocks.base import DataBlock from pydatalab.models import ITEM_MODELS @@ -16,15 +16,16 @@ class ChatBlockResponse(DataBlockResponse): messages: list[dict] = Field(default_factory=list) - prompt: str | None + prompt: str | None = None model: str available_models: dict[str, ModelCard] | None = Field( - datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, validate_default=True, datalab_exclude_from_db=True, datalab_exclude_from_load=True ) - token_count: int | None + token_count: int | None = None temperature: float - @validator("available_models", pre=True, always=True) + @field_validator("available_models", mode="before") + @classmethod def set_available_models(cls, _): return AVAILABLE_MODELS @@ -204,8 +205,8 @@ def _prepare_item_json_for_chat(self, item_id: str, item_data: dict | None = Non item_model.blocks_obj = { k: block for k, block in item_model.blocks_obj.items() if block.blocktype != "chat" } - item_data = item_model.dict(exclude_none=True, exclude_unset=True) - item_data["type"] = item_model.type + item_info = item_model.model_dump(exclude_none=True, exclude_unset=True) + item_info["type"] = item_model.type # strip irrelevant or large fields item_filenames = { diff --git a/pydatalab/src/pydatalab/apps/chat/models.py b/pydatalab/src/pydatalab/apps/chat/models.py index 03efc7246..756ca1970 100644 --- a/pydatalab/src/pydatalab/apps/chat/models.py +++ b/pydatalab/src/pydatalab/apps/chat/models.py @@ -9,7 +9,7 @@ class ModelCard(BaseModel): context_window: int input_cost_usd_per_MTok: float output_cost_usd_per_MTok: float - chat_client: type[BaseChatModel] | None = Field(exclude=True) + chat_client: type[BaseChatModel] | None = Field(None, exclude=True) __all__ = ("AVAILABLE_MODELS", "ModelCard") diff --git a/pydatalab/src/pydatalab/apps/nmr/models.py b/pydatalab/src/pydatalab/apps/nmr/models.py index 269684c26..73a981b4a 100644 --- a/pydatalab/src/pydatalab/apps/nmr/models.py +++ b/pydatalab/src/pydatalab/apps/nmr/models.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from pydatalab.models.blocks import DataBlockResponse @@ -46,8 +46,7 @@ class NMRMetadata(BaseModel): title: str | None = None """Title or description of the NMR experiment.""" - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") class NMRModel(DataBlockResponse): diff --git a/pydatalab/src/pydatalab/apps/xrd/blocks.py b/pydatalab/src/pydatalab/apps/xrd/blocks.py index cf7f327a3..6393464f4 100644 --- a/pydatalab/src/pydatalab/apps/xrd/blocks.py +++ b/pydatalab/src/pydatalab/apps/xrd/blocks.py @@ -264,7 +264,7 @@ def generate_xrd_plot(self, filenames: list[str | Path] | None = None) -> None: pattern_df.attrs["wavelength"] = ( f"{self.data.get('wavelength', self.defaults['wavelength'])} Å" ) - peak_information[str(f["immutable_id"])] = PeakInformation(**peak_data).dict() + peak_information[str(f["immutable_id"])] = PeakInformation(**peak_data).model_dump() if len(all_files) > 1: pattern_df["normalized intensity (staggered)"] += ind pattern_dfs.append(pattern_df) diff --git a/pydatalab/src/pydatalab/blocks/base.py b/pydatalab/src/pydatalab/blocks/base.py index 04b475fd7..2e696d70e 100644 --- a/pydatalab/src/pydatalab/blocks/base.py +++ b/pydatalab/src/pydatalab/blocks/base.py @@ -196,7 +196,7 @@ def to_db(self) -> dict: for (f, s) in self.block_db_model.schema()["properties"].items() if s.get("datalab_exclude_from_db") } - return self.block_db_model(**self.data).dict( + return self.block_db_model(**self.data).model_dump( exclude=exclude_fields, exclude_unset=True, exclude_none=True, @@ -248,7 +248,7 @@ def to_web(self) -> dict[str, Any]: else: self.data.pop("warnings", None) - return self.block_db_model(**self.data).dict(exclude_unset=True, exclude_none=True) + return self.block_db_model(**self.data).model_dump(exclude_unset=True, exclude_none=True) def process_events(self, events: list[dict] | dict): """Handle any supported events passed to the block.""" @@ -343,5 +343,5 @@ def update_from_web(self, data: dict): if s.get("datalab_exclude_from_load") } [data.pop(f, None) for f in exclude_fields] - self.data.update(self.block_db_model(**data).dict()) + self.data.update(self.block_db_model(**data).model_dump(exclude_unset=True)) return self diff --git a/pydatalab/src/pydatalab/routes/v0_1/auth.py b/pydatalab/src/pydatalab/routes/v0_1/auth.py index c303206bb..ef5f2d228 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/auth.py +++ b/pydatalab/src/pydatalab/routes/v0_1/auth.py @@ -505,7 +505,7 @@ def find_user_with_identity( if verify and not person.identities[identity_index].verified: flask_mongo.db.users.update_one( - {"_id": person.immutable_id}, + {"_id": ObjectId(person.immutable_id)}, {"$set": {f"identities.{identity_index}.verified": True}}, ) @@ -557,7 +557,7 @@ def attach_identity_to_user( RuntimeError: If the update was unsuccessful. """ - update = {"$push": {"identities": identity.dict()}} + update = {"$push": {"identities": identity.model_dump()}} if use_display_name and identity and identity.display_name: update["$set"] = {"display_name": identity.display_name} @@ -612,12 +612,11 @@ def attach_identity_to_user( identity, use_display_name=True, account_status=account_status ) LOGGER.debug("Inserting new user model %s into database", user) - insert_pydantic_model_fork_safe(user, "users") - user_model = get_by_id(str(user.immutable_id)) + inserted_id = insert_pydantic_model_fork_safe(user, "users") + user = get_by_id(inserted_id) if user is None: raise RuntimeError("Failed to insert user into database") - wrapped_login_user(user_model) # Send email notification to admins _send_admin_email_notification(user) @@ -626,7 +625,7 @@ def attach_identity_to_user( # Log the user into the session with this identity if user is not None: - wrapped_login_user(get_by_id(str(user.immutable_id))) + wrapped_login_user(user) def _validate_magic_link_request(email: str, referrer: str) -> None: @@ -1056,7 +1055,7 @@ def redirect_to_ui(blueprint, token): # pylint: disable=unused-argument def get_authenticated_user_info(): """Returns metadata associated with the currently authenticated user.""" if current_user.is_authenticated: - current_user_response = json.loads(current_user.person.json()) + current_user_response = json.loads(current_user.person.model_dump_json()) current_user_response["role"] = current_user.role.value return jsonify(current_user_response), 200 else: diff --git a/pydatalab/src/pydatalab/routes/v0_1/collections.py b/pydatalab/src/pydatalab/routes/v0_1/collections.py index a8afffc3d..fd3f6dc6f 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/collections.py +++ b/pydatalab/src/pydatalab/routes/v0_1/collections.py @@ -87,7 +87,7 @@ def get_collection(collection_id): { "status": "success", "collection_id": collection_id, - "data": json.loads(collection.json(exclude_unset=True)), + "data": json.loads(collection.model_dump_json(exclude_unset=True)), "child_items": list(samples), } ) @@ -168,7 +168,7 @@ def create_collection(): ) result: InsertOneResult = flask_mongo.db.collections.insert_one( - data_model.dict(exclude={"creators", "groups"}) + data_model.model_dump(exclude={"creators", "groups"}) ) if not result.acknowledged: return ( @@ -218,7 +218,7 @@ def create_collection(): response = { "status": "success", - "data": json.loads(data_model.json()), + "data": json.loads(data_model.model_dump_json()), } if errors: @@ -278,7 +278,7 @@ def save_collection(collection_id): collection.update(updated_data) try: - collection = Collection(**collection).dict() + collection = Collection(**collection).model_dump() except ValidationError as exc: return ( jsonify( @@ -534,7 +534,7 @@ def search_collections(): ) cursor = [ - json.loads(Collection(**doc).json(exclude_unset=True)) + json.loads(Collection(**doc).model_dump_json(exclude_unset=True)) for doc in flask_mongo.db.collections.aggregate(pipeline) ] diff --git a/pydatalab/src/pydatalab/routes/v0_1/graphs.py b/pydatalab/src/pydatalab/routes/v0_1/graphs.py index 1299bfd09..72fb68317 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/graphs.py +++ b/pydatalab/src/pydatalab/routes/v0_1/graphs.py @@ -217,7 +217,9 @@ def add_related_items(current_item_id: str, current_depth: int): { "data": { "id": document["item_id"], - "name": document["name"] if document["name"] else document["item_id"], + "name": document.get("name") + if document.get("name") + else document["item_id"], "type": document["type"], "special": document["item_id"] == item_id, } diff --git a/pydatalab/src/pydatalab/routes/v0_1/info.py b/pydatalab/src/pydatalab/routes/v0_1/info.py index a2212ea87..40e4fc6a5 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/info.py +++ b/pydatalab/src/pydatalab/routes/v0_1/info.py @@ -7,14 +7,20 @@ from functools import lru_cache from flask import Blueprint, jsonify, request -from pydantic import AnyUrl, BaseModel, Field, validator +from pydantic import ( + AnyUrl, + BaseModel, + ConfigDict, + Field, + field_validator, + model_validator, +) from pydatalab import __version__ from pydatalab.apps import BLOCK_TYPES from pydatalab.config import CONFIG from pydatalab.feature_flags import FEATURE_FLAGS, FeatureFlags -from pydatalab.models import Collection, Person -from pydatalab.models.items import Item +from pydatalab.models import ITEM_SCHEMAS, Person from pydatalab.mongo import flask_mongo from pydatalab.permissions import active_users_or_get_only @@ -24,8 +30,7 @@ class Attributes(BaseModel): - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class Meta(BaseModel): @@ -39,9 +44,7 @@ class Meta(BaseModel): class Links(BaseModel): self: AnyUrl - - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class Data(BaseModel): @@ -53,39 +56,58 @@ class Data(BaseModel): class JSONAPIResponse(BaseModel): data: Data | list[Data] meta: Meta - links: Links | None + links: Links | None = None class MetaPerson(BaseModel): - dislay_name: str | None + display_name: str | None = None contact_email: str class Info(Attributes, Meta): - maintainer: MetaPerson | None - issue_tracker: AnyUrl | None - homepage: AnyUrl | None - source_repository: AnyUrl | None + maintainer: MetaPerson | None = None + issue_tracker: AnyUrl | None = None + homepage: AnyUrl | None = None + source_repository: AnyUrl | None = None identifier_prefix: str features: FeatureFlags = FEATURE_FLAGS max_upload_bytes: int - @validator("maintainer") + @field_validator("maintainer", mode="before") + @classmethod def strip_maintainer_fields(cls, v): if isinstance(v, Person): return MetaPerson(contact_email=v.contact_email, display_name=v.display_name) return v + @model_validator(mode="after") + def ensure_features_serialization(self): + """Ensure features are properly serialized for frontend consumption.""" + if hasattr(self.features, "model_dump"): + features_dict = self.features.model_dump() + else: + features_dict = self.features + + if not isinstance(self.features, FeatureFlags): + self.features = FeatureFlags(**features_dict) + return self + @lru_cache(maxsize=1) def _get_deployment_metadata_once() -> dict: identifier_prefix = CONFIG.IDENTIFIER_PREFIX metadata = ( - CONFIG.DEPLOYMENT_METADATA.dict(exclude_none=True) if CONFIG.DEPLOYMENT_METADATA else {} + CONFIG.DEPLOYMENT_METADATA.model_dump(exclude_none=True) + if CONFIG.DEPLOYMENT_METADATA + else {} + ) + metadata.update( + { + "identifier_prefix": identifier_prefix, + "max_upload_bytes": CONFIG.MAX_CONTENT_LENGTH, + "features": FEATURE_FLAGS, + } ) - metadata.update({"identifier_prefix": identifier_prefix}) - metadata.update({"max_upload_bytes": CONFIG.MAX_CONTENT_LENGTH}) - return metadata @@ -95,18 +117,15 @@ def get_info(): versions, features and so on. """ - metadata = _get_deployment_metadata_once() + + response_data = JSONAPIResponse( + data=Data(id="/", type="info", attributes=Info(**_get_deployment_metadata_once())), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), + links=Links(self=request.url), + ) return ( - jsonify( - json.loads( - JSONAPIResponse( - data=Data(id="/", type="info", attributes=Info(**metadata)), - meta=Meta(query=request.query_string), - links=Links(self=request.url), - ).json() - ) - ), + jsonify(json.loads(response_data.model_dump_json())), 200, ) @@ -149,30 +168,11 @@ def list_block_types(): for block_type, block in BLOCK_TYPES.items() ], meta=Meta(query=request.query_string), - ).json() + ).model_dump_json() ) ) -def get_all_items_models(): - return Item.__subclasses__() - - -def generate_schemas(): - schemas: dict[str, dict] = {} - - for model_class in get_all_items_models() + [Collection]: - model_type = model_class.schema()["properties"]["type"]["default"] - - schemas[model_type] = model_class.schema(by_alias=False) - - return schemas - - -# Generate once on import -SCHEMAS = generate_schemas() - - @INFO.route("/info/types", methods=["GET"]) def list_supported_types(): """Returns a list of supported schemas.""" @@ -190,10 +190,10 @@ def list_supported_types(): "schema": schema, }, ) - for item_type, schema in SCHEMAS.items() + for item_type, schema in ITEM_SCHEMAS.items() ], meta=Meta(query=request.query_string), - ).json() + ).model_dump_json() ) ) @@ -201,7 +201,7 @@ def list_supported_types(): @INFO.route("/info/types/", methods=["GET"]) def get_schema_type(item_type): """Returns the schema of the given type.""" - if item_type not in SCHEMAS: + if item_type not in ITEM_SCHEMAS: return jsonify( {"status": "error", "detail": f"Item type {item_type} not found for this deployment"} ), 404 @@ -215,11 +215,11 @@ def get_schema_type(item_type): attributes={ "version": __version__, "api_version": __api_version__, - "schema": SCHEMAS[item_type], + "schema": ITEM_SCHEMAS[item_type], }, ), meta=Meta(query=request.query_string), - ).json() + ).model_dump_json() ) ) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 1c4db0314..bf68bef9f 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -514,8 +514,8 @@ def search_items(): Returns: response list of dictionaries containing the matching items in order of descending match score. - """ + """ query = request.args.get("query", type=str) nresults = request.args.get("nresults", default=100, type=int) types = request.args.get("types", default=None) @@ -672,7 +672,7 @@ def _create_sample( elif CONFIG.TESTING and not current_user.is_authenticated: # Set fake ID to ObjectId("000000000000000000000000") so a dummy user can be created # locally for testing creator UI elements - new_sample["creator_ids"] = [PUBLIC_USER_ID] + new_sample["creator_ids"] = [str(PUBLIC_USER_ID)] new_sample["creators"] = [ { "display_name": "Public testing user", @@ -716,7 +716,6 @@ def _create_sample( try: # Check on relationship fields and prefill new_sample = entry_reference_lookup(new_sample) - data_model: Item = model(**new_sample) except ValidationError as error: @@ -730,7 +729,9 @@ def _create_sample( # the `Entry` model. try: result = flask_mongo.db.items.insert_one( - data_model.dict(exclude={"creators", "collections", "groups"}) + data_model.model_dump( + exclude={"creators", "collections", "groups"}, exclude_none=True, by_alias=True + ) ) except DuplicateKeyError as error: raise Conflict(f"Duplicate key error: {str(error)}.") @@ -765,7 +766,7 @@ def _create_sample( data = { "status": "success", "item_id": data_model.item_id, - "sample_list_entry": data_model.dict(), + "sample_list_entry": data_model.model_dump(), } return (data, 201) # 201 Created @@ -1136,7 +1137,6 @@ def get_item_data( elif refcode: if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" - match = {"refcode": refcode} else: raise BadRequest("No item_id or refcode provided.") @@ -1161,6 +1161,7 @@ def get_item_data( try: doc = list(cursor)[0] + except IndexError: doc = None @@ -1236,7 +1237,7 @@ def get_item_data( ) # loop over and collect all 'outer' relationships presented by other items - incoming_relationships: dict[RelationshipType, set[str]] = {} + incoming_relationships: dict[RelationshipType, set] = {} for d in relationships_query_results: for k in d["relationships"]: if k["relation"] not in incoming_relationships: @@ -1246,7 +1247,7 @@ def get_item_data( ) # loop over and aggregate all 'inner' relationships presented by this item - inlined_relationships: dict[RelationshipType, set[str]] = {} + inlined_relationships: dict[RelationshipType, set] = {} if doc.relationships is not None: inlined_relationships = { relation: { @@ -1266,7 +1267,7 @@ def get_item_data( ) # Must be exported to JSON first to apply the custom pydantic JSON encoders - return_dict = json.loads(doc.json(exclude_unset=True)) + return_dict = doc.model_dump(mode="json", exclude_unset=True) if item_id is None: item_id = return_dict["item_id"] @@ -1574,7 +1575,7 @@ def restore_version(refcode): # Insert validated data flask_mongo.db.item_versions.insert_one( - validated_restored_version.dict(by_alias=True, exclude_none=True) + validated_restored_version.model_dump(by_alias=True, exclude_none=True) ) return jsonify( @@ -1654,6 +1655,7 @@ def save_item(): # These keys should not be updated here and cannot be modified by the user through this endpoint for k in ( "_id", + "immutable_id", "file_ObjectIds", "files", "creators", @@ -1761,6 +1763,42 @@ def save_item(): ] updated_data["collections"] = inaccessible + existing_item = flask_mongo.db.items.find_one({"item_id": item_id}) + if existing_item: + existing_relationships = existing_item.get("relationships", []) + non_collection_relationships = [ + rel for rel in existing_relationships if rel.get("type") != "collections" + ] + + collection_relationships = [] + for coll in updated_data.get("collections", []): + immutable_id = coll.get("immutable_id") + collection_id = coll.get("collection_id") + + if immutable_id: + if isinstance(immutable_id, str): + from bson import ObjectId + + immutable_id = ObjectId(immutable_id) + elif collection_id: + collection_doc = flask_mongo.db.collections.find_one( + {"collection_id": collection_id} + ) + if collection_doc: + immutable_id = collection_doc["_id"] + + if immutable_id: + collection_relationships.append( + { + "relation": None, + "immutable_id": immutable_id, + "type": "collections", + "description": "Is a member of", + } + ) + + updated_data["relationships"] = non_collection_relationships + collection_relationships + item_type = item["type"] preserve_relationships = "collections" not in updated_data @@ -1770,7 +1808,13 @@ def save_item(): try: item = entry_reference_lookup(item) - item = ITEM_MODELS[item_type](**item).dict() + item = ITEM_MODELS[item_type](**item).model_dump( + exclude_none=True, + exclude_unset=True, + by_alias=True, + exclude={"collections", "creators", "immutable_id"}, + ) + except ValidationError as exc: return ( jsonify( @@ -1785,8 +1829,10 @@ def save_item(): item["relationships"] = original_relationships # remove collections and creators and any other reference fields - item.pop("collections") - item.pop("creators") + item.pop("collections", None) + item.pop("creators", None) + item.pop("immutable_id", None) + item.pop("files", None) # `last_modified` is controlled by the versioning branch below: only bump it when a # snapshot is actually saved, so re-submitting identical data leaves it untouched. diff --git a/pydatalab/src/pydatalab/routes/v0_1/remotes.py b/pydatalab/src/pydatalab/routes/v0_1/remotes.py index be50502f9..4e69952a2 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/remotes.py +++ b/pydatalab/src/pydatalab/routes/v0_1/remotes.py @@ -76,7 +76,9 @@ def list_remote_directories(): response = {} response["meta"] = {} - response["meta"]["remotes"] = [json.loads(d.json()) for d in CONFIG.REMOTE_FILESYSTEMS] + response["meta"]["remotes"] = [ + json.loads(d.model_dump_json()) for d in CONFIG.REMOTE_FILESYSTEMS + ] if all_directory_structures: oldest_update = min(d["last_updated"] for d in all_directory_structures) response["meta"]["oldest_cache_update"] = oldest_update.isoformat() @@ -139,7 +141,7 @@ def get_remote_directory(remote_id: str): response: dict[str, Any] = {} response["meta"] = {} - response["meta"]["remote"] = json.loads(d.json()) + response["meta"]["remote"] = json.loads(d.model_dump_json()) response["data"] = directory_structure return jsonify(response), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/users.py b/pydatalab/src/pydatalab/routes/v0_1/users.py index b59aca49d..f790a12bf 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/users.py +++ b/pydatalab/src/pydatalab/routes/v0_1/users.py @@ -6,11 +6,12 @@ from bson import ObjectId from flask import Blueprint, jsonify, request from flask_login import current_user +from pydantic import ValidationError from werkzeug.exceptions import BadRequest, Forbidden, Unauthorized from pydatalab.config import CONFIG from pydatalab.logger import LOGGER -from pydatalab.models.people import AccountStatus, DisplayName, EmailStr, Person +from pydatalab.models.people import Person from pydatalab.mongo import ( USERS_FTS_FIELDS, build_search_pipeline, @@ -49,22 +50,22 @@ def save_user(user_id): update = {} - try: - if display_name: - update["display_name"] = DisplayName(display_name) + if display_name: + update["display_name"] = display_name - except ValueError: - raise BadRequest(f"Invalid display name {display_name!r} was passed") + if contact_email or contact_email in (None, ""): + if contact_email in ("", None): + update["contact_email"] = None + else: + update["contact_email"] = contact_email - try: - if contact_email or contact_email in (None, ""): - if contact_email in ("", None): - update["contact_email"] = None - else: - update["contact_email"] = EmailStr(contact_email) + if account_status: + update["account_status"] = account_status - except ValueError: - raise BadRequest(f"Invalid email address {contact_email!r} was passed") + try: + _ = Person(**update) + except ValidationError as e: + raise BadRequest(f"Invalid user data: {e.errors()}") from e if "contact_email" in update or "display_name" in update: existing = ( @@ -128,12 +129,6 @@ def save_user(user_id): trigger_email_verification = False LOGGER.critical("Unable to send verification email on this deployment: %s", e) - try: - if account_status: - update["account_status"] = AccountStatus(account_status) - except ValueError: - raise BadRequest(f"Invalid account status {account_status!r} was passed") - if not update: return jsonify({"status": "success", "message": "No update to perform."}), 200 From 785d241ed15d5a15f8e9af7069e0fb9c3f9fe993 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 20:34:24 +0100 Subject: [PATCH 09/31] More model updates More updates for models Fixes for models Simplify item versioning model Remove unecessary by_alias=True --- pydatalab/schemas/cell.json | 46 +++++++++++----- pydatalab/schemas/equipment.json | 50 ++++++++++++----- pydatalab/schemas/sample.json | 46 +++++++++++----- pydatalab/schemas/startingmaterial.json | 54 +++++++++++++------ pydatalab/src/pydatalab/models/blocks.py | 6 ++- pydatalab/src/pydatalab/models/equipment.py | 2 +- pydatalab/src/pydatalab/models/export_task.py | 5 +- pydatalab/src/pydatalab/models/files.py | 4 +- .../pydatalab/models/starting_materials.py | 6 +-- pydatalab/src/pydatalab/models/tasks.py | 12 ++--- pydatalab/src/pydatalab/models/traits.py | 4 +- pydatalab/src/pydatalab/models/versions.py | 52 +++++------------- pydatalab/src/pydatalab/mongo.py | 2 +- pydatalab/src/pydatalab/routes/v0_1/items.py | 7 +-- pydatalab/src/pydatalab/versioning.py | 4 +- 15 files changed, 179 insertions(+), 121 deletions(-) diff --git a/pydatalab/schemas/cell.json b/pydatalab/schemas/cell.json index 771d40fb0..c66e9dcb3 100644 --- a/pydatalab/schemas/cell.json +++ b/pydatalab/schemas/cell.json @@ -84,14 +84,15 @@ "additionalProperties": true, "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "collection_id": { + "immutable_id": { "description": "The immutable ID of the collection.", - "title": "Collection Id", + "format": "objectid", + "title": "Immutable Id", "type": "string" } }, "required": [ - "collection_id" + "immutable_id" ], "title": "CollectionReference", "type": "object" @@ -239,6 +240,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", "title": "B64 Encoded Image" }, @@ -254,6 +256,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "A JSON-encoded string containing the Bokeh plot data, if any.", "title": "Bokeh Plot Data" }, @@ -272,6 +275,20 @@ "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", "title": "Computed" }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, "metadata": { "anyOf": [ { @@ -290,9 +307,7 @@ }, "required": [ "blocktype", - "block_id", - "b64_encoded_image", - "bokeh_plot_data" + "block_id" ], "title": "DataBlockResponse", "type": "object" @@ -609,10 +624,18 @@ "title": "Source" }, "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The timestamp for the original file upload.", - "format": "datetime", - "title": "Time Added", - "type": "string" + "title": "Time Added" }, "metadata": { "anyOf": [ @@ -678,15 +701,14 @@ "type": "null" } ], + "default": null, "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ "name", "extension", - "time_added", - "is_live", - "checksums" + "is_live" ], "title": "File", "type": "object" diff --git a/pydatalab/schemas/equipment.json b/pydatalab/schemas/equipment.json index 5482268c6..e0312f30a 100644 --- a/pydatalab/schemas/equipment.json +++ b/pydatalab/schemas/equipment.json @@ -14,14 +14,15 @@ "additionalProperties": true, "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "collection_id": { + "immutable_id": { "description": "The immutable ID of the collection.", - "title": "Collection Id", + "format": "objectid", + "title": "Immutable Id", "type": "string" } }, "required": [ - "collection_id" + "immutable_id" ], "title": "CollectionReference", "type": "object" @@ -169,6 +170,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", "title": "B64 Encoded Image" }, @@ -184,6 +186,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "A JSON-encoded string containing the Bokeh plot data, if any.", "title": "Bokeh Plot Data" }, @@ -202,6 +205,20 @@ "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", "title": "Computed" }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, "metadata": { "anyOf": [ { @@ -220,9 +237,7 @@ }, "required": [ "blocktype", - "block_id", - "b64_encoded_image", - "bokeh_plot_data" + "block_id" ], "title": "DataBlockResponse", "type": "object" @@ -470,10 +485,18 @@ "title": "Source" }, "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The timestamp for the original file upload.", - "format": "datetime", - "title": "Time Added", - "type": "string" + "title": "Time Added" }, "metadata": { "anyOf": [ @@ -539,15 +562,14 @@ "type": "null" } ], + "default": null, "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ "name", "extension", - "time_added", - "is_live", - "checksums" + "is_live" ], "title": "File", "type": "object" @@ -1306,13 +1328,13 @@ "type": "null" } ], + "default": null, "description": "Contact information for equipment (e.g., email address or phone number).", "title": "Contact" } }, "required": [ - "item_id", - "contact" + "item_id" ], "title": "Equipment", "type": "object" diff --git a/pydatalab/schemas/sample.json b/pydatalab/schemas/sample.json index 57b0fb8ce..2d85b667b 100644 --- a/pydatalab/schemas/sample.json +++ b/pydatalab/schemas/sample.json @@ -14,14 +14,15 @@ "additionalProperties": true, "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "collection_id": { + "immutable_id": { "description": "The immutable ID of the collection.", - "title": "Collection Id", + "format": "objectid", + "title": "Immutable Id", "type": "string" } }, "required": [ - "collection_id" + "immutable_id" ], "title": "CollectionReference", "type": "object" @@ -211,6 +212,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", "title": "B64 Encoded Image" }, @@ -226,6 +228,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "A JSON-encoded string containing the Bokeh plot data, if any.", "title": "Bokeh Plot Data" }, @@ -244,6 +247,20 @@ "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", "title": "Computed" }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, "metadata": { "anyOf": [ { @@ -262,9 +279,7 @@ }, "required": [ "blocktype", - "block_id", - "b64_encoded_image", - "bokeh_plot_data" + "block_id" ], "title": "DataBlockResponse", "type": "object" @@ -581,10 +596,18 @@ "title": "Source" }, "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The timestamp for the original file upload.", - "format": "datetime", - "title": "Time Added", - "type": "string" + "title": "Time Added" }, "metadata": { "anyOf": [ @@ -650,15 +673,14 @@ "type": "null" } ], + "default": null, "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ "name", "extension", - "time_added", - "is_live", - "checksums" + "is_live" ], "title": "File", "type": "object" diff --git a/pydatalab/schemas/startingmaterial.json b/pydatalab/schemas/startingmaterial.json index 40ea9e4c1..a096ffb55 100644 --- a/pydatalab/schemas/startingmaterial.json +++ b/pydatalab/schemas/startingmaterial.json @@ -14,14 +14,15 @@ "additionalProperties": true, "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "collection_id": { + "immutable_id": { "description": "The immutable ID of the collection.", - "title": "Collection Id", + "format": "objectid", + "title": "Immutable Id", "type": "string" } }, "required": [ - "collection_id" + "immutable_id" ], "title": "CollectionReference", "type": "object" @@ -211,6 +212,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", "title": "B64 Encoded Image" }, @@ -226,6 +228,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "A JSON-encoded string containing the Bokeh plot data, if any.", "title": "Bokeh Plot Data" }, @@ -244,6 +247,20 @@ "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", "title": "Computed" }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, "metadata": { "anyOf": [ { @@ -262,9 +279,7 @@ }, "required": [ "blocktype", - "block_id", - "b64_encoded_image", - "bokeh_plot_data" + "block_id" ], "title": "DataBlockResponse", "type": "object" @@ -581,10 +596,18 @@ "title": "Source" }, "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The timestamp for the original file upload.", - "format": "datetime", - "title": "Time Added", - "type": "string" + "title": "Time Added" }, "metadata": { "anyOf": [ @@ -650,15 +673,14 @@ "type": "null" } ], + "default": null, "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ "name", "extension", - "time_added", - "is_live", - "checksums" + "is_live" ], "title": "File", "type": "object" @@ -1498,6 +1520,7 @@ "type": "null" } ], + "default": null, "description": "The name of the substance in the container.", "title": "Name" }, @@ -1567,6 +1590,7 @@ "type": "null" } ], + "default": null, "description": "The chemical purity of this container with regards to the defined substance.", "title": "Chemical Purity" }, @@ -1618,6 +1642,7 @@ "type": "null" } ], + "default": null, "description": "Supplier or manufacturer of the chemical.", "title": "Supplier" }, @@ -1649,10 +1674,7 @@ } }, "required": [ - "item_id", - "name", - "chemical_purity", - "supplier" + "item_id" ], "title": "StartingMaterial", "type": "object" diff --git a/pydatalab/src/pydatalab/models/blocks.py b/pydatalab/src/pydatalab/models/blocks.py index a89af0ffc..f1e02a4d4 100644 --- a/pydatalab/src/pydatalab/models/blocks.py +++ b/pydatalab/src/pydatalab/models/blocks.py @@ -44,12 +44,12 @@ class DataBlockResponse(BaseModel): """Any warnings that occurred during block processing.""" b64_encoded_image: dict[str, str] | None = Field( - datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, datalab_exclude_from_db=True, datalab_exclude_from_load=True ) """Any base64-encoded image data associated with the block, keyed by `file_id`.""" bokeh_plot_data: dict | None = Field( - datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, datalab_exclude_from_db=True, datalab_exclude_from_load=True ) """A JSON-encoded string containing the Bokeh plot data, if any.""" @@ -59,6 +59,8 @@ class DataBlockResponse(BaseModel): Examples could include peak positions, and widths, but not the full spectrum. """ + processed: dict | None = Field(default=None, datalab_exclude_from_load=True) + metadata: dict | None = Field(default=None, datalab_exclude_from_load=True) """Any structured metadata associated with the block, for example, experimental acquisition parameters.""" diff --git a/pydatalab/src/pydatalab/models/equipment.py b/pydatalab/src/pydatalab/models/equipment.py index 820961905..fb2dbd9d5 100644 --- a/pydatalab/src/pydatalab/models/equipment.py +++ b/pydatalab/src/pydatalab/models/equipment.py @@ -22,7 +22,7 @@ class Equipment(Item): location: str | None = None """Place where the equipment is located""" - contact: str | None + contact: str | None = None """Contact information for equipment (e.g., email address or phone number).""" status: EquipmentStatus = Field(default=EquipmentStatus.WORKING) diff --git a/pydatalab/src/pydatalab/models/export_task.py b/pydatalab/src/pydatalab/models/export_task.py index d2085ffde..e2728888d 100644 --- a/pydatalab/src/pydatalab/models/export_task.py +++ b/pydatalab/src/pydatalab/models/export_task.py @@ -1,7 +1,7 @@ from datetime import datetime, timezone from enum import Enum -from pydantic import Field +from pydantic import ConfigDict, Field from pydatalab.models.utils import BaseModel, PyObjectId @@ -48,5 +48,4 @@ class ExportTask(BaseModel): error_message: str | None = None """Error message if status is ERROR""" - class Config: - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) diff --git a/pydatalab/src/pydatalab/models/files.py b/pydatalab/src/pydatalab/models/files.py index ca17eadd5..898b5f986 100644 --- a/pydatalab/src/pydatalab/models/files.py +++ b/pydatalab/src/pydatalab/models/files.py @@ -52,7 +52,7 @@ class File(Entry, HasOwner, HasRevisionControl): source: str | None = None """The source of the file, e.g. 'remote' or 'uploaded'.""" - time_added: IsoformatDateTime + time_added: IsoformatDateTime | None = None """The timestamp for the original file upload.""" metadata: dict[Any, Any] | None = None @@ -69,5 +69,5 @@ class File(Entry, HasOwner, HasRevisionControl): is_live: bool """Whether or not the file should be watched for future updates.""" - checksums: FileChecksums | None + checksums: FileChecksums | None = None """Content checksums (MD5 and SHA-256) of the file.""" diff --git a/pydatalab/src/pydatalab/models/starting_materials.py b/pydatalab/src/pydatalab/models/starting_materials.py index 845bd6e5a..8bbdad870 100644 --- a/pydatalab/src/pydatalab/models/starting_materials.py +++ b/pydatalab/src/pydatalab/models/starting_materials.py @@ -24,13 +24,13 @@ class StartingMaterial(Item, HasSynthesisInfo, HasSubstanceInfo): date_opened: IsoformatDateTime | None = Field(None, alias="Date opened") """The date the item was opened""" - chemical_purity: str | None = Field(alias="Chemical purity") + chemical_purity: str | None = Field(None, alias="Chemical purity") """The chemical purity of this container with regards to the defined substance.""" full_percent: str | None = Field(None, alias="Full %") """The amount of the defined substance remaining in the container, expressed as a percentage.""" - name: str | None = Field(alias="Container Name") + name: str | None = Field(None, alias="Container Name") """The name of the substance in the container.""" size: str | None = Field(None, alias="Container Size") @@ -39,7 +39,7 @@ class StartingMaterial(Item, HasSynthesisInfo, HasSubstanceInfo): size_unit: str | None = Field(None, alias="Unit") """Units for the 'size' field.""" - supplier: str | None = Field(alias="Supplier") + supplier: str | None = Field(None, alias="Supplier") """Supplier or manufacturer of the chemical.""" location: str | None = Field(None, alias="Location") diff --git a/pydatalab/src/pydatalab/models/tasks.py b/pydatalab/src/pydatalab/models/tasks.py index 14728df26..ab21d39c8 100644 --- a/pydatalab/src/pydatalab/models/tasks.py +++ b/pydatalab/src/pydatalab/models/tasks.py @@ -2,7 +2,7 @@ from enum import Enum from typing import Literal -from pydantic import Field, validator +from pydantic import ConfigDict, Field, field_validator from pydatalab.models.utils import BaseModel, PyObjectId @@ -90,9 +90,10 @@ class Task(BaseModel): spec: ExportTaskSpec | BlockProcessingTaskSpec """Task-specific data""" - @validator("spec", pre=True, always=True) - def validate_spec_type(cls, v, values): - task_type = values.get("type") + @field_validator("spec", mode="before") + @classmethod + def validate_spec_type(cls, v, info): + task_type = info.data.get("type") if task_type == TaskType.EXPORT: if not isinstance(v, ExportTaskSpec): @@ -103,5 +104,4 @@ def validate_spec_type(cls, v, values): return v - class Config: - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) diff --git a/pydatalab/src/pydatalab/models/traits.py b/pydatalab/src/pydatalab/models/traits.py index 8cc0f9deb..b7805c984 100644 --- a/pydatalab/src/pydatalab/models/traits.py +++ b/pydatalab/src/pydatalab/models/traits.py @@ -52,9 +52,9 @@ class HasBlocks(BaseModel): class CollectionReference(BaseModel): """A reference to a collection, used for inlining collection info within other models.""" - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", populate_by_name=True) - collection_id: str + immutable_id: PyObjectId = Field(alias="_id") """The immutable ID of the collection.""" diff --git a/pydatalab/src/pydatalab/models/versions.py b/pydatalab/src/pydatalab/models/versions.py index 6efce3a2c..4063ebdc7 100644 --- a/pydatalab/src/pydatalab/models/versions.py +++ b/pydatalab/src/pydatalab/models/versions.py @@ -3,7 +3,7 @@ from datetime import datetime from enum import Enum -from pydantic import Field, validator +from pydantic import ConfigDict, Field, model_validator from pydatalab.models.utils import BaseModel, PyObjectId, Refcode @@ -56,17 +56,16 @@ class ItemVersion(BaseModel): user_agent: str | None = None """User agent string of the client that triggered this version. Will only be stored if it matches a known value from the datalab ecosystem.""" - @validator("restored_from_version") - def validate_restored_from_version(cls, v, values): + @model_validator(mode="after") + def validate_restored_from_version(self): """Ensure restored_from_version is only present when action='restored'.""" - action = values.get("action") - if action == VersionAction.RESTORED and v is None: + if self.action == VersionAction.RESTORED and self.restored_from_version is None: raise ValueError("restored_from_version must be provided when action='restored'") - if action != VersionAction.RESTORED and v is not None: + if self.action != VersionAction.RESTORED and self.restored_from_version is not None: raise ValueError( - f"restored_from_version should only be present when action='restored', got action='{action}'" + f"restored_from_version should only be present when action='restored', got action='{self.action}'" ) - return v + return self class VersionCounter(BaseModel): @@ -82,50 +81,25 @@ class VersionCounter(BaseModel): counter: int = Field(1, ge=1) """Current version counter value (1-indexed, matches version numbers)""" - class Config: - extra = "ignore" # Allow MongoDB's _id field and other internal fields + model_config = ConfigDict(extra="ignore") class RestoreVersionRequest(BaseModel): """Request body for restoring a version.""" - version_id: str + version_id: PyObjectId """ObjectId string of the version to restore to""" - @validator("version_id") - def validate_version_id_format(cls, v): - """Validate that version_id is a valid ObjectId string.""" - try: - from bson import ObjectId - - ObjectId(v) - except Exception as e: - raise ValueError(f"version_id must be a valid ObjectId string: {e}") - return v - - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") class CompareVersionsQuery(BaseModel): """Query parameters for comparing two versions.""" - v1: str + v1: PyObjectId """ObjectId string of the first version""" - v2: str + v2: PyObjectId """ObjectId string of the second version""" - @validator("v1", "v2") - def validate_version_ids(cls, v): - """Validate that version IDs are valid ObjectId strings.""" - try: - from bson import ObjectId - - ObjectId(v) - except Exception as e: - raise ValueError(f"Version ID must be a valid ObjectId string: {e}") - return v - - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") diff --git a/pydatalab/src/pydatalab/mongo.py b/pydatalab/src/pydatalab/mongo.py index a8b7334af..09215c3e6 100644 --- a/pydatalab/src/pydatalab/mongo.py +++ b/pydatalab/src/pydatalab/mongo.py @@ -167,7 +167,7 @@ def insert_pydantic_model_fork_safe(model: BaseModel, collection: str) -> str: """Inserts a Pydantic model into chosen collection, returning the inserted ID.""" return ( get_database()[collection] - .insert_one(model.model_dump(by_alias=True, exclude_none=True)) + .insert_one(model.model_dump(by_alias=False, exclude_none=True)) .inserted_id ) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index bf68bef9f..7b0c73343 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -729,9 +729,7 @@ def _create_sample( # the `Entry` model. try: result = flask_mongo.db.items.insert_one( - data_model.model_dump( - exclude={"creators", "collections", "groups"}, exclude_none=True, by_alias=True - ) + data_model.model_dump(exclude={"creators", "collections", "groups"}, exclude_none=True) ) except DuplicateKeyError as error: raise Conflict(f"Duplicate key error: {str(error)}.") @@ -1575,7 +1573,7 @@ def restore_version(refcode): # Insert validated data flask_mongo.db.item_versions.insert_one( - validated_restored_version.model_dump(by_alias=True, exclude_none=True) + validated_restored_version.model_dump(exclude_none=True) ) return jsonify( @@ -1811,7 +1809,6 @@ def save_item(): item = ITEM_MODELS[item_type](**item).model_dump( exclude_none=True, exclude_unset=True, - by_alias=True, exclude={"collections", "creators", "immutable_id"}, ) diff --git a/pydatalab/src/pydatalab/versioning.py b/pydatalab/src/pydatalab/versioning.py index 1bdf8568d..13433bb25 100644 --- a/pydatalab/src/pydatalab/versioning.py +++ b/pydatalab/src/pydatalab/versioning.py @@ -192,9 +192,7 @@ def save_version_snapshot( ) # Insert validated data (convert to dict and exclude None values) - flask_mongo.db.item_versions.insert_one( - validated_version.dict(by_alias=True, exclude_none=True) - ) + flask_mongo.db.item_versions.insert_one(validated_version.model_dump(exclude_none=True)) return ( {"status": "success", "message": "Version saved.", "version": next_version_number}, 200, From 15d92aee136d50babcc4e01bde6a9309309680e9 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 20:57:45 +0100 Subject: [PATCH 10/31] Minimal test updates --- pydatalab/tests/server/conftest.py | 22 +++---- pydatalab/tests/server/test_item_versions.py | 48 ++------------- pydatalab/tests/server/test_samples.py | 22 +++---- pydatalab/tests/test_models.py | 61 +++++++++++++++----- 4 files changed, 72 insertions(+), 81 deletions(-) diff --git a/pydatalab/tests/server/conftest.py b/pydatalab/tests/server/conftest.py index 4cc1c7938..b00855b89 100644 --- a/pydatalab/tests/server/conftest.py +++ b/pydatalab/tests/server/conftest.py @@ -479,7 +479,7 @@ def fixture_insert_complicated_sample_constituents(user_id): creator_ids=[user_id], refcode=generate_unique_refcode(), ) - flask_mongo.db.items.insert_one(sm.dict(exclude_unset=False)) + flask_mongo.db.items.insert_one(sm.model_dump(exclude_unset=False)) items.append(sm) yield items @@ -492,7 +492,7 @@ def fixture_insert_complicated_sample_constituents(user_id): def example_items(user_id, admin_user_id): """Create a collection of samples with mixed ownership between the user and admin.""" return [ - d.dict(exclude_unset=False) + d.model_dump(exclude_unset=False) for d in [ Sample( **{ @@ -579,23 +579,23 @@ def example_items(user_id, admin_user_id): @pytest.fixture(scope="module", name="default_sample_dict") -def fixture_default_sample_dict(default_sample): - return default_sample.dict(exclude_unset=True) +def fixture_default_sample_model_dump(default_sample): + return default_sample.model_dump(exclude_unset=True) @pytest.fixture(scope="module", name="default_cell_dict") -def fixture_default_cell_dict(default_cell): - return default_cell.dict(exclude_unset=True) +def fixture_default_cell_model_dump(default_cell): + return default_cell.model_dump(exclude_unset=True) @pytest.fixture(scope="module", name="default_starting_material_dict") -def fixture_default_starting_material_dict(default_starting_material): - return default_starting_material.dict(exclude_unset=True) +def fixture_default_starting_material_model_dump(default_starting_material): + return default_starting_material.model_dump(exclude_unset=True) @pytest.fixture(scope="module", name="default_equipment_dict") -def fixture_default_equipment_dict(default_equipment): - return default_equipment.dict(exclude_unset=True) +def fixture_default_equipment_model_dump(default_equipment): + return default_equipment.model_dump(exclude_unset=True) def _insert_and_cleanup_item_from_model(model): @@ -604,7 +604,7 @@ def _insert_and_cleanup_item_from_model(model): refcode = generate_unique_refcode() model.refcode = refcode - flask_mongo.db.items.insert_one(model.dict(exclude_unset=False)) + flask_mongo.db.items.insert_one(model.model_dump(exclude_unset=False)) yield model flask_mongo.db.items.delete_one({"refcode": model.refcode}) diff --git a/pydatalab/tests/server/test_item_versions.py b/pydatalab/tests/server/test_item_versions.py index e3b3201f5..4a61a84ec 100644 --- a/pydatalab/tests/server/test_item_versions.py +++ b/pydatalab/tests/server/test_item_versions.py @@ -88,7 +88,6 @@ def test_save_version_nonexistent_item(self, client): response = client.post("/items/nonexistent/save-version/") assert response.status_code == 404 - assert response.json["status"] == "error" class TestListVersions: @@ -232,8 +231,6 @@ def test_get_version_nonexistent(self, client, sample_with_version): response = client.get(f"/items/{refcode}/versions/{fake_id}/") assert response.status_code == 404 - assert response.json["status"] == "error" - assert "not found" in response.json["message"].lower() class TestCompareVersions: @@ -282,24 +279,12 @@ def test_compare_versions_missing_parameters(self, client, sample_with_version): # Missing v2 - request.args.get() returns "" for missing params, which fails ObjectId validation response = client.get(f"/items/{refcode}/compare-versions/?v1=some_id") assert response.status_code == 400 - assert response.json["message"] == "Invalid query parameters" - assert "errors" in response.json - errors = response.json["errors"] - # Should have error for v2 (empty string is invalid ObjectId) - v2_errors = [e for e in errors if "v2" in str(e["loc"])] - assert len(v2_errors) == 1 - assert "valid objectid" in v2_errors[0]["msg"].lower() + assert "Invalid query parameters" in response.json["message"] # Missing v1 - same behavior response = client.get(f"/items/{refcode}/compare-versions/?v2=some_id") assert response.status_code == 400 - assert response.json["message"] == "Invalid query parameters" - assert "errors" in response.json - errors = response.json["errors"] - # Should have error for v1 (empty string is invalid ObjectId) - v1_errors = [e for e in errors if "v1" in str(e["loc"])] - assert len(v1_errors) == 1 - assert "valid objectid" in v1_errors[0]["msg"].lower() + assert "Invalid query parameters" in response.json["message"] def test_compare_versions_invalid_id(self, client, sample_with_version): """Test comparing versions with invalid ID format.""" @@ -307,15 +292,7 @@ def test_compare_versions_invalid_id(self, client, sample_with_version): response = client.get(f"/items/{refcode}/compare-versions/?v1=invalid&v2=invalid") assert response.status_code == 400 - assert response.json["message"] == "Invalid query parameters" - # Check Pydantic's structured error response - assert "errors" in response.json - errors = response.json["errors"] - # Should have errors for both v1 and v2 - assert len(errors) == 2 - for error in errors: - assert error["loc"][0] in ["v1", "v2"] - assert "valid ObjectId" in error["msg"] + assert "Invalid query parameters" in response.json["message"] def test_compare_versions_detects_changes(self, client, sample_with_version): """Test that compare_versions properly detects changes using DeepDiff.""" @@ -483,13 +460,7 @@ def test_restore_version_missing_version_id(self, client, sample_with_version): response = client.post(f"/items/{refcode}/restore-version/", json={}) assert response.status_code == 400 - assert response.json["message"] == "Invalid request body" - # Check Pydantic's structured error response - assert "errors" in response.json - errors = response.json["errors"] - assert len(errors) == 1 - assert errors[0]["loc"] == ["version_id"] - assert "required" in errors[0]["msg"].lower() + assert "Invalid request body" in response.json["message"] def test_restore_version_invalid_id(self, client, sample_with_version): """Test restoring with invalid version ID.""" @@ -497,13 +468,7 @@ def test_restore_version_invalid_id(self, client, sample_with_version): response = client.post(f"/items/{refcode}/restore-version/", json={"version_id": "invalid"}) assert response.status_code == 400 - assert response.json["message"] == "Invalid request body" - # Check Pydantic's structured error response - assert "errors" in response.json - errors = response.json["errors"] - assert len(errors) == 1 - assert errors[0]["loc"] == ["version_id"] - assert "valid ObjectId" in errors[0]["msg"] + assert "Invalid request body" in response.json["message"] def test_restore_version_nonexistent(self, client, sample_with_version): """Test restoring non-existent version.""" @@ -512,7 +477,6 @@ def test_restore_version_nonexistent(self, client, sample_with_version): response = client.post(f"/items/{refcode}/restore-version/", json={"version_id": fake_id}) assert response.status_code == 404 - assert "not found" in response.json["message"].lower() def test_restore_version_increments_version_number(self, client, sample_with_version): """Test that restore increments the item version number.""" @@ -575,7 +539,6 @@ def test_delete_version_nonexistent(self, client, sample_with_version): response = client.delete(f"/items/{refcode}/versions/{fake_id}/") assert response.status_code == 404 - assert response.json["status"] == "error" def test_delete_version_invalid_id(self, client, sample_with_version): """Test deleting with invalid ID format.""" @@ -955,7 +918,6 @@ def test_restore_requires_write_permission(self, another_client, sample_with_ver # Should fail because another_client doesn't have access to this item assert response.status_code == 404 - assert "not found or insufficient permissions" in response.json["message"].lower() class TestEdgeCases: diff --git a/pydatalab/tests/server/test_samples.py b/pydatalab/tests/server/test_samples.py index c840c42dd..4c288f9e9 100644 --- a/pydatalab/tests/server/test_samples.py +++ b/pydatalab/tests/server/test_samples.py @@ -624,8 +624,8 @@ def test_create_cell(client, default_cell): assert cell["electrolyte"][1]["item"]["chemform"] == "NaCl" assert ( - cell["positive_electrode"][0]["item"]["name"] - == default_cell.positive_electrode[0].item.name + cell["positive_electrode"][0]["item"]["item_id"] + == default_cell.positive_electrode[0].item.item_id ) # The anode has a "real" entry in the db, so it should be looked up properly # with dereferenced chemical formula and name, even if it was not provided at link time @@ -924,7 +924,7 @@ def test_remove_items_from_collection_success( response = client.post("/new-sample/", json=sample_dict) assert response.status_code == 201 - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_remove" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -996,7 +996,7 @@ def test_remove_items_from_collection_not_found(client): @pytest.mark.dependency() def test_remove_items_from_collection_no_items_provided(client, default_collection): """Test removing with no item IDs provided.""" - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_empty_items" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1012,7 +1012,7 @@ def test_remove_items_from_collection_no_items_provided(client, default_collecti @pytest.mark.dependency() def test_remove_items_from_collection_no_matching_items(client, default_collection): """Test removing items that don't exist.""" - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_no_match" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1041,7 +1041,7 @@ def test_remove_items_from_collection_partial_success( response = client.post("/new-sample/", json=sample_dict) assert response.status_code == 201 - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_partial" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1100,7 +1100,7 @@ def test_copy_sample_and_add_to_collection(client, default_sample_dict, default_ assert response.status_code == 201 assert response.json["status"] == "success" - collection_dict = default_collection.dict().copy() + collection_dict = default_collection.model_dump().copy() collection_dict["collection_id"] = "test_copy_collection" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1132,12 +1132,12 @@ def test_copy_sample_and_add_to_collection(client, default_sample_dict, default_ def test_copy_sample_from_collection_to_different_collection( client, default_sample_dict, default_collection ): - collection1_dict = default_collection.dict().copy() + collection1_dict = default_collection.model_dump().copy() collection1_dict["collection_id"] = "collection_1" response = client.put("/collections", json={"data": collection1_dict}) assert response.status_code == 201 - collection2_dict = default_collection.dict().copy() + collection2_dict = default_collection.model_dump().copy() collection2_dict["collection_id"] = "collection_2" response = client.put("/collections", json={"data": collection2_dict}) assert response.status_code == 201 @@ -1177,7 +1177,7 @@ def test_copy_sample_from_collection_to_different_collection( @pytest.mark.dependency(depends=["test_copy_sample_from_collection_to_different_collection"]) def test_copy_sample_without_copying_collections(client, default_sample_dict, default_collection): - collection_dict = default_collection.dict().copy() + collection_dict = default_collection.model_dump().copy() collection_dict["collection_id"] = "test_no_auto_copy_collection" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1217,7 +1217,7 @@ def test_collections_permissions(client, admin_client, default_sample_dict, defa assert response.json["status"] == "success" refcode = response.json["sample_list_entry"]["refcode"] - admin_collection = default_collection.dict().copy() + admin_collection = default_collection.model_dump().copy() admin_collection["collection_id"] = "admin_only_collection" response = admin_client.put("/collections", json={"data": admin_collection}) diff --git a/pydatalab/tests/test_models.py b/pydatalab/tests/test_models.py index c2ba5959b..c92dbb7ce 100644 --- a/pydatalab/tests/test_models.py +++ b/pydatalab/tests/test_models.py @@ -81,7 +81,7 @@ def test_sample_with_inlined_reference(): @pytest.mark.parametrize("model", ITEM_MODELS.values()) def test_generate_schemas(model): """Test that all item model schemas can be generated.""" - assert model.schema() + assert model.model_json_schema() def test_attribute_docstrings_in_schema(): @@ -187,6 +187,9 @@ def test_file(): def test_custom_and_inherited_items(): class TestItem(Item): type: str = "items_custom" + new_field: str + + TestItem.model_rebuild() item = TestItem( type="items_custom", @@ -195,9 +198,10 @@ class TestItem(Item): creators=None, date="2020-01-01 00:00", item_id="1234", + new_field="This is a new field", ) - item_dict = item.dict() + item_dict = item.model_dump() assert item_dict["type"] == "items_custom" assert item_dict["creator_ids"][0] == ObjectId("0123456789ab0123456789ab") assert item_dict["creator_ids"][1] == ObjectId("1023456789ab0123456789ab") @@ -207,8 +211,8 @@ class TestItem(Item): item_json = json.loads(item.json()) assert item_json["type"] == "items_custom" - assert item_json["creator_ids"][0] == str(ObjectId("0123456789ab0123456789ab")) - assert item_json["creator_ids"][1] == str(ObjectId("1023456789ab0123456789ab")) + assert item_json["creator_ids"][0] == "0123456789ab0123456789ab" + assert item_json["creator_ids"][1] == "1023456789ab0123456789ab" assert ( item_json["date"] == datetime.datetime.fromisoformat("2020-01-01 00:00") @@ -224,7 +228,7 @@ class TestItem(Item): item_id="1234", ) - sample_dict = sample.dict() + sample_dict = sample.model_dump() assert sample_dict["type"] == "samples" assert sample_dict["creator_ids"][0] == ObjectId("0123456789ab0123456789ab") assert sample_dict["creator_ids"][1] == ObjectId("1023456789ab0123456789ab") @@ -261,13 +265,17 @@ class TestItem(Item): "MP2018_TEST_COMMERCIAL", "MP2018_TEST_COMMERCIAL_4.5V_hold", "AAAAAA", - 111111111, + "111111111", ], ) def test_good_ids(id): """Test good human-readable IDs for validity.""" - assert HumanReadableIdentifier(id) + class TestModel(pydantic.BaseModel): + test_id: HumanReadableIdentifier + + model = TestModel(test_id=id) + assert model.test_id == id @pytest.mark.parametrize( @@ -277,6 +285,7 @@ def test_good_ids(id): "mp 1 2 3 4 5 6", "lithium & sodium", "me388-123456789-123456789-really-long-descriptive-identifier-that-should-be-the-name-but-is-otherwise-valid", + 111111111, 1111111111111111111111111111111111111111111111111, "_AAAA", "AAA_", @@ -288,8 +297,11 @@ def test_good_ids(id): def test_bad_ids(id): """Test bad human-readable IDs for invalidity.""" + class TestModel(pydantic.BaseModel): + test_id: HumanReadableIdentifier + with pytest.raises(pydantic.ValidationError): - HumanReadableIdentifier(id) + TestModel(test_id=id) def test_cell_with_inlined_reference(): @@ -312,7 +324,7 @@ def test_cell_with_inlined_reference(): assert cell assert len(cell.relationships) == 1 - cell = Cell(**json.loads(cell.json())) + cell = Cell(**json.loads(cell.model_dump_json())) assert cell assert len(cell.relationships) == 1 @@ -432,7 +444,7 @@ def test_cell_relationship_deduplication(): assert parthood[0].item_id == "test_cathode" # Re-validating an already-clean cell must not grow the relationships list. - cell = Cell(**json.loads(cell.json())) + cell = Cell(**json.loads(cell.model_dump_json())) parthood = [r for r in cell.relationships if r.relation == RelationshipType.PARTHOOD] assert len(parthood) == 1 assert parthood[0].refcode == "grey:ABCDEF" @@ -527,7 +539,7 @@ def test_sample_synthesis_relationship_deduplication(): assert parents[0].item_id == "sm_1" # Re-validating an already-clean sample must not grow the relationships list. - sample = Sample(**json.loads(sample.json())) + sample = Sample(**json.loads(sample.model_dump_json())) parents = [r for r in sample.relationships if r.relation == RelationshipType.PARENT] assert len(parents) == 1 assert parents[0].refcode == "grey:ABCDEF" @@ -590,8 +602,11 @@ def test_good_refcodes(refcode): def test_bad_refcodes(refcode): """Test bad refcodes for invalidity.""" + class TestModel(pydantic.BaseModel): + test_refcode: Refcode + with pytest.raises(pydantic.ValidationError): - Refcode(refcode) + TestModel(test_refcode=refcode) @pytest.mark.parametrize( @@ -606,7 +621,10 @@ def test_bad_refcodes(refcode): def test_good_display_name(display_name): """Test good display name for validity.""" - assert DisplayName(display_name) + class TestModel(pydantic.BaseModel): + name: DisplayName + + assert TestModel(name=display_name) @pytest.mark.parametrize( @@ -620,8 +638,11 @@ def test_good_display_name(display_name): def test_bad_display_name(display_name): """Test bad display_name for invalidity.""" + class TestModel(pydantic.BaseModel): + name: DisplayName + with pytest.raises(ValueError): - DisplayName(display_name) + TestModel(name=display_name) @pytest.mark.parametrize( @@ -631,7 +652,11 @@ def test_bad_display_name(display_name): ], ) def test_good_email(contact_email): - assert EmailStr(contact_email) + + class TestModel(pydantic.BaseModel): + email: EmailStr + + assert TestModel(email=contact_email) @pytest.mark.parametrize( @@ -644,5 +669,9 @@ def test_good_email(contact_email): ], ) def test_bad_email(contact_email): + + class TestModel(pydantic.BaseModel): + email: EmailStr + with pytest.raises(ValueError): - assert EmailStr(contact_email) + TestModel(email=contact_email) From ee520221ca94e37179750b8287297bad9be6f816 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 22:30:36 +0100 Subject: [PATCH 11/31] Patch optional vs default value for plugin data blocks --- pydatalab/src/pydatalab/models/utils.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/pydatalab/src/pydatalab/models/utils.py b/pydatalab/src/pydatalab/models/utils.py index 96df03119..696ec620c 100644 --- a/pydatalab/src/pydatalab/models/utils.py +++ b/pydatalab/src/pydatalab/models/utils.py @@ -4,7 +4,7 @@ from collections.abc import Callable from enum import Enum from functools import partial -from typing import Annotated, Any, TypeAlias +from typing import Annotated, Any, TypeAlias, get_args import pint from bson import ObjectId @@ -29,6 +29,25 @@ class BaseModel(PydanticBaseModel): model_config = ConfigDict(use_attribute_docstrings=True) + @classmethod + def __pydantic_init_subclass__(cls, **kwargs): + """Restore pydantic v1's implicit `None` default for optional fields. + + In pydantic v2 a field annotated ``T | None`` with no assigned default is + *required*, whereas in v1 it implicitly defaulted to ``None``. This back-fills + a ``None`` default for any required field that allows ``None``, so that models + (including those in plugins not yet migrated to v2) keep their v1 semantics. + Fields that already declare a default are untouched. + """ + super().__pydantic_init_subclass__(**kwargs) + changed = False + for field in cls.model_fields.values(): + if field.is_required() and type(None) in get_args(field.annotation): + field.default = None + changed = True + if changed: + cls.model_rebuild(force=True) + class ItemType(str, Enum): """An enumeration of the types of items known by this implementation, should be made dynamic in the future.""" From ea36008d16a9086c762e38ce5411874c6327173a Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 22:31:50 +0100 Subject: [PATCH 12/31] Simplify error handling of all item routes --- pydatalab/src/pydatalab/routes/v0_1/items.py | 206 +++++-------------- 1 file changed, 46 insertions(+), 160 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 7b0c73343..6cd43f1f6 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -11,7 +11,7 @@ from flask_login import current_user from pydantic import ValidationError from pymongo.errors import DuplicateKeyError -from werkzeug.exceptions import BadRequest, Conflict, NotFound +from werkzeug.exceptions import BadRequest, Conflict, InternalServerError, NotFound from pydatalab.apps import BLOCK_TYPES from pydatalab.config import CONFIG @@ -523,7 +523,7 @@ def search_items(): types = types.split(",") if not query: - return jsonify({"status": "error", "message": "No query provided."}), 400 + raise BadRequest("No query provided.") permissions = get_default_permissions(user_only=False) pipeline = build_search_pipeline(query, get_items_fts_fields(), permissions) @@ -796,12 +796,9 @@ def create_samples(): sample_jsons = request_json["new_sample_datas"] if len(sample_jsons) > CONFIG.MAX_BATCH_CREATE_SIZE: - return jsonify( - { - "status": "error", - "message": f"Batch size limit exceeded. Maximum allowed: {CONFIG.MAX_BATCH_CREATE_SIZE}, requested: {len(sample_jsons)}", - } - ), 400 + raise BadRequest( + f"Batch size limit exceeded. Maximum allowed: {CONFIG.MAX_BATCH_CREATE_SIZE}, requested: {len(sample_jsons)}" + ) copy_from_item_ids = request_json.get("copy_from_item_ids") generate_ids_automatically = request_json.get("generate_ids_automatically") @@ -888,13 +885,7 @@ def _process_item_permissions( ] if not groups_requested and not creators_requested: - return ( - { - "status": "error", - "message": "No valid creator or group IDs found in the request.", - }, - 400, - ) + raise BadRequest("No valid creator or group IDs found in the request.") # Validate all creator IDs are present in the database if creator_ids: @@ -902,13 +893,7 @@ def _process_item_permissions( d for d in flask_mongo.db.users.find({"_id": {"$in": creator_ids}}, {"_id": 1}) ] if len(found_creator_ids) != len(creator_ids): - return ( - { - "status": "error", - "message": "One or more creator IDs not found in the database.", - }, - 400, - ) + raise BadRequest("One or more creator IDs not found in the database.") if group_ids: # Validate all group IDs are present in the database @@ -916,13 +901,7 @@ def _process_item_permissions( d for d in flask_mongo.db.groups.find({"_id": {"$in": group_ids}}, {"_id": 1}) ] if len(found_group_ids) != len(group_ids): - return ( - { - "status": "error", - "message": "One or more group IDs not found in the database.", - }, - 400, - ) + raise BadRequest("One or more group IDs not found in the database.") if append_mode: if creators_requested: @@ -984,12 +963,8 @@ def _process_item_permissions( ) if result.modified_count != 1: - return ( - { - "status": "error", - "message": "Failed to update permissions: you cannot remove yourself or the base owner as a creator.", - }, - 400, + raise BadRequest( + "Failed to update permissions: you cannot remove yourself or the base owner as a creator." ) return {"status": "success"}, 200 @@ -1028,12 +1003,7 @@ def issue_physical_token(refcode: str): ) if not current_item: - return jsonify( - { - "status": "error", - "message": f"No valid item found with the given {refcode=}.", - } - ), 404 + raise NotFound token = secrets.token_urlsafe(16) access_document = { @@ -1088,15 +1058,7 @@ def delete_sample(): ) if result.deleted_count != 1: - return ( - jsonify( - { - "status": "error", - "message": f"Failed to delete item with {item_id=}.", - } - ), - 400, - ) + raise BadRequest(f"Failed to delete item with {item_id=}.") flask_mongo.db.api_keys.delete_many({"refcode": item["refcode"], "type": "access_token"}) @@ -1164,15 +1126,7 @@ def get_item_data( doc = None if not doc: - return ( - jsonify( - { - "status": "error", - "message": f"No matching items for {match=} with current authorization.", - } - ), - 404, - ) + raise NotFound # See LAST_MODIFIED_PROJECTION: same backfill, applied outside an aggregation if not doc.get("last_modified") and isinstance(doc.get("_id"), ObjectId): @@ -1290,9 +1244,7 @@ def list_versions(refcode): # Check if user has access to the item (read access) has_access, _ = check_version_access(refcode, user_only=False) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1344,9 +1296,7 @@ def get_version(refcode, version_id): # Check if user has access to the item (read access) has_access, _ = check_version_access(refcode, user_only=False) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1354,7 +1304,7 @@ def get_version(refcode, version_id): try: version_object_id = ObjectId(version_id) except (InvalidId, TypeError): - return jsonify({"status": "error", "message": f"Invalid version_id: {version_id}"}), 400 + raise BadRequest(f"Invalid version_id: {version_id}") version = list( flask_mongo.db.item_versions.aggregate( @@ -1381,7 +1331,7 @@ def get_version(refcode, version_id): version = version[0] if not version: - return jsonify({"status": "error", "message": "Version not found"}), 404 + raise NotFound return jsonify({"status": "success", "version": version}), 200 @@ -1395,9 +1345,7 @@ def compare_versions(refcode): # Check if user has access to the item (read access) has_access, _ = check_version_access(refcode, user_only=False) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1408,20 +1356,12 @@ def compare_versions(refcode): v1=request.args.get("v1", ""), v2=request.args.get("v2", "") ) except ValidationError as exc: - return jsonify( - {"status": "error", "message": "Invalid query parameters", "errors": exc.errors()} - ), 400 + raise BadRequest(f"Invalid query parameters: {exc}") - try: - v1_object_id = ObjectId(query_params.v1) - v2_object_id = ObjectId(query_params.v2) - except (InvalidId, TypeError) as e: - return jsonify({"status": "error", "message": f"Invalid version ID format: {str(e)}"}), 400 - - v1 = flask_mongo.db.item_versions.find_one({"_id": v1_object_id, "refcode": refcode}) - v2 = flask_mongo.db.item_versions.find_one({"_id": v2_object_id, "refcode": refcode}) + v1 = flask_mongo.db.item_versions.find_one({"_id": query_params.v1, "refcode": refcode}) + v2 = flask_mongo.db.item_versions.find_one({"_id": query_params.v2, "refcode": refcode}) if not v1 or not v2: - return jsonify({"status": "error", "message": "One or both versions not found"}), 404 + return NotFound("One or both versions not found") # Use DeepDiff for proper nested structure comparison # This handles nested dicts, lists, type changes, and provides detailed change information @@ -1466,29 +1406,23 @@ def restore_version(refcode): try: restore_request = RestoreVersionRequest(**request.get_json()) except ValidationError as exc: - return jsonify( - {"status": "error", "message": "Invalid request body", "errors": exc.errors()} - ), 400 + raise BadRequest(f"Invalid request body: {exc}") try: version_object_id = ObjectId(restore_request.version_id) except (InvalidId, TypeError): - return jsonify( - {"status": "error", "message": f"Invalid version_id: {restore_request.version_id}"} - ), 400 + raise BadRequest(f"Invalid version_id: {restore_request.version_id}") # Check permissions - user must have write access current_item = flask_mongo.db.items.find_one( {"refcode": refcode, **get_default_permissions(user_only=True)} ) if not current_item: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound version = flask_mongo.db.item_versions.find_one({"_id": version_object_id, "refcode": refcode}) if not version: - return jsonify({"status": "error", "message": "Version not found"}), 404 + raise NotFound restored_data = version["data"].copy() @@ -1497,12 +1431,9 @@ def restore_version(refcode): # Ensure type consistency if restored_data.get("type") != current_item.get("type"): - return jsonify( - { - "status": "error", - "message": f"Cannot restore version with different type. Current: {current_item.get('type')}, Version: {restored_data.get('type')}", - } - ), 400 + raise BadRequest( + f"Cannot restore version with different type. Current: {current_item.get('type')}, Version: {restored_data.get('type')}" + ) # Atomically get the next version number (used for both version in item_versions and item.version) next_version_number = get_next_version_number(refcode) @@ -1514,19 +1445,15 @@ def restore_version(refcode): # Validate restored data against the item model item_type = current_item["type"] if item_type not in ITEM_MODELS: - return jsonify({"status": "error", "message": f"Invalid item type: {item_type}"}), 400 + raise BadRequest(f"Invalid item type: {item_type}") try: # Validate using the appropriate model ITEM_MODELS[item_type](**restored_data) except ValidationError as exc: - return jsonify( - { - "status": "error", - "message": f"Restored data failed validation: {str(exc)}", - "output": str(exc), - } - ), 400 + raise BadRequest( + f"Restored data failed validation against schema for type {item_type}: {exc}" + ) # Perform the restore first flask_mongo.db.items.update_one({"refcode": refcode}, {"$set": restored_data}) @@ -1563,13 +1490,7 @@ def restore_version(refcode): refcode, str(exc), ) - return jsonify( - { - "status": "error", - "message": f"Restored version data validation failed: {str(exc)}", - "output": str(exc), - } - ), 400 + raise BadRequest(f"Restored version data validation failed: {exc}") # Insert validated data flask_mongo.db.item_versions.insert_one( @@ -1594,9 +1515,7 @@ def delete_version(refcode, version_id): # Check if user has write access to the item (write access required) has_access, _ = check_version_access(refcode, user_only=True) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1604,13 +1523,13 @@ def delete_version(refcode, version_id): try: version_object_id = ObjectId(version_id) except (InvalidId, TypeError): - return jsonify({"status": "error", "message": f"Invalid version_id: {version_id}"}), 400 + raise BadRequest(f"Invalid version_id: {version_id}") result = flask_mongo.db.item_versions.delete_one({"_id": version_object_id, "refcode": refcode}) if result.deleted_count == 1: return jsonify({"status": "success"}), 200 else: - return jsonify({"status": "error", "message": "Version not found"}), 404 + raise NotFound @ITEMS.route("/items//save-version/", methods=["POST"]) @@ -1682,23 +1601,13 @@ def save_item(): ) if not item: - return ( - jsonify( - status="error", - message=f"Unable to find item with appropriate permissions and {item_id=}.", - ), - 404, - ) + raise NotFound # Store refcode for version saving after successful update refcode = item.get("refcode") if not refcode: - return ( - jsonify( - status="error", - message=f"Item {item_id} does not have a refcode.", - ), - 400, + raise InternalServerError( + f"Item {item_id} does not have a refcode; please report this issue." ) user_only = item["type"] not in ("starting_materials", "equipment") @@ -1708,13 +1617,7 @@ def save_item(): ) if not item: - return ( - jsonify( - status="error", - message=f"Unable to find item with appropriate permissions and {item_id=}.", - ), - 404, - ) + raise NotFound if "collections" in updated_data: requested_collections = updated_data["collections"] @@ -1812,14 +1715,9 @@ def save_item(): exclude={"collections", "creators", "immutable_id"}, ) - except ValidationError as exc: - return ( - jsonify( - status="error", - message=f"Unable to update item {item_id=} ({item_type=}) with new data {updated_data}", - output=str(exc), - ), - 400, + except ValidationError: + raise BadRequest( + f"Unable to update item {item_id=} ({item_type=}) with new data {updated_data}" ) if preserve_relationships and original_relationships is not None: @@ -1844,14 +1742,7 @@ def save_item(): ) if result.matched_count != 1: - return ( - jsonify( - status="error", - message=f"{item_id} item update failed. no subdocument matched", - output=result.raw_result, - ), - 400, - ) + raise BadRequest(f"{item_id} item update failed. no subdocument matched") # Now save a version AFTER successful item update. # Only increment item.version and bump last_modified when content actually changed @@ -1908,12 +1799,7 @@ def get_access_token_info(refcode: str): ) if not current_item: - return jsonify( - { - "status": "error", - "message": f"No valid item found with the given {refcode=}.", - } - ), 404 + raise NotFound existing_token = flask_mongo.db.api_keys.find_one( {"refcode": refcode, "active": True, "type": "access_token"}, From 8d7c0469ddeb8f0c1685ecd3da9d51772c0d325e Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 23:02:52 +0100 Subject: [PATCH 13/31] Fix exports for pydantic 2 --- pydatalab/src/pydatalab/export.py | 6 +++--- pydatalab/src/pydatalab/routes/v0_1/export.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pydatalab/src/pydatalab/export.py b/pydatalab/src/pydatalab/export.py index 8835ba341..1a83a32b5 100644 --- a/pydatalab/src/pydatalab/export.py +++ b/pydatalab/src/pydatalab/export.py @@ -112,7 +112,7 @@ def write_eln_file( item_folder = root_folder / item[primary_key] - item_metadata = ITEM_MODELS[item.get("type")](**item).json(indent=2) + item_metadata = ITEM_MODELS[item.get("type")](**item).model_dump_json(indent=2) zipf.writestr(str(item_folder / "metadata.json"), item_metadata) for file in item.get("files", []): @@ -378,7 +378,7 @@ def create_eln_file( item_data = list(cursor)[0] ItemModel = ITEM_MODELS[item_data["type"]] - item_data = ItemModel(**item_data).dict() + item_data = ItemModel(**item_data).model_dump() except IndexError: item_data = None @@ -420,7 +420,7 @@ def create_eln_file( for ind, item in enumerate(all_items): ItemModel = ITEM_MODELS[item["type"]] - _all_items.append(ItemModel(**item).dict()) + _all_items.append(ItemModel(**item).model_dump()) all_items = _all_items diff --git a/pydatalab/src/pydatalab/routes/v0_1/export.py b/pydatalab/src/pydatalab/routes/v0_1/export.py index 14f4194d7..fe51ccb0e 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/export.py +++ b/pydatalab/src/pydatalab/routes/v0_1/export.py @@ -134,7 +134,7 @@ def _do_export( def add_stage(message: str, level: str = "info"): stage = TaskStage(timestamp=datetime.now(tz=timezone.utc), message=message, level=level) flask_mongo.db.tasks.update_one( - {"task_id": task_id}, {"$push": {"spec.stages": stage.dict()}} + {"task_id": task_id}, {"$push": {"spec.stages": stage.model_dump()}} ) try: @@ -221,7 +221,7 @@ def start_collection_export(collection_id: str): ), ) - flask_mongo.db.tasks.insert_one(export_task.dict(exclude_none=False)) + flask_mongo.db.tasks.insert_one(export_task.model_dump(exclude_none=False)) task_scheduler.add_job( func=_generate_export_in_background, @@ -390,7 +390,7 @@ def start_item_export(item_id: str): ), ) - flask_mongo.db.tasks.insert_one(export_task.dict(exclude_none=False)) + flask_mongo.db.tasks.insert_one(export_task.model_dump(exclude_none=False)) task_scheduler.add_job( func=_generate_export_in_background, From bc5c0a35269bbcec841501bfe67708fc0eb0c8db Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 23:22:56 +0100 Subject: [PATCH 14/31] Fix info endpoint for pydantic 2 --- pydatalab/src/pydatalab/routes/v0_1/info.py | 37 +++++++++------------ 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/info.py b/pydatalab/src/pydatalab/routes/v0_1/info.py index 40e4fc6a5..4bf7bd6e1 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/info.py +++ b/pydatalab/src/pydatalab/routes/v0_1/info.py @@ -5,6 +5,7 @@ from datetime import timedelta as td from datetime import timezone as tz from functools import lru_cache +from typing import Any, Generic, TypeVar from flask import Blueprint, jsonify, request from pydantic import ( @@ -13,7 +14,6 @@ ConfigDict, Field, field_validator, - model_validator, ) from pydatalab import __version__ @@ -47,14 +47,19 @@ class Links(BaseModel): model_config = ConfigDict(extra="allow") -class Data(BaseModel): +AttributesT = TypeVar("AttributesT") + + +class Data(BaseModel, Generic[AttributesT]): id: str type: str - attributes: Attributes + attributes: AttributesT + """The attributes payload, serialized as whatever concrete type the envelope is + parametrised with (e.g. ``Data[Info]``) so subclass fields are not stripped.""" -class JSONAPIResponse(BaseModel): - data: Data | list[Data] +class JSONAPIResponse(BaseModel, Generic[AttributesT]): + data: Data[AttributesT] | list[Data[AttributesT]] meta: Meta links: Links | None = None @@ -70,7 +75,7 @@ class Info(Attributes, Meta): homepage: AnyUrl | None = None source_repository: AnyUrl | None = None identifier_prefix: str - features: FeatureFlags = FEATURE_FLAGS + features: FeatureFlags | None = None max_upload_bytes: int @field_validator("maintainer", mode="before") @@ -80,18 +85,6 @@ def strip_maintainer_fields(cls, v): return MetaPerson(contact_email=v.contact_email, display_name=v.display_name) return v - @model_validator(mode="after") - def ensure_features_serialization(self): - """Ensure features are properly serialized for frontend consumption.""" - if hasattr(self.features, "model_dump"): - features_dict = self.features.model_dump() - else: - features_dict = self.features - - if not isinstance(self.features, FeatureFlags): - self.features = FeatureFlags(**features_dict) - return self - @lru_cache(maxsize=1) def _get_deployment_metadata_once() -> dict: @@ -118,7 +111,7 @@ def get_info(): """ - response_data = JSONAPIResponse( + response_data = JSONAPIResponse[Info]( data=Data(id="/", type="info", attributes=Info(**_get_deployment_metadata_once())), meta=Meta(query=request.query_string.decode() if request.query_string else ""), links=Links(self=request.url), @@ -150,7 +143,7 @@ def list_block_types(): """Returns a list of all blocks implemented in this server.""" return jsonify( json.loads( - JSONAPIResponse( + JSONAPIResponse[dict[str, Any]]( data=[ Data( id=block_type, @@ -179,7 +172,7 @@ def list_supported_types(): return jsonify( json.loads( - JSONAPIResponse( + JSONAPIResponse[dict[str, Any]]( data=[ Data( id=item_type, @@ -208,7 +201,7 @@ def get_schema_type(item_type): return jsonify( json.loads( - JSONAPIResponse( + JSONAPIResponse[dict[str, Any]]( data=Data( id=item_type, type="item_type", From 71d23a93f00fdcf43f852d2d929fa797f541b80c Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 23:31:59 +0100 Subject: [PATCH 15/31] Unpick circular dependencies between export/items routes by moving lookups to mongo module --- pydatalab/src/pydatalab/export.py | 4 +- pydatalab/src/pydatalab/mongo.py | 40 +++++++++++++++++ pydatalab/src/pydatalab/routes/v0_1/items.py | 46 ++++---------------- 3 files changed, 50 insertions(+), 40 deletions(-) diff --git a/pydatalab/src/pydatalab/export.py b/pydatalab/src/pydatalab/export.py index 1a83a32b5..a06551765 100644 --- a/pydatalab/src/pydatalab/export.py +++ b/pydatalab/src/pydatalab/export.py @@ -12,10 +12,10 @@ from pydatalab.config import CONFIG from pydatalab.logger import LOGGER from pydatalab.models import ITEM_MODELS -from pydatalab.mongo import flask_mongo -from pydatalab.routes.v0_1.items import ( +from pydatalab.mongo import ( creators_lookup, files_lookup, + flask_mongo, groups_lookup, ) diff --git a/pydatalab/src/pydatalab/mongo.py b/pydatalab/src/pydatalab/mongo.py index 09215c3e6..509bb2d2c 100644 --- a/pydatalab/src/pydatalab/mongo.py +++ b/pydatalab/src/pydatalab/mongo.py @@ -25,6 +25,9 @@ "GROUPS_FTS_FIELDS", "generate_heuristic_regex_search", "build_search_pipeline", + "creators_lookup", + "groups_lookup", + "files_lookup", ) flask_mongo = PyMongo() @@ -35,6 +38,43 @@ """ +def creators_lookup() -> dict: + return { + "from": "users", + "let": {"creator_ids": "$creator_ids"}, + "pipeline": [ + {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$creator_ids", []]}]}}}, + {"$addFields": {"__order": {"$indexOfArray": ["$$creator_ids", "$_id"]}}}, + {"$sort": {"__order": 1}}, + {"$project": {"_id": 1, "display_name": 1, "gravatar_hash": 1}}, + ], + "as": "creators", + } + + +def groups_lookup() -> dict: + return { + "from": "groups", + "let": {"group_ids": "$group_ids"}, + "pipeline": [ + {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$group_ids", []]}]}}}, + {"$addFields": {"__order": {"$indexOfArray": ["$$group_ids", "$_id"]}}}, + {"$sort": {"__order": 1}}, + {"$project": {"_id": 1, "display_name": 1, "group_id": 1}}, + ], + "as": "groups", + } + + +def files_lookup() -> dict: + return { + "from": "files", + "localField": "file_ObjectIds", + "foreignField": "_id", + "as": "files", + } + + @lru_cache(maxsize=1) def get_items_fts_fields() -> set[str]: """Get all string fields from item models for full-text search.""" diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 6cd43f1f6..b8934ef17 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -25,7 +25,14 @@ RestoreVersionRequest, VersionAction, ) -from pydatalab.mongo import build_search_pipeline, flask_mongo, get_items_fts_fields +from pydatalab.mongo import ( + build_search_pipeline, + creators_lookup, + files_lookup, + flask_mongo, + get_items_fts_fields, + groups_lookup, +) from pydatalab.permissions import ( PUBLIC_USER_ID, access_token_or_active_users, @@ -302,34 +309,6 @@ def get_samples_summary(match: dict | None = None, project: dict | None = None) ) -def creators_lookup() -> dict: - return { - "from": "users", - "let": {"creator_ids": "$creator_ids"}, - "pipeline": [ - {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$creator_ids", []]}]}}}, - {"$addFields": {"__order": {"$indexOfArray": ["$$creator_ids", "$_id"]}}}, - {"$sort": {"__order": 1}}, - {"$project": {"_id": 1, "display_name": 1, "gravatar_hash": 1}}, - ], - "as": "creators", - } - - -def groups_lookup() -> dict: - return { - "from": "groups", - "let": {"group_ids": "$group_ids"}, - "pipeline": [ - {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$group_ids", []]}]}}}, - {"$addFields": {"__order": {"$indexOfArray": ["$$group_ids", "$_id"]}}}, - {"$sort": {"__order": 1}}, - {"$project": {"_id": 1, "display_name": 1, "group_id": 1}}, - ], - "as": "groups", - } - - def entry_reference_lookup(item_doc: dict) -> dict: """Looks up any field that contains an entry reference and resolves it to the item data.""" @@ -409,15 +388,6 @@ def entry_reference_lookup(item_doc: dict) -> dict: return item_doc -def files_lookup() -> dict: - return { - "from": "files", - "localField": "file_ObjectIds", - "foreignField": "_id", - "as": "files", - } - - def collections_lookup() -> dict: """Looks inside the relationships of the item, searches for IDs in the collections table and then projects only the collection ID and name for the response. From b3d5e66e07089302c2a2bada7c060f46c2e2d7f6 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 23:53:20 +0100 Subject: [PATCH 16/31] Move `datalab_*` fields in schema to `json_schema_extra` and allow carve out for plugins --- pydatalab/src/pydatalab/apps/__init__.py | 11 ++++++++++- pydatalab/src/pydatalab/apps/chat/blocks.py | 4 +++- pydatalab/src/pydatalab/blocks/base.py | 4 ++-- pydatalab/src/pydatalab/models/blocks.py | 18 +++++++++++++----- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/pydatalab/src/pydatalab/apps/__init__.py b/pydatalab/src/pydatalab/apps/__init__.py index 9f49a0950..f48a6427e 100644 --- a/pydatalab/src/pydatalab/apps/__init__.py +++ b/pydatalab/src/pydatalab/apps/__init__.py @@ -3,8 +3,11 @@ """ +import warnings from typing import TYPE_CHECKING +from pydantic.warnings import PydanticDeprecatedSince20 + if TYPE_CHECKING: # This import is required to prevent circular imports for application-specific blocks from pydatalab.blocks.base import DataBlock # noqa @@ -116,7 +119,13 @@ def load_block_plugins(): block_plugins: dict[str, type[DataBlock]] = {} for entry_point in entry_points(group="pydatalab.apps.plugins"): - block = entry_point.load() + # Plugins may not yet be migrated to pydantic v2, so tolerate the v2 + # deprecation warnings their models emit at import time (e.g. extra `Field` + # kwargs like `datalab_exclude_*`) rather than letting them error out plugin + # loading. datalab's own deprecations are still surfaced as errors. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", PydanticDeprecatedSince20) + block = entry_point.load() if not issubclass(block, DataBlock): raise ValueError(f"Plugin {block} must be a subclass of DataBlock") diff --git a/pydatalab/src/pydatalab/apps/chat/blocks.py b/pydatalab/src/pydatalab/apps/chat/blocks.py index b95294faa..78b53b1b2 100644 --- a/pydatalab/src/pydatalab/apps/chat/blocks.py +++ b/pydatalab/src/pydatalab/apps/chat/blocks.py @@ -19,7 +19,9 @@ class ChatBlockResponse(DataBlockResponse): prompt: str | None = None model: str available_models: dict[str, ModelCard] | None = Field( - None, validate_default=True, datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, + validate_default=True, + json_schema_extra={"datalab_exclude_from_db": True, "datalab_exclude_from_load": True}, ) token_count: int | None = None temperature: float diff --git a/pydatalab/src/pydatalab/blocks/base.py b/pydatalab/src/pydatalab/blocks/base.py index 2e696d70e..37333b59e 100644 --- a/pydatalab/src/pydatalab/blocks/base.py +++ b/pydatalab/src/pydatalab/blocks/base.py @@ -193,7 +193,7 @@ def to_db(self) -> dict: LOGGER.debug("Casting block %s to database object.", self.__class__.__name__) exclude_fields: set[str] = { f - for (f, s) in self.block_db_model.schema()["properties"].items() + for (f, s) in self.block_db_model.model_json_schema()["properties"].items() if s.get("datalab_exclude_from_db") } return self.block_db_model(**self.data).model_dump( @@ -339,7 +339,7 @@ def update_from_web(self, data: dict): ) exclude_fields: set[str] = { f - for (f, s) in self.block_db_model.schema()["properties"].items() + for (f, s) in self.block_db_model.model_json_schema()["properties"].items() if s.get("datalab_exclude_from_load") } [data.pop(f, None) for f in exclude_fields] diff --git a/pydatalab/src/pydatalab/models/blocks.py b/pydatalab/src/pydatalab/models/blocks.py index f1e02a4d4..e6d9244e3 100644 --- a/pydatalab/src/pydatalab/models/blocks.py +++ b/pydatalab/src/pydatalab/models/blocks.py @@ -44,23 +44,31 @@ class DataBlockResponse(BaseModel): """Any warnings that occurred during block processing.""" b64_encoded_image: dict[str, str] | None = Field( - None, datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, + json_schema_extra={"datalab_exclude_from_db": True, "datalab_exclude_from_load": True}, ) """Any base64-encoded image data associated with the block, keyed by `file_id`.""" bokeh_plot_data: dict | None = Field( - None, datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, + json_schema_extra={"datalab_exclude_from_db": True, "datalab_exclude_from_load": True}, ) """A JSON-encoded string containing the Bokeh plot data, if any.""" - computed: dict | None = Field(default=None, datalab_exclude_from_load=True) + computed: dict | None = Field( + default=None, json_schema_extra={"datalab_exclude_from_load": True} + ) """Any processed or computed data associated with the block, small enough to store and filter directly in the database, i.e., strings or a few hundred numbers not exceeding 16KB in size. Examples could include peak positions, and widths, but not the full spectrum. """ - processed: dict | None = Field(default=None, datalab_exclude_from_load=True) + processed: dict | None = Field( + default=None, json_schema_extra={"datalab_exclude_from_load": True} + ) - metadata: dict | None = Field(default=None, datalab_exclude_from_load=True) + metadata: dict | None = Field( + default=None, json_schema_extra={"datalab_exclude_from_load": True} + ) """Any structured metadata associated with the block, for example, experimental acquisition parameters.""" From 50b55fe4247362eafc93e8a2b01967e726566d18 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 00:16:59 +0100 Subject: [PATCH 17/31] Final wave of model_dump/dict changes --- pydatalab/src/pydatalab/apps/nmr/blocks.py | 6 ++-- pydatalab/src/pydatalab/routes/v0_1/admin.py | 8 ++++-- pydatalab/src/pydatalab/routes/v0_1/blocks.py | 4 +-- pydatalab/src/pydatalab/routes/v0_1/groups.py | 5 +++- pydatalab/src/pydatalab/routes/v0_1/users.py | 2 +- pydatalab/tests/server/test_async_blocks.py | 26 ++++++++--------- pydatalab/tests/server/test_export.py | 28 +++++++++---------- pydatalab/tests/server/test_graph.py | 2 +- pydatalab/tests/server/test_item_graph.py | 13 +++++---- pydatalab/tests/server/test_item_versions.py | 12 ++++---- pydatalab/tests/server/test_samples.py | 21 ++++++++------ pydatalab/tests/server/test_search.py | 2 +- .../tests/server/test_starting_materials.py | 2 +- pydatalab/tests/test_models.py | 6 ++-- 14 files changed, 73 insertions(+), 64 deletions(-) diff --git a/pydatalab/src/pydatalab/apps/nmr/blocks.py b/pydatalab/src/pydatalab/apps/nmr/blocks.py index 028b4b16d..c2975b875 100644 --- a/pydatalab/src/pydatalab/apps/nmr/blocks.py +++ b/pydatalab/src/pydatalab/apps/nmr/blocks.py @@ -130,7 +130,7 @@ def read_bruker_nmr_data( metadata["pulse_program_name"] = a_dic["acqus"]["PULPROG"] metadata["title"] = topspin_title - self.data["metadata"] = NMRMetadata(**metadata).dict() + self.data["metadata"] = NMRMetadata(**metadata).model_dump() return serialized_df, metadata @@ -209,7 +209,7 @@ def read_jcamp_nmr_data( pass serialized_df = df.to_dict() if (df is not None) else None - self.data["metadata"] = NMRMetadata(**metadata).dict() + self.data["metadata"] = NMRMetadata(**metadata).model_dump() return serialized_df, metadata @@ -293,7 +293,7 @@ def read_jeol_nmr_data( metadata["nscans"] = nscans serialized_df = df.to_dict() if (df is not None) else None - self.data["metadata"] = NMRMetadata(**metadata).dict() + self.data["metadata"] = NMRMetadata(**metadata).model_dump() return serialized_df, metadata diff --git a/pydatalab/src/pydatalab/routes/v0_1/admin.py b/pydatalab/src/pydatalab/routes/v0_1/admin.py index 55e886fae..eea5a6478 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/admin.py +++ b/pydatalab/src/pydatalab/routes/v0_1/admin.py @@ -125,7 +125,7 @@ def get_users(): elif not isinstance(user["managers"], list): user["managers"] = [] - return jsonify({"status": "success", "data": [Person(**d).dict() for d in users_list]}) + return jsonify({"status": "success", "data": [Person(**d).model_dump() for d in users_list]}) @ADMIN.route("/roles/", methods=["PATCH"]) @@ -357,7 +357,9 @@ def get_groups(): [{"$match": {}}, {"$lookup": members_lookup}, {"$lookup": managers_lookup}] ) - return jsonify({"status": "success", "data": [Group(**d).dict() for d in group_docs]}), 200 + return jsonify( + {"status": "success", "data": [Group(**d).model_dump() for d in group_docs]} + ), 200 @ADMIN.route("/groups", methods=["PUT"]) @@ -392,7 +394,7 @@ def create_group(): try: group_immutable_id = flask_mongo.db.groups.insert_one( - group.dict(exclude_unset=True) + group.model_dump(exclude_unset=True) ).inserted_id except pymongo.errors.DuplicateKeyError: return jsonify( diff --git a/pydatalab/src/pydatalab/routes/v0_1/blocks.py b/pydatalab/src/pydatalab/routes/v0_1/blocks.py index dfdbb038e..360e610f6 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/blocks.py +++ b/pydatalab/src/pydatalab/routes/v0_1/blocks.py @@ -54,7 +54,7 @@ def add_stage(message: str, level: str = "info", traceback: str | None = None): timestamp=datetime.now(tz=timezone.utc), message=message, level=level, detail=traceback ) flask_mongo.db.tasks.update_one( - {"task_id": task_id}, {"$push": {"spec.stages": stage.dict()}} + {"task_id": task_id}, {"$push": {"spec.stages": stage.model_dump()}} ) with app_ctx, req_ctx: @@ -369,7 +369,7 @@ def update_block(): ), ) - flask_mongo.db.tasks.insert_one(block_task.dict()) + flask_mongo.db.tasks.insert_one(block_task.model_dump()) task_scheduler.add_job( func=_process_block_async, diff --git a/pydatalab/src/pydatalab/routes/v0_1/groups.py b/pydatalab/src/pydatalab/routes/v0_1/groups.py index e1a54d5ef..d5b4e5648 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/groups.py +++ b/pydatalab/src/pydatalab/routes/v0_1/groups.py @@ -42,5 +42,8 @@ def search_groups(): cursor = flask_mongo.db.groups.aggregate(pipeline) return jsonify( - {"status": "success", "data": list(json.loads(Group(**d).json()) for d in cursor)} + { + "status": "success", + "data": list(json.loads(Group(**d).model_dump_json()) for d in cursor), + } ), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/users.py b/pydatalab/src/pydatalab/routes/v0_1/users.py index f790a12bf..ab9e737f0 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/users.py +++ b/pydatalab/src/pydatalab/routes/v0_1/users.py @@ -218,5 +218,5 @@ def search_users(): cursor = flask_mongo.db.users.aggregate(pipeline) return jsonify( - {"status": "success", "users": [json.loads(Person(**d).json()) for d in cursor]} + {"status": "success", "users": [json.loads(Person(**d).model_dump_json()) for d in cursor]} ), 200 diff --git a/pydatalab/tests/server/test_async_blocks.py b/pydatalab/tests/server/test_async_blocks.py index 7d143e14d..226ec9fff 100644 --- a/pydatalab/tests/server/test_async_blocks.py +++ b/pydatalab/tests/server/test_async_blocks.py @@ -147,7 +147,7 @@ def test_status_pending(self, client, user_id, database): block_id="test_block", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -182,7 +182,7 @@ def test_status_processing_with_stages(self, client, user_id, database): stages=stages, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -226,7 +226,7 @@ def test_status_ready_with_gridfs_data(self, client, user_id, database, app): block_id=block_id, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) # Write block data to GridFS from pydatalab.mongo import get_database @@ -276,7 +276,7 @@ def test_status_ready_no_gridfs_data(self, client, user_id, database): block_id=block_id, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -304,7 +304,7 @@ def test_status_error(self, client, user_id, database): block_id="test_block", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -334,7 +334,7 @@ def test_status_ignores_export_tasks(self, client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 404 @@ -369,7 +369,7 @@ def test_status_ready_respects_permissions(self, another_client, user_id, databa block_id=block_id, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) from pydatalab.mongo import get_database @@ -428,7 +428,7 @@ def test_successful_processing(self, app, user_id, database): status=TaskStatus.PENDING, spec=BlockProcessingTaskSpec(item_id=item_id, block_id=block_id), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) block_data = { "blocktype": "comment", @@ -470,7 +470,7 @@ def test_processing_error_handling(self, app, user_id, database): status=TaskStatus.PENDING, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) # Pass an invalid block type to trigger an error block_data = { @@ -509,7 +509,7 @@ def test_marks_timed_out_tasks_as_error(self, app, user_id, database): status=TaskStatus.PROCESSING, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - task_dict = task.dict() + task_dict = task.model_dump() task_dict["created_at"] = old_time database.tasks.insert_one(task_dict) @@ -534,7 +534,7 @@ def test_does_not_timeout_recent_tasks(self, app, user_id, database): status=TaskStatus.PROCESSING, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _cleanup_stale_tasks() @@ -558,7 +558,7 @@ def test_purges_old_completed_tasks_and_gridfs(self, app, user_id, database): completed_at=old_time, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - task_dict = task.dict() + task_dict = task.model_dump() task_dict["created_at"] = old_time database.tasks.insert_one(task_dict) @@ -586,7 +586,7 @@ def test_does_not_purge_recent_completed_tasks(self, app, user_id, database): completed_at=datetime.now(tz=timezone.utc), spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _cleanup_stale_tasks() diff --git a/pydatalab/tests/server/test_export.py b/pydatalab/tests/server/test_export.py index 9ef6ef757..7e8d4c7d3 100644 --- a/pydatalab/tests/server/test_export.py +++ b/pydatalab/tests/server/test_export.py @@ -78,7 +78,7 @@ def test_get_export_status_pending(client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/status") assert response.status_code == 200 @@ -107,7 +107,7 @@ def test_get_export_status_ready(client, user_id, database, tmp_path): file_path=str(file_path), ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/status") assert response.status_code == 200 @@ -136,7 +136,7 @@ def test_get_export_status_error(client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/status") assert response.status_code == 200 @@ -180,7 +180,7 @@ def test_download_export_success(client, user_id, database, tmp_path, monkeypatc file_path=str(file_path), ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/download") assert response.status_code == 200 @@ -203,7 +203,7 @@ def test_not_users_export_download(client, user_id, another_client, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = another_client.get(f"/exports/{task_id}/download") assert response.status_code == 404 @@ -223,7 +223,7 @@ def test_download_export_not_ready(client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/download") assert response.status_code == 400 @@ -248,7 +248,7 @@ def test_download_export_file_missing(client, user_id, database): file_path="/nonexistent/path.eln.zip", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/download") assert response.status_code == 404 @@ -275,7 +275,7 @@ def test_do_export_success(database, sample_collection, insert_default_sample, u export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _do_export(task_id, collection_id=collection_id, export_type="collection") @@ -299,7 +299,7 @@ def test_do_export_success(database, sample_collection, insert_default_sample, u export_type="item", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _do_export(task_id, item_id=item_id, export_type="item") @@ -326,7 +326,7 @@ def test_do_export_error_handling(database, user_id): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _do_export(task_id, collection_id="nonexistent_collection", export_type="collection") @@ -367,7 +367,7 @@ def test_cleanup_old_exports(database, user_id, tmp_path, monkeypatch): file_path=str(old_file), ), ) - database.tasks.insert_one(old_task.dict()) + database.tasks.insert_one(old_task.model_dump()) # A recent export: created within the window, must be retained. recent_task_id = "recent-export-to-keep" @@ -385,7 +385,7 @@ def test_cleanup_old_exports(database, user_id, tmp_path, monkeypatch): file_path=str(recent_file), ), ) - database.tasks.insert_one(recent_task.dict()) + database.tasks.insert_one(recent_task.model_dump()) try: _cleanup_old_exports() @@ -422,7 +422,7 @@ def test_cleanup_old_exports_missing_file(database, user_id): file_path="/nonexistent/already-deleted.eln.zip", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) try: _cleanup_old_exports() @@ -447,7 +447,7 @@ def test_do_export_status_transitions(database, sample_collection, user_id): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) status_during_export = [] diff --git a/pydatalab/tests/server/test_graph.py b/pydatalab/tests/server/test_graph.py index 3e10a9dc9..9c1f30e3d 100644 --- a/pydatalab/tests/server/test_graph.py +++ b/pydatalab/tests/server/test_graph.py @@ -61,7 +61,7 @@ def test_simple_graph(admin_client): ) new_samples = [ - json.loads(d.json()) + json.loads(d.model_dump_json()) for d in [parent, child_1, child_2, child_3, child_4, missing_child, cell] ] diff --git a/pydatalab/tests/server/test_item_graph.py b/pydatalab/tests/server/test_item_graph.py index 08ffa7db4..f04169aa6 100644 --- a/pydatalab/tests/server/test_item_graph.py +++ b/pydatalab/tests/server/test_item_graph.py @@ -15,7 +15,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(material.json())}, + json={"new_sample_data": json.loads(material.model_dump_json())}, ) assert creation.status_code == 201 @@ -39,7 +39,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(parent.json())}, + json={"new_sample_data": json.loads(parent.model_dump_json())}, ) assert creation.status_code == 201 @@ -68,7 +68,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(child.json())}, + json={"new_sample_data": json.loads(child.model_dump_json())}, ) grandchild = Sample( @@ -80,7 +80,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(grandchild.json())}, + json={"new_sample_data": json.loads(grandchild.model_dump_json())}, ) great_grandchild = Sample( @@ -92,7 +92,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(great_grandchild.json())}, + json={"new_sample_data": json.loads(great_grandchild.model_dump_json())}, ) graph = client.get("/item-graph").json @@ -114,7 +114,8 @@ def test_single_starting_material(admin_client, client): ) creation = admin_client.post( - "/new-sample/", json={"new_sample_data": json.loads(admin_great_great_grandchild.json())} + "/new-sample/", + json={"new_sample_data": json.loads(admin_great_great_grandchild.model_dump_json())}, ) graph = admin_client.get("/item-graph/great-grandchild?sudo=1").json diff --git a/pydatalab/tests/server/test_item_versions.py b/pydatalab/tests/server/test_item_versions.py index 4a61a84ec..4c6fd3c3d 100644 --- a/pydatalab/tests/server/test_item_versions.py +++ b/pydatalab/tests/server/test_item_versions.py @@ -24,7 +24,7 @@ def sample_with_version(client, user_id): "synthesis_description": "Initial synthesis", } ) - flask_mongo.db.items.insert_one(sample.dict(exclude_unset=False)) + flask_mongo.db.items.insert_one(sample.model_dump(exclude_unset=False)) yield sample @@ -559,7 +559,7 @@ def test_save_item_creates_version(self, client, sample_with_version): refcode_short = sample_with_version.refcode.split(":")[1] # Modify and save the item using save-item endpoint - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "Updated via save-item" response = client.post("/save-item/", json={"item_id": item_id, "data": item_data}) @@ -583,7 +583,7 @@ def test_save_item_increments_version(self, client, sample_with_version): refcode = sample_with_version.refcode - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "First real change" client.post("/save-item/", json={"item_id": sample_with_version.item_id, "data": item_data}) @@ -651,7 +651,7 @@ def test_save_item_endpoint_creates_manual_save_action(self, client, sample_with refcode_short = sample_with_version.refcode.split(":")[1] # Save via save-item endpoint (user clicking save button) - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "Updated via save-item" client.post("/save-item/", json={"item_id": sample_with_version.item_id, "data": item_data}) @@ -819,7 +819,7 @@ def test_save_item_identical_does_not_create_version(self, client, sample_with_v refcode = sample_with_version.refcode.split(":")[1] full_refcode = sample_with_version.refcode - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "A new description" client.post("/save-item/", json={"item_id": item_id, "data": item_data}) @@ -839,7 +839,7 @@ def test_save_item_with_change_creates_new_version(self, client, sample_with_ver refcode = sample_with_version.refcode.split(":")[1] full_refcode = sample_with_version.refcode - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "First real change" client.post("/save-item/", json={"item_id": item_id, "data": item_data}) diff --git a/pydatalab/tests/server/test_samples.py b/pydatalab/tests/server/test_samples.py index 4c288f9e9..9edb53e4f 100644 --- a/pydatalab/tests/server/test_samples.py +++ b/pydatalab/tests/server/test_samples.py @@ -360,7 +360,7 @@ def test_item_regex_search( def test_new_sample_with_relationships( client, complicated_sample, insert_complicated_sample_constituents ): - complicated_sample_json = json.loads(complicated_sample.json()) + complicated_sample_json = json.loads(complicated_sample.model_dump_json()) response = client.post("/new-sample/", json=complicated_sample_json) # Test that 201: Created is emitted assert response.status_code == 201, response.json @@ -424,7 +424,7 @@ def test_new_sample_with_relationships( description="This is a new relationship", ) ) - derived_sample_json = json.loads(derived_sample.json()) + derived_sample_json = json.loads(derived_sample.model_dump_json()) response = client.post("/new-sample/", json=derived_sample_json) # Test that 201: Created is emitted @@ -528,7 +528,7 @@ def test_copy_from_sample(client, complicated_sample): """ complicated_sample.item_id = "new_complicated_sample" - complicated_sample_json = json.loads(complicated_sample.json()) + complicated_sample_json = json.loads(complicated_sample.model_dump_json()) response = client.post("/new-sample/", json=complicated_sample_json) # Test that 201: Created is emitted @@ -558,12 +558,13 @@ def test_copy_from_sample(client, complicated_sample): @pytest.mark.dependency(depends=["test_copy_from_sample"]) def test_create_multiple_samples(client, complicated_sample): - samples = [complicated_sample, complicated_sample.copy()] + samples = [complicated_sample, complicated_sample.model_copy()] samples[0].item_id = "another_new_complicated_sample" samples[1].item_id = "additional_new_complicated_sample" response = client.post( - "/new-samples/", json={"new_sample_datas": [json.loads(s.json()) for s in samples]} + "/new-samples/", + json={"new_sample_datas": [json.loads(s.model_dump_json()) for s in samples]}, ) assert response.status_code == 207, response.json assert response.json["nsuccess"] == 2, response.json @@ -577,7 +578,7 @@ def test_create_multiple_samples(client, complicated_sample): response = client.post( "/new-samples/", json={ - "new_sample_datas": [json.loads(s.json()) for s in samples], + "new_sample_datas": [json.loads(s.model_dump_json()) for s in samples], "copy_from_item_ids": [ "another_new_complicated_sample", "additional_new_complicated_sample", @@ -599,7 +600,7 @@ def test_create_multiple_samples(client, complicated_sample): @pytest.mark.dependency(depends=["test_create_multiple_samples"]) def test_create_cell(client, default_cell): - response = client.post("/new-sample/", json=json.loads(default_cell.json())) + response = client.post("/new-sample/", json=json.loads(default_cell.model_dump_json())) assert response.status_code == 201, response.json assert response.json["status"] == "success" @@ -672,7 +673,9 @@ def test_create_collections(client, default_collection, database): assert response.status_code == 200 # Create an empty collection - response = client.put("/collections", json={"data": json.loads(default_collection.json())}) + response = client.put( + "/collections", json={"data": json.loads(default_collection.model_dump_json())} + ) assert response.status_code == 201, response.json assert response.json["status"] == "success" assert response.json["data"]["collection_id"] == "test_collection" @@ -691,7 +694,7 @@ def test_create_collections(client, default_collection, database): new_collection = copy.deepcopy(default_collection) new_collection.collection_id = "test_collection_2" - data = json.loads(new_collection.json()) + data = json.loads(new_collection.model_dump_json()) data.update( { "starting_members": [ diff --git a/pydatalab/tests/server/test_search.py b/pydatalab/tests/server/test_search.py index 40e4771dd..d9b0cabe9 100644 --- a/pydatalab/tests/server/test_search.py +++ b/pydatalab/tests/server/test_search.py @@ -52,7 +52,7 @@ def test_collection_regex_search(query, should_find, real_mongo_client, client, if real_mongo_client is None: pytest.skip("Skipping search tests, not connected to real MongoDB") - test_collection = default_collection.dict() + test_collection = default_collection.model_dump() clean_query = query.replace("=", "_").replace("%23", "hash_").replace("'", "") test_collection["collection_id"] = f"test_coll_{clean_query}" test_collection["title"] = "test collection" diff --git a/pydatalab/tests/server/test_starting_materials.py b/pydatalab/tests/server/test_starting_materials.py index 3ee28080a..080559e5d 100644 --- a/pydatalab/tests/server/test_starting_materials.py +++ b/pydatalab/tests/server/test_starting_materials.py @@ -133,7 +133,7 @@ def test_starting_materials_list_includes_collections( client, default_starting_material_dict, default_collection ): """GET /starting-materials/ should include a collections array with collection_id for each item.""" - collection_data = json.loads(default_collection.json()) + collection_data = json.loads(default_collection.model_dump_json()) collection_data["starting_members"] = [{"item_id": default_starting_material_dict["item_id"]}] response = client.put("/collections", json={"data": collection_data}) assert response.status_code == 201, response.json diff --git a/pydatalab/tests/test_models.py b/pydatalab/tests/test_models.py index c92dbb7ce..625ae5d08 100644 --- a/pydatalab/tests/test_models.py +++ b/pydatalab/tests/test_models.py @@ -52,7 +52,7 @@ def test_sample_with_inlined_reference(): assert parents[0].item_id == a.item_id assert parents[0].refcode == a.refcode - b_both = Sample(**json.loads(b_both.json())) + b_both = Sample(**json.loads(b_both.model_dump_json())) parents = [r for r in b_both.relationships if r.relation == RelationshipType.PARENT] assert len(parents) == 1 @@ -209,7 +209,7 @@ class TestItem(Item): tzinfo=datetime.timezone.utc ) - item_json = json.loads(item.json()) + item_json = json.loads(item.model_dump_json()) assert item_json["type"] == "items_custom" assert item_json["creator_ids"][0] == "0123456789ab0123456789ab" assert item_json["creator_ids"][1] == "1023456789ab0123456789ab" @@ -239,7 +239,7 @@ class TestItem(Item): "2020-01-01 00:00" ).replace(tzinfo=datetime.timezone.utc) - sample_json = json.loads(sample.json()) + sample_json = json.loads(sample.model_dump_json()) assert sample_json["type"] == "samples" assert sample_json["creator_ids"][0] == str(ObjectId("0123456789ab0123456789ab")) assert sample_json["creator_ids"][1] == str(ObjectId("1023456789ab0123456789ab")) From c5698a030d6ff4eda1f07b16de473d92fc3d0e98 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 00:19:53 +0100 Subject: [PATCH 18/31] Minor tweaks suggested by copilot --- pydatalab/src/pydatalab/models/people.py | 4 ---- pydatalab/src/pydatalab/routes/v0_1/info.py | 6 +++--- pydatalab/src/pydatalab/routes/v0_1/items.py | 8 ++++++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pydatalab/src/pydatalab/models/people.py b/pydatalab/src/pydatalab/models/people.py index ec3485ade..5a3897419 100644 --- a/pydatalab/src/pydatalab/models/people.py +++ b/pydatalab/src/pydatalab/models/people.py @@ -190,10 +190,6 @@ def add_missing_type(cls, v): v = "people" return v - @classmethod - def set_default_type(cls, _): - return "people" - @staticmethod def new_user_from_identity( identity: Identity, diff --git a/pydatalab/src/pydatalab/routes/v0_1/info.py b/pydatalab/src/pydatalab/routes/v0_1/info.py index 4bf7bd6e1..dd74ebb90 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/info.py +++ b/pydatalab/src/pydatalab/routes/v0_1/info.py @@ -160,7 +160,7 @@ def list_block_types(): ) for block_type, block in BLOCK_TYPES.items() ], - meta=Meta(query=request.query_string), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), ).model_dump_json() ) ) @@ -185,7 +185,7 @@ def list_supported_types(): ) for item_type, schema in ITEM_SCHEMAS.items() ], - meta=Meta(query=request.query_string), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), ).model_dump_json() ) ) @@ -211,7 +211,7 @@ def get_schema_type(item_type): "schema": ITEM_SCHEMAS[item_type], }, ), - meta=Meta(query=request.query_string), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), ).model_dump_json() ) ) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index b8934ef17..32b7d9241 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -708,9 +708,13 @@ def _create_sample( raise BadRequest(f"Failed to add new item {new_sample['item_id']!r} to database.") # Save initial version snapshot after successful item creation + refcode = data_model.refcode + if not isinstance(refcode, str): + raise InternalServerError(f"Invalid refcode generated for new item: {refcode!r}") try: version_resp, version_status = save_version_snapshot( - data_model.refcode, action=VersionAction.CREATED + refcode, + action=VersionAction.CREATED, # type: ignore ) if version_status != 200: LOGGER.error( @@ -1331,7 +1335,7 @@ def compare_versions(refcode): v1 = flask_mongo.db.item_versions.find_one({"_id": query_params.v1, "refcode": refcode}) v2 = flask_mongo.db.item_versions.find_one({"_id": query_params.v2, "refcode": refcode}) if not v1 or not v2: - return NotFound("One or both versions not found") + raise NotFound("One or both versions not found") # Use DeepDiff for proper nested structure comparison # This handles nested dicts, lists, type changes, and provides detailed change information From 62c150314ea91dc5a1e3e07d85d2ea067ca9334e Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 00:28:35 +0100 Subject: [PATCH 19/31] Fix docker build by reordering tasks imports --- pydatalab/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index aeedcd452..e526501e2 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -9,8 +9,6 @@ import tomlkit from invoke import Collection, task -from pydatalab.models.utils import UserRole - ns = Collection() dev = Collection("dev") admin = Collection("admin") @@ -274,13 +272,15 @@ def create_mongo_indices(_): @task -def change_user_role(_, display_name: str, role: "UserRole"): +def change_user_role(_, display_name: str, role: str): """This task takes a user's name and gives them the desired role.""" from bson import ObjectId from pydatalab.models.utils import UserRole from pydatalab.mongo import _get_active_mongo_client + role = UserRole(role.upper()) + try: role = getattr(UserRole, role.upper()) except AttributeError: From 2d9337b49dc2cccac7ff5f9f673a2beefa61ee77 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 10:05:05 +0100 Subject: [PATCH 20/31] Fix for auth routes --- pydatalab/src/pydatalab/login.py | 2 +- pydatalab/src/pydatalab/routes/v0_1/auth.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pydatalab/src/pydatalab/login.py b/pydatalab/src/pydatalab/login.py index 948d057a8..7accd84ac 100644 --- a/pydatalab/src/pydatalab/login.py +++ b/pydatalab/src/pydatalab/login.py @@ -105,7 +105,7 @@ def groups_lookup() -> dict: } -def get_by_id(user_id: str) -> LoginUser | None: +def get_by_id(user_id: str | ObjectId) -> LoginUser | None: """Lookup the user database ID and create a new `LoginUser` with the relevant metadata. diff --git a/pydatalab/src/pydatalab/routes/v0_1/auth.py b/pydatalab/src/pydatalab/routes/v0_1/auth.py index ef5f2d228..f5d36aedc 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/auth.py +++ b/pydatalab/src/pydatalab/routes/v0_1/auth.py @@ -613,10 +613,12 @@ def attach_identity_to_user( ) LOGGER.debug("Inserting new user model %s into database", user) inserted_id = insert_pydantic_model_fork_safe(user, "users") - user = get_by_id(inserted_id) - if user is None: + login_user = get_by_id(inserted_id) + if login_user is None: raise RuntimeError("Failed to insert user into database") + user = login_user.person + # Send email notification to admins _send_admin_email_notification(user) @@ -625,7 +627,8 @@ def attach_identity_to_user( # Log the user into the session with this identity if user is not None: - wrapped_login_user(user) + login_user = get_by_id(user.immutable_id) + wrapped_login_user(login_user) def _validate_magic_link_request(email: str, referrer: str) -> None: From 43e2ab37d8e2812d55f43d48660ac67975f43f51 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 10:05:22 +0100 Subject: [PATCH 21/31] Update plugin config validator for pydantic 2 --- pydatalab/tasks.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index e526501e2..63f7844a0 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -64,17 +64,16 @@ class PluginConfigModel(BaseModel): dependencies: list[str] = [] tool: ToolSection = ToolSection() - @model_validator(mode="before") - @classmethod - def _sources_must_match_dependencies(cls, values): - deps = {d.split("[")[0].strip() for d in values.get("dependencies", [])} - sources = values.get("tool", ToolSection()).uv.sources + @model_validator(mode="after") + def _sources_must_match_dependencies(self): + deps = {d.split("[")[0].strip() for d in self.dependencies} + sources = self.tool.uv.sources orphans = sorted(set(sources) - deps) if orphans: raise ValueError( f"[tool.uv.sources] entries have no matching dependency: {orphans}" ) - return values + return self return PluginConfigModel From e8b38e538fab5fcae74a63d56ef68f57e91b5b19 Mon Sep 17 00:00:00 2001 From: BenSmithGreyGroup Date: Thu, 25 Jun 2026 12:33:14 +0100 Subject: [PATCH 22/31] Added test to check if None fields are preserved with saving items and versions (pydantic 1 to 2 change) --- pydatalab/tests/server/test_item_versions.py | 129 +++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/pydatalab/tests/server/test_item_versions.py b/pydatalab/tests/server/test_item_versions.py index 4c6fd3c3d..dbf70396b 100644 --- a/pydatalab/tests/server/test_item_versions.py +++ b/pydatalab/tests/server/test_item_versions.py @@ -1352,3 +1352,132 @@ def test_sample_lifecycle(client, sample_with_version): flask_mongo.db.version_counters.delete_one({"refcode": refcode}) print("[TEST] ✓ Lifecycle test completed successfully") + + +class TestNoneFieldPreservation: + """Tests that None-valued fields are preserved through item creation, versioning, and restore. + + Covers two bugs introduced during the pydantic v1->v2 migration: + + Bug 1 (items.py): model_dump(exclude_none=True) on item creation strips None fields from + MongoDB entirely, so newly created items are missing keys like 'description', 'chemform', etc. + + Bug 2 (versioning.py): model_dump(exclude_none=True) on ItemVersion now recursively strips + None values from inside the 'data' dict (pydantic v2 changed behaviour vs v1), so version + snapshots also lose those keys. When restoring, $set cannot clear a key that is absent + from the restore payload. + """ + + def test_item_creation_preserves_none_fields_in_db(self, client): + """Creating an item with no description should store description=None in MongoDB, + not omit the key entirely.""" + from pydatalab.mongo import flask_mongo + + item_id = "none_field_test_create" + response = client.post( + "/new-sample/", + json={"name": "None field test", "item_id": item_id, "type": "samples"}, + ) + assert response.status_code == 201, response.json + + refcode = response.json["sample_list_entry"]["refcode"] + item = flask_mongo.db.items.find_one({"refcode": refcode}) + + # 'description' is a defined field on Sample — it must be present as a key + # even when the user did not supply a value. + assert "description" in item, ( + "'description' key missing from newly-created item; " + "model_dump(exclude_none=True) is incorrectly stripping None fields on creation" + ) + + # Cleanup + flask_mongo.db.items.delete_one({"refcode": refcode}) + flask_mongo.db.item_versions.delete_many({"refcode": refcode}) + flask_mongo.db.version_counters.delete_one({"refcode": refcode}) + + def test_version_snapshot_preserves_none_fields(self, client): + """Version snapshot stored in item_versions must retain None-valued fields from the + item document; pydantic v2 model_dump(exclude_none=True) must not strip them from + inside the nested 'data' dict.""" + from pydatalab.mongo import flask_mongo + + item_id = "none_field_test_version" + response = client.post( + "/new-sample/", + json={"name": "None field version test", "item_id": item_id, "type": "samples"}, + ) + assert response.status_code == 201, response.json + refcode = response.json["sample_list_entry"]["refcode"] + refcode_short = refcode.split(":")[-1] + + # The initial version is created on item creation; retrieve it. + list_resp = client.get(f"/items/{refcode_short}/versions/") + assert len(list_resp.json["versions"]) == 1 + version_id = list_resp.json["versions"][0]["_id"] + + version_resp = client.get(f"/items/{refcode_short}/versions/{version_id}/") + snapshot_data = version_resp.json["version"]["data"] + + assert "description" in snapshot_data, ( + "'description' key missing from version snapshot 'data'; " + "model_dump(exclude_none=True) on ItemVersion is recursively stripping None " + "values from the nested 'data' dict (pydantic v2 regression)" + ) + + # Cleanup + flask_mongo.db.items.delete_one({"refcode": refcode}) + flask_mongo.db.item_versions.delete_many({"refcode": refcode}) + flask_mongo.db.version_counters.delete_one({"refcode": refcode}) + + def test_restore_to_version_with_none_field_clears_populated_field(self, client): + """Restoring to a version where a field was None must clear that field in the current + item, even if the field has since been populated. + + Failure mode: the version snapshot is missing the key entirely (due to Bug 1/2), so + $set has nothing to write and the field retains its populated value after restore. + """ + from pydatalab.mongo import flask_mongo + + item_id = "none_field_test_restore" + create_resp = client.post( + "/new-sample/", + json={"name": "Restore none test", "item_id": item_id, "type": "samples"}, + ) + assert create_resp.status_code == 201, create_resp.json + refcode = create_resp.json["sample_list_entry"]["refcode"] + refcode_short = refcode.split(":")[-1] + + # Confirm the initial version has no description set (None / absent). + list_resp = client.get(f"/items/{refcode_short}/versions/") + v1_id = list_resp.json["versions"][0]["_id"] + + # Now populate description and save — this creates version 2. + get_resp = client.get(f"/get-item-data/{item_id}") + item_data = get_resp.json["item_data"] + item_data["description"] = "A description added later" + save_resp = client.post("/save-item/", json={"item_id": item_id, "data": item_data}) + assert save_resp.status_code == 200, save_resp.json + + # Verify description is now set in the DB. + current = flask_mongo.db.items.find_one({"refcode": refcode}) + assert current.get("description") == "A description added later" + + # Restore to version 1 (description was None/absent). + restore_resp = client.post( + f"/items/{refcode_short}/restore-version/", json={"version_id": v1_id} + ) + assert restore_resp.status_code == 200, restore_resp.json + + # After restore, description must be None (or absent), not the later value. + restored = flask_mongo.db.items.find_one({"refcode": refcode}) + assert restored.get("description") is None, ( + f"description should be None after restoring to the version where it was unset, " + f"but got {restored.get('description')!r}. " + "This happens because the version snapshot is missing the 'description' key, " + "so $set does not overwrite the populated value." + ) + + # Cleanup + flask_mongo.db.items.delete_one({"refcode": refcode}) + flask_mongo.db.item_versions.delete_many({"refcode": refcode}) + flask_mongo.db.version_counters.delete_one({"refcode": refcode}) From 18893b8f04f6fc963854705de3da09fed88cd558 Mon Sep 17 00:00:00 2001 From: BenSmithGreyGroup Date: Thu, 25 Jun 2026 12:36:57 +0100 Subject: [PATCH 23/31] Fix None field stripping in item creation and version snapshots (pydantic v2). --- pydatalab/src/pydatalab/routes/v0_1/items.py | 11 ++++++----- pydatalab/src/pydatalab/versioning.py | 7 +++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 32b7d9241..dd8ee691a 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -699,7 +699,7 @@ def _create_sample( # the `Entry` model. try: result = flask_mongo.db.items.insert_one( - data_model.model_dump(exclude={"creators", "collections", "groups"}, exclude_none=True) + data_model.model_dump(exclude={"creators", "collections", "groups"}) ) except DuplicateKeyError as error: raise Conflict(f"Duplicate key error: {str(error)}.") @@ -1466,10 +1466,11 @@ def restore_version(refcode): ) raise BadRequest(f"Restored version data validation failed: {exc}") - # Insert validated data - flask_mongo.db.item_versions.insert_one( - validated_restored_version.model_dump(exclude_none=True) - ) + # Insert validated data, restoring original restored_data into 'data' so None-valued + # fields are not stripped by model_dump(exclude_none=True) recursing into the dict. + restored_version_doc = validated_restored_version.model_dump(exclude_none=True) + restored_version_doc["data"] = restored_data + flask_mongo.db.item_versions.insert_one(restored_version_doc) return jsonify( { diff --git a/pydatalab/src/pydatalab/versioning.py b/pydatalab/src/pydatalab/versioning.py index 13433bb25..c658b94f4 100644 --- a/pydatalab/src/pydatalab/versioning.py +++ b/pydatalab/src/pydatalab/versioning.py @@ -191,8 +191,11 @@ def save_version_snapshot( 400, ) - # Insert validated data (convert to dict and exclude None values) - flask_mongo.db.item_versions.insert_one(validated_version.model_dump(exclude_none=True)) + # Insert validated data, but restore the original item dict into 'data' so that + # None-valued fields are not stripped by model_dump(exclude_none=True) recursing into it. + version_doc = validated_version.model_dump(exclude_none=True) + version_doc["data"] = item + flask_mongo.db.item_versions.insert_one(version_doc) return ( {"status": "success", "message": "Version saved.", "version": next_version_number}, 200, From 3b0c96dc5d0aeebfb230386fd3464cd00f99384d Mon Sep 17 00:00:00 2001 From: BenSmithGreyGroup Date: Thu, 25 Jun 2026 15:44:58 +0100 Subject: [PATCH 24/31] Add failing test. Checking if saving None overwrites a previously saved value. --- pydatalab/tests/server/test_samples.py | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/pydatalab/tests/server/test_samples.py b/pydatalab/tests/server/test_samples.py index 9edb53e4f..19622649b 100644 --- a/pydatalab/tests/server/test_samples.py +++ b/pydatalab/tests/server/test_samples.py @@ -1293,6 +1293,40 @@ def test_save_item_with_malformed_constituent_returns_400(client, default_sample assert response.json["status"] == "error" +def test_clearing_field_via_save_item_persists_to_db(client): + """Setting a field then clearing it (setting to None) via save-item must + actually clear it in MongoDB. Fails if exclude_none=True strips the None + from the $set payload, leaving the old value untouched.""" + item_id = "clear_field_test" + response = client.post( + "/new-sample/", + json={"name": "Clear field test", "item_id": item_id, "type": "samples"}, + ) + assert response.status_code == 201, response.json + + # Set molar_mass and save. + item_data = client.get(f"/get-item-data/{item_id}").json["item_data"] + item_data["molar_mass"] = 123.45 + assert ( + client.post("/save-item/", json={"item_id": item_id, "data": item_data}).status_code == 200 + ) + + assert client.get(f"/get-item-data/{item_id}").json["item_data"]["molar_mass"] == 123.45 + + # Now clear molar_mass and save again. + item_data = client.get(f"/get-item-data/{item_id}").json["item_data"] + item_data["molar_mass"] = None + assert ( + client.post("/save-item/", json={"item_id": item_id, "data": item_data}).status_code == 200 + ) + + result = client.get(f"/get-item-data/{item_id}").json["item_data"].get("molar_mass") + assert result is None, ( + f"molar_mass should be None after being cleared but got {result!r}; " + "exclude_none=True in save_item is stripping the None from the $set payload" + ) + + def test_get_item_with_malformed_stored_constituent_returns_500(client, database, user_id): """If an item with a malformed constituent ended up in the database (e.g. from an older write path), GET should surface it as a server-side From c3fdafdc1a3cb9f7e73a9d891df69a19d343e710 Mon Sep 17 00:00:00 2001 From: BenSmithGreyGroup Date: Thu, 25 Jun 2026 15:45:15 +0100 Subject: [PATCH 25/31] Remove exclude None from the save_item route --- pydatalab/src/pydatalab/routes/v0_1/items.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index dd8ee691a..cee1b698c 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -1685,7 +1685,6 @@ def save_item(): try: item = entry_reference_lookup(item) item = ITEM_MODELS[item_type](**item).model_dump( - exclude_none=True, exclude_unset=True, exclude={"collections", "creators", "immutable_id"}, ) From 2e9d9d74d1627195bb089f13506a5704e5d25cf0 Mon Sep 17 00:00:00 2001 From: Julien Bouquiaux Date: Fri, 12 Jun 2026 09:50:52 +0200 Subject: [PATCH 26/31] Fix status dropdown broken by Pydantic v2 schema migration Pydantic v2's model_json_schema() uses "$defs" instead of "definitions" (Pydantic v1). Update the four item-information components to use the correct key so possibleItemStatuses is no longer undefined. --- webapp/src/components/CellInformation.vue | 2 +- webapp/src/components/EquipmentInformation.vue | 2 +- webapp/src/components/SampleInformation.vue | 2 +- webapp/src/components/StartingMaterialInformation.vue | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/webapp/src/components/CellInformation.vue b/webapp/src/components/CellInformation.vue index 1d1416675..8c0b34a5b 100644 --- a/webapp/src/components/CellInformation.vue +++ b/webapp/src/components/CellInformation.vue @@ -179,7 +179,7 @@ export default { return this.$store.state.schemas[this.item?.type]; }, possibleItemStatuses() { - return this.schema?.attributes?.schema?.definitions?.CellStatus?.enum; + return this.schema?.attributes?.schema?.["$defs"]?.CellStatus?.enum; }, }, }; diff --git a/webapp/src/components/EquipmentInformation.vue b/webapp/src/components/EquipmentInformation.vue index 66401d1a6..7668b3201 100644 --- a/webapp/src/components/EquipmentInformation.vue +++ b/webapp/src/components/EquipmentInformation.vue @@ -135,7 +135,7 @@ export default { return this.$store.state.schemas[this.item?.type]; }, possibleItemStatuses() { - return this.schema?.attributes?.schema?.definitions?.EquipmentStatus?.enum; + return this.schema?.attributes?.schema?.["$defs"]?.EquipmentStatus?.enum; }, uniqueLocations() { return [ diff --git a/webapp/src/components/SampleInformation.vue b/webapp/src/components/SampleInformation.vue index d47c8b8f1..29f2fcded 100644 --- a/webapp/src/components/SampleInformation.vue +++ b/webapp/src/components/SampleInformation.vue @@ -126,7 +126,7 @@ export default { return this.$store.state.schemas[this.item?.type]; }, possibleItemStatuses() { - return this.schema?.attributes?.schema?.definitions?.ItemStatus?.enum; + return this.schema?.attributes?.schema?.["$defs"]?.ItemStatus?.enum; }, }, }; diff --git a/webapp/src/components/StartingMaterialInformation.vue b/webapp/src/components/StartingMaterialInformation.vue index 26a78494f..7bb2bdc3f 100644 --- a/webapp/src/components/StartingMaterialInformation.vue +++ b/webapp/src/components/StartingMaterialInformation.vue @@ -184,7 +184,7 @@ export default { return this.$store.state.schemas[this.item?.type]; }, possibleItemStatuses() { - return this.schema?.attributes?.schema?.definitions?.StartingMaterialsStatus?.enum; + return this.schema?.attributes?.schema?.["$defs"]?.StartingMaterialsStatus?.enum; }, Barcode: createComputedSetterForItemField("barcode"), uniqueSuppliers() { From ebfd51131559c19cf33642fdbb4a0af9f5806ed9 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 4 Aug 2026 22:54:53 +0100 Subject: [PATCH 27/31] Fix enum value sanitisation in change-user-role --- pydatalab/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index 63f7844a0..f40f8a3f7 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -278,7 +278,7 @@ def change_user_role(_, display_name: str, role: str): from pydatalab.models.utils import UserRole from pydatalab.mongo import _get_active_mongo_client - role = UserRole(role.upper()) + role = UserRole(role.lower()) try: role = getattr(UserRole, role.upper()) From 6d809c7a5ee5f9ee2347a86ea8d01047879ccc11 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 4 Aug 2026 23:26:44 +0100 Subject: [PATCH 28/31] Use `model_dump(mode=json)` to prevent repeated deserialization --- pydatalab/src/pydatalab/routes/v0_1/auth.py | 3 +- .../src/pydatalab/routes/v0_1/collections.py | 7 +- pydatalab/src/pydatalab/routes/v0_1/groups.py | 4 +- pydatalab/src/pydatalab/routes/v0_1/info.py | 99 +++++++++---------- .../src/pydatalab/routes/v0_1/remotes.py | 7 +- pydatalab/src/pydatalab/routes/v0_1/users.py | 3 +- 6 files changed, 53 insertions(+), 70 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/auth.py b/pydatalab/src/pydatalab/routes/v0_1/auth.py index f5d36aedc..43a289e6f 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/auth.py +++ b/pydatalab/src/pydatalab/routes/v0_1/auth.py @@ -5,7 +5,6 @@ """ import datetime -import json import os import random import re @@ -1058,7 +1057,7 @@ def redirect_to_ui(blueprint, token): # pylint: disable=unused-argument def get_authenticated_user_info(): """Returns metadata associated with the currently authenticated user.""" if current_user.is_authenticated: - current_user_response = json.loads(current_user.person.model_dump_json()) + current_user_response = current_user.person.model_dump(mode="json") current_user_response["role"] = current_user.role.value return jsonify(current_user_response), 200 else: diff --git a/pydatalab/src/pydatalab/routes/v0_1/collections.py b/pydatalab/src/pydatalab/routes/v0_1/collections.py index fd3f6dc6f..a0b3c4f6e 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/collections.py +++ b/pydatalab/src/pydatalab/routes/v0_1/collections.py @@ -1,5 +1,4 @@ import datetime -import json from bson import ObjectId from flask import Blueprint, jsonify, request @@ -87,7 +86,7 @@ def get_collection(collection_id): { "status": "success", "collection_id": collection_id, - "data": json.loads(collection.model_dump_json(exclude_unset=True)), + "data": collection.model_dump(mode="json", exclude_unset=True), "child_items": list(samples), } ) @@ -218,7 +217,7 @@ def create_collection(): response = { "status": "success", - "data": json.loads(data_model.model_dump_json()), + "data": data_model.model_dump(mode="json"), } if errors: @@ -534,7 +533,7 @@ def search_collections(): ) cursor = [ - json.loads(Collection(**doc).model_dump_json(exclude_unset=True)) + Collection(**doc).model_dump(mode="json", exclude_unset=True) for doc in flask_mongo.db.collections.aggregate(pipeline) ] diff --git a/pydatalab/src/pydatalab/routes/v0_1/groups.py b/pydatalab/src/pydatalab/routes/v0_1/groups.py index d5b4e5648..779166452 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/groups.py +++ b/pydatalab/src/pydatalab/routes/v0_1/groups.py @@ -1,5 +1,3 @@ -import json - from flask import Blueprint, jsonify, request from pydatalab.models.people import Group @@ -44,6 +42,6 @@ def search_groups(): return jsonify( { "status": "success", - "data": list(json.loads(Group(**d).model_dump_json()) for d in cursor), + "data": [Group(**d).model_dump(mode="json") for d in cursor], } ), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/info.py b/pydatalab/src/pydatalab/routes/v0_1/info.py index dd74ebb90..b2b45aafb 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/info.py +++ b/pydatalab/src/pydatalab/routes/v0_1/info.py @@ -1,6 +1,5 @@ """This submodule defines introspective info endpoints of the API.""" -import json from datetime import datetime from datetime import timedelta as td from datetime import timezone as tz @@ -118,7 +117,7 @@ def get_info(): ) return ( - jsonify(json.loads(response_data.model_dump_json())), + jsonify(response_data.model_dump(mode="json")), 200, ) @@ -142,27 +141,23 @@ def get_stats(): def list_block_types(): """Returns a list of all blocks implemented in this server.""" return jsonify( - json.loads( - JSONAPIResponse[dict[str, Any]]( - data=[ - Data( - id=block_type, - type="block_type", - attributes={ - "name": getattr(block, "name", ""), - "description": getattr(block, "description", ""), - "version": getattr(block, "version", __version__), - "accepted_file_extensions": getattr( - block, "accepted_file_extensions", [] - ), - "multi_file": getattr(block, "multi_file", False), - }, - ) - for block_type, block in BLOCK_TYPES.items() - ], - meta=Meta(query=request.query_string.decode() if request.query_string else ""), - ).model_dump_json() - ) + JSONAPIResponse[dict[str, Any]]( + data=[ + Data( + id=block_type, + type="block_type", + attributes={ + "name": getattr(block, "name", ""), + "description": getattr(block, "description", ""), + "version": getattr(block, "version", __version__), + "accepted_file_extensions": getattr(block, "accepted_file_extensions", []), + "multi_file": getattr(block, "multi_file", False), + }, + ) + for block_type, block in BLOCK_TYPES.items() + ], + meta=Meta(query=request.query_string.decode() if request.query_string else ""), + ).model_dump(mode="json") ) @@ -171,23 +166,21 @@ def list_supported_types(): """Returns a list of supported schemas.""" return jsonify( - json.loads( - JSONAPIResponse[dict[str, Any]]( - data=[ - Data( - id=item_type, - type="item_type", - attributes={ - "version": __version__, - "api_version": __api_version__, - "schema": schema, - }, - ) - for item_type, schema in ITEM_SCHEMAS.items() - ], - meta=Meta(query=request.query_string.decode() if request.query_string else ""), - ).model_dump_json() - ) + JSONAPIResponse[dict[str, Any]]( + data=[ + Data( + id=item_type, + type="item_type", + attributes={ + "version": __version__, + "api_version": __api_version__, + "schema": schema, + }, + ) + for item_type, schema in ITEM_SCHEMAS.items() + ], + meta=Meta(query=request.query_string.decode() if request.query_string else ""), + ).model_dump(mode="json") ) @@ -200,20 +193,18 @@ def get_schema_type(item_type): ), 404 return jsonify( - json.loads( - JSONAPIResponse[dict[str, Any]]( - data=Data( - id=item_type, - type="item_type", - attributes={ - "version": __version__, - "api_version": __api_version__, - "schema": ITEM_SCHEMAS[item_type], - }, - ), - meta=Meta(query=request.query_string.decode() if request.query_string else ""), - ).model_dump_json() - ) + JSONAPIResponse[dict[str, Any]]( + data=Data( + id=item_type, + type="item_type", + attributes={ + "version": __version__, + "api_version": __api_version__, + "schema": ITEM_SCHEMAS[item_type], + }, + ), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), + ).model_dump(mode="json") ) diff --git a/pydatalab/src/pydatalab/routes/v0_1/remotes.py b/pydatalab/src/pydatalab/routes/v0_1/remotes.py index 4e69952a2..dab3fa00f 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/remotes.py +++ b/pydatalab/src/pydatalab/routes/v0_1/remotes.py @@ -1,4 +1,3 @@ -import json from typing import Any from flask import Blueprint, jsonify, request @@ -76,9 +75,7 @@ def list_remote_directories(): response = {} response["meta"] = {} - response["meta"]["remotes"] = [ - json.loads(d.model_dump_json()) for d in CONFIG.REMOTE_FILESYSTEMS - ] + response["meta"]["remotes"] = [d.model_dump(mode="json") for d in CONFIG.REMOTE_FILESYSTEMS] if all_directory_structures: oldest_update = min(d["last_updated"] for d in all_directory_structures) response["meta"]["oldest_cache_update"] = oldest_update.isoformat() @@ -141,7 +138,7 @@ def get_remote_directory(remote_id: str): response: dict[str, Any] = {} response["meta"] = {} - response["meta"]["remote"] = json.loads(d.model_dump_json()) + response["meta"]["remote"] = d.model_dump(mode="json") response["data"] = directory_structure return jsonify(response), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/users.py b/pydatalab/src/pydatalab/routes/v0_1/users.py index ab9e737f0..15cc3ca48 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/users.py +++ b/pydatalab/src/pydatalab/routes/v0_1/users.py @@ -1,4 +1,3 @@ -import json from datetime import datetime from datetime import timedelta as td from datetime import timezone as tz @@ -218,5 +217,5 @@ def search_users(): cursor = flask_mongo.db.users.aggregate(pipeline) return jsonify( - {"status": "success", "users": [json.loads(Person(**d).model_dump_json()) for d in cursor]} + {"status": "success", "users": [Person(**d).model_dump(mode="json") for d in cursor]} ), 200 From 725bedf98b5744f782e0110f3208faffaa42808d Mon Sep 17 00:00:00 2001 From: BenSmithGreyGroup Date: Fri, 17 Jul 2026 14:35:48 +0100 Subject: [PATCH 29/31] Add theoretical capacity and calculation of nominal capacity to cell model. --- pydatalab/schemas/cell.json | 49 +++++++++++++++++ pydatalab/src/pydatalab/models/cells.py | 38 +++++++++++++ webapp/src/components/CellInformation.vue | 65 +++++++++++++++++++++++ 3 files changed, 152 insertions(+) diff --git a/pydatalab/schemas/cell.json b/pydatalab/schemas/cell.json index c66e9dcb3..7edf2ccc7 100644 --- a/pydatalab/schemas/cell.json +++ b/pydatalab/schemas/cell.json @@ -1559,6 +1559,55 @@ "default": 1, "title": "Active Ion Charge", "type": "number" + }, + "theoretical_capacity": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The theoretical specific capacity of the active material, in mAh/g.", + "title": "Theoretical Capacity" + }, + "nominal_capacity_unit": { + "default": "mAh", + "description": "The unit that `nominal_capacity` is given in.", + "enum": [ + "mAh", + "Ah" + ], + "title": "Nominal Capacity Unit", + "type": "string" + }, + "nominal_capacity": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The nominal capacity of the cell, computed as\n`theoretical_capacity * characteristic_mass`. See `set_nominal_capacity`.", + "title": "Nominal Capacity" + }, + "nominal_capacity_mah": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "`nominal_capacity` normalized to mAh, regardless of the unit currently selected\non this item (`nominal_capacity_unit`). Read-only/derived: always recomputed on\nsave (see `set_nominal_capacity`). Prefer this field over `nominal_capacity`\nwhenever comparing or aggregating across cells, since `nominal_capacity_unit` can\ndiffer from item to item.", + "title": "Nominal Capacity Mah" } }, "required": [ diff --git a/pydatalab/src/pydatalab/models/cells.py b/pydatalab/src/pydatalab/models/cells.py index e8690db9a..ad62ed2d1 100644 --- a/pydatalab/src/pydatalab/models/cells.py +++ b/pydatalab/src/pydatalab/models/cells.py @@ -11,6 +11,11 @@ from pydatalab.models.items import Item from pydatalab.models.utils import CellStatus, Constituent +# Conversion factor from nominal_capacity_unit to the base unit (mAh). Single source of +# truth: used by `set_nominal_capacity`, so any downstream consumer normalizes +# consistently regardless of which unit is currently selected on a given item. +NOMINAL_CAPACITY_TO_MAH = {"mAh": 1, "Ah": 1e3} + class CellComponent(Constituent): ... @@ -59,6 +64,23 @@ class Cell(Item): status: CellStatus = Field(default=CellStatus.ACTIVE) """The status of the cells, indicating its current state.""" + theoretical_capacity: float | None = None + """The theoretical specific capacity of the active material, in mAh/g.""" + + nominal_capacity_unit: Literal["mAh", "Ah"] = "mAh" + """The unit that `nominal_capacity` is given in.""" + + nominal_capacity: float | None = None + """The nominal capacity of the cell, computed as + `theoretical_capacity * characteristic_mass`. See `set_nominal_capacity`.""" + + nominal_capacity_mah: float | None = None + """`nominal_capacity` normalized to mAh, regardless of the unit currently selected + on this item (`nominal_capacity_unit`). Read-only/derived: always recomputed on + save (see `set_nominal_capacity`). Prefer this field over `nominal_capacity` + whenever comparing or aggregating across cells, since `nominal_capacity_unit` can + differ from item to item.""" + @field_validator("characteristic_molar_mass", mode="before") @classmethod def set_molar_mass(cls, v, info): @@ -73,6 +95,22 @@ def set_molar_mass(cls, v, info): return None return v + @model_validator(mode="after") + def set_nominal_capacity(self): + if self.theoretical_capacity is None or self.characteristic_mass is None: + self.nominal_capacity_mah = None + return self + + # theoretical_capacity is in mAh/g; characteristic_mass is in mg (divide by + # 1000 to get grams). + nominal_capacity_mah = self.theoretical_capacity * self.characteristic_mass / 1000 + + self.nominal_capacity_mah = nominal_capacity_mah + self.nominal_capacity = ( + nominal_capacity_mah / NOMINAL_CAPACITY_TO_MAH[self.nominal_capacity_unit] + ) + return self + @model_validator(mode="after") def add_missing_electrode_relationships(self): """Add any missing cell component constituents to parent relationships""" diff --git a/webapp/src/components/CellInformation.vue b/webapp/src/components/CellInformation.vue index 8c0b34a5b..55ba37d0e 100644 --- a/webapp/src/components/CellInformation.vue +++ b/webapp/src/components/CellInformation.vue @@ -93,6 +93,37 @@ /> +
+
+ + +
+
+ +
+
+ {{ NominalCapacityDisplay }} +
+
+ +
+
+
+
@@ -175,12 +206,46 @@ export default { CharacteristicMass: createComputedSetterForItemField("characteristic_mass"), Collections: createComputedSetterForItemField("collections"), Status: createComputedSetterForItemField("status"), + TheoreticalCapacity: createComputedSetterForItemField("theoretical_capacity"), + NominalCapacityUnit: createComputedSetterForItemField("nominal_capacity_unit"), schema() { return this.$store.state.schemas[this.item?.type]; }, possibleItemStatuses() { return this.schema?.attributes?.schema?.["$defs"]?.CellStatus?.enum; }, + // Recomputed live from the store on every render — no save round-trip needed. + NominalCapacity() { + const nominalCapacityToMah = { mAh: 1, Ah: 1e3 }; + + // theoretical_capacity is always mAh/g. + const theoreticalCapacity = Number(this.TheoreticalCapacity); + const characteristicMass = Number(this.CharacteristicMass); + if (!Number.isFinite(theoreticalCapacity) || !Number.isFinite(characteristicMass)) { + return null; + } + + const nominalCapacityUnit = this.NominalCapacityUnit || "mAh"; + + // characteristic_mass is stored in mg; divide by 1000 to get grams. + const mAh = (theoreticalCapacity * characteristicMass) / 1000; + return mAh / nominalCapacityToMah[nominalCapacityUnit]; + }, + NominalCapacityDisplay() { + return this.NominalCapacity === null ? "—" : this.NominalCapacity.toFixed(4); + }, + }, + watch: { + // Persist the live-computed value too, so it's saved without relying on a + // server round-trip (the backend validator recomputes it again on save). + NominalCapacity(value) { + if (value !== null && value !== this.item?.nominal_capacity) { + this.$store.commit("updateItemData", { + item_id: this.item_id, + item_data: { nominal_capacity: value }, + }); + } + }, }, }; From cd6495219387f28b7a9b24aba8d148d746e411ec Mon Sep 17 00:00:00 2001 From: BenSmithGreyGroup Date: Fri, 17 Jul 2026 14:39:15 +0100 Subject: [PATCH 30/31] Added unit test for nominal capacity calculation --- pydatalab/tests/test_models.py | 53 ++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/pydatalab/tests/test_models.py b/pydatalab/tests/test_models.py index 625ae5d08..71749207b 100644 --- a/pydatalab/tests/test_models.py +++ b/pydatalab/tests/test_models.py @@ -472,6 +472,59 @@ def test_cell_relationship_deduplication(): assert parthood[0].item_id == "test_cathode" +def test_cell_nominal_capacity(): + """`nominal_capacity` (and its mAh-normalized counterpart `nominal_capacity_mah`) + are derived from `theoretical_capacity` (mAh/g) * `characteristic_mass` (mg), + unit-aware with respect to `nominal_capacity_unit`.""" + from pydatalab.models.cells import Cell + + # Neither input supplied: no capacity can be computed. + cell = Cell(item_id="abcd-1-2-3") + assert cell.nominal_capacity is None + assert cell.nominal_capacity_mah is None + + # Only one of the two inputs supplied: still no capacity can be computed. + cell = Cell(item_id="abcd-1-2-3", theoretical_capacity=200.0) + assert cell.nominal_capacity is None + assert cell.nominal_capacity_mah is None + + cell = Cell(item_id="abcd-1-2-3", characteristic_mass=5.0) + assert cell.nominal_capacity is None + assert cell.nominal_capacity_mah is None + + # Default unit (mAh): 200 mAh/g * 5 mg = 1 mAh. + cell = Cell(item_id="abcd-1-2-3", characteristic_mass=5.0, theoretical_capacity=200.0) + assert cell.nominal_capacity_unit == "mAh" + assert cell.nominal_capacity == pytest.approx(1.0) + assert cell.nominal_capacity_mah == pytest.approx(1.0) + + # Requesting the result in Ah: same underlying quantity, different display unit. + cell = Cell( + item_id="abcd-1-2-3", + characteristic_mass=5.0, + theoretical_capacity=200.0, + nominal_capacity_unit="Ah", + ) + assert cell.nominal_capacity == pytest.approx(0.001) + # nominal_capacity_mah is unit-independent and always mAh. + assert cell.nominal_capacity_mah == pytest.approx(1.0) + + # A client-supplied nominal_capacity is not trusted: it is always recomputed + # from theoretical_capacity * characteristic_mass. + cell = Cell( + item_id="abcd-1-2-3", + characteristic_mass=5.0, + theoretical_capacity=200.0, + nominal_capacity=999, + ) + assert cell.nominal_capacity == pytest.approx(1.0) + + # Round-tripping through JSON (as happens on save/load) preserves the computed values. + cell = Cell(**json.loads(cell.model_dump_json())) + assert cell.nominal_capacity == pytest.approx(1.0) + assert cell.nominal_capacity_mah == pytest.approx(1.0) + + def test_sample_synthesis_relationship_deduplication(): """Regression test for duplicated parent relationships on synthesis constituents. From fd3bdc32f71c47512ac314bac36698ec077a3ac0 Mon Sep 17 00:00:00 2001 From: BenSmithGreyGroup Date: Fri, 17 Jul 2026 14:49:39 +0100 Subject: [PATCH 31/31] Add post-plot hooks so plugins can compute derived metrics from CycleBlock data --- pydatalab/src/pydatalab/apps/echem/blocks.py | 50 +++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/pydatalab/src/pydatalab/apps/echem/blocks.py b/pydatalab/src/pydatalab/apps/echem/blocks.py index 8775ff00d..44fa1476f 100644 --- a/pydatalab/src/pydatalab/apps/echem/blocks.py +++ b/pydatalab/src/pydatalab/apps/echem/blocks.py @@ -1,7 +1,8 @@ import hashlib import warnings +from collections.abc import Callable from pathlib import Path -from typing import Any +from typing import Any, ClassVar import bokeh import pandas as pd @@ -69,6 +70,30 @@ class CycleBlock(DataBlock): "derivative_mode": None, } + post_plot_hooks: ClassVar[ + list[Callable[["CycleBlock", pd.DataFrame, pd.DataFrame | None], None]] + ] = [] + """Hooks called after plot_cycle completes, receiving (block, unfiltered_df, cycle_summary_df). + + Plugins can append callables here to extract derived metrics into ``block.data["computed"]`` + without monkey-patching. Each hook is called once per plot update with the primary + (non-comparison) DataFrames. + + ``unfiltered_df`` is the full navani DataFrame re-parsed directly from the source file, + bypassing the BDF cache, so instrument-specific columns (e.g. ``unknown_colID_249``) that + are not part of the BDF schema are available. Note this incurs an extra file parse when + hooks are registered. + + Example:: + + from pydatalab.apps.echem import CycleBlock + + def my_hook(block, unfiltered_df, cycle_summary_df): + block.data["computed"] = {"my_metric": unfiltered_df["unknown_colID_249"].max()} + + CycleBlock.post_plot_hooks.append(my_hook) + """ + def _get_characteristic_mass_g(self): doc = flask_mongo.db.items.find_one( {"item_id": self.data["item_id"]}, {"characteristic_mass": 1} @@ -424,6 +449,8 @@ def plot_cycle(self): raw_dfs = {} cycle_summary_dfs = {} + raw_df = None + cycle_summary_df = None if self.data.get("mode") is None: self.data["mode"] = "single" @@ -529,6 +556,27 @@ def plot_cycle(self): self.data["bokeh_plot_data"] = bokeh.embed.json_item( layout, theme=bokeh_plots.DATALAB_BOKEH_THEME ) + + if raw_df is not None and self.post_plot_hooks: + # Re-parse from source to get the full unfiltered DataFrame, since the BDF cache + # only preserves BDF-schema columns and drops instrument-specific ones. + file_infos = [get_file_info_by_id(fid, update_if_live=False) for fid in file_ids] + locations = [Path(info["location"]) for info in file_infos] + try: + unfiltered_df = self._parse_echem_files( + locations[0], locations if len(locations) > 1 else None + ) + except Exception as exc: + warnings.warn(f"post_plot_hook source re-parse failed, hooks will not run: {exc}") + unfiltered_df = None + + if unfiltered_df is not None: + for hook in self.post_plot_hooks: + try: + hook(self, unfiltered_df, cycle_summary_df) + except Exception as exc: + warnings.warn(f"post_plot_hook {hook!r} failed: {exc}") + return @property