From c169b3edea9d74ab075e7ff65672ba18c9458aa1 Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Sun, 7 Jun 2026 15:40:18 +0100 Subject: [PATCH 01/24] Update pydantic version across various files --- .github/dependabot.yml | 2 - .pre-commit-config.yaml | 2 +- pydatalab/pyproject.toml | 4 +- pydatalab/uv.lock | 123 +++++++++++++++++++++++++++++++-------- 4 files changed, 104 insertions(+), 27 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1bf622ebd..785ee179c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -25,8 +25,6 @@ updates: - minor - patch ignore: - - dependency-name: "pydantic" - versions: [ ">=2" ] - dependency-name: "bokeh" versions: [ ">=3" ] - dependency-name: "langchain" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5e95d91bd..f2670933a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -67,7 +67,7 @@ repos: hooks: - id: mypy additional_dependencies: - ["types-python-dateutil", "types-requests", "types-paramiko", "pydantic~=1.10"] + ["types-python-dateutil", "types-requests", "types-paramiko", "pydantic", "pydantic-settings"] args: ["--config-file", "pydatalab/pyproject.toml"] - repo: local diff --git a/pydatalab/pyproject.toml b/pydatalab/pyproject.toml index f31989343..a864456cf 100644 --- a/pydatalab/pyproject.toml +++ b/pydatalab/pyproject.toml @@ -21,7 +21,9 @@ dependencies = [ "bokeh ~= 2.4, < 3.0", "matplotlib ~= 3.8", "periodictable ~= 1.7", - "pydantic[email, dotenv] < 2.0", + "pydantic[email] >= 2.11, < 3.0", + "pydantic-settings ~= 2.10", + "python-dotenv ~= 1.1", "pint ~= 0.24", "pandas[excel] ~= 2.2", "pymongo ~= 4.7", diff --git a/pydatalab/uv.lock b/pydatalab/uv.lock index 24aba88be..2e30294ad 100644 --- a/pydatalab/uv.lock +++ b/pydatalab/uv.lock @@ -99,6 +99,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, ] +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + [[package]] name = "anthropic" version = "0.60.0" @@ -621,8 +630,10 @@ dependencies = [ { name = "pandas", extra = ["excel"] }, { name = "periodictable" }, { name = "pint" }, - { name = "pydantic", extra = ["dotenv", "email"] }, + { name = "pydantic", extra = ["email"] }, + { name = "pydantic-settings" }, { name = "pymongo" }, + { name = "python-dotenv" }, { name = "tomlkit" }, ] @@ -746,10 +757,12 @@ requires-dist = [ { name = "psutil", marker = "extra == 'apps'", specifier = ">=7.0.0" }, { name = "pyarrow", marker = "extra == 'apps'", specifier = "~=23.0.1" }, { name = "pybaselines", marker = "extra == 'apps'", specifier = "~=1.1" }, - { name = "pydantic", extras = ["dotenv", "email"], specifier = "<2.0" }, + { name = "pydantic", extras = ["email"], specifier = ">=2.11,<3.0" }, + { name = "pydantic-settings", specifier = "~=2.10" }, { name = "pyjwt", marker = "extra == 'server'", specifier = "~=2.9" }, { name = "pymongo", specifier = "~=4.7" }, { name = "python-dateutil", marker = "extra == 'apps'", specifier = "~=2.9" }, + { name = "python-dotenv", specifier = "~=1.1" }, { name = "python-dotenv", marker = "extra == 'server'", specifier = "~=1.0" }, { name = "renishawwire", marker = "extra == 'apps'", specifier = ">=0.1.16" }, { name = "scipy", marker = "extra == 'apps'", specifier = "~=1.13" }, @@ -2331,38 +2344,90 @@ wheels = [ [[package]] name = "pydantic" -version = "1.10.22" +version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, { name = "typing-extensions" }, + { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9a/57/5996c63f0deec09e9e901a2b838247c97c6844999562eac4e435bcb83938/pydantic-1.10.22.tar.gz", hash = "sha256:ee1006cebd43a8e7158fb7190bb8f4e2da9649719bff65d0c287282ec38dec6d", size = 356771, upload-time = "2025-04-24T13:38:43.605Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/92/91eb5c75a1460292e1f2f3e577122574ebb942fbac19ad2369ff00b9eb24/pydantic-1.10.22-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:57889565ccc1e5b7b73343329bbe6198ebc472e3ee874af2fa1865cfe7048228", size = 2852481, upload-time = "2025-04-24T13:36:55.045Z" }, - { url = "https://files.pythonhosted.org/packages/08/f3/dd54b49fc5caaed06f5a0d0a5ec35a81cf722cd6b42455f408dad1ef3f7d/pydantic-1.10.22-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90729e22426de79bc6a3526b4c45ec4400caf0d4f10d7181ba7f12c01bb3897d", size = 2585586, upload-time = "2025-04-24T13:36:58.453Z" }, - { url = "https://files.pythonhosted.org/packages/ec/9b/48d10180cc614ffb66da486e99bc1f8b639fb44edf322864f2fb161e2351/pydantic-1.10.22-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8684d347f351554ec94fdcb507983d3116dc4577fb8799fed63c65869a2d10", size = 3336974, upload-time = "2025-04-24T13:37:00.652Z" }, - { url = "https://files.pythonhosted.org/packages/ff/80/b55ad0029ae8e7b8b5c81ad7c4e800774a52107d26f70c6696857dc733d5/pydantic-1.10.22-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8dad498ceff2d9ef1d2e2bc6608f5b59b8e1ba2031759b22dfb8c16608e1802", size = 3362338, upload-time = "2025-04-24T13:37:02.42Z" }, - { url = "https://files.pythonhosted.org/packages/65/e0/8a5cd2cd29a5632581ba466f5792194b2a568aa052ce9da9ba98b634debf/pydantic-1.10.22-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fac529cc654d4575cf8de191cce354b12ba705f528a0a5c654de6d01f76cd818", size = 3519505, upload-time = "2025-04-24T13:37:04.322Z" }, - { url = "https://files.pythonhosted.org/packages/38/c5/c776d03ec374f22860802b2cee057b41e866be3c80826b53d4c001692db3/pydantic-1.10.22-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4148232aded8dd1dd13cf910a01b32a763c34bd79a0ab4d1ee66164fcb0b7b9d", size = 3485878, upload-time = "2025-04-24T13:37:06.102Z" }, - { url = "https://files.pythonhosted.org/packages/d1/a2/1efd064513a2c1bcb5c2b0e022cdf77d132ef7f7f20d91bb439d759f6a88/pydantic-1.10.22-cp310-cp310-win_amd64.whl", hash = "sha256:ece68105d9e436db45d8650dc375c760cc85a6793ae019c08769052902dca7db", size = 2299673, upload-time = "2025-04-24T13:37:07.969Z" }, - { url = "https://files.pythonhosted.org/packages/42/03/e435ed85a9abda29e3fbdb49c572fe4131a68c6daf3855a01eebda9e1b27/pydantic-1.10.22-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8e530a8da353f791ad89e701c35787418605d35085f4bdda51b416946070e938", size = 2845682, upload-time = "2025-04-24T13:37:10.142Z" }, - { url = "https://files.pythonhosted.org/packages/72/ea/4a625035672f6c06d3f1c7e33aa0af6bf1929991e27017e98b9c2064ae0b/pydantic-1.10.22-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:654322b85642e9439d7de4c83cb4084ddd513df7ff8706005dada43b34544946", size = 2553286, upload-time = "2025-04-24T13:37:11.946Z" }, - { url = "https://files.pythonhosted.org/packages/a4/f0/424ad837746e69e9f061ba9be68c2a97aef7376d1911692904d8efbcd322/pydantic-1.10.22-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8bece75bd1b9fc1c32b57a32831517943b1159ba18b4ba32c0d431d76a120ae", size = 3141232, upload-time = "2025-04-24T13:37:14.394Z" }, - { url = "https://files.pythonhosted.org/packages/14/67/4979c19e8cfd092085a292485e0b42d74e4eeefbb8cd726aa8ba38d06294/pydantic-1.10.22-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eccb58767f13c6963dcf96d02cb8723ebb98b16692030803ac075d2439c07b0f", size = 3214272, upload-time = "2025-04-24T13:37:16.201Z" }, - { url = "https://files.pythonhosted.org/packages/1a/04/32339ce43e97519d19e7759902515c750edbf4832a13063a4ab157f83f42/pydantic-1.10.22-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7778e6200ff8ed5f7052c1516617423d22517ad36cc7a3aedd51428168e3e5e8", size = 3321646, upload-time = "2025-04-24T13:37:19.086Z" }, - { url = "https://files.pythonhosted.org/packages/92/35/dffc1b29cb7198aadab68d75447191e59bdbc1f1d2d51826c9a4460d372f/pydantic-1.10.22-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bffe02767d27c39af9ca7dc7cd479c00dda6346bb62ffc89e306f665108317a2", size = 3244258, upload-time = "2025-04-24T13:37:20.929Z" }, - { url = "https://files.pythonhosted.org/packages/11/c5/c4ce6ebe7f528a879441eabd2c6dd9e2e4c54f320a8c9344ba93b3aa8701/pydantic-1.10.22-cp311-cp311-win_amd64.whl", hash = "sha256:23bc19c55427091b8e589bc08f635ab90005f2dc99518f1233386f46462c550a", size = 2309702, upload-time = "2025-04-24T13:37:23.296Z" }, - { url = "https://files.pythonhosted.org/packages/e9/e0/1ed151a56869be1588ad2d8cda9f8c1d95b16f74f09a7cea879ca9b63a8b/pydantic-1.10.22-py3-none-any.whl", hash = "sha256:343037d608bcbd34df937ac259708bfc83664dadf88afe8516c4f282d7d471a9", size = 166503, upload-time = "2025-04-24T13:38:41.374Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, ] [package.optional-dependencies] -dotenv = [ - { name = "python-dotenv" }, -] email = [ { name = "email-validator" }, ] +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/08/f1ba952f1c8ae5581c70fa9c6da89f247b83e3dd8c09c035d5d7931fc23d/pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4", size = 2113146, upload-time = "2026-05-06T13:37:36.537Z" }, + { url = "https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5", size = 1949769, upload-time = "2026-05-06T13:37:46.365Z" }, + { url = "https://files.pythonhosted.org/packages/64/ba/bfb1d928fd5b49e1258935ff104ae356e9fd89384a55bf9f847e9193ad40/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba", size = 1974958, upload-time = "2026-05-06T13:37:28.611Z" }, + { url = "https://files.pythonhosted.org/packages/4e/74/76223bfb117b64af743c9b6670d1364516f5c0604f96b48f3272f6af6cc6/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b", size = 2042118, upload-time = "2026-05-06T13:36:55.216Z" }, + { url = "https://files.pythonhosted.org/packages/cb/7b/848732968bc8f48f3187542f08358b9d842db564147b256669426ebb1652/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c", size = 2222876, upload-time = "2026-05-06T13:38:25.455Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2f/e90b63ee2e14bd8d3db8f705a6d75d64e6ee1b7c2c8833747ce706e1e0ce/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50", size = 2286703, upload-time = "2026-05-06T13:37:53.304Z" }, + { url = "https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd", size = 2092042, upload-time = "2026-05-06T13:38:46.981Z" }, + { url = "https://files.pythonhosted.org/packages/a9/da/0a422b57bf8504102bf3c4ccea9c41bab5a5cee6a54650acf8faf67f5a24/pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01", size = 2117231, upload-time = "2026-05-06T13:39:23.146Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2a/2ac13c3af305843e23c5078c53d135656b3f05a2fd78cb7bbbb12e97b473/pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d", size = 2168388, upload-time = "2026-05-06T13:40:08.06Z" }, + { url = "https://files.pythonhosted.org/packages/72/04/2beacf7e1607e93eefe4aed1b4709f079b905fb77530179d4f7c71745f22/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4", size = 2184769, upload-time = "2026-05-06T13:38:13.901Z" }, + { url = "https://files.pythonhosted.org/packages/9e/29/d2b9fd9f539133548eaf622c06a4ce176cb46ac59f32d0359c4abc0de047/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f", size = 2319312, upload-time = "2026-05-06T13:39:08.24Z" }, + { url = "https://files.pythonhosted.org/packages/7c/af/0f7a5b85fec6075bea96e3ef9187de38fccced0de92c1e7feda8d5cc7bb9/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39", size = 2361817, upload-time = "2026-05-06T13:38:43.2Z" }, + { url = "https://files.pythonhosted.org/packages/25/a4/73363fec545fd3ec025490bdda2743c56d0dd5b6266b1a53bbe9e4265375/pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d", size = 1987085, upload-time = "2026-05-06T13:39:25.497Z" }, + { url = "https://files.pythonhosted.org/packages/01/aa/62f082da2c91fac1c234bc9ee0066257ce83f0604abd72e4c9d5991f2d84/pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf", size = 2074311, upload-time = "2026-05-06T13:39:59.922Z" }, + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", size = 235551, upload-time = "2026-05-08T13:40:06.542Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964, upload-time = "2026-05-08T13:40:04.958Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -3065,6 +3130,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, ] +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + [[package]] name = "tzdata" version = "2025.2" From 7abcda2e0c21201a5843c1416e12762e686bb3b5 Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Tue, 11 Nov 2025 01:06:15 +0000 Subject: [PATCH 02/24] Refactor server config for pydantic v2 --- pydatalab/src/pydatalab/config.py | 116 ++++++++++++++++++------------ 1 file changed, 71 insertions(+), 45 deletions(-) diff --git a/pydatalab/src/pydatalab/config.py b/pydatalab/src/pydatalab/config.py index 4f1b58ae9..8ee80d11f 100644 --- a/pydatalab/src/pydatalab/config.py +++ b/pydatalab/src/pydatalab/config.py @@ -9,12 +9,13 @@ from pydantic import ( AnyUrl, BaseModel, - BaseSettings, + ConfigDict, Field, ValidationError, - root_validator, - validator, + field_validator, + model_validator, ) +from pydantic_settings import BaseSettings, SettingsConfigDict from pydatalab.models import Person from pydatalab.models.utils import RandomAlphabeticalRefcodeFactory, RefCodeFactory @@ -24,7 +25,7 @@ config_logger = logging.getLogger("pydatalab.config") -def config_file_settings(settings: BaseSettings) -> dict[str, Any]: +def config_file_settings(settings_cls: type[BaseSettings] | None = None) -> dict[str, Any]: """Returns a dictionary of server settings loaded from the default or specified JSON config file location (via the env var `PYDATALAB_CONFIG_FILE`). @@ -34,7 +35,7 @@ def config_file_settings(settings: BaseSettings) -> dict[str, Any]: res = {} if config_file.is_file(): config_logger.debug("Loading from config file at %s", config_file) - config_file_content = config_file.read_text(encoding=settings.__config__.env_file_encoding) + config_file_content = config_file.read_text(encoding="utf-8") try: res = json.loads(config_file_content) @@ -51,20 +52,20 @@ def config_file_settings(settings: BaseSettings) -> dict[str, Any]: class DeploymentMetadata(BaseModel): """A model for specifying metadata about a datalab deployment.""" - maintainer: Person | None + maintainer: Person | None = None issue_tracker: AnyUrl | None = Field("https://github.com/datalab-org/datalab/issues") - homepage: AnyUrl | None + homepage: AnyUrl | None = None source_repository: AnyUrl | None = Field("https://github.com/datalab-org/datalab") - @validator("maintainer") + @field_validator("maintainer") + @classmethod def strip_fields_from_person(cls, v): if not v.contact_email: raise ValueError("Must provide contact email for maintainer.") return Person(contact_email=v.contact_email, display_name=v.display_name) - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class BackupStrategy(BaseModel): @@ -75,7 +76,8 @@ class BackupStrategy(BaseModel): description="Whether this backup strategy is active; i.e., whether it is actually used. All strategies will be disabled in testing scenarios.", ) hostname: str | None = Field( - description="The hostname of the SSH-accessible server on which to store the backup (`None` indicates local backups)." + None, + description="The hostname of the SSH-accessible server on which to store the backup (`None` indicates local backups).", ) location: Path = Field( description="The location under which to store the backups on the host. Each backup will be date-stamped and stored in a subdirectory of this location." @@ -87,7 +89,7 @@ class BackupStrategy(BaseModel): frequency: str | None = Field( None, description="The frequency of the backup, described in the crontab syntax.", - pattern=r"^(?:\*|\d+(?:-\d+)?)(?:\/\d+)?(?:,\d+(?:-\d+)?(?:\/\d+)?)*$", + examples=["5 4 * * *", "5 2 1 1,4,7,10 *"], ) notification_email_address: str | None = Field( None, description="An email address to send backup notifications to." @@ -143,7 +145,7 @@ class ServerConfig(BaseSettings): False, description="Whether to run the server in testing mode, i.e., without user auth." ) - SECRET_KEY: str = Field( + SECRET_KEY: str | None = Field( None, description="The secret key to use for Flask. This value should be changed and/or loaded from an environment variable for production deployments.", ) @@ -180,7 +182,7 @@ class ServerConfig(BaseSettings): REMOTE_FILESYSTEMS: list[RemoteFilesystem] = Field( [], - descripton="A list of dictionaries describing remote filesystems to be accessible from the server.", + description="A list of dictionaries describing remote filesystems to be accessible from the server.", ) REMOTE_CACHE_MAX_AGE: int = Field( @@ -275,18 +277,24 @@ class ServerConfig(BaseSettings): description="The desired backup configuration.", ) - @root_validator + @model_validator(mode="before") + @classmethod def validate_cache_ages(cls, values): - if values.get("REMOTE_CACHE_MIN_AGE") > values.get("REMOTE_CACHE_MAX_AGE"): + min_age = values.get("REMOTE_CACHE_MIN_AGE") + max_age = values.get("REMOTE_CACHE_MAX_AGE") + + if min_age is not None and max_age is not None and min_age > max_age: raise RuntimeError( - f"The maximum cache age must be greater than the minimum cache age: min {values.get('REMOTE_CACHE_MIN_AGE')=}, max {values.get('REMOTE_CACHE_MAX_AGE')=}" + f"The maximum cache age must be greater than the minimum cache age: min {min_age=}, max {max_age=}" ) return values - @validator("SECRET_KEY", pre=True, always=True) - def validate_secret_key(cls, v, values): + @field_validator("SECRET_KEY", mode="before") + @classmethod + def validate_secret_key(cls, v, info): if v is None: - if values.get("TESTING"): + data = info.data if hasattr(info, "data") else {} + if data.get("TESTING"): config_logger.error( "`CONFIG.TESTING` is enabled - generating a deterministic secret key for testing purposes. This MUST be updated for production deployments." ) @@ -296,7 +304,8 @@ def validate_secret_key(cls, v, values): return v - @validator("ROOT_PATH") + @field_validator("ROOT_PATH", mode="before") + @classmethod def validate_root_path(cls, v): if not v.startswith("/"): v = "/" + v @@ -306,20 +315,22 @@ def validate_root_path(cls, v): return v - @validator("IDENTIFIER_PREFIX", pre=True, always=True) - def validate_identifier_prefix(cls, v, values): + @field_validator("IDENTIFIER_PREFIX", mode="before") + @classmethod + def validate_identifier_prefix(cls, v, info): """Make sure that the identifier prefix is set and is valid, raising clear error messages if not. If in testing mode, then set the prefix to 'test' too. The app startup will test for this value and should also warn aggressively that this is unset. """ - if values.get("TESTING") or v is None: + data = info.data if hasattr(info, "data") else {} + if data.get("TESTING") or v is None: return "test" if len(v) > 12: raise RuntimeError( - "Identifier prefix must be less than 12 characters long, received {v=}" + f"Identifier prefix must be less than 12 characters long, received {v=}" ) # test a trial refcode @@ -331,18 +342,18 @@ def validate_identifier_prefix(cls, v, values): raise RuntimeError( f"Invalid identifier prefix: {v}. Validation with refcode `AAAAAA` returned error: {exc}" ) - return v - @root_validator + @model_validator(mode="before") + @classmethod def deactivate_backup_strategies_during_testing(cls, values): if values.get("TESTING"): for name in values.get("BACKUP_STRATEGIES", {}): values["BACKUP_STRATEGIES"][name].active = False - return values - @validator("LOG_FILE") + @field_validator("LOG_FILE", mode="before") + @classmethod def make_missing_log_directory(cls, v): """Make sure that the log directory exists and is writable.""" if v is None: @@ -355,25 +366,40 @@ def make_missing_log_directory(cls, v): raise RuntimeError(f"Unable to create log file at {v}") from exc return v - class Config: - env_prefix = "pydatalab_" - extra = "allow" - env_file = ".env" - env_file_encoding = "utf-8" - validate_assignment = True - - @classmethod - def customise_sources( - cls, + def update(self, values: dict): + """Update the configuration with new values, following Pydantic v1 behavior.""" + for key, value in values.items(): + key_upper = key.upper() + if hasattr(self, key_upper): + setattr(self, key_upper, value) + else: + setattr(self, key_upper, value) + + model_config = SettingsConfigDict( + env_prefix="PYDATALAB_", + extra="allow", + env_file=".env", + env_file_encoding="utf-8", + validate_assignment=True, + case_sensitive=False, + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings, + env_settings, + dotenv_settings, + file_secret_settings, + ): + return ( init_settings, env_settings, + dotenv_settings, + config_file_settings, file_secret_settings, - ): - return (init_settings, env_settings, config_file_settings, file_secret_settings) - - def update(self, mapping): - for key in mapping: - setattr(self, key.upper(), mapping[key]) + ) CONFIG: ServerConfig = ServerConfig() From fd4755e0223ed791a009c062fa11bb19ee15491c Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 15:51:35 +0100 Subject: [PATCH 03/24] Update traits module to pydantic v2 --- pydatalab/src/pydatalab/models/traits.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/pydatalab/src/pydatalab/models/traits.py b/pydatalab/src/pydatalab/models/traits.py index 140f5182c..435b39d34 100644 --- a/pydatalab/src/pydatalab/models/traits.py +++ b/pydatalab/src/pydatalab/models/traits.py @@ -1,11 +1,20 @@ from typing import Any -from pydantic import BaseModel, Field, root_validator, validator +from pydantic import BaseModel, Field, field_validator, model_validator from pydatalab.models.blocks import DataBlockResponse from pydatalab.models.people import Group, Person from pydatalab.models.utils import Constituent, InlineSubstance, PyObjectId +__all__ = ( + "HasOwner", + "HasRevisionControl", + "HasBlocks", + "IsCollectable", + "HasSynthesisInfo", + "HasSubstanceInfo", +) + class HasOwner(BaseModel): creator_ids: list[PyObjectId] = Field([]) @@ -50,7 +59,8 @@ class IsCollectable(BaseModel): collections: list[Collection] = Field([]) """Inlined info for the collections associated with this item.""" - @root_validator + @model_validator(mode="before") + @classmethod def add_missing_collection_relationships(cls, values): from pydatalab.models.relationships import TypedRelationship @@ -99,7 +109,8 @@ class HasSynthesisInfo(BaseModel): synthesis_description: str | None = None """Free-text details of the procedure applied to synthesise the sample""" - @root_validator + @model_validator(mode="before") + @classmethod def add_missing_synthesis_relationships(cls, values): """Add any missing sample synthesis constituents to parent relationships""" from pydatalab.models.relationships import RelationshipType, TypedRelationship @@ -205,7 +216,8 @@ class HasSubstanceInfo(BaseModel): CAS: str | None = Field(alias="Substance CAS") """The CAS Registry Number for the substance described by this entry.""" - @validator("molar_mass") + @field_validator("molar_mass", mode="before") + @classmethod def add_molar_mass(cls, v, values): from periodictable import formula From 77f234531948f00c7c5408e5ace6b2ff9888a635 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 11 Nov 2025 01:06:39 +0000 Subject: [PATCH 04/24] Refactor model imports to avoid circular deps --- pydatalab/src/pydatalab/models/__init__.py | 29 ++++++++++++++++------ 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/pydatalab/src/pydatalab/models/__init__.py b/pydatalab/src/pydatalab/models/__init__.py index 92db5f475..ea76f4514 100644 --- a/pydatalab/src/pydatalab/models/__init__.py +++ b/pydatalab/src/pydatalab/models/__init__.py @@ -1,20 +1,34 @@ -from pydantic import BaseModel +import functools from pydatalab.models.cells import Cell from pydatalab.models.collections import Collection from pydatalab.models.equipment import Equipment from pydatalab.models.files import File +from pydatalab.models.items import Item from pydatalab.models.people import Person from pydatalab.models.samples import Sample from pydatalab.models.starting_materials import StartingMaterial from pydatalab.models.versions import ItemVersion -ITEM_MODELS: dict[str, type[BaseModel]] = { - "samples": Sample, - "starting_materials": StartingMaterial, - "cells": Cell, - "equipment": Equipment, -} + +@functools.lru_cache(maxsize=1) +def get_item_models() -> dict[str, type[Item]]: + """ + Returns a dictionary of item models keyed by their type. + """ + return { + model.model_json_schema()["properties"]["type"]["default"]: model + for model in Item.__subclasses__() + } + + +@functools.lru_cache(maxsize=1) +def generate_schemas() -> dict[str, dict]: + return {t: model.model_json_schema(by_alias=False) for t, model in get_item_models().items()} + + +ITEM_MODELS: dict[str, type[Item]] = get_item_models() +ITEM_SCHEMAS = generate_schemas() __all__ = ( "File", @@ -26,4 +40,5 @@ "Equipment", "ItemVersion", "ITEM_MODELS", + "ITEM_SCHEMAS", ) From 358e27ccf4a33044f6ba4083d8ec882c66c9b8c2 Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Tue, 11 Nov 2025 01:07:23 +0000 Subject: [PATCH 05/24] Rework all models and tasks into pydantic v2 amenable form Move back to descriptions in field docstrings and configure this with customised BaseModel More model updates for pydantic v2 --- pydatalab/schemas/cell.json | 2147 ++++++++++------ pydatalab/schemas/equipment.json | 1771 ++++++++----- pydatalab/schemas/plugin_config.json | 172 +- pydatalab/schemas/sample.json | 2104 ++++++++++------ pydatalab/schemas/startingmaterial.json | 2241 +++++++++++------ pydatalab/src/pydatalab/models/blocks.py | 11 +- pydatalab/src/pydatalab/models/cells.py | 79 +- pydatalab/src/pydatalab/models/collections.py | 19 +- pydatalab/src/pydatalab/models/entries.py | 22 +- pydatalab/src/pydatalab/models/equipment.py | 10 +- pydatalab/src/pydatalab/models/export_task.py | 50 +- pydatalab/src/pydatalab/models/files.py | 32 +- pydatalab/src/pydatalab/models/items.py | 23 +- pydatalab/src/pydatalab/models/people.py | 87 +- .../src/pydatalab/models/relationships.py | 43 +- pydatalab/src/pydatalab/models/samples.py | 4 +- .../pydatalab/models/starting_materials.py | 25 +- pydatalab/src/pydatalab/models/tasks.py | 89 +- pydatalab/src/pydatalab/models/traits.py | 117 +- pydatalab/src/pydatalab/models/utils.py | 262 +- pydatalab/src/pydatalab/models/versions.py | 80 +- pydatalab/tasks.py | 8 +- pydatalab/tests/test_models.py | 21 + 23 files changed, 5979 insertions(+), 3438 deletions(-) diff --git a/pydatalab/schemas/cell.json b/pydatalab/schemas/cell.json index 94dbbb862..771d40fb0 100644 --- a/pydatalab/schemas/cell.json +++ b/pydatalab/schemas/cell.json @@ -1,674 +1,706 @@ { - "title": "Cell", - "description": "A model for representing electrochemical cells.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "cells", - "const": "cells", - "pattern": "^cells$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Name", + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" + ], + "title": "AccountStatus", "type": "string" }, - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "default": "active", - "allOf": [ - { - "$ref": "#/definitions/CellStatus" + "CellComponent": { + "properties": { + "item": { + "anyOf": [ + { + "$ref": "#/$defs/EntryReference" + }, + { + "$ref": "#/$defs/InlineSubstance" + } + ], + "description": "A reference to item (sample or starting material) entry for the constituent substance.", + "title": "Item" + }, + "quantity": { + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the constituent material used to create the sample.", + "title": "Quantity" + }, + "unit": { + "default": "g", + "description": "The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).", + "title": "Unit", + "type": "string" } - ] - }, - "cell_format": { - "$ref": "#/definitions/CellFormat" - }, - "cell_format_description": { - "title": "Cell Format Description", - "type": "string" + }, + "required": [ + "item" + ], + "title": "CellComponent", + "type": "object" }, - "cell_preparation_description": { - "title": "Cell Preparation Description", + "CellFormat": { + "enum": [ + "coin", + "pouch", + "in situ (XRD)", + "in situ (NMR)", + "in situ (SQUID)", + "in situ (optical)", + "swagelok", + "cylindrical", + "other" + ], + "title": "CellFormat", "type": "string" }, - "characteristic_mass": { - "title": "Characteristic Mass", - "type": "number" - }, - "characteristic_chemical_formula": { - "title": "Characteristic Chemical Formula", + "CellStatus": { + "description": "An enumeration of the status of cells", + "enum": [ + "active", + "planned", + "disposed", + "cycled", + "shorted", + "dismantled", + "other" + ], + "title": "CellStatus", "type": "string" }, - "characteristic_molar_mass": { - "title": "Characteristic Molar Mass", - "type": "number" - }, - "positive_electrode": { - "title": "Positive Electrode", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/CellComponent" - } - }, - "negative_electrode": { - "title": "Negative Electrode", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/CellComponent" - } - }, - "electrolyte": { - "title": "Electrolyte", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/CellComponent" - } + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", + "properties": { + "collection_id": { + "description": "The immutable ID of the collection.", + "title": "Collection Id", + "type": "string" + } + }, + "required": [ + "collection_id" + ], + "title": "CollectionReference", + "type": "object" }, - "active_ion_charge": { - "title": "Active Ion Charge", - "default": 1, - "type": "number" - } - }, - "required": [ - "item_id" - ], - "definitions": { "DataBlockResponse": { - "title": "DataBlockResponse", + "additionalProperties": true, "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", "properties": { "blocktype": { + "description": "The type of the block.", "title": "Blocktype", "type": "string" }, "block_id": { + "description": "A shorthand random ID for the block.", "title": "Block Id", "type": "string" }, "item_id": { - "title": "Item Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" }, "collection_id": { - "title": "Collection Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" }, "title": { - "title": "Title", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" }, "freeform_comment": { - "title": "Freeform Comment", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" }, "file_id": { - "title": "File Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" }, "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" }, "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" }, "warnings": { - "title": "Warnings", - "type": "array", - "items": { - "type": "string" - } - }, - "b64_encoded_image": { - "title": "B64 Encoded Image", - "datalab_exclude_from_db": true, - "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" + }, + "b64_encoded_image": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_db": true, + "datalab_exclude_from_load": true, + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" }, "bokeh_plot_data": { - "title": "Bokeh Plot Data", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object" + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" }, "computed": { - "title": "Computed", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" }, "metadata": { - "title": "Metadata", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" } }, "required": [ "blocktype", - "block_id" - ] - }, - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" - ], - "type": "string" - }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", - "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" + "block_id", + "b64_encoded_image", + "bokeh_plot_data" ], - "type": "string" + "title": "DataBlockResponse", + "type": "object" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "EntryReference": { + "additionalProperties": true, + "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", "properties": { - "description": { - "title": "Description", + "type": { + "title": "Type", "type": "string" }, - "relation": { - "$ref": "#/definitions/RelationshipType" - }, - "type": { - "$ref": "#/definitions/KnownType" + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" }, "immutable_id": { - "title": "Immutable Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Immutable Id" }, "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Item Id" }, "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Refcode" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" } }, "required": [ "type" - ] - }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" ], - "type": "string" + "title": "EntryReference", + "type": "object" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" }, - "identifier": { - "title": "Identifier", - "type": "string" + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" }, - "name": { - "title": "Name", - "type": "string" + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" }, - "display_name": { - "title": "Display Name", - "type": "string" - } - }, - "required": [ - "identity_type", - "identifier", - "name" - ] - }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" - ], - "type": "string" - }, - "Group": { - "title": "Group", - "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", - "properties": { "type": { + "const": "files", + "default": "files", "title": "Type", - "default": "groups", - "const": "groups", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" }, - "members": { - "title": "Members", - "type": "array", + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" + }, + "item_ids": { + "description": "A list of item IDs associated with this file.", "items": { - "type": "object" - } + "type": "string" + }, + "title": "Item Ids", + "type": "array" }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, + "blocks": { + "description": "A list of block IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Blocks", + "type": "array" + }, + "name": { + "description": "The filename on disk.", + "title": "Name", "type": "string" }, - "description": { - "title": "Description", + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", "type": "string" }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object" - } - ] - } - } - } - }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", - "enum": [ - "active", - "unverified", - "deactivated" - ], - "type": "string" - }, - "Person": { - "title": "Person", - "description": "A model that describes an individual and their digital identities.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "default": "people", - "const": "people", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "identities": { - "title": "Identities", - "type": "array", - "items": { - "$ref": "#/definitions/Identity" - } - }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" - }, - "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" - }, - "gravatar_hash": { - "title": "Gravatar Hash", - "type": "string" - }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } - }, - "role": { - "default": "user", - "allOf": [ + "original_name": { + "anyOf": [ + { + "type": "string" + }, { - "$ref": "#/definitions/UserRole" + "type": "null" } - ] - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" }, - "account_status": { - "default": "unverified", - "allOf": [ + "location": { + "anyOf": [ + { + "type": "string" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] - } - } - }, - "Collection": { - "title": "Collection", - "description": "An Entry is an abstract base class for any model that can be\ndeserialized and stored in the database.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", - "type": "string" + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + "time_added": { + "description": "The timestamp for the original file upload.", + "format": "datetime", + "title": "Time Added", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" }, - "title": { - "title": "Title", - "type": "string" + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" }, - "description": { - "title": "Description", - "type": "string" + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", + "type": "boolean" }, - "num_items": { - "title": "Num Items", - "type": "integer" + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "description": "Content checksums (MD5 and SHA-256) of the file." } - } + }, + "required": [ + "name", + "extension", + "time_added", + "is_live", + "checksums" + ], + "title": "File", + "type": "object" }, "FileChecksums": { - "title": "FileChecksums", "description": "Content checksums for a file.", - "type": "object", "properties": { "md5": { + "description": "The MD5 hex digest of the file contents.", "title": "Md5", "type": "string" }, "sha256": { + "description": "The SHA-256 hex digest of the file contents.", "title": "Sha256", "type": "string" } @@ -676,283 +708,840 @@ "required": [ "md5", "sha256" - ] + ], + "title": "FileChecksums", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", + "Group": { + "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, "type": { + "const": "groups", + "default": "groups", + "description": "The entry type as a string.", "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "size": { - "title": "Size", - "type": "integer" + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" + "group_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, - "item_ids": { - "title": "Item Ids", - "type": "array", + "members": { + "description": "A list of people that belong to this group; stored on the user objects.", "items": { - "type": "string" - } + "additionalProperties": true, + "type": "object" + }, + "title": "Members", + "type": "array" }, - "blocks": { - "title": "Blocks", - "type": "array", - "items": { - "type": "string" - } + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" }, - "name": { - "title": "Name", - "type": "string" + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" + }, + "managers": { + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", + "items": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + } + ] + }, + "title": "Managers", + "type": "array" + } + }, + "title": "Group", + "type": "object" + }, + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", + "properties": { + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." + }, + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", + "type": "string" + }, + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", + "type": "string" + }, + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" + }, + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" + } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "InlineSubstance": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "name" + ], + "title": "InlineSubstance", + "type": "object" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" + ], + "title": "KnownType", + "type": "string" + }, + "Person": { + "description": "A model that describes an individual and their digital identities.", + "properties": { + "type": { + "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "identities": { + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", + "items": { + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" + }, + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" + }, + "contact_email": { + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" + }, + "gravatar_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" + }, + "managers": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" + }, + "role": { + "$ref": "#/$defs/UserRole", + "default": "user", + "description": "The role assigned to this person." + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" + }, + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." + } + }, + "title": "Person", + "type": "object" + }, + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" + }, + "TypedRelationship": { + "additionalProperties": true, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" + }, + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." + }, + "type": { + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" + }, + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" + } + }, + "required": [ + "type" + ], + "title": "TypedRelationship", + "type": "object" + }, + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing electrochemical cells.", + "properties": { + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" }, - "extension": { - "title": "Extension", - "type": "string" + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" }, - "original_name": { - "title": "Original Name", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "cells", + "default": "cells", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", "type": "string" }, - "location": { - "title": "Location", + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "url_path": { - "title": "Url Path", + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", "type": "string" }, - "source": { - "title": "Source", + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { "type": "string" }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" - }, - "metadata": { - "title": "Metadata", - "type": "object" - }, - "representation": { - "title": "Representation" - }, - "source_server_name": { - "title": "Source Server Name", + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "source_path": { - "title": "Source Path", + { + "type": "null" + } + ], + "default": null, + "description": "A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).", + "title": "Date" + }, + "name": { + "anyOf": [ + { "type": "string" }, - "is_live": { - "title": "Is Live", - "type": "boolean" + { + "type": "null" + } + ], + "default": null, + "description": "An optional human-readable/usable name for the entry.", + "title": "Name" + }, + "files": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" }, - "checksums": { - "$ref": "#/definitions/FileChecksums" + { + "type": "null" } + ], + "default": null, + "description": "Any files attached to this sample.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" }, - "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + "title": "File Objectids", + "type": "array" }, - "CellStatus": { - "title": "CellStatus", - "description": "An enumeration of the status of cells", - "enum": [ - "active", - "planned", - "disposed", - "cycled", - "shorted", - "dismantled", - "other" - ], - "type": "string" + "status": { + "$ref": "#/$defs/CellStatus", + "default": "active", + "description": "The status of the cells, indicating its current state." }, - "CellFormat": { - "title": "CellFormat", - "description": "An enumeration.", - "enum": [ - "coin", - "pouch", - "in situ (XRD)", - "in situ (NMR)", - "in situ (SQUID)", - "in situ (optical)", - "swagelok", - "cylindrical", - "other" + "cell_format": { + "anyOf": [ + { + "$ref": "#/$defs/CellFormat" + }, + { + "type": "null" + } ], - "type": "string" + "default": null, + "description": "The form factor of the cell, e.g., coin, pouch, in situ or otherwise." }, - "EntryReference": { - "title": "EntryReference", - "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "immutable_id": { - "title": "Immutable Id", + "cell_format_description": { + "anyOf": [ + { "type": "string" }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + { + "type": "null" + } + ], + "default": null, + "description": "Additional human-readable description of the cell form factor, e.g., 18650, AMPIX, CAMPIX", + "title": "Cell Format Description" + }, + "cell_preparation_description": { + "anyOf": [ + { "type": "string" }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + { + "type": "null" + } + ], + "default": null, + "description": "Description of how the cell was prepared.", + "title": "Cell Preparation Description" + }, + "characteristic_mass": { + "anyOf": [ + { + "type": "number" }, - "chemform": { - "title": "Chemform", - "type": "string" + { + "type": "null" } - }, - "required": [ - "type" - ] + ], + "default": null, + "description": "The characteristic mass of the cell in milligrams. Can be used to normalize capacities.", + "title": "Characteristic Mass" }, - "InlineSubstance": { - "title": "InlineSubstance", - "type": "object", - "properties": { - "name": { - "title": "Name", + "characteristic_chemical_formula": { + "anyOf": [ + { "type": "string" }, - "chemform": { - "title": "Chemform", - "type": "string" + { + "type": "null" } - }, - "required": [ - "name" - ] + ], + "default": null, + "description": "The chemical formula of the active material. Can be used to calculated molar mass in g/mol for normalizing capacities.", + "title": "Characteristic Chemical Formula" }, - "CellComponent": { - "title": "CellComponent", - "description": "A constituent of a sample.", - "type": "object", - "properties": { - "item": { - "title": "Item", - "anyOf": [ - { - "$ref": "#/definitions/EntryReference" - }, - { - "$ref": "#/definitions/InlineSubstance" - } - ] - }, - "quantity": { - "title": "Quantity", - "minimum": 0, + "characteristic_molar_mass": { + "anyOf": [ + { "type": "number" }, - "unit": { - "title": "Unit", - "default": "g", - "type": "string" + { + "type": "null" } + ], + "default": null, + "description": "The molar mass of the active material, in g/mol. Will be inferred from the chemical formula, or can be supplied if it cannot be supplied", + "title": "Characteristic Molar Mass" + }, + "positive_electrode": { + "items": { + "$ref": "#/$defs/CellComponent" }, - "required": [ - "item", - "quantity" - ] + "title": "Positive Electrode", + "type": "array" + }, + "negative_electrode": { + "items": { + "$ref": "#/$defs/CellComponent" + }, + "title": "Negative Electrode", + "type": "array" + }, + "electrolyte": { + "items": { + "$ref": "#/$defs/CellComponent" + }, + "title": "Electrolyte", + "type": "array" + }, + "active_ion": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The active ion species.", + "title": "Active Ion" + }, + "active_ion_charge": { + "default": 1, + "title": "Active Ion Charge", + "type": "number" } - } + }, + "required": [ + "item_id" + ], + "title": "Cell", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/equipment.json b/pydatalab/schemas/equipment.json index d8c82722a..5482268c6 100644 --- a/pydatalab/schemas/equipment.json +++ b/pydatalab/schemas/equipment.json @@ -1,816 +1,1319 @@ { - "title": "Equipment", - "description": "A model for representing an experimental sample.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "equipment", - "const": "equipment", - "pattern": "^equipment$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Name", + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" + ], + "title": "AccountStatus", "type": "string" }, - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "default": "working", - "allOf": [ - { - "$ref": "#/definitions/EquipmentStatus" + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", + "properties": { + "collection_id": { + "description": "The immutable ID of the collection.", + "title": "Collection Id", + "type": "string" } - ] - }, - "serial_numbers": { - "title": "Serial Numbers", - "type": "string" - }, - "manufacturer": { - "title": "Manufacturer", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" + }, + "required": [ + "collection_id" + ], + "title": "CollectionReference", + "type": "object" }, - "contact": { - "title": "Contact", - "type": "string" - } - }, - "required": [ - "item_id" - ], - "definitions": { "DataBlockResponse": { - "title": "DataBlockResponse", + "additionalProperties": true, "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", "properties": { "blocktype": { + "description": "The type of the block.", "title": "Blocktype", "type": "string" }, "block_id": { + "description": "A shorthand random ID for the block.", "title": "Block Id", "type": "string" }, "item_id": { - "title": "Item Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" }, "collection_id": { - "title": "Collection Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" }, "title": { - "title": "Title", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" }, "freeform_comment": { - "title": "Freeform Comment", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" }, "file_id": { - "title": "File Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" }, "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" }, "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" }, "warnings": { - "title": "Warnings", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" }, "b64_encoded_image": { - "title": "B64 Encoded Image", + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" }, "bokeh_plot_data": { - "title": "Bokeh Plot Data", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object" + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" }, "computed": { - "title": "Computed", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" }, "metadata": { - "title": "Metadata", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" } }, "required": [ "blocktype", - "block_id" - ] - }, - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" + "block_id", + "b64_encoded_image", + "bokeh_plot_data" ], - "type": "string" + "title": "DataBlockResponse", + "type": "object" }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "EquipmentStatus": { + "description": "An enumeration of the status of equipments", "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" + "working", + "broken", + "being_fixed", + "defunct", + "not_being_fixed", + "other" ], + "title": "EquipmentStatus", "type": "string" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", "properties": { - "description": { - "title": "Description", - "type": "string" + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" }, - "relation": { - "$ref": "#/definitions/RelationshipType" + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" }, - "type": { - "$ref": "#/definitions/KnownType" + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" }, - "immutable_id": { - "title": "Immutable Id", - "type": "string" + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - } - }, - "required": [ - "type" - ] - }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" - ], - "type": "string" - }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", - "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" }, - "identifier": { - "title": "Identifier", + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "files", + "default": "files", + "title": "Type", "type": "string" }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" + }, + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" + }, + "item_ids": { + "description": "A list of item IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Item Ids", + "type": "array" + }, + "blocks": { + "description": "A list of block IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Blocks", + "type": "array" + }, "name": { + "description": "The filename on disk.", "title": "Name", "type": "string" }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", + "type": "string" }, - "display_name": { - "title": "Display Name", + "original_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" + }, + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" + }, + "time_added": { + "description": "The timestamp for the original file upload.", + "format": "datetime", + "title": "Time Added", "type": "string" + }, + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" + }, + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" + }, + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" + }, + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" + }, + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", + "type": "boolean" + }, + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ - "identity_type", - "identifier", - "name" - ] + "name", + "extension", + "time_added", + "is_live", + "checksums" + ], + "title": "File", + "type": "object" }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" + "FileChecksums": { + "description": "Content checksums for a file.", + "properties": { + "md5": { + "description": "The MD5 hex digest of the file contents.", + "title": "Md5", + "type": "string" + }, + "sha256": { + "description": "The SHA-256 hex digest of the file contents.", + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "md5", + "sha256" ], - "type": "string" + "title": "FileChecksums", + "type": "object" }, "Group": { - "title": "Group", "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", "properties": { "type": { - "title": "Type", - "default": "groups", "const": "groups", + "default": "groups", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, "members": { - "title": "Members", - "type": "array", + "description": "A list of people that belong to this group; stored on the user objects.", "items": { + "additionalProperties": true, "type": "object" - } + }, + "title": "Members", + "type": "array" }, "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" }, "description": { - "title": "Description", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" }, "managers": { - "title": "Managers", - "type": "array", + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", "items": { "anyOf": [ { + "format": "objectid", "type": "string" }, { + "additionalProperties": true, "type": "object" } ] - } + }, + "title": "Managers", + "type": "array" } - } - }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", - "enum": [ - "active", - "unverified", - "deactivated" - ], - "type": "string" + }, + "title": "Group", + "type": "object" }, - "Person": { - "title": "Person", - "description": "A model that describes an individual and their digital identities.", - "type": "object", + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", "properties": { - "type": { - "title": "Type", - "default": "people", - "const": "people", - "type": "string" + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "identities": { - "title": "Identities", - "type": "array", - "items": { - "$ref": "#/definitions/Identity" - } - }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", "type": "string" }, - "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" - }, - "gravatar_hash": { - "title": "Gravatar Hash", - "type": "string" + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } - }, - "role": { - "default": "user", - "allOf": [ + "display_name": { + "anyOf": [ { - "$ref": "#/definitions/UserRole" - } - ] - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "account_status": { - "default": "unverified", - "allOf": [ + "type": "string" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" } - } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" + ], + "title": "KnownType", + "type": "string" }, - "Collection": { - "title": "Collection", - "description": "An Entry is an abstract base class for any model that can be\ndeserialized and stored in the database.", - "type": "object", + "Person": { + "description": "A model that describes an individual and their digital identities.", "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, "type": { + "const": "people", + "default": "people", + "description": "The entry type as a string.", "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, "title": "Relationships", - "type": "array", + "type": "array" + }, + "identities": { + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" }, - "title": { - "title": "Title", - "type": "string" + "contact_email": { + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" }, - "description": { - "title": "Description", - "type": "string" + "gravatar_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" }, - "num_items": { - "title": "Num Items", - "type": "integer" - } - } - }, - "FileChecksums": { - "title": "FileChecksums", - "description": "Content checksums for a file.", - "type": "object", - "properties": { - "md5": { - "title": "Md5", - "type": "string" + "managers": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" }, - "sha256": { - "title": "Sha256", - "type": "string" + "role": { + "$ref": "#/$defs/UserRole", + "default": "user", + "description": "The role assigned to this person." + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" + }, + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." } }, - "required": [ - "md5", - "sha256" - ] + "title": "Person", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" + }, + "TypedRelationship": { + "additionalProperties": true, "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." }, "type": { - "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", - "type": "string" + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." }, "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" }, - "size": { - "title": "Size", - "type": "integer" + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" + } + }, + "required": [ + "type" + ], + "title": "TypedRelationship", + "type": "object" + }, + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing an experimental sample.", + "properties": { + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" }, - "item_ids": { - "title": "Item Ids", - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { "items": { - "type": "string" - } + "$ref": "#/$defs/Person" + }, + "type": "array" }, - "blocks": { - "title": "Blocks", - "type": "array", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { "items": { - "type": "string" - } + "$ref": "#/$defs/Group" + }, + "type": "array" }, - "name": { - "title": "Name", + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "equipment", + "default": "equipment", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", "type": "string" }, - "extension": { - "title": "Extension", + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "original_name": { - "title": "Original Name", + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", "type": "string" }, - "location": { - "title": "Location", + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { "type": "string" }, - "url_path": { - "title": "Url Path", + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", "type": "string" }, - "source": { - "title": "Source", + { + "type": "null" + } + ], + "default": null, + "description": "A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).", + "title": "Date" + }, + "name": { + "anyOf": [ + { "type": "string" }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" - }, - "metadata": { - "title": "Metadata", - "type": "object" - }, - "representation": { - "title": "Representation" + { + "type": "null" + } + ], + "default": null, + "description": "An optional human-readable/usable name for the entry.", + "title": "Name" + }, + "files": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" }, - "source_server_name": { - "title": "Source Server Name", + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this sample.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "status": { + "$ref": "#/$defs/EquipmentStatus", + "default": "working", + "description": "The status of the equipment, indicating its current state." + }, + "serial_numbers": { + "anyOf": [ + { "type": "string" }, - "source_path": { - "title": "Source Path", + { + "type": "null" + } + ], + "default": null, + "description": "A string describing one or more serial numbers for the instrument.", + "title": "Serial Numbers" + }, + "manufacturer": { + "anyOf": [ + { "type": "string" }, - "is_live": { - "title": "Is Live", - "type": "boolean" + { + "type": "null" + } + ], + "default": null, + "description": "The manufacturer of this piece of equipment", + "title": "Manufacturer" + }, + "location": { + "anyOf": [ + { + "type": "string" }, - "checksums": { - "$ref": "#/definitions/FileChecksums" + { + "type": "null" } - }, - "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + ], + "default": null, + "description": "Place where the equipment is located", + "title": "Location" }, - "EquipmentStatus": { - "title": "EquipmentStatus", - "description": "An enumeration of the status of equipments", - "enum": [ - "working", - "broken", - "being_fixed", - "defunct", - "not_being_fixed", - "other" + "contact": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } ], - "type": "string" + "description": "Contact information for equipment (e.g., email address or phone number).", + "title": "Contact" } - } + }, + "required": [ + "item_id", + "contact" + ], + "title": "Equipment", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/plugin_config.json b/pydatalab/schemas/plugin_config.json index ae64bcb3c..57975833d 100644 --- a/pydatalab/schemas/plugin_config.json +++ b/pydatalab/schemas/plugin_config.json @@ -1,96 +1,130 @@ { - "title": "PluginConfigModel", - "description": "The schema for the top-level plugins.toml file.", - "type": "object", - "properties": { - "dependencies": { - "title": "Dependencies", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "tool": { - "title": "Tool", - "default": { + "$defs": { + "ToolSection": { + "properties": { "uv": { - "sources": {} + "$ref": "#/$defs/UvSection", + "default": { + "sources": {} + } } }, - "allOf": [ - { - "$ref": "#/definitions/ToolSection" + "title": "ToolSection", + "type": "object" + }, + "UvSection": { + "properties": { + "sources": { + "additionalProperties": { + "$ref": "#/$defs/UvSource" + }, + "default": {}, + "title": "Sources", + "type": "object" } - ] - } - }, - "additionalProperties": false, - "definitions": { + }, + "title": "UvSection", + "type": "object" + }, "UvSource": { - "title": "UvSource", "description": "A single entry under `[tool.uv.sources]` in plugins.toml.", - "type": "object", "properties": { "git": { - "title": "Git", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Git" }, "rev": { - "title": "Rev", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Rev" }, "branch": { - "title": "Branch", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Branch" }, "tag": { - "title": "Tag", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tag" }, "path": { - "title": "Path", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" }, "editable": { - "title": "Editable", - "type": "boolean" + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Editable" } }, - "additionalProperties": false - }, - "UvSection": { - "title": "UvSection", - "type": "object", - "properties": { - "sources": { - "title": "Sources", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/UvSource" - } - } + "title": "UvSource", + "type": "object" + } + }, + "description": "The schema for the top-level plugins.toml file.", + "properties": { + "dependencies": { + "default": [], + "items": { + "type": "string" }, - "additionalProperties": false + "title": "Dependencies", + "type": "array" }, - "ToolSection": { - "title": "ToolSection", - "type": "object", - "properties": { + "tool": { + "$ref": "#/$defs/ToolSection", + "default": { "uv": { - "title": "Uv", - "default": { - "sources": {} - }, - "allOf": [ - { - "$ref": "#/definitions/UvSection" - } - ] + "sources": {} } - }, - "additionalProperties": false + } } - } + }, + "title": "PluginConfigModel", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/sample.json b/pydatalab/schemas/sample.json index 92488c652..57b0fb8ce 100644 --- a/pydatalab/schemas/sample.json +++ b/pydatalab/schemas/sample.json @@ -1,766 +1,678 @@ { - "title": "Sample", - "description": "A model for representing an experimental sample.", - "type": "object", - "properties": { - "chemform": { - "title": "Chemform", - "example": [ - "Na3P", - "Na3P", - "LiNiO2@C", - "Na3+xP", - "LiNi1/3Co0.1Mn0.1O2" - ], - "type": "string" - }, - "smiles": { - "title": "Smiles", - "aliases": [ - "SMILES", - "smiles_representation" - ], - "type": "string" - }, - "inchi": { - "title": "Inchi", - "type": "string" - }, - "inchi_key": { - "title": "Inchi Key", - "type": "string" - }, - "GHS_codes": { - "title": "Ghs H-Codes", - "examples": [ - "H224", - "H303, H316, H319" + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" ], + "title": "AccountStatus", "type": "string" }, - "molar_mass": { - "title": "Molecular Weight", - "type": "number" - }, - "CAS": { - "title": "Substance Cas", - "type": "string" - }, - "synthesis_constituents": { - "title": "Synthesis Constituents", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Constituent" - } - }, - "synthesis_description": { - "title": "Synthesis Description", - "type": "string" - }, - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "samples", - "const": "samples", - "pattern": "^samples$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Name", - "type": "string" - }, - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "default": "active", - "allOf": [ - { - "$ref": "#/definitions/ItemStatus" - } - ] - } - }, - "required": [ - "item_id" - ], - "definitions": { - "EntryReference": { - "title": "EntryReference", - "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "immutable_id": { - "title": "Immutable Id", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "chemform": { - "title": "Chemform", - "type": "string" - } - }, - "required": [ - "type" - ] - }, - "InlineSubstance": { - "title": "InlineSubstance", - "type": "object", + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "name": { - "title": "Name", - "type": "string" - }, - "chemform": { - "title": "Chemform", + "collection_id": { + "description": "The immutable ID of the collection.", + "title": "Collection Id", "type": "string" } }, "required": [ - "name" - ] + "collection_id" + ], + "title": "CollectionReference", + "type": "object" }, "Constituent": { - "title": "Constituent", "description": "A constituent of a sample.", - "type": "object", "properties": { "item": { - "title": "Item", "anyOf": [ { - "$ref": "#/definitions/EntryReference" + "$ref": "#/$defs/EntryReference" }, { - "$ref": "#/definitions/InlineSubstance" + "$ref": "#/$defs/InlineSubstance" } - ] + ], + "description": "A reference to item (sample or starting material) entry for the constituent substance.", + "title": "Item" }, "quantity": { - "title": "Quantity", - "minimum": 0, - "type": "number" + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the constituent material used to create the sample.", + "title": "Quantity" }, "unit": { - "title": "Unit", "default": "g", + "description": "The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).", + "title": "Unit", "type": "string" } }, "required": [ - "item", - "quantity" - ] + "item" + ], + "title": "Constituent", + "type": "object" }, "DataBlockResponse": { - "title": "DataBlockResponse", + "additionalProperties": true, "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", "properties": { "blocktype": { + "description": "The type of the block.", "title": "Blocktype", "type": "string" }, "block_id": { + "description": "A shorthand random ID for the block.", "title": "Block Id", "type": "string" }, "item_id": { - "title": "Item Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" }, "collection_id": { - "title": "Collection Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" }, "title": { - "title": "Title", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" }, "freeform_comment": { - "title": "Freeform Comment", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" }, "file_id": { - "title": "File Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" }, "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" }, "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" }, "warnings": { - "title": "Warnings", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" }, "b64_encoded_image": { - "title": "B64 Encoded Image", + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" }, "bokeh_plot_data": { - "title": "Bokeh Plot Data", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object" + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" }, "computed": { - "title": "Computed", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" }, "metadata": { - "title": "Metadata", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" } }, "required": [ "blocktype", - "block_id" - ] - }, - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" - ], - "type": "string" - }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", - "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" + "block_id", + "b64_encoded_image", + "bokeh_plot_data" ], - "type": "string" + "title": "DataBlockResponse", + "type": "object" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "EntryReference": { + "additionalProperties": true, + "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", "properties": { - "description": { - "title": "Description", + "type": { + "title": "Type", "type": "string" }, - "relation": { - "$ref": "#/definitions/RelationshipType" - }, - "type": { - "$ref": "#/definitions/KnownType" + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" }, "immutable_id": { - "title": "Immutable Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Immutable Id" }, "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Item Id" }, "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Refcode" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" } }, "required": [ "type" - ] - }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" ], - "type": "string" + "title": "EntryReference", + "type": "object" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" }, - "identifier": { - "title": "Identifier", - "type": "string" + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" }, - "name": { - "title": "Name", - "type": "string" + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" }, - "display_name": { - "title": "Display Name", - "type": "string" - } - }, - "required": [ - "identity_type", - "identifier", - "name" - ] - }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" - ], - "type": "string" - }, - "Group": { - "title": "Group", - "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", - "properties": { "type": { + "const": "files", + "default": "files", "title": "Type", - "default": "groups", - "const": "groups", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" }, - "members": { - "title": "Members", - "type": "array", + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" + }, + "item_ids": { + "description": "A list of item IDs associated with this file.", "items": { - "type": "object" - } + "type": "string" + }, + "title": "Item Ids", + "type": "array" }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, + "blocks": { + "description": "A list of block IDs associated with this file.", + "items": { + "type": "string" + }, + "title": "Blocks", + "type": "array" + }, + "name": { + "description": "The filename on disk.", + "title": "Name", "type": "string" }, - "description": { - "title": "Description", + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", "type": "string" }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object" - } - ] - } - } - } - }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", - "enum": [ - "active", - "unverified", - "deactivated" - ], - "type": "string" - }, - "Person": { - "title": "Person", - "description": "A model that describes an individual and their digital identities.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "default": "people", - "const": "people", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "identities": { - "title": "Identities", - "type": "array", - "items": { - "$ref": "#/definitions/Identity" - } - }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" - }, - "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" - }, - "gravatar_hash": { - "title": "Gravatar Hash", - "type": "string" - }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } - }, - "role": { - "default": "user", - "allOf": [ + "original_name": { + "anyOf": [ + { + "type": "string" + }, { - "$ref": "#/definitions/UserRole" + "type": "null" } - ] + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "account_status": { - "default": "unverified", - "allOf": [ + "location": { + "anyOf": [ + { + "type": "string" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] - } - } - }, - "Collection": { - "title": "Collection", - "description": "An Entry is an abstract base class for any model that can be\ndeserialized and stored in the database.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", - "type": "string" + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + "time_added": { + "description": "The timestamp for the original file upload.", + "format": "datetime", + "title": "Time Added", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" }, - "title": { - "title": "Title", - "type": "string" + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" }, - "description": { - "title": "Description", - "type": "string" + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", + "type": "boolean" }, - "num_items": { - "title": "Num Items", - "type": "integer" + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "description": "Content checksums (MD5 and SHA-256) of the file." } - } + }, + "required": [ + "name", + "extension", + "time_added", + "is_live", + "checksums" + ], + "title": "File", + "type": "object" }, "FileChecksums": { - "title": "FileChecksums", "description": "Content checksums for a file.", - "type": "object", "properties": { "md5": { + "description": "The MD5 hex digest of the file contents.", "title": "Md5", "type": "string" }, "sha256": { + "description": "The SHA-256 hex digest of the file contents.", "title": "Sha256", "type": "string" } @@ -768,167 +680,214 @@ "required": [ "md5", "sha256" - ] + ], + "title": "FileChecksums", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", + "Group": { + "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, "type": { + "const": "groups", + "default": "groups", + "description": "The entry type as a string.", "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "size": { - "title": "Size", - "type": "integer" - }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "item_ids": { - "title": "Item Ids", - "type": "array", - "items": { - "type": "string" - } + "group_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, - "blocks": { - "title": "Blocks", - "type": "array", + "members": { + "description": "A list of people that belong to this group; stored on the user objects.", "items": { - "type": "string" - } - }, - "name": { - "title": "Name", - "type": "string" + "additionalProperties": true, + "type": "object" + }, + "title": "Members", + "type": "array" }, - "extension": { - "title": "Extension", - "type": "string" + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" }, - "original_name": { - "title": "Original Name", - "type": "string" + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" }, - "location": { - "title": "Location", - "type": "string" + "managers": { + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", + "items": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + } + ] + }, + "title": "Managers", + "type": "array" + } + }, + "title": "Group", + "type": "object" + }, + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", + "properties": { + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." }, - "url_path": { - "title": "Url Path", + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", "type": "string" }, - "source": { - "title": "Source", + "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", + "title": "Name", "type": "string" }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" - }, - "metadata": { - "title": "Metadata", - "type": "object" - }, - "representation": { - "title": "Representation" - }, - "source_server_name": { - "title": "Source Server Name", - "type": "string" + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" }, - "source_path": { - "title": "Source Path", + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" + } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "InlineSubstance": { + "properties": { + "name": { + "title": "Name", "type": "string" }, - "is_live": { - "title": "Is Live", - "type": "boolean" - }, - "checksums": { - "$ref": "#/definitions/FileChecksums" + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" } }, "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + "name" + ], + "title": "InlineSubstance", + "type": "object" }, "ItemStatus": { - "title": "ItemStatus", "description": "An enumeration of the status of items", "enum": [ "active", @@ -938,7 +897,644 @@ "failed", "other" ], + "title": "ItemStatus", + "type": "string" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" + ], + "title": "KnownType", + "type": "string" + }, + "Person": { + "description": "A model that describes an individual and their digital identities.", + "properties": { + "type": { + "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "identities": { + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", + "items": { + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" + }, + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" + }, + "contact_email": { + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" + }, + "gravatar_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" + }, + "managers": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" + }, + "role": { + "$ref": "#/$defs/UserRole", + "default": "user", + "description": "The role assigned to this person." + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" + }, + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." + } + }, + "title": "Person", + "type": "object" + }, + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" + }, + "TypedRelationship": { + "additionalProperties": true, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" + }, + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." + }, + "type": { + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" + }, + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" + } + }, + "required": [ + "type" + ], + "title": "TypedRelationship", + "type": "object" + }, + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing an experimental sample.", + "properties": { + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A string representation of the chemical formula or composition associated with this sample.\n\nThe representation is relatively free-form; clients are expected parse and interpret HTML markup for subscripts\nand accept unicode characters for greek letters.", + "examples": [ + "Na3P", + "Na3P", + "LiNiO2@C", + "Na3+xP", + "LiNi1/3Co0.1Mn0.1O2" + ], + "title": "Chemform" + }, + "smiles": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A SMILES string representation of the chemical structure associated with this sample.", + "title": "Smiles" + }, + "inchi": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An International Chemical Identifier (InChI) string representation of chemicals/molecules associated with this sample.", + "title": "Inchi" + }, + "inchi_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A unique key derived from the InChI.", + "title": "Inchi Key" + }, + "GHS_codes": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A string describing any GHS hazard codes associated with this item. See https://pubchem.ncbi.nlm.nih.gov/ghs/ for code definitions.", + "examples": [ + "H224", + "H303, H316, H319" + ], + "title": "Ghs Codes" + }, + "molar_mass": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Mass per formula unit, in g/mol.", + "title": "Molar Mass" + }, + "CAS": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The CAS Registry Number for the substance described by this entry.", + "title": "Cas" + }, + "synthesis_constituents": { + "default": [], + "description": "A list of references to constituent materials giving the amount and relevant inlined details of consituent items.", + "items": { + "$ref": "#/$defs/Constituent" + }, + "title": "Synthesis Constituents", + "type": "array" + }, + "synthesis_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Free-text details of the procedure applied to synthesise the sample", + "title": "Synthesis Description" + }, + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "samples", + "default": "samples", + "title": "Type", "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).", + "title": "Date" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional human-readable/usable name for the entry.", + "title": "Name" + }, + "files": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this sample.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "status": { + "$ref": "#/$defs/ItemStatus", + "default": "active", + "description": "The status of the sample, indicating its current state." } - } + }, + "required": [ + "item_id" + ], + "title": "Sample", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/schemas/startingmaterial.json b/pydatalab/schemas/startingmaterial.json index b51586d6b..40ea9e4c1 100644 --- a/pydatalab/schemas/startingmaterial.json +++ b/pydatalab/schemas/startingmaterial.json @@ -1,803 +1,678 @@ { - "title": "StartingMaterial", - "description": "A model for representing an experimental sample, based on the connection\nwith cheminventory.net, which mixes container-level and substance-level\ninformation.", - "type": "object", - "properties": { - "chemform": { - "title": "Chemform", - "example": [ - "Na3P", - "Na3P", - "LiNiO2@C", - "Na3+xP", - "LiNi1/3Co0.1Mn0.1O2" - ], - "type": "string" - }, - "smiles": { - "title": "Smiles", - "aliases": [ - "SMILES", - "smiles_representation" - ], - "type": "string" - }, - "inchi": { - "title": "Inchi", - "type": "string" - }, - "inchi_key": { - "title": "Inchi Key", - "type": "string" - }, - "GHS_codes": { - "title": "Ghs H-Codes", - "examples": [ - "H224", - "H303, H316, H319" + "$defs": { + "AccountStatus": { + "description": "A string enum representing the account status.", + "enum": [ + "active", + "unverified", + "deactivated" ], + "title": "AccountStatus", "type": "string" }, - "molar_mass": { - "title": "Molecular Weight", - "type": "number" - }, - "CAS": { - "title": "Substance Cas", - "type": "string" - }, - "synthesis_constituents": { - "title": "Synthesis Constituents", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Constituent" - } - }, - "synthesis_description": { - "title": "Synthesis Description", - "type": "string" - }, - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "collections": { - "title": "Collections", - "default": [], - "type": "array", - "items": { - "$ref": "#/definitions/Collection" - } - }, - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, - "type": { - "title": "Type", - "default": "starting_materials", - "const": "starting_materials", - "pattern": "^starting_materials$", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" - }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "date": { - "title": "Date Acquired", - "type": "string", - "format": "date-time" - }, - "name": { - "title": "Container Name", - "type": "string" - }, - "files": { - "title": "Files", - "type": "array", - "items": { - "$ref": "#/definitions/File" - } - }, - "file_ObjectIds": { - "title": "File Objectids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "default": "available", - "allOf": [ - { - "$ref": "#/definitions/StartingMaterialsStatus" - } - ] - }, - "barcode": { - "title": "Barcode", - "type": "string" - }, - "date_opened": { - "title": "Date Opened", - "type": "string", - "format": "date-time" - }, - "chemical_purity": { - "title": "Chemical Purity", - "type": "string" - }, - "full_percent": { - "title": "Full %", - "type": "string" - }, - "size": { - "title": "Container Size", - "type": "string" - }, - "size_unit": { - "title": "Unit", - "type": "string" - }, - "supplier": { - "title": "Supplier", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" - }, - "comment": { - "title": "Comments", - "type": "string" - } - }, - "required": [ - "item_id" - ], - "definitions": { - "EntryReference": { - "title": "EntryReference", - "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "immutable_id": { - "title": "Immutable Id", - "type": "string" - }, - "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" - }, - "chemform": { - "title": "Chemform", - "type": "string" - } - }, - "required": [ - "type" - ] - }, - "InlineSubstance": { - "title": "InlineSubstance", - "type": "object", + "CollectionReference": { + "additionalProperties": true, + "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "name": { - "title": "Name", - "type": "string" - }, - "chemform": { - "title": "Chemform", + "collection_id": { + "description": "The immutable ID of the collection.", + "title": "Collection Id", "type": "string" } }, "required": [ - "name" - ] + "collection_id" + ], + "title": "CollectionReference", + "type": "object" }, "Constituent": { - "title": "Constituent", "description": "A constituent of a sample.", - "type": "object", "properties": { "item": { - "title": "Item", "anyOf": [ { - "$ref": "#/definitions/EntryReference" + "$ref": "#/$defs/EntryReference" }, { - "$ref": "#/definitions/InlineSubstance" + "$ref": "#/$defs/InlineSubstance" } - ] + ], + "description": "A reference to item (sample or starting material) entry for the constituent substance.", + "title": "Item" }, "quantity": { - "title": "Quantity", - "minimum": 0, - "type": "number" + "anyOf": [ + { + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the constituent material used to create the sample.", + "title": "Quantity" }, "unit": { - "title": "Unit", "default": "g", + "description": "The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).", + "title": "Unit", "type": "string" } }, "required": [ - "item", - "quantity" - ] + "item" + ], + "title": "Constituent", + "type": "object" }, "DataBlockResponse": { - "title": "DataBlockResponse", + "additionalProperties": true, "description": "A generic response model for a block, i.e., what is stored in `self.data`\nin the corresponding DataBlock class.\n\nIt is expected but not mandatory that this model will be extended by the specific block type\nwhere possible.", - "type": "object", "properties": { "blocktype": { + "description": "The type of the block.", "title": "Blocktype", "type": "string" }, "block_id": { + "description": "A shorthand random ID for the block.", "title": "Block Id", "type": "string" }, "item_id": { - "title": "Item Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The item that the block is attached to, if any.", + "title": "Item Id" }, "collection_id": { - "title": "Collection Id", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The collection that the block is attached to, if any.", + "title": "Collection Id" }, "title": { - "title": "Title", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The title of the block, if any.", + "title": "Title" }, "freeform_comment": { - "title": "Freeform Comment", - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A freeform comment for the block, if any.", + "title": "Freeform Comment" }, "file_id": { - "title": "File Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the file associated with the block, if any.", + "title": "File Id" }, "file_ids": { - "title": "File Ids", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of file IDs associated with the block, if any.", + "title": "File Ids" }, "errors": { - "title": "Errors", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any errors that occurred during block processing.", + "title": "Errors" }, "warnings": { - "title": "Warnings", - "type": "array", - "items": { - "type": "string" - } + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any warnings that occurred during block processing.", + "title": "Warnings" }, "b64_encoded_image": { - "title": "B64 Encoded Image", + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object", - "additionalProperties": { - "type": "string" - } + "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", + "title": "B64 Encoded Image" }, "bokeh_plot_data": { - "title": "Bokeh Plot Data", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, - "type": "object" + "description": "A JSON-encoded string containing the Bokeh plot data, if any.", + "title": "Bokeh Plot Data" }, "computed": { - "title": "Computed", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", + "title": "Computed" }, "metadata": { - "title": "Metadata", + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], "datalab_exclude_from_load": true, - "type": "object" + "default": null, + "description": "Any structured metadata associated with the block, for example,\nexperimental acquisition parameters.", + "title": "Metadata" } }, "required": [ "blocktype", - "block_id" - ] - }, - "RelationshipType": { - "title": "RelationshipType", - "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", - "enum": [ - "parent", - "child", - "sibling", - "is_part_of", - "other" - ], - "type": "string" - }, - "KnownType": { - "title": "KnownType", - "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", - "enum": [ - "samples", - "starting_materials", - "blocks", - "files", - "people", - "collections" + "block_id", + "b64_encoded_image", + "bokeh_plot_data" ], - "type": "string" + "title": "DataBlockResponse", + "type": "object" }, - "TypedRelationship": { - "title": "TypedRelationship", - "type": "object", + "EntryReference": { + "additionalProperties": true, + "description": "A reference to a database entry by ID and type.\n\nCan include additional arbitarary metadata useful for\ninlining the item data.", "properties": { - "description": { - "title": "Description", + "type": { + "title": "Type", "type": "string" }, - "relation": { - "$ref": "#/definitions/RelationshipType" - }, - "type": { - "$ref": "#/definitions/KnownType" + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" }, "immutable_id": { - "title": "Immutable Id", - "type": "string" + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Immutable Id" }, "item_id": { - "title": "Item Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Item Id" }, "refcode": { - "title": "Refcode", - "minLength": 1, - "maxLength": 40, - "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Refcode" + }, + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" } }, "required": [ "type" - ] - }, - "IdentityType": { - "title": "IdentityType", - "description": "A string enum representing the supported verifiable identity types.", - "enum": [ - "email", - "orcid", - "github", - "google", - "microsoft" ], - "type": "string" + "title": "EntryReference", + "type": "object" }, - "Identity": { - "title": "Identity", - "description": "A model for identities that can be provided by external systems\nand associated with a given user.", - "type": "object", + "File": { + "description": "A model for representing a file that has been tracked or uploaded to datalab.", "properties": { - "identity_type": { - "$ref": "#/definitions/IdentityType" + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" }, - "identifier": { - "title": "Identifier", - "type": "string" + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" }, - "name": { - "title": "Name", - "type": "string" + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" }, - "verified": { - "title": "Verified", - "default": false, - "type": "boolean" + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" }, - "display_name": { - "title": "Display Name", - "type": "string" - } - }, - "required": [ - "identity_type", - "identifier", - "name" - ] - }, - "UserRole": { - "title": "UserRole", - "description": "An enumeration.", - "enum": [ - "user", - "admin", - "manager" - ], - "type": "string" - }, - "Group": { - "title": "Group", - "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", - "type": "object", - "properties": { "type": { + "const": "files", + "default": "files", "title": "Type", - "default": "groups", - "const": "groups", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "group_id": { - "title": "Group Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "members": { - "title": "Members", - "type": "array", - "items": { - "type": "object" - } + "size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The size of the file on disk in bytes.", + "title": "Size" }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, - "type": "string" - }, - "description": { - "title": "Description", - "type": "string" - }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "object" - } - ] - } - } - } - }, - "AccountStatus": { - "title": "AccountStatus", - "description": "A string enum representing the account status.", - "enum": [ - "active", - "unverified", - "deactivated" - ], - "type": "string" - }, - "Person": { - "title": "Person", - "description": "A model that describes an individual and their digital identities.", - "type": "object", - "properties": { - "type": { - "title": "Type", - "default": "people", - "const": "people", - "type": "string" - }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", - "type": "string" - }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "last_modified_remote": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The last date/time at which the remote file was modified.", + "title": "Last Modified Remote" }, - "relationships": { - "title": "Relationships", - "type": "array", + "item_ids": { + "description": "A list of item IDs associated with this file.", "items": { - "$ref": "#/definitions/TypedRelationship" - } + "type": "string" + }, + "title": "Item Ids", + "type": "array" }, - "identities": { - "title": "Identities", - "type": "array", + "blocks": { + "description": "A list of block IDs associated with this file.", "items": { - "$ref": "#/definitions/Identity" - } + "type": "string" + }, + "title": "Blocks", + "type": "array" }, - "display_name": { - "title": "Display Name", - "minLength": 1, - "maxLength": 150, + "name": { + "description": "The filename on disk.", + "title": "Name", "type": "string" }, - "contact_email": { - "title": "Contact Email", - "type": "string", - "format": "email" - }, - "gravatar_hash": { - "title": "Gravatar Hash", + "extension": { + "description": "The file extension that the file was uploaded with.", + "title": "Extension", "type": "string" }, - "managers": { - "title": "Managers", - "type": "array", - "items": { - "type": "string" - } - }, - "role": { - "default": "user", - "allOf": [ + "original_name": { + "anyOf": [ + { + "type": "string" + }, { - "$ref": "#/definitions/UserRole" + "type": "null" } - ] - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + ], + "default": null, + "description": "The raw filename as uploaded.", + "title": "Original Name" }, - "account_status": { - "default": "unverified", - "allOf": [ + "location": { + "anyOf": [ + { + "type": "string" + }, { - "$ref": "#/definitions/AccountStatus" + "type": "null" } - ] - } - } - }, - "Collection": { - "title": "Collection", - "description": "An Entry is an abstract base class for any model that can be\ndeserialized and stored in the database.", - "type": "object", - "properties": { - "blocks_obj": { - "title": "Blocks Obj", - "default": {}, - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/DataBlockResponse" - } - }, - "display_order": { - "title": "Display Order", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } + ], + "default": null, + "description": "The location of the file on disk.", + "title": "Location" }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } + "url_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to a remote file.", + "title": "Url Path" }, - "type": { - "title": "Type", - "default": "collections", - "const": "collections", - "pattern": "^collections$", - "type": "string" + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The source of the file, e.g. 'remote' or 'uploaded'.", + "title": "Source" }, - "immutable_id": { - "title": "Immutable ID", - "format": "uuid", + "time_added": { + "description": "The timestamp for the original file upload.", + "format": "datetime", + "title": "Time Added", "type": "string" }, - "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional metadata.", + "title": "Metadata" }, - "relationships": { - "title": "Relationships", - "type": "array", - "items": { - "$ref": "#/definitions/TypedRelationship" - } + "representation": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "title": "Representation" }, - "collection_id": { - "title": "Collection Id", - "minLength": 1, - "maxLength": 40, - "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", - "type": "string" + "source_server_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The server name at which the file is stored.", + "title": "Source Server Name" }, - "title": { - "title": "Title", - "type": "string" + "source_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the file on the remote resource.", + "title": "Source Path" }, - "description": { - "title": "Description", - "type": "string" + "is_live": { + "description": "Whether or not the file should be watched for future updates.", + "title": "Is Live", + "type": "boolean" }, - "num_items": { - "title": "Num Items", - "type": "integer" + "checksums": { + "anyOf": [ + { + "$ref": "#/$defs/FileChecksums" + }, + { + "type": "null" + } + ], + "description": "Content checksums (MD5 and SHA-256) of the file." } - } + }, + "required": [ + "name", + "extension", + "time_added", + "is_live", + "checksums" + ], + "title": "File", + "type": "object" }, "FileChecksums": { - "title": "FileChecksums", "description": "Content checksums for a file.", - "type": "object", "properties": { "md5": { + "description": "The MD5 hex digest of the file contents.", "title": "Md5", "type": "string" }, "sha256": { + "description": "The SHA-256 hex digest of the file contents.", "title": "Sha256", "type": "string" } @@ -805,167 +680,383 @@ "required": [ "md5", "sha256" - ] + ], + "title": "FileChecksums", + "type": "object" }, - "File": { - "title": "File", - "description": "A model for representing a file that has been tracked or uploaded to datalab.", - "type": "object", + "Group": { + "description": "A model that describes a group of users, for the sake\nof applying group permissions.\n\nEach `Person` can point to multiple groups.\n\nRelationships between groups can be described via the `relationships`\nfield inherited from `Entry`.", "properties": { - "revision": { - "title": "Revision", - "default": 1, - "type": "integer" - }, - "revisions": { - "title": "Revisions", - "type": "object" - }, - "version": { - "title": "Version", - "default": 1, - "type": "integer" - }, - "creator_ids": { - "title": "Creator Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "creators": { - "title": "Creators", - "type": "array", - "items": { - "$ref": "#/definitions/Person" - } - }, - "group_ids": { - "title": "Group Ids", - "default": [], - "type": "array", - "items": { - "type": "string" - } - }, - "groups": { - "title": "Groups", - "type": "array", - "items": { - "$ref": "#/definitions/Group" - } - }, "type": { + "const": "groups", + "default": "groups", + "description": "The entry type as a string.", "title": "Type", - "default": "files", - "const": "files", - "pattern": "^files$", "type": "string" }, "immutable_id": { - "title": "Immutable ID", + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", "format": "uuid", - "type": "string" + "title": "Immutable ID" }, "last_modified": { - "title": "Last Modified", - "type": "string", - "format": "date-time" + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, "relationships": { - "title": "Relationships", - "type": "array", + "description": "A list of related entries and their types.", "items": { - "$ref": "#/definitions/TypedRelationship" - } - }, - "size": { - "title": "Size", - "type": "integer" + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "last_modified_remote": { - "title": "Last Modified Remote", - "type": "string", - "format": "date-time" + "group_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A short, locally-unique ID for the group.", + "title": "Group Id" }, - "item_ids": { - "title": "Item Ids", - "type": "array", + "members": { + "description": "A list of people that belong to this group; stored on the user objects.", "items": { - "type": "string" - } - }, - "blocks": { - "title": "Blocks", - "type": "array", + "additionalProperties": true, + "type": "object" + }, + "title": "Members", + "type": "array" + }, + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The chosen display name for the group", + "title": "Display Name" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the group", + "title": "Description" + }, + "managers": { + "description": "A list of user IDs that can manage this group; stored in db as list of IDs.", "items": { - "type": "string" - } + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + } + ] + }, + "title": "Managers", + "type": "array" + } + }, + "title": "Group", + "type": "object" + }, + "Identity": { + "description": "A model for identities that can be provided by external systems\nand associated with a given user.", + "properties": { + "identity_type": { + "$ref": "#/$defs/IdentityType", + "description": "The type or provider of the identity." + }, + "identifier": { + "description": "The identifier for the identity, e.g., an email address, an ORCID, a GitHub user ID.", + "title": "Identifier", + "type": "string" }, "name": { + "description": "The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.", "title": "Name", "type": "string" }, - "extension": { - "title": "Extension", - "type": "string" + "verified": { + "default": false, + "description": "Whether the identity has been verified (by some means, e.g., OAuth2 or email)", + "title": "Verified", + "type": "boolean" }, - "original_name": { - "title": "Original Name", + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user's display name associated with the identity, also to be exposed in free text searches.", + "title": "Display Name" + } + }, + "required": [ + "identity_type", + "identifier", + "name" + ], + "title": "Identity", + "type": "object" + }, + "IdentityType": { + "description": "A string enum representing the supported verifiable identity types.", + "enum": [ + "email", + "orcid", + "github", + "google", + "microsoft" + ], + "title": "IdentityType", + "type": "string" + }, + "InlineSubstance": { + "properties": { + "name": { + "title": "Name", "type": "string" }, - "location": { - "title": "Location", + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Chemform" + } + }, + "required": [ + "name" + ], + "title": "InlineSubstance", + "type": "object" + }, + "KnownType": { + "description": "An enumeration of the types of entry known by this implementation, should be made dynamic in the future.", + "enum": [ + "samples", + "starting_materials", + "blocks", + "files", + "people", + "collections" + ], + "title": "KnownType", + "type": "string" + }, + "Person": { + "description": "A model that describes an individual and their digital identities.", + "properties": { + "type": { + "const": "people", + "default": "people", + "description": "The entry type as a string.", + "title": "Type", "type": "string" }, - "url_path": { - "title": "Url Path", - "type": "string" + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" }, - "source": { - "title": "Source", - "type": "string" + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" }, - "time_added": { - "title": "Time Added", - "type": "string", - "format": "date-time" + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" }, - "metadata": { - "title": "Metadata", - "type": "object" + "identities": { + "description": "A list of identities attached to this person, e.g., email addresses, OAuth accounts.", + "items": { + "$ref": "#/$defs/Identity" + }, + "title": "Identities", + "type": "array" }, - "representation": { - "title": "Representation" + "display_name": { + "anyOf": [ + { + "maxLength": 150, + "minLength": 1, + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The user-chosen display name.", + "title": "Display Name" }, - "source_server_name": { - "title": "Source Server Name", - "type": "string" + "contact_email": { + "anyOf": [ + { + "format": "email", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "In the case of multiple *verified* email identities, this email will be used as the primary contact.", + "title": "Contact Email" }, - "source_path": { - "title": "Source Path", - "type": "string" + "gravatar_hash": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "MD5 hash used by the frontend to fetch a Gravatar avatar without exposing the raw email.", + "title": "Gravatar Hash" }, - "is_live": { - "title": "Is Live", - "type": "boolean" + "managers": { + "anyOf": [ + { + "items": { + "format": "objectid", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of user IDs that can manage this person's items.", + "title": "Managers" }, - "checksums": { - "$ref": "#/definitions/FileChecksums" + "role": { + "$ref": "#/$defs/UserRole", + "default": "user", + "description": "The role assigned to this person." + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "A list of groups that this person belongs to.", + "title": "Groups" + }, + "account_status": { + "$ref": "#/$defs/AccountStatus", + "default": "unverified", + "description": "The status of the user's account." } }, - "required": [ - "item_ids", - "blocks", - "name", - "extension", - "time_added", - "is_live" - ] + "title": "Person", + "type": "object" + }, + "RelationshipType": { + "description": "An enumeration of the possible types of relationship between two entries.\n\n```mermaid\nclassDiagram\nclass entryC\nentryC --|> entryA: parent\nentryC ..|> entryD\nentryA <..> entryD: sibling\nentryA --|> entryB : child\n```", + "enum": [ + "parent", + "child", + "sibling", + "is_part_of", + "other", + "collections" + ], + "title": "RelationshipType", + "type": "string" }, "StartingMaterialsStatus": { - "title": "StartingMaterialsStatus", "description": "An enumeration of the status of starting materials", "enum": [ "ordered", @@ -976,7 +1067,593 @@ "exhausted", "other" ], + "title": "StartingMaterialsStatus", "type": "string" - } - } + }, + "TypedRelationship": { + "additionalProperties": true, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the relationship.", + "title": "Description" + }, + "relation": { + "anyOf": [ + { + "$ref": "#/$defs/RelationshipType" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided." + }, + "type": { + "$ref": "#/$defs/KnownType", + "description": "The type of the related resource." + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable ID of the entry that is related to this entry.", + "title": "Immutable Id" + }, + "item_id": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the entry that is related to this entry.", + "title": "Item Id" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The refcode of the entry that is related to this entry.", + "title": "Refcode" + } + }, + "required": [ + "type" + ], + "title": "TypedRelationship", + "type": "object" + }, + "UserRole": { + "enum": [ + "user", + "admin", + "manager" + ], + "title": "UserRole", + "type": "string" + } + }, + "description": "A model for representing an experimental sample, based on the connection\nwith cheminventory.net, which mixes container-level and substance-level\ninformation.", + "properties": { + "chemform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A string representation of the chemical formula or composition associated with this sample.\n\nThe representation is relatively free-form; clients are expected parse and interpret HTML markup for subscripts\nand accept unicode characters for greek letters.", + "examples": [ + "Na3P", + "Na3P", + "LiNiO2@C", + "Na3+xP", + "LiNi1/3Co0.1Mn0.1O2" + ], + "title": "Chemform" + }, + "smiles": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A SMILES string representation of the chemical structure associated with this sample.", + "title": "Smiles" + }, + "inchi": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An International Chemical Identifier (InChI) string representation of chemicals/molecules associated with this sample.", + "title": "Inchi" + }, + "inchi_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A unique key derived from the InChI.", + "title": "Inchi Key" + }, + "GHS_codes": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A string describing any GHS hazard codes associated with this item. See https://pubchem.ncbi.nlm.nih.gov/ghs/ for code definitions.", + "examples": [ + "H224", + "H303, H316, H319" + ], + "title": "Ghs Codes" + }, + "molar_mass": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Mass per formula unit, in g/mol.", + "title": "Molar Mass" + }, + "CAS": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The CAS Registry Number for the substance described by this entry.", + "title": "Cas" + }, + "synthesis_constituents": { + "default": [], + "description": "A list of references to constituent materials giving the amount and relevant inlined details of consituent items.", + "items": { + "$ref": "#/$defs/Constituent" + }, + "title": "Synthesis Constituents", + "type": "array" + }, + "synthesis_description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Free-text details of the procedure applied to synthesise the sample", + "title": "Synthesis Description" + }, + "blocks_obj": { + "additionalProperties": { + "$ref": "#/$defs/DataBlockResponse" + }, + "default": {}, + "description": "A mapping from block ID to block data.", + "title": "Blocks Obj", + "type": "object" + }, + "display_order": { + "default": [], + "description": "The order in which to display block data in the UI.", + "items": { + "type": "string" + }, + "title": "Display Order", + "type": "array" + }, + "collections": { + "description": "Inlined info for the collections associated with this item.", + "items": { + "$ref": "#/$defs/CollectionReference" + }, + "title": "Collections", + "type": "array" + }, + "revision": { + "default": 1, + "description": "The revision number of the entry.", + "title": "Revision", + "type": "integer" + }, + "revisions": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "An optional mapping from old revision numbers to the model state at that revision.", + "title": "Revisions" + }, + "version": { + "default": 1, + "description": "The version number used by the version control system for tracking snapshots.", + "title": "Version", + "type": "integer" + }, + "creator_ids": { + "default": [], + "description": "The database IDs of the user(s) who created the item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Creator Ids", + "type": "array" + }, + "creators": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Person" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the people associated with this item.", + "title": "Creators" + }, + "group_ids": { + "default": [], + "description": "The database IDs of the group(s) that have read-access to this item.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "Group Ids", + "type": "array" + }, + "groups": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/Group" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Inlined info for the groups with access to this item.", + "title": "Groups" + }, + "type": { + "const": "starting_materials", + "default": "starting_materials", + "title": "Type", + "type": "string" + }, + "immutable_id": { + "anyOf": [ + { + "format": "objectid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The immutable database ID of the entry.", + "format": "uuid", + "title": "Immutable ID" + }, + "last_modified": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The timestamp at which the entry was last modified.", + "title": "Last Modified" + }, + "relationships": { + "description": "A list of related entries and their types.", + "items": { + "$ref": "#/$defs/TypedRelationship" + }, + "title": "Relationships", + "type": "array" + }, + "refcode": { + "anyOf": [ + { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z]{2,10}:(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.", + "title": "Refcode" + }, + "item_id": { + "description": "A locally unique, human-readable identifier for the entry. This ID is mutable.", + "maxLength": 40, + "minLength": 1, + "pattern": "^(?:[a-zA-Z0-9]+|[a-zA-Z0-9][a-zA-Z0-9._-]+[a-zA-Z0-9])$", + "title": "Item Id", + "type": "string" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A description of the item, either in plain-text or a markup language.", + "title": "Description" + }, + "date": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The date the item was acquired", + "title": "Date" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The name of the substance in the container.", + "title": "Name" + }, + "files": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/File" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any files attached to this sample.", + "title": "Files" + }, + "file_ObjectIds": { + "description": "Links to object IDs of files stored within the database.", + "items": { + "format": "objectid", + "type": "string" + }, + "title": "File Objectids", + "type": "array" + }, + "status": { + "$ref": "#/$defs/StartingMaterialsStatus", + "default": "available", + "description": "The status of the starting materials, indicating its current state." + }, + "barcode": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A unique barcode provided by an external source, e.g., cheminventory.", + "title": "Barcode" + }, + "date_opened": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The date the item was opened", + "title": "Date Opened" + }, + "chemical_purity": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The chemical purity of this container with regards to the defined substance.", + "title": "Chemical Purity" + }, + "full_percent": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The amount of the defined substance remaining in the container, expressed as a percentage.", + "title": "Full Percent" + }, + "size": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The total size of the container, in units of `size_unit`.", + "title": "Size" + }, + "size_unit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Units for the 'size' field.", + "title": "Size Unit" + }, + "supplier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Supplier or manufacturer of the chemical.", + "title": "Supplier" + }, + "location": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The place where the container is located.", + "title": "Location" + }, + "comment": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Any additional comments or notes about the container.", + "title": "Comment" + } + }, + "required": [ + "item_id", + "name", + "chemical_purity", + "supplier" + ], + "title": "StartingMaterial", + "type": "object" } \ No newline at end of file diff --git a/pydatalab/src/pydatalab/models/blocks.py b/pydatalab/src/pydatalab/models/blocks.py index 9e33aec65..a89af0ffc 100644 --- a/pydatalab/src/pydatalab/models/blocks.py +++ b/pydatalab/src/pydatalab/models/blocks.py @@ -1,6 +1,6 @@ -from pydantic import BaseModel, Field +from pydantic import ConfigDict, Field -from pydatalab.models.utils import JSON_ENCODERS, PyObjectId +from pydatalab.models.utils import BaseModel, PyObjectId class DataBlockResponse(BaseModel): @@ -11,6 +11,8 @@ class DataBlockResponse(BaseModel): where possible. """ + model_config = ConfigDict(validate_by_name=True, extra="allow") + blocktype: str """The type of the block.""" @@ -60,8 +62,3 @@ class DataBlockResponse(BaseModel): metadata: dict | None = Field(default=None, datalab_exclude_from_load=True) """Any structured metadata associated with the block, for example, experimental acquisition parameters.""" - - class Config: - allow_population_by_field_name = True - json_encoders = JSON_ENCODERS - extra = "allow" diff --git a/pydatalab/src/pydatalab/models/cells.py b/pydatalab/src/pydatalab/models/cells.py index 8474ad056..e8690db9a 100644 --- a/pydatalab/src/pydatalab/models/cells.py +++ b/pydatalab/src/pydatalab/models/cells.py @@ -1,13 +1,16 @@ from enum import Enum +from typing import Literal -from pydantic import Field, root_validator, validator +from pydantic import ( + Field, + field_validator, + model_validator, +) from pydatalab.models.entries import EntryReference from pydatalab.models.items import Item from pydatalab.models.utils import CellStatus, Constituent -# from pydatalab.logger import LOGGER - class CellComponent(Constituent): ... @@ -27,72 +30,66 @@ class CellFormat(str, Enum): class Cell(Item): """A model for representing electrochemical cells.""" - type: str = Field("cells", const="cells", pattern="^cells$") + type: Literal["cells"] = "cells" - cell_format: CellFormat | None + cell_format: CellFormat | None = None """The form factor of the cell, e.g., coin, pouch, in situ or otherwise.""" - cell_format_description: str | None + cell_format_description: str | None = None """Additional human-readable description of the cell form factor, e.g., 18650, AMPIX, CAMPIX""" - cell_preparation_description: str | None + cell_preparation_description: str | None = None + """Description of how the cell was prepared.""" - characteristic_mass: float | None + characteristic_mass: float | None = None """The characteristic mass of the cell in milligrams. Can be used to normalize capacities.""" - characteristic_chemical_formula: str | None + characteristic_chemical_formula: str | None = None """The chemical formula of the active material. Can be used to calculated molar mass in g/mol for normalizing capacities.""" - characteristic_molar_mass: float | None + characteristic_molar_mass: float | None = None """The molar mass of the active material, in g/mol. Will be inferred from the chemical formula, or can be supplied if it cannot be supplied""" - positive_electrode: list[CellComponent] = [] - - negative_electrode: list[CellComponent] = [] - - electrolyte: list[CellComponent] = [] - + positive_electrode: list[CellComponent] = Field(default_factory=list) + negative_electrode: list[CellComponent] = Field(default_factory=list) + electrolyte: list[CellComponent] = Field(default_factory=list) + active_ion: str | None = None + """The active ion species.""" active_ion_charge: float = 1 - status: CellStatus = Field(default=CellStatus.ACTIVE) """The status of the cells, indicating its current state.""" - @validator("characteristic_molar_mass", always=True, pre=True) - def set_molar_mass(cls, v, values): + @field_validator("characteristic_molar_mass", mode="before") + @classmethod + def set_molar_mass(cls, v, info): from periodictable import formula - if not v: - chemical_formula = values.get("characteristic_chemical_formula") - + if not v and hasattr(info, "data") and info.data: + chemical_formula = info.data.get("characteristic_chemical_formula") if chemical_formula: try: return formula(chemical_formula).mass except Exception: return None - return v - @root_validator - def add_missing_electrode_relationships(cls, values): + @model_validator(mode="after") + def add_missing_electrode_relationships(self): """Add any missing cell component constituents to parent relationships""" from pydatalab.models.relationships import RelationshipType, TypedRelationship - existing_parthood_relationships = {} - if values.get("relationships") is not None: - # Index by refcode *and* item_id so a stored relationship carrying an - # item_id still matches a refcode-enriched constituent (and vice-versa). - existing_parthood_relationships = { - identifier: relationship - for relationship in values["relationships"] - if relationship.relation == RelationshipType.PARTHOOD - for identifier in (relationship.refcode, relationship.item_id) - if identifier - } - else: - values["relationships"] = [] + # Index by refcode *and* item_id so a stored relationship carrying an + # item_id still matches a refcode-enriched constituent (and vice-versa). + existing_parthood_relationships = { + identifier: relationship + for relationship in self.relationships + if relationship.relation == RelationshipType.PARTHOOD + for identifier in (relationship.refcode, relationship.item_id) + if identifier + } for component in ("positive_electrode", "negative_electrode", "electrolyte"): - for constituent in values.get(component, []): + for constituent in getattr(self, component): if not isinstance(constituent.item, EntryReference): continue @@ -110,7 +107,7 @@ def add_missing_electrode_relationships(cls, values): type=constituent.item.type, description="Is a constituent of", ) - values["relationships"].append(relationship) + self.relationships.append(relationship) else: # Back-fill any identifier missing from the stored relationship relationship.refcode = relationship.refcode or refcode @@ -123,4 +120,4 @@ def add_missing_electrode_relationships(cls, values): if identifier: existing_parthood_relationships[identifier] = relationship - return values + return self diff --git a/pydatalab/src/pydatalab/models/collections.py b/pydatalab/src/pydatalab/models/collections.py index 4ce6b7b5b..be47e0631 100644 --- a/pydatalab/src/pydatalab/models/collections.py +++ b/pydatalab/src/pydatalab/models/collections.py @@ -1,4 +1,8 @@ -from pydantic import Field, root_validator +from typing import Literal + +from pydantic import ( + model_validator, +) from pydatalab.models.entries import Entry from pydatalab.models.traits import HasBlocks, HasOwner @@ -6,21 +10,22 @@ class Collection(Entry, HasOwner, HasBlocks): - type: str = Field("collections", const="collections", pattern="^collections$") + type: Literal["collections"] = "collections" - collection_id: HumanReadableIdentifier = Field(None) + collection_id: HumanReadableIdentifier """A short human-readable/usable name for the collection.""" - title: str | None + title: str | None = None """A descriptive title for the collection.""" - description: str | None + description: str | None = None """A description of the collection, either in plain-text or a markup language.""" - num_items: int | None = Field(None) + num_items: int | None = None """Inlined number of items associated with this collection.""" - @root_validator + @model_validator(mode="before") + @classmethod def check_ids(cls, values): if not any(values.get(k) is not None for k in ("collection_id", "immutable_id")): raise ValueError("Collection must have at least collection_id or immutable_id") diff --git a/pydatalab/src/pydatalab/models/entries.py b/pydatalab/src/pydatalab/models/entries.py index 694a312ad..a5b61f9de 100644 --- a/pydatalab/src/pydatalab/models/entries.py +++ b/pydatalab/src/pydatalab/models/entries.py @@ -1,10 +1,10 @@ import abc -from pydantic import BaseModel, Field, root_validator +from pydantic import ConfigDict, Field, model_validator from pydatalab.models.relationships import TypedRelationship from pydatalab.models.utils import ( - JSON_ENCODERS, + BaseModel, EntryReference, IsoformatDateTime, PyObjectId, @@ -20,21 +20,22 @@ class Entry(BaseModel, abc.ABC): type: str """The resource type of the entry.""" - immutable_id: PyObjectId = Field( + immutable_id: PyObjectId | None = Field( None, title="Immutable ID", alias="_id", - format="uuid", + json_schema_extra={"format": "uuid"}, ) """The immutable database ID of the entry.""" last_modified: IsoformatDateTime | None = None """The timestamp at which the entry was last modified.""" - relationships: list[TypedRelationship] | None = None + relationships: list[TypedRelationship] = Field(default_factory=list) """A list of related entries and their types.""" - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def check_id_names(cls, values): """Slightly upsetting hack: this case *should* be covered by the pydantic setting for populating fields by alias names. @@ -42,6 +43,10 @@ def check_id_names(cls, values): if "_id" in values: values["immutable_id"] = values.pop("_id") + # Coerce a stored/explicit null into an empty list so the field is always a list. + if values.get("relationships") is None: + values["relationships"] = [] + return values def to_reference(self, additional_fields: list[str] | None = None) -> "EntryReference": @@ -63,7 +68,4 @@ def to_reference(self, additional_fields: list[str] | None = None) -> "EntryRefe return EntryReference(**data) - class Config: - allow_population_by_field_name = True - json_encoders = JSON_ENCODERS - extra = "ignore" + model_config = ConfigDict(populate_by_name=True, extra="ignore") diff --git a/pydatalab/src/pydatalab/models/equipment.py b/pydatalab/src/pydatalab/models/equipment.py index 1d60a1f5c..820961905 100644 --- a/pydatalab/src/pydatalab/models/equipment.py +++ b/pydatalab/src/pydatalab/models/equipment.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import Field from pydatalab.models.items import Item @@ -9,15 +11,15 @@ class Equipment(Item): """A model for representing an experimental sample.""" - type: str = Field("equipment", const="equipment", pattern="^equipment$") + type: Literal["equipment"] = "equipment" - serial_numbers: str | None + serial_numbers: str | None = None """A string describing one or more serial numbers for the instrument.""" - manufacturer: str | None + manufacturer: str | None = None """The manufacturer of this piece of equipment""" - location: str | None + location: str | None = None """Place where the equipment is located""" contact: str | None diff --git a/pydatalab/src/pydatalab/models/export_task.py b/pydatalab/src/pydatalab/models/export_task.py index 171567038..d2085ffde 100644 --- a/pydatalab/src/pydatalab/models/export_task.py +++ b/pydatalab/src/pydatalab/models/export_task.py @@ -1,9 +1,9 @@ from datetime import datetime, timezone from enum import Enum -from pydantic import BaseModel, Field +from pydantic import Field -from pydatalab.models.utils import PyObjectId +from pydatalab.models.utils import BaseModel, PyObjectId class ExportStatus(str, Enum): @@ -18,23 +18,35 @@ class ExportStatus(str, Enum): class ExportTask(BaseModel): """Model for an export task.""" - task_id: str = Field(..., description="Unique identifier for the export task") - collection_id: str | None = Field(None, description="ID of the collection being exported") - item_id: str | None = Field(None, description="ID of the item being exported") - export_type: str = Field( - default="collection", description="Type of export: 'collection' or 'item' or 'graph'" - ) - status: ExportStatus = Field( - default=ExportStatus.PENDING, description="Current status of the task" - ) - creator_id: PyObjectId = Field(..., description="ID of the user who created the export") - created_at: datetime = Field( - default_factory=lambda: datetime.now(tz=timezone.utc), - description="When the task was created", - ) - completed_at: datetime | None = Field(None, description="When the task was completed") - file_path: str | None = Field(None, description="Path to the generated .eln file") - error_message: str | None = Field(None, description="Error message if status is ERROR") + task_id: str + """Unique identifier for the export task""" + + collection_id: str | None = None + """ID of the collection being exported""" + + item_id: str | None = None + """ID of the item being exported""" + + export_type: str = "collection" + """Type of export: 'collection' or 'item' or 'graph'""" + + status: ExportStatus = ExportStatus.PENDING + """Current status of the task""" + + creator_id: PyObjectId + """ID of the user who created the export""" + + created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc)) + """When the task was created""" + + completed_at: datetime | None = None + """When the task was completed""" + + file_path: str | None = None + """Path to the generated .eln file""" + + error_message: str | None = None + """Error message if status is ERROR""" class Config: use_enum_values = True diff --git a/pydatalab/src/pydatalab/models/files.py b/pydatalab/src/pydatalab/models/files.py index 1cdc744af..ca17eadd5 100644 --- a/pydatalab/src/pydatalab/models/files.py +++ b/pydatalab/src/pydatalab/models/files.py @@ -1,10 +1,10 @@ -from typing import Any +from typing import Any, Literal -from pydantic import BaseModel, Field +from pydantic import Field from pydatalab.models.entries import Entry from pydatalab.models.traits import HasOwner, HasRevisionControl -from pydatalab.models.utils import IsoformatDateTime +from pydatalab.models.utils import BaseModel, IsoformatDateTime class FileChecksums(BaseModel): @@ -20,18 +20,18 @@ class FileChecksums(BaseModel): class File(Entry, HasOwner, HasRevisionControl): """A model for representing a file that has been tracked or uploaded to datalab.""" - type: str = Field("files", const="files", pattern="^files$") + type: Literal["files"] = "files" - size: int | None + size: int | None = None """The size of the file on disk in bytes.""" - last_modified_remote: IsoformatDateTime | None + last_modified_remote: IsoformatDateTime | None = None """The last date/time at which the remote file was modified.""" - item_ids: list[str] + item_ids: list[str] = Field(default_factory=list) """A list of item IDs associated with this file.""" - blocks: list[str] + blocks: list[str] = Field(default_factory=list) """A list of block IDs associated with this file.""" name: str @@ -40,30 +40,30 @@ class File(Entry, HasOwner, HasRevisionControl): extension: str """The file extension that the file was uploaded with.""" - original_name: str | None + original_name: str | None = None """The raw filename as uploaded.""" - location: str | None + location: str | None = None """The location of the file on disk.""" - url_path: str | None + url_path: str | None = None """The path to a remote file.""" - source: str | None + source: str | None = None """The source of the file, e.g. 'remote' or 'uploaded'.""" time_added: IsoformatDateTime """The timestamp for the original file upload.""" - metadata: dict[Any, Any] | None + metadata: dict[Any, Any] | None = None """Any additional metadata.""" - representation: Any | None + representation: Any | None = None - source_server_name: str | None + source_server_name: str | None = None """The server name at which the file is stored.""" - source_path: str | None + source_path: str | None = None """The path to the file on the remote resource.""" is_live: bool diff --git a/pydatalab/src/pydatalab/models/items.py b/pydatalab/src/pydatalab/models/items.py index f3eb1799b..d1295d180 100644 --- a/pydatalab/src/pydatalab/models/items.py +++ b/pydatalab/src/pydatalab/models/items.py @@ -1,6 +1,6 @@ import abc -from pydantic import Field, validator +from pydantic import Field, field_validator from pydatalab.models.entries import Entry from pydatalab.models.files import File @@ -21,33 +21,32 @@ class Item(Entry, HasOwner, HasRevisionControl, IsCollectable, HasBlocks, abc.ABC): """The generic model for data types that will be exposed with their own named endpoints.""" - refcode: Refcode = None # type: ignore - """A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) - and a locally unique string, ideally created with some consistent scheme. - """ + refcode: Refcode | None = None + """A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) and a locally unique string, ideally created with some consistent scheme.""" item_id: HumanReadableIdentifier """A locally unique, human-readable identifier for the entry. This ID is mutable.""" - description: str | None + description: str | None = None """A description of the item, either in plain-text or a markup language.""" - date: IsoformatDateTime | None + date: IsoformatDateTime | None = None """A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).""" - name: str | None + name: str | None = None """An optional human-readable/usable name for the entry.""" - files: list[File] | None + files: list[File] | None = None """Any files attached to this sample.""" - file_ObjectIds: list[PyObjectId] = Field([]) + file_ObjectIds: list[PyObjectId] = Field(default_factory=list) """Links to object IDs of files stored within the database.""" - status: str | None + status: str | None = None """The status of the item, with allowed values defined by the specific item class.""" - @validator("refcode", pre=True, always=True) + @field_validator("refcode", mode="before") + @classmethod def refcode_validator(cls, v): """Generate a refcode if not provided.""" diff --git a/pydatalab/src/pydatalab/models/people.py b/pydatalab/src/pydatalab/models/people.py index a86a2d4a2..ec3485ade 100644 --- a/pydatalab/src/pydatalab/models/people.py +++ b/pydatalab/src/pydatalab/models/people.py @@ -1,14 +1,17 @@ from __future__ import annotations from enum import Enum +from typing import Annotated, Literal -import bson -import bson.errors -from pydantic import BaseModel, ConstrainedStr, Field, parse_obj_as, validator from pydantic import EmailStr as PydanticEmailStr +from pydantic import ( + Field, + StringConstraints, + field_validator, +) from pydatalab.models.entries import Entry -from pydatalab.models.utils import HumanReadableIdentifier, PyObjectId, UserRole +from pydatalab.models.utils import BaseModel, HumanReadableIdentifier, PyObjectId, UserRole class IdentityType(str, Enum): @@ -36,29 +39,32 @@ class Identity(BaseModel): name: str """The name associated with the identity to be exposed in free-text searches over people, e.g., an institutional username, a GitHub username.""" - verified: bool = Field(False) + verified: bool = False """Whether the identity has been verified (by some means, e.g., OAuth2 or email)""" - display_name: str | None + display_name: str | None = None """The user's display name associated with the identity, also to be exposed in free text searches.""" - @validator("name", pre=True, always=True) - def add_missing_name(cls, v, values): + @field_validator("name", mode="before") + @classmethod + def add_missing_name(cls, v, info): """If the identity is created without a free-text 'name', then for certain providers, populate this field so that it can appear in the free text index, e.g., an ORCID, or an institutional username from an email address. """ - if v is None: - if values["identity_type"] == IdentityType.ORCID: - return values["identifier"] - if values["identity_type"] == IdentityType.EMAIL: - return values["identifier"].split("@")[0] - + if v is None and hasattr(info, "data") and info.data: + data = info.data + if data.get("identity_type") == IdentityType.ORCID: + return data.get("identifier") + if data.get("identity_type") == IdentityType.EMAIL: + identifier = data.get("identifier", "") + return identifier.split("@")[0] if "@" in identifier else identifier return v - @validator("verified", pre=True, always=True) + @field_validator("verified", mode="before") + @classmethod def add_missing_verification(cls, v): """Fills in missing value for `verified` if not given.""" if not v: @@ -66,18 +72,11 @@ def add_missing_verification(cls, v): return v -class DisplayName(ConstrainedStr): - """A constrained string less than 150 characters long but with - non-empty content, intended to be entered by the user. - - """ - - max_length = 150 - min_length = 1 - strip_whitespace = True - - def __new__(cls, value): - return parse_obj_as(cls, value) +DisplayName = Annotated[ + str, + StringConstraints(min_length=1, max_length=150, strip_whitespace=True), +] +"""A constrained string less than 150 characters long but with non-empty content, intended to be entered by the user.""" class EmailStr(PydanticEmailStr): @@ -112,38 +111,40 @@ class Group(Entry): """ - type: str = Field("groups", const=True) + type: Literal["groups"] = "groups" """The entry type as a string.""" - group_id: HumanReadableIdentifier | None = Field(None) + group_id: HumanReadableIdentifier | None = None """A short, locally-unique ID for the group.""" - members: list[dict] = Field(None) - """A list of people that belong to this group.""" + members: list[dict] = Field(default_factory=list) + """A list of people that belong to this group; stored on the user objects.""" - display_name: DisplayName | None = Field(None) + display_name: DisplayName | None = None """The chosen display name for the group""" - description: str | None = Field(None) + description: str | None = None """A description of the group""" managers: list[PyObjectId | dict] = Field(default_factory=list) - """A list of user IDs that can manage this group.""" + """A list of user IDs that can manage this group; stored in db as list of IDs.""" - @validator("members", pre=True, always=True) + @field_validator("members", mode="before") + @classmethod def cast_members_to_people(cls, v): """Casts members to list of people if not None.""" if v is not None: - return [Person(**member).dict(exclude_unset=True) for member in v] + return [Person(**member).model_dump(exclude_unset=True) for member in v] return v - @validator("managers", pre=True, always=True) + @field_validator("managers", mode="before") + @classmethod def cast_managers_to_people(cls, v): """Casts managers to list of people if not None.""" if v and isinstance(v[0], dict): return [ - Person(**member).dict(exclude_unset=True) + Person(**member).model_dump(exclude_unset=True) for member in v if isinstance(member, dict) ] @@ -154,7 +155,7 @@ def cast_managers_to_people(cls, v): class Person(Entry): """A model that describes an individual and their digital identities.""" - type: str = Field("people", const=True) + type: Literal["people"] = "people" """The entry type as a string.""" identities: list[Identity] = Field(default_factory=list) @@ -181,14 +182,15 @@ class Person(Entry): account_status: AccountStatus = Field(AccountStatus.UNVERIFIED) """The status of the user's account.""" - @validator("type", pre=True, always=True) + @field_validator("type", mode="before") + @classmethod def add_missing_type(cls, v): """Fill in missing `type` field if not provided.""" if v is None: v = "people" return v - @validator("type", pre=True) + @classmethod def set_default_type(cls, _): return "people" @@ -214,8 +216,6 @@ def new_user_from_identity( A `Person` object with only the provided identity. """ - user_id = bson.ObjectId() - display_name = None if use_display_name: display_name = identity.display_name @@ -227,7 +227,6 @@ def new_user_from_identity( from pydatalab.mongo import gravatar_hash_for return Person( - immutable_id=user_id, identities=[identity], display_name=display_name, contact_email=contact_email, diff --git a/pydatalab/src/pydatalab/models/relationships.py b/pydatalab/src/pydatalab/models/relationships.py index e844c2498..e4c53ec33 100644 --- a/pydatalab/src/pydatalab/models/relationships.py +++ b/pydatalab/src/pydatalab/models/relationships.py @@ -1,8 +1,13 @@ from enum import Enum -from pydantic import BaseModel, root_validator, validator +from pydantic import ( + ConfigDict, + field_validator, + model_validator, +) from pydatalab.models.utils import ( + BaseModel, HumanReadableIdentifier, KnownType, PyObjectId, @@ -29,46 +34,52 @@ class entryC SIBLING = "sibling" PARTHOOD = "is_part_of" OTHER = "other" + COLLECTIONS = "collections" class TypedRelationship(BaseModel): - description: str | None + description: str | None = None """A description of the relationship.""" - relation: RelationshipType | None + relation: RelationshipType | None = None """The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided.""" type: KnownType """The type of the related resource.""" - immutable_id: PyObjectId | None + immutable_id: PyObjectId | None = None """The immutable ID of the entry that is related to this entry.""" - item_id: HumanReadableIdentifier | None + item_id: HumanReadableIdentifier | None = None """The ID of the entry that is related to this entry.""" - refcode: Refcode | None + refcode: Refcode | None = None """The refcode of the entry that is related to this entry.""" - @validator("relation") - def check_for_description(cls, v, values): - if v == RelationshipType.OTHER and values.get("description") is None: - raise ValueError( - f"A description must be provided if the relationship type is {RelationshipType.OTHER.value!r}." - ) - + @field_validator("relation") + @classmethod + def check_for_description(cls, v, info): + if v == RelationshipType.OTHER: + data = info.data if hasattr(info, "data") and info.data else {} + if data.get("description") is None: + raise ValueError( + f"A description must be provided if the relationship type is {RelationshipType.OTHER.value!r}." + ) return v - @root_validator + @model_validator(mode="before") + @classmethod def check_id_fields(cls, values): """Check that at least one of the possible identifier fields is provided.""" id_fields = ("immutable_id", "item_id", "refcode") - if all(values[f] is None for f in id_fields): + if all(values.get(f) is None for f in id_fields): raise ValueError(f"Must provide at least one of {id_fields!r}") if values.get("refcode") and values.get("item_id"): pass elif values.get("immutable_id") and (values.get("refcode") or values.get("item_id")): - raise ValueError("Must provide only one of {id_fields!r}") + raise ValueError(f"Must provide only one of {id_fields!r}") return values + + model_config = ConfigDict(extra="allow") diff --git a/pydatalab/src/pydatalab/models/samples.py b/pydatalab/src/pydatalab/models/samples.py index 25a2eda93..1664e7874 100644 --- a/pydatalab/src/pydatalab/models/samples.py +++ b/pydatalab/src/pydatalab/models/samples.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import Field from pydatalab.models.items import Item @@ -8,7 +10,7 @@ class Sample(Item, HasSynthesisInfo, HasSubstanceInfo): """A model for representing an experimental sample.""" - type: str = Field("samples", const="samples", pattern="^samples$") + type: Literal["samples"] = "samples" status: SampleStatus = Field(default=SampleStatus.ACTIVE) """The status of the sample, indicating its current state.""" diff --git a/pydatalab/src/pydatalab/models/starting_materials.py b/pydatalab/src/pydatalab/models/starting_materials.py index c44ec4408..845bd6e5a 100644 --- a/pydatalab/src/pydatalab/models/starting_materials.py +++ b/pydatalab/src/pydatalab/models/starting_materials.py @@ -1,3 +1,5 @@ +from typing import Literal + from pydantic import Field from pydatalab.models.items import Item @@ -9,46 +11,41 @@ class StartingMaterial(Item, HasSynthesisInfo, HasSubstanceInfo): """A model for representing an experimental sample, based on the connection with cheminventory.net, which mixes container-level and substance-level information. - """ - type: str = Field( - "starting_materials", const="starting_materials", pattern="^starting_materials$" - ) + type: Literal["starting_materials"] = "starting_materials" - barcode: str | None = Field( - alias="Barcode", - ) + barcode: str | None = Field(None, alias="Barcode") """A unique barcode provided by an external source, e.g., cheminventory.""" - date: IsoformatDateTime | None = Field(alias="Date Acquired") + date: IsoformatDateTime | None = Field(None, alias="Date Acquired") """The date the item was acquired""" - date_opened: IsoformatDateTime | None = Field(alias="Date opened") + date_opened: IsoformatDateTime | None = Field(None, alias="Date opened") """The date the item was opened""" chemical_purity: str | None = Field(alias="Chemical purity") """The chemical purity of this container with regards to the defined substance.""" - full_percent: str | None = Field(alias="Full %") + full_percent: str | None = Field(None, alias="Full %") """The amount of the defined substance remaining in the container, expressed as a percentage.""" name: str | None = Field(alias="Container Name") """The name of the substance in the container.""" - size: str | None = Field(alias="Container Size") + size: str | None = Field(None, alias="Container Size") """The total size of the container, in units of `size_unit`.""" - size_unit: str | None = Field(alias="Unit") + size_unit: str | None = Field(None, alias="Unit") """Units for the 'size' field.""" supplier: str | None = Field(alias="Supplier") """Supplier or manufacturer of the chemical.""" - location: str | None = Field(alias="Location") + location: str | None = Field(None, alias="Location") """The place where the container is located.""" - comment: str | None = Field(alias="Comments") + comment: str | None = Field(None, alias="Comments") """Any additional comments or notes about the container.""" status: StartingMaterialsStatus = Field(default=StartingMaterialsStatus.AVAILABLE) diff --git a/pydatalab/src/pydatalab/models/tasks.py b/pydatalab/src/pydatalab/models/tasks.py index fa64714c3..14728df26 100644 --- a/pydatalab/src/pydatalab/models/tasks.py +++ b/pydatalab/src/pydatalab/models/tasks.py @@ -2,9 +2,9 @@ from enum import Enum from typing import Literal -from pydantic import BaseModel, Field, validator +from pydantic import Field, validator -from pydatalab.models.utils import PyObjectId +from pydatalab.models.utils import BaseModel, PyObjectId class TaskStatus(str, Enum): @@ -20,12 +20,17 @@ class TaskType(str, Enum): class TaskStage(BaseModel): - timestamp: datetime = Field(..., description="When this stage occurred") - message: str = Field(..., description="Description of this processing stage") - level: Literal["info", "warning", "error"] = Field( - default="info", description="Severity level of this stage" - ) - detail: str | None = Field(None, description="Optional detailed information about this stage") + timestamp: datetime + """When this stage occurred""" + + message: str + """Description of this processing stage""" + + level: Literal["info", "warning", "error"] = "info" + """Severity level of this stage""" + + detail: str | None = None + """Optional detailed information about this stage""" class TaskSpec(BaseModel): @@ -33,35 +38,57 @@ class TaskSpec(BaseModel): class ExportTaskSpec(TaskSpec): - collection_id: str | None = Field(None, description="Collection ID being exported") - item_id: str | None = Field(None, description="Item ID being exported") - export_type: str = Field(..., description="Type of export: collection/item/graph") - file_path: str | None = Field(None, description="Path to generated .eln file") - stages: list[TaskStage] = Field( - default_factory=list, description="Timestamped processing stages" - ) + collection_id: str | None = None + """Collection ID being exported""" + + item_id: str | None = None + """Item ID being exported""" + + export_type: str + """Type of export: collection/item/graph""" + + file_path: str | None = None + """Path to generated .eln file""" + + stages: list[TaskStage] = Field(default_factory=list) + """Timestamped processing stages""" class BlockProcessingTaskSpec(TaskSpec): - item_id: str = Field(..., description="Item ID containing the block") - block_id: str = Field(..., description="Block ID being processed") - stages: list[TaskStage] = Field( - default_factory=list, description="Timestamped processing stages" - ) + item_id: str + """Item ID containing the block""" + + block_id: str + """Block ID being processed""" + + stages: list[TaskStage] = Field(default_factory=list) + """Timestamped processing stages""" class Task(BaseModel): - task_id: str = Field(..., description="Unique identifier for the task") - type: TaskType = Field(..., description="Type of task") - status: TaskStatus = Field(default=TaskStatus.PENDING, description="Current status") - creator_id: PyObjectId = Field(..., description="ID of the user who created the task") - created_at: datetime = Field( - default_factory=lambda: datetime.now(tz=timezone.utc), - description="When the task was created", - ) - completed_at: datetime | None = Field(None, description="When completed") - error_message: str | None = Field(None, description="Error message if status is ERROR") - spec: ExportTaskSpec | BlockProcessingTaskSpec = Field(..., description="Task-specific data") + task_id: str + """Unique identifier for the task""" + + type: TaskType + """Type of task""" + + status: TaskStatus = TaskStatus.PENDING + """Current status""" + + creator_id: PyObjectId + """ID of the user who created the task""" + + created_at: datetime = Field(default_factory=lambda: datetime.now(tz=timezone.utc)) + """When the task was created""" + + completed_at: datetime | None = None + """When completed""" + + error_message: str | None = None + """Error message if status is ERROR""" + + spec: ExportTaskSpec | BlockProcessingTaskSpec + """Task-specific data""" @validator("spec", pre=True, always=True) def validate_spec_type(cls, v, values): diff --git a/pydatalab/src/pydatalab/models/traits.py b/pydatalab/src/pydatalab/models/traits.py index 435b39d34..8cc0f9deb 100644 --- a/pydatalab/src/pydatalab/models/traits.py +++ b/pydatalab/src/pydatalab/models/traits.py @@ -1,10 +1,10 @@ from typing import Any -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import AliasChoices, ConfigDict, Field, field_validator, model_validator from pydatalab.models.blocks import DataBlockResponse from pydatalab.models.people import Group, Person -from pydatalab.models.utils import Constituent, InlineSubstance, PyObjectId +from pydatalab.models.utils import BaseModel, Constituent, InlineSubstance, PyObjectId __all__ = ( "HasOwner", @@ -49,34 +49,36 @@ class HasBlocks(BaseModel): """The order in which to display block data in the UI.""" +class CollectionReference(BaseModel): + """A reference to a collection, used for inlining collection info within other models.""" + + model_config = ConfigDict(extra="allow") + + collection_id: str + """The immutable ID of the collection.""" + + class IsCollectable(BaseModel): """Trait mixin for models that can be added to collections. """ - from pydatalab.models.collections import Collection - - collections: list[Collection] = Field([]) + collections: list[CollectionReference] = Field(default_factory=list) """Inlined info for the collections associated with this item.""" - @model_validator(mode="before") - @classmethod - def add_missing_collection_relationships(cls, values): + @model_validator(mode="after") + def add_missing_collection_relationships(self): from pydatalab.models.relationships import TypedRelationship - if values.get("collections") is not None: - new_ids = {coll.immutable_id for coll in values["collections"]} - existing_collection_relationship_ids = set() - if values.get("relationships") is not None: - existing_collection_relationship_ids = { - relationship.immutable_id - for relationship in values["relationships"] - if relationship.type == "collections" - } - else: - values["relationships"] = [] - - for collection in values.get("collections", []): + if self.collections is not None: + new_ids = {coll.immutable_id for coll in self.collections} + existing_collection_relationship_ids = { + relationship.immutable_id + for relationship in self.relationships + if relationship.type == "collections" + } + + for collection in self.collections: if collection.immutable_id not in existing_collection_relationship_ids: relationship = TypedRelationship( relation=None, @@ -84,20 +86,18 @@ def add_missing_collection_relationships(cls, values): type="collections", description="Is a member of", ) - values["relationships"].append(relationship) + self.relationships.append(relationship) - values["relationships"] = [ + self.relationships = [ d - for d in values.get("relationships", []) + for d in self.relationships if d.type != "collections" or d.immutable_id in new_ids ] - if len([d for d in values.get("relationships", []) if d.type == "collections"]) != len( - values.get("collections", []) - ): + if len([d for d in self.relationships if d.type == "collections"]) != len(self.collections): raise RuntimeError("Relationships and collections mismatch") - return values + return self class HasSynthesisInfo(BaseModel): @@ -109,29 +109,24 @@ class HasSynthesisInfo(BaseModel): synthesis_description: str | None = None """Free-text details of the procedure applied to synthesise the sample""" - @model_validator(mode="before") - @classmethod - def add_missing_synthesis_relationships(cls, values): + @model_validator(mode="after") + def add_missing_synthesis_relationships(self): """Add any missing sample synthesis constituents to parent relationships""" from pydatalab.models.relationships import RelationshipType, TypedRelationship constituents_set = set() - if values.get("synthesis_constituents") is not None: - existing_parent_relationships = {} - if values.get("relationships") is not None: - # Index by refcode *and* item_id so a stored relationship carrying an - # item_id still matches a refcode-enriched constituent (and vice-versa). - existing_parent_relationships = { - identifier: relationship - for relationship in values["relationships"] - if relationship.relation == RelationshipType.PARENT - for identifier in (relationship.refcode, relationship.item_id) - if identifier - } - else: - values["relationships"] = [] - - for constituent in values.get("synthesis_constituents", []): + if self.synthesis_constituents is not None: + # Index by refcode *and* item_id so a stored relationship carrying an + # item_id still matches a refcode-enriched constituent (and vice-versa). + existing_parent_relationships = { + identifier: relationship + for relationship in self.relationships + if relationship.relation == RelationshipType.PARENT + for identifier in (relationship.refcode, relationship.item_id) + if identifier + } + + for constituent in self.synthesis_constituents: # If this is an inline relationship, just skip it if isinstance(constituent.item, InlineSubstance): continue @@ -150,7 +145,7 @@ def add_missing_synthesis_relationships(cls, values): type=constituent.item.type, description="Is a constituent of", ) - values["relationships"].append(relationship) + self.relationships.append(relationship) else: # Back-fill any identifier missing from the stored relationship relationship.refcode = relationship.refcode or refcode @@ -168,9 +163,9 @@ def add_missing_synthesis_relationships(cls, values): # Finally, filter out any parent relationships with item that were removed # from the synthesis constituents - values["relationships"] = [ + self.relationships = [ rel - for rel in values["relationships"] + for rel in self.relationships if not ( rel.refcode not in constituents_set and rel.item_id not in constituents_set @@ -179,14 +174,15 @@ def add_missing_synthesis_relationships(cls, values): ) ] - return values + return self class HasSubstanceInfo(BaseModel): """Trait mixin for models that have substance information.""" chemform: str | None = Field( - example=["Na3P", "Na3P", "LiNiO2@C", "Na3+xP", "LiNi1/3Co0.1Mn0.1O2"], + None, + examples=["Na3P", "Na3P", "LiNiO2@C", "Na3+xP", "LiNi1/3Co0.1Mn0.1O2"], ) """A string representation of the chemical formula or composition associated with this sample. @@ -195,7 +191,9 @@ class HasSubstanceInfo(BaseModel): """ - smiles: str | None = Field(None, aliases=["SMILES", "smiles_representation"]) + smiles: str | None = Field( + None, validation_alias=AliasChoices("smiles_representation", "SMILES") + ) """A SMILES string representation of the chemical structure associated with this sample.""" inchi: str | None = Field(None) @@ -205,25 +203,28 @@ class HasSubstanceInfo(BaseModel): """A unique key derived from the InChI.""" GHS_codes: str | None = Field( + None, alias="GHS H-codes", examples=["H224", "H303, H316, H319"], ) """A string describing any GHS hazard codes associated with this item. See https://pubchem.ncbi.nlm.nih.gov/ghs/ for code definitions.""" - molar_mass: float | None = Field(alias="Molecular Weight") + molar_mass: float | None = Field(None, alias="Molecular Weight", validate_default=True) """Mass per formula unit, in g/mol.""" - CAS: str | None = Field(alias="Substance CAS") + CAS: str | None = Field(None, alias="Substance CAS") """The CAS Registry Number for the substance described by this entry.""" @field_validator("molar_mass", mode="before") @classmethod - def add_molar_mass(cls, v, values): + def add_molar_mass(cls, v, info): + """Fill in the molar mass if not already set and a chemical formula is provided.""" from periodictable import formula - if v is None and values.get("chemform"): + chemform = info.data.get("chemform") + if v is None and chemform: try: - return formula(values.get("chemform")).mass + return formula(chemform).mass except Exception: return None diff --git a/pydatalab/src/pydatalab/models/utils.py b/pydatalab/src/pydatalab/models/utils.py index 58230b184..96df03119 100644 --- a/pydatalab/src/pydatalab/models/utils.py +++ b/pydatalab/src/pydatalab/models/utils.py @@ -4,18 +4,30 @@ from collections.abc import Callable from enum import Enum from functools import partial -from typing import TypeAlias +from typing import Annotated, Any, TypeAlias import pint -from bson.objectid import ObjectId +from bson import ObjectId from pydantic import ( - BaseModel, - ConstrainedStr, + BaseModel as PydanticBaseModel, +) +from pydantic import ( + ConfigDict, Field, - parse_obj_as, - root_validator, - validator, + StringConstraints, + field_validator, + model_validator, ) +from pydantic_core import core_schema + + +class BaseModel(PydanticBaseModel): + """The base model for all pydatalab models, configured to pull field + descriptions from attribute docstrings into the generated JSON schema. + + """ + + model_config = ConfigDict(use_attribute_docstrings=True) class ItemType(str, Enum): @@ -92,43 +104,30 @@ class KnownType(str, Enum): """ -class HumanReadableIdentifier(ConstrainedStr): - """Used to constrain human-readable and URL-safe identifiers for items.""" - - min_length = 1 - max_length = 40 - strip_whitespace = True - to_lower = False - strict = False - regex = IDENTIFIER_REGEX - - def __init__(self, value): - self.value = parse_obj_as(type(self), value) - - def __str__(self): - return self.value - - def __repr__(self): - return self.value - - def __bool__(self): - return bool(self.value) - - -class Refcode(HumanReadableIdentifier): - regex = r"^[a-z]{2,10}:" + IDENTIFIER_REGEX[1:] - """A regex to match refcodes that have a lower-case prefix between 2-10 chars, followed by a colon, - and then the normal rules for an ID (url-safe etc.). - - """ - - @property - def prefix(self): - return self.value.split(":")[0] - - @property - def identifier(self): - return self.value.split(":")[1] +HumanReadableIdentifier = Annotated[ + str, + StringConstraints( + min_length=1, + max_length=40, + strip_whitespace=True, + to_lower=False, + strict=False, + pattern=IDENTIFIER_REGEX, + ), +] +"""Used to constrain human-readable and URL-safe identifiers for items.""" + + +Refcode = Annotated[ + str, + StringConstraints( + min_length=1, + max_length=40, + strip_whitespace=True, + pattern=r"^[a-z]{2,10}:" + IDENTIFIER_REGEX[1:], + ), +] +"""A regex to match refcodes that have a lower-case prefix between 2-10 chars, followed by a colon, and then the normal rules for an ID (url-safe etc.).""" class UserRole(str, Enum): @@ -149,8 +148,12 @@ def __init__(self, dimensions: str): self._dimensions = dimensions @classmethod - def __get_validators__(self): - yield self.validate + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls.validate, + core_schema.str_schema(), + serialization=core_schema.plain_serializer_function_ser_schema(str, when_used="json"), + ) @classmethod def validate(self, v): @@ -159,77 +162,120 @@ def validate(self, v): raise ValueError("Value {v} must have dimensions of mass, not {v.dimensions}") return q - @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="string") - Mass: TypeAlias = PintType("[mass]") # type: ignore # noqa Volume: TypeAlias = PintType("[volume]") # type: ignore # noqa class PyObjectId(ObjectId): - """A wrapper class for a BSON ObjectId that can be used as a Pydantic field type. - - Modified from "Getting started iwth MongoDB and FastAPI": - https://www.mongodb.com/developer/languages/python/python-quickstart-fastapi/. - - """ + @classmethod + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls.validate, + core_schema.union_schema( + [ + core_schema.str_schema(), + core_schema.is_instance_schema(ObjectId), + core_schema.is_instance_schema(cls), + core_schema.dict_schema(), + core_schema.none_schema(), + ] + ), + serialization=core_schema.plain_serializer_function_ser_schema( + lambda x: str(x) if x else None, when_used="json" + ), + ) @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_json_schema__(cls, core_schema, handler): + return { + "type": "string", + "format": "objectid", + } @classmethod def validate(cls, v): - if isinstance(v, dict) and "$oid" in v: - v = v["$oid"] + if v is None: + return None + if isinstance(v, cls): + return v + if isinstance(v, ObjectId): + return cls(v) - if not ObjectId.is_valid(v): - raise ValueError("Invalid ObjectId") + if isinstance(v, dict): + if "$oid" in v: + return cls(ObjectId(v["$oid"])) + elif "_id" in v and isinstance(v["_id"], (str, ObjectId)): + return cls(ObjectId(v["_id"])) + elif len(v) == 1: + first_val = next(iter(v.values())) + if isinstance(first_val, str) and ObjectId.is_valid(first_val): + return cls(ObjectId(first_val)) + raise ValueError(f"Cannot convert dict to ObjectId: {v}") - return ObjectId(v) + if isinstance(v, str): + if not ObjectId.is_valid(v): + raise ValueError("Invalid ObjectId string") + return cls(ObjectId(v)) - @classmethod - def __modify_schema__(cls, field_schema): - field_schema.update(type="string") + raise ValueError(f"Cannot convert {type(v)} to ObjectId: {v}") class IsoformatDateTime(datetime.datetime): """A datetime container that is more flexible than the pydantic default.""" @classmethod - def __get_validators__(cls): - yield cls.validate + def __get_pydantic_core_schema__(cls, source_type: Any, handler: Any) -> core_schema.CoreSchema: + return core_schema.no_info_after_validator_function( + cls.validate, + core_schema.union_schema( + [ + core_schema.str_schema(), + core_schema.is_instance_schema(datetime.datetime), + ] + ), + serialization=core_schema.plain_serializer_function_ser_schema( + lambda x: x.isoformat() if x else None, when_used="json" + ), + ) + + @classmethod + def __get_pydantic_json_schema__(cls, core_schema, handler): + return { + "type": "string", + "format": "datetime", + } @classmethod def validate(cls, v) -> datetime.datetime | None: """Cast isoformat strings to datetimes and enforce UTC if tzinfo is missing.""" + if v is None: + return None + + if isinstance(v, datetime.datetime): + if v.tzinfo is None: + v = v.replace(tzinfo=datetime.timezone.utc) + return v + if isinstance(v, str): - if v in ["0", " "]: + if v in ["0", " ", ""]: return None v = datetime.datetime.fromisoformat(v) + if v.tzinfo is None: + v = v.replace(tzinfo=datetime.timezone.utc) + return v - if v.tzinfo is None: - v = v.replace(tzinfo=datetime.timezone.utc) - - return v - - -JSON_ENCODERS = { - pint.Quantity: str, - ObjectId: str, -} + raise ValueError(f"Invalid datetime value: {v}") class RefCodeFactory: refcode_generator: Callable @classmethod - def generate(self): + def generate(cls): from pydatalab.config import CONFIG - return f"{CONFIG.IDENTIFIER_PREFIX}:{self.refcode_generator()}" + return f"{CONFIG.IDENTIFIER_PREFIX}:{cls.refcode_generator()}" def random_uppercase(length: int = 6): @@ -257,7 +303,7 @@ def generate_unique_refcode(): class InlineSubstance(BaseModel): name: str - chemform: str | None + chemform: str | None = None class EntryReference(BaseModel): @@ -269,15 +315,19 @@ class EntryReference(BaseModel): """ type: str - name: str | None - immutable_id: PyObjectId | None - item_id: HumanReadableIdentifier | None - refcode: Refcode | None - chemform: str | None + name: str | None = None + immutable_id: PyObjectId | None = None + item_id: HumanReadableIdentifier | None = None + refcode: Refcode | None = None + chemform: str | None = None - @root_validator + @model_validator(mode="before") + @classmethod def check_id_fields(cls, values): """Check that at least one of the possible identifier fields is provided.""" + if not isinstance(values, dict): + return values + id_fields = ("immutable_id", "item_id", "refcode") if all(values.get(f) is None for f in id_fields): @@ -285,8 +335,7 @@ def check_id_fields(cls, values): return values - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class Constituent(BaseModel): @@ -295,23 +344,22 @@ class Constituent(BaseModel): item: EntryReference | InlineSubstance """A reference to item (sample or starting material) entry for the constituent substance.""" - quantity: float | None = Field(..., ge=0) + quantity: float | None = Field(default=None, ge=0) """The amount of the constituent material used to create the sample.""" - unit: str = Field("g") - """The unit symbol for the value provided in `quantity`, default is mass - in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol). - """ + unit: str = "g" + """The unit symbol for the value provided in `quantity`, default is mass in grams (g) but could also refer to volumes (mL, L, etc.) or moles (mol).""" - @validator("item") + @field_validator("item") + @classmethod def check_itemhood(cls, v): """Check that the reference within the constituent is to an item type.""" - if "type" in (v.value for v in ItemType): - raise ValueError(f"`type` must be one of {ItemType!r}") - + if hasattr(v, "type") and v.type not in [item_type.value for item_type in ItemType]: + raise ValueError(f"`type` must be one of {[t.value for t in ItemType]!r}") return v - @validator("item", pre=True, always=True) + @field_validator("item", mode="before") + @classmethod def coerce_reference(cls, v): if isinstance(v, dict): refcode = v.pop("refcode", None) @@ -327,4 +375,18 @@ def coerce_reference(cls, v): if not name: raise ValueError("Inline substance must have a name!") return InlineSubstance(name=name, chemform=chemform) + elif hasattr(v, "model_dump"): + item_id = getattr(v, "item_id", None) + refcode = getattr(v, "refcode", None) + item_type = getattr(v, "type", None) + name = getattr(v, "name", None) + chemform = getattr(v, "chemform", None) + + if item_id or refcode: + return EntryReference( + item_id=item_id, refcode=refcode, type=item_type, name=name, chemform=chemform + ) + else: + return InlineSubstance(name=name or str(v), chemform=chemform) + return v diff --git a/pydatalab/src/pydatalab/models/versions.py b/pydatalab/src/pydatalab/models/versions.py index f62914df5..6efce3a2c 100644 --- a/pydatalab/src/pydatalab/models/versions.py +++ b/pydatalab/src/pydatalab/models/versions.py @@ -3,9 +3,9 @@ from datetime import datetime from enum import Enum -from pydantic import BaseModel, Field, validator +from pydantic import Field, validator -from pydatalab.models.utils import PyObjectId, Refcode +from pydatalab.models.utils import BaseModel, PyObjectId, Refcode class VersionAction(str, Enum): @@ -26,34 +26,35 @@ class ItemVersion(BaseModel): view history and restore previous states. """ - refcode: Refcode = Field(..., description="The refcode of the item this version belongs to") - version: int = Field(..., ge=1, description="Sequential version number (1-indexed)") - timestamp: datetime = Field( - ..., description="When this version was created (ISO format with timezone)" - ) - action: VersionAction = Field( - ..., - description="The action that triggered this version: 'created' (item creation), " - "'manual_save' (user save), 'auto_save' (system save), or 'restored' (version restore)", - ) - user_id: PyObjectId | None = Field( - None, description="User's ObjectId for efficient querying and indexing" - ) - creator: dict | None = Field( - None, description="Inlined information about the user who created this version (e.g., name)" - ) - datalab_version: str = Field( - ..., description="Version of datalab-server that created this snapshot" - ) - data: dict = Field(..., description="Complete snapshot of the item data at this version") - restored_from_version: PyObjectId | None = Field( - None, - description="ObjectId of the version that was restored from (only present if action='restored')", - ) - user_agent: str | None = Field( - None, - description="User agent string of the client that triggered this version. Will only be stored if it matches a known value from the datalab ecosystem.", - ) + refcode: Refcode + """The refcode of the item this version belongs to""" + + version: int = Field(ge=1) + """Sequential version number (1-indexed)""" + + timestamp: datetime + """When this version was created (ISO format with timezone)""" + + action: VersionAction + """The action that triggered this version: 'created' (item creation), 'manual_save' (user save), 'auto_save' (system save), or 'restored' (version restore)""" + + user_id: PyObjectId | None = None + """User's ObjectId for efficient querying and indexing""" + + creator: dict | None = None + """Inlined information about the user who created this version (e.g., name)""" + + datalab_version: str + """Version of datalab-server that created this snapshot""" + + data: dict + """Complete snapshot of the item data at this version""" + + restored_from_version: PyObjectId | None = None + """ObjectId of the version that was restored from (only present if action='restored')""" + + user_agent: str | None = None + """User agent string of the client that triggered this version. Will only be stored if it matches a known value from the datalab ecosystem.""" @validator("restored_from_version") def validate_restored_from_version(cls, v, values): @@ -75,10 +76,11 @@ class VersionCounter(BaseModel): It ensures atomic increment of version numbers to prevent race conditions. """ - refcode: Refcode = Field(..., description="The refcode this counter belongs to") - counter: int = Field( - 1, ge=1, description="Current version counter value (1-indexed, matches version numbers)" - ) + refcode: Refcode + """The refcode this counter belongs to""" + + counter: int = Field(1, ge=1) + """Current version counter value (1-indexed, matches version numbers)""" class Config: extra = "ignore" # Allow MongoDB's _id field and other internal fields @@ -87,7 +89,8 @@ class Config: class RestoreVersionRequest(BaseModel): """Request body for restoring a version.""" - version_id: str = Field(..., description="ObjectId string of the version to restore to") + version_id: str + """ObjectId string of the version to restore to""" @validator("version_id") def validate_version_id_format(cls, v): @@ -107,8 +110,11 @@ class Config: class CompareVersionsQuery(BaseModel): """Query parameters for comparing two versions.""" - v1: str = Field(..., description="ObjectId string of the first version") - v2: str = Field(..., description="ObjectId string of the second version") + v1: str + """ObjectId string of the first version""" + + v2: str + """ObjectId string of the second version""" @validator("v1", "v2") def validate_version_ids(cls, v): diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index c6131dc9f..eabf7c1a8 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -33,7 +33,7 @@ def load_plugin_schema(): need them. A JSON Schema generated from the returned model is emitted to `pydatalab/schemas/plugin_config.json` by `invoke dev.generate-schemas`. """ - from pydantic import BaseModel, root_validator + from pydantic import BaseModel, model_validator class UvSource(BaseModel): """A single entry under `[tool.uv.sources]` in plugins.toml.""" @@ -48,7 +48,8 @@ class UvSource(BaseModel): class Config: extra = "forbid" - @root_validator + @model_validator(mode="before") + @classmethod def _exactly_one_source(cls, values): has_git = values.get("git") is not None has_path = values.get("path") is not None @@ -79,7 +80,8 @@ class PluginConfigModel(BaseModel): class Config: extra = "forbid" - @root_validator + @model_validator(mode="before") + @classmethod def _sources_must_match_dependencies(cls, values): deps = {d.split("[")[0].strip() for d in values.get("dependencies", [])} sources = values.get("tool", ToolSection()).uv.sources diff --git a/pydatalab/tests/test_models.py b/pydatalab/tests/test_models.py index 1e59bfbce..c2ba5959b 100644 --- a/pydatalab/tests/test_models.py +++ b/pydatalab/tests/test_models.py @@ -84,6 +84,27 @@ def test_generate_schemas(model): assert model.schema() +def test_attribute_docstrings_in_schema(): + """Field docstrings should be pulled into the JSON schema as descriptions. + + Relies on ``use_attribute_docstrings`` being set on the shared ``BaseModel`` + in ``pydatalab.models.utils``; covers fields defined across several different + trait mixins to confirm the config propagates through inheritance. + """ + properties = Sample.model_json_schema(by_alias=False)["properties"] + + expected = { + "creators": "Inlined info for the people associated with this item.", # HasOwner + "blocks_obj": "A mapping from block ID to block data.", # HasBlocks + "revision": "The revision number of the entry.", # HasRevisionControl + "collections": "Inlined info for the collections associated with this item.", # IsCollectable + "name": "An optional human-readable/usable name for the entry.", # Item + } + + for field, description in expected.items(): + assert properties[field]["description"] == description + + def test_relationship_with_custom_type(): """Test that a relationship with a custom type can be created.""" relationship = TypedRelationship( From 8addb328a1fb9b202d0aaa10db27a17360b048c9 Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Tue, 11 Nov 2025 01:08:22 +0000 Subject: [PATCH 06/24] Load FTS fields for MongoDB from model schemas directly Rework ITEMS_FTS_FIELDS lookup --- pydatalab/src/pydatalab/mongo.py | 60 ++++++++++++++------ pydatalab/src/pydatalab/routes/v0_1/items.py | 4 +- pydatalab/tests/server/test_items.py | 4 +- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/pydatalab/src/pydatalab/mongo.py b/pydatalab/src/pydatalab/mongo.py index 04ae93010..022b9af92 100644 --- a/pydatalab/src/pydatalab/mongo.py +++ b/pydatalab/src/pydatalab/mongo.py @@ -10,7 +10,6 @@ from pymongo.errors import ConnectionFailure from pydatalab.logger import LOGGER -from pydatalab.models import ITEM_MODELS __all__ = ( "flask_mongo", @@ -33,20 +32,40 @@ """One-liner that pulls all non-semantic string fields out of all item models implemented for this server. """ -ITEMS_FTS_FIELDS: set[str] = set().union( - *( - { - f - for f, p in model.schema(by_alias=False)["properties"].items() - if ( - p.get("type") == "string" - and p.get("format") not in ("date-time", "uuid") - and f != "type" - ) - } - for model in ITEM_MODELS.values() - ) -) + + +@lru_cache(maxsize=1) +def get_items_fts_fields() -> set[str]: + """Get all string fields from item models for full-text search.""" + from pydatalab.models import ITEM_MODELS + + fields = set() + + for model_name, model in ITEM_MODELS.items(): + schema = model.model_json_schema(by_alias=False) + + model_fields = set() + for f, p in schema.get("properties", {}).items(): + if f == "type": + continue + + if p.get("type") == "string" and p.get("format") not in ("date-time", "uuid"): + model_fields.add(f) + elif "anyOf" in p: + for option in p["anyOf"]: + if option.get("type") == "string" and option.get("format") not in ( + "date-time", + "uuid", + ): + model_fields.add(f) + break + + fields.update(model_fields) + + return fields + + +ITEMS_FTS_FIELDS: set[str] = set() USERS_FTS_FIELDS: set[str] = {"identities.name", "display_name", "contact_email"} """Fields to search for users.""" @@ -144,7 +163,7 @@ def insert_pydantic_model_fork_safe(model: BaseModel, collection: str) -> str: """Inserts a Pydantic model into chosen collection, returning the inserted ID.""" return ( get_database()[collection] - .insert_one(model.dict(by_alias=True, exclude_none=True)) + .insert_one(model.model_dump(by_alias=True, exclude_none=True)) .inserted_id ) @@ -163,7 +182,6 @@ def _get_active_mongo_client(timeoutMS: int = 1000) -> pymongo.MongoClient: """ from pydatalab.config import CONFIG - from pydatalab.logger import LOGGER try: client = pymongo.MongoClient( @@ -231,6 +249,14 @@ def create_default_indices( """ + global ITEMS_FTS_FIELDS + + if not ITEMS_FTS_FIELDS: + ITEMS_FTS_FIELDS = get_items_fts_fields() + + if not ITEMS_FTS_FIELDS: + raise ValueError("Cannot create text indices: no fields available for full-text search") + if client is None: client = _get_active_mongo_client() db = client.get_database() diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index f1b9a6866..6f8fc8bc7 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -25,7 +25,7 @@ RestoreVersionRequest, VersionAction, ) -from pydatalab.mongo import ITEMS_FTS_FIELDS, build_search_pipeline, flask_mongo +from pydatalab.mongo import build_search_pipeline, flask_mongo, get_items_fts_fields from pydatalab.permissions import ( PUBLIC_USER_ID, access_token_or_active_users, @@ -518,7 +518,7 @@ def search_items(): return jsonify({"status": "error", "message": "No query provided."}), 400 permissions = get_default_permissions(user_only=False) - pipeline = build_search_pipeline(query, ITEMS_FTS_FIELDS, permissions) + pipeline = build_search_pipeline(query, get_items_fts_fields(), permissions) if types is not None: if pipeline and "$match" in pipeline[0]: diff --git a/pydatalab/tests/server/test_items.py b/pydatalab/tests/server/test_items.py index e58bafd2d..9c9ad1598 100644 --- a/pydatalab/tests/server/test_items.py +++ b/pydatalab/tests/server/test_items.py @@ -22,7 +22,7 @@ def test_single_item_endpoints(client, inserted_default_items): def test_fts_fields(): """Test non-exhaustively that certain fields make it into the fts index.""" - from pydatalab.mongo import ITEMS_FTS_FIELDS + from pydatalab.mongo import get_items_fts_fields fields = ("item_id", "name", "description", "refcode", "synthesis_description", "supplier") - assert all(field in ITEMS_FTS_FIELDS for field in fields) + assert all(field in get_items_fts_fields() for field in fields) From 8a9205fb0f439cc528d4c353e6fbe85e76167ce2 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Tue, 11 Nov 2025 01:09:59 +0000 Subject: [PATCH 07/24] Misc pydantic v2 bumps around code --- pydatalab/src/pydatalab/backups.py | 2 +- pydatalab/src/pydatalab/errors.py | 1 + pydatalab/src/pydatalab/file_utils.py | 10 +++++----- pydatalab/src/pydatalab/main.py | 4 ++-- pydatalab/tasks.py | 23 +++++------------------ 5 files changed, 14 insertions(+), 26 deletions(-) diff --git a/pydatalab/src/pydatalab/backups.py b/pydatalab/src/pydatalab/backups.py index bea81e84d..f058a62a6 100644 --- a/pydatalab/src/pydatalab/backups.py +++ b/pydatalab/src/pydatalab/backups.py @@ -92,7 +92,7 @@ def take_snapshot(snapshot_path: Path, encrypt: bool = False) -> None: LOGGER.debug("Dumping server config.") with tempfile.TemporaryDirectory() as temp_dir: with open(tmp_config := Path(temp_dir) / "config.json", "w") as f: - data = CONFIG.json(indent=2, exclude_unset=True) + data = CONFIG.model_dump_json(indent=2, exclude_unset=True) f.write(data) tar.add( diff --git a/pydatalab/src/pydatalab/errors.py b/pydatalab/src/pydatalab/errors.py index 675856781..da092968d 100644 --- a/pydatalab/src/pydatalab/errors.py +++ b/pydatalab/src/pydatalab/errors.py @@ -70,6 +70,7 @@ def handle_pydantic_validation_error(exc: ValidationError) -> tuple[Response, in These always come from malformed data, so should not necessarily trigger the Flask debugger. """ + LOGGER.critical("Pydantic validation error: %s", exc, exc_info=True) response = { "title": exc.__class__.__name__, "message": str(exc.args[:]) if exc.args else "", diff --git a/pydatalab/src/pydatalab/file_utils.py b/pydatalab/src/pydatalab/file_utils.py index 33bb8958b..5bda4eabc 100644 --- a/pydatalab/src/pydatalab/file_utils.py +++ b/pydatalab/src/pydatalab/file_utils.py @@ -336,7 +336,7 @@ def get_file_info_by_id(file_id: str | ObjectId, update_if_live: bool = True) -> if update_if_live and file_info.is_live: file_info = _check_and_sync_file(file_info, file_id) - return file_info.dict() + return file_info.model_dump() def update_uploaded_file(file: FileStorage, file_id: ObjectId, size_bytes: int | None = None): @@ -405,7 +405,7 @@ def update_uploaded_file(file: FileStorage, file_id: ObjectId, size_bytes: int | file.save(updated_file_entry.location) - ret = updated_file_entry.dict() + ret = updated_file_entry.model_dump() ret.update({"_id": file_id}) return ret @@ -498,7 +498,7 @@ def save_uploaded_file( f"Cannot store file: insufficient space available on disk (required: {size_bytes // 1024**3} GB). Please contact your datalab administrator." ) file_collection = client.get_database().files - result = file_collection.insert_one(new_file_document.dict(), session=session) + result = file_collection.insert_one(new_file_document.model_dump(), session=session) if not result.acknowledged: raise RuntimeError( f"db operation failed when trying to insert new file. Result: {result}" @@ -534,7 +534,7 @@ def save_uploaded_file( f"db operation failed when trying to insert new file ObjectId into sample: {item_id}" ) - ret = updated_file_entry.dict() + ret = updated_file_entry.model_dump() ret.update({"_id": inserted_id}) return ret @@ -610,7 +610,7 @@ def add_file_from_remote_directory( creator_ids=creator_ids if creator_ids is not None else [], ) - result = file_collection.insert_one(new_file_document.dict()) + result = file_collection.insert_one(new_file_document.model_dump()) if not result.acknowledged: raise OSError(f"db operation failed when trying to insert new file. Result: {result}") diff --git a/pydatalab/src/pydatalab/main.py b/pydatalab/src/pydatalab/main.py index d8ae01bba..d326ac1a8 100644 --- a/pydatalab/src/pydatalab/main.py +++ b/pydatalab/src/pydatalab/main.py @@ -45,7 +45,7 @@ def create_app( CONFIG.update(config_override) app.config.from_prefixed_env() - app.config.update(CONFIG.dict()) + app.config.update(CONFIG.model_dump()) # This value will still be overwritten by any dotenv values app.config["MAIL_DEBUG"] = app.config.get("MAIL_DEBUG") or CONFIG.TESTING @@ -53,7 +53,7 @@ def create_app( # percolate datalab mail settings up to the `MAIL_` env vars/app config # for use by Flask Mail if CONFIG.EMAIL_AUTH_SMTP_SETTINGS is not None: - mail_settings = CONFIG.EMAIL_AUTH_SMTP_SETTINGS.dict() + mail_settings = CONFIG.EMAIL_AUTH_SMTP_SETTINGS.model_dump() for key in mail_settings: app.config[key] = mail_settings[key] diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index eabf7c1a8..dad1da782 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -5,13 +5,11 @@ import shutil import subprocess import time -import typing import tomlkit from invoke import Collection, task -if typing.TYPE_CHECKING: - from pydatalab.models.utils import UserRole +from pydatalab.models.utils import UserRole ns = Collection() dev = Collection("dev") @@ -45,9 +43,6 @@ class UvSource(BaseModel): path: str | None = None editable: bool | None = None - class Config: - extra = "forbid" - @model_validator(mode="before") @classmethod def _exactly_one_source(cls, values): @@ -62,24 +57,15 @@ def _exactly_one_source(cls, values): class UvSection(BaseModel): sources: dict[str, UvSource] = {} - class Config: - extra = "forbid" - class ToolSection(BaseModel): uv: UvSection = UvSection() - class Config: - extra = "forbid" - class PluginConfigModel(BaseModel): """The schema for the top-level plugins.toml file.""" dependencies: list[str] = [] tool: ToolSection = ToolSection() - class Config: - extra = "forbid" - @model_validator(mode="before") @classmethod def _sources_must_match_dependencies(cls, values): @@ -117,7 +103,7 @@ def generate_schemas(_): schemas_path = pathlib.Path(__file__).parent / "schemas" for model in ITEM_MODELS.values(): - schema = model.schema(by_alias=False) + schema = model.model_json_schema(by_alias=False) with open(schemas_path / f"{model.__name__.lower()}.json", "w") as f: json.dump(schema, f, indent=2) @@ -418,11 +404,12 @@ def add_missing_refcodes(_): def _check_id(id=None, base_url=None, api_key=None): - from pydatalab.logger import setup_log - """Checks the given item ID served at the base URL and logs the result.""" + import requests + from pydatalab.logger import setup_log + log = setup_log("check_item_validity") response = requests.get( f"{base_url}/get-item-data/{id}", headers={"DATALAB-API-KEY": api_key}, timeout=30 From f64fba38175383c382e2aafbf6fca02137491d75 Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Tue, 11 Nov 2025 01:12:51 +0000 Subject: [PATCH 08/24] Update various routes for pydantic v2 Fixes for routes --- pydatalab/src/pydatalab/routes/v0_1/auth.py | 13 +-- .../src/pydatalab/routes/v0_1/collections.py | 10 +- pydatalab/src/pydatalab/routes/v0_1/graphs.py | 4 +- pydatalab/src/pydatalab/routes/v0_1/info.py | 108 +++++++++--------- pydatalab/src/pydatalab/routes/v0_1/items.py | 73 +++++++++--- .../src/pydatalab/routes/v0_1/remotes.py | 6 +- pydatalab/src/pydatalab/routes/v0_1/users.py | 35 +++--- 7 files changed, 146 insertions(+), 103 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/auth.py b/pydatalab/src/pydatalab/routes/v0_1/auth.py index 549d96f99..fe814ae18 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/auth.py +++ b/pydatalab/src/pydatalab/routes/v0_1/auth.py @@ -505,7 +505,7 @@ def find_user_with_identity( if verify and not person.identities[identity_index].verified: flask_mongo.db.users.update_one( - {"_id": person.immutable_id}, + {"_id": ObjectId(person.immutable_id)}, {"$set": {f"identities.{identity_index}.verified": True}}, ) @@ -557,7 +557,7 @@ def attach_identity_to_user( RuntimeError: If the update was unsuccessful. """ - update = {"$push": {"identities": identity.dict()}} + update = {"$push": {"identities": identity.model_dump()}} if use_display_name and identity and identity.display_name: update["$set"] = {"display_name": identity.display_name} @@ -612,12 +612,11 @@ def attach_identity_to_user( identity, use_display_name=True, account_status=account_status ) LOGGER.debug("Inserting new user model %s into database", user) - insert_pydantic_model_fork_safe(user, "users") - user_model = get_by_id(str(user.immutable_id)) + inserted_id = insert_pydantic_model_fork_safe(user, "users") + user = get_by_id(inserted_id) if user is None: raise RuntimeError("Failed to insert user into database") - wrapped_login_user(user_model) # Send email notification to admins _send_admin_email_notification(user) @@ -626,7 +625,7 @@ def attach_identity_to_user( # Log the user into the session with this identity if user is not None: - wrapped_login_user(get_by_id(str(user.immutable_id))) + wrapped_login_user(user) def _validate_magic_link_request(email: str, referrer: str) -> None: @@ -1050,7 +1049,7 @@ def redirect_to_ui(blueprint, token): # pylint: disable=unused-argument def get_authenticated_user_info(): """Returns metadata associated with the currently authenticated user.""" if current_user.is_authenticated: - current_user_response = json.loads(current_user.person.json()) + current_user_response = json.loads(current_user.person.model_dump_json()) current_user_response["role"] = current_user.role.value return jsonify(current_user_response), 200 else: diff --git a/pydatalab/src/pydatalab/routes/v0_1/collections.py b/pydatalab/src/pydatalab/routes/v0_1/collections.py index a8afffc3d..fd3f6dc6f 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/collections.py +++ b/pydatalab/src/pydatalab/routes/v0_1/collections.py @@ -87,7 +87,7 @@ def get_collection(collection_id): { "status": "success", "collection_id": collection_id, - "data": json.loads(collection.json(exclude_unset=True)), + "data": json.loads(collection.model_dump_json(exclude_unset=True)), "child_items": list(samples), } ) @@ -168,7 +168,7 @@ def create_collection(): ) result: InsertOneResult = flask_mongo.db.collections.insert_one( - data_model.dict(exclude={"creators", "groups"}) + data_model.model_dump(exclude={"creators", "groups"}) ) if not result.acknowledged: return ( @@ -218,7 +218,7 @@ def create_collection(): response = { "status": "success", - "data": json.loads(data_model.json()), + "data": json.loads(data_model.model_dump_json()), } if errors: @@ -278,7 +278,7 @@ def save_collection(collection_id): collection.update(updated_data) try: - collection = Collection(**collection).dict() + collection = Collection(**collection).model_dump() except ValidationError as exc: return ( jsonify( @@ -534,7 +534,7 @@ def search_collections(): ) cursor = [ - json.loads(Collection(**doc).json(exclude_unset=True)) + json.loads(Collection(**doc).model_dump_json(exclude_unset=True)) for doc in flask_mongo.db.collections.aggregate(pipeline) ] diff --git a/pydatalab/src/pydatalab/routes/v0_1/graphs.py b/pydatalab/src/pydatalab/routes/v0_1/graphs.py index 1299bfd09..72fb68317 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/graphs.py +++ b/pydatalab/src/pydatalab/routes/v0_1/graphs.py @@ -217,7 +217,9 @@ def add_related_items(current_item_id: str, current_depth: int): { "data": { "id": document["item_id"], - "name": document["name"] if document["name"] else document["item_id"], + "name": document.get("name") + if document.get("name") + else document["item_id"], "type": document["type"], "special": document["item_id"] == item_id, } diff --git a/pydatalab/src/pydatalab/routes/v0_1/info.py b/pydatalab/src/pydatalab/routes/v0_1/info.py index a2212ea87..40e4fc6a5 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/info.py +++ b/pydatalab/src/pydatalab/routes/v0_1/info.py @@ -7,14 +7,20 @@ from functools import lru_cache from flask import Blueprint, jsonify, request -from pydantic import AnyUrl, BaseModel, Field, validator +from pydantic import ( + AnyUrl, + BaseModel, + ConfigDict, + Field, + field_validator, + model_validator, +) from pydatalab import __version__ from pydatalab.apps import BLOCK_TYPES from pydatalab.config import CONFIG from pydatalab.feature_flags import FEATURE_FLAGS, FeatureFlags -from pydatalab.models import Collection, Person -from pydatalab.models.items import Item +from pydatalab.models import ITEM_SCHEMAS, Person from pydatalab.mongo import flask_mongo from pydatalab.permissions import active_users_or_get_only @@ -24,8 +30,7 @@ class Attributes(BaseModel): - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class Meta(BaseModel): @@ -39,9 +44,7 @@ class Meta(BaseModel): class Links(BaseModel): self: AnyUrl - - class Config: - extra = "allow" + model_config = ConfigDict(extra="allow") class Data(BaseModel): @@ -53,39 +56,58 @@ class Data(BaseModel): class JSONAPIResponse(BaseModel): data: Data | list[Data] meta: Meta - links: Links | None + links: Links | None = None class MetaPerson(BaseModel): - dislay_name: str | None + display_name: str | None = None contact_email: str class Info(Attributes, Meta): - maintainer: MetaPerson | None - issue_tracker: AnyUrl | None - homepage: AnyUrl | None - source_repository: AnyUrl | None + maintainer: MetaPerson | None = None + issue_tracker: AnyUrl | None = None + homepage: AnyUrl | None = None + source_repository: AnyUrl | None = None identifier_prefix: str features: FeatureFlags = FEATURE_FLAGS max_upload_bytes: int - @validator("maintainer") + @field_validator("maintainer", mode="before") + @classmethod def strip_maintainer_fields(cls, v): if isinstance(v, Person): return MetaPerson(contact_email=v.contact_email, display_name=v.display_name) return v + @model_validator(mode="after") + def ensure_features_serialization(self): + """Ensure features are properly serialized for frontend consumption.""" + if hasattr(self.features, "model_dump"): + features_dict = self.features.model_dump() + else: + features_dict = self.features + + if not isinstance(self.features, FeatureFlags): + self.features = FeatureFlags(**features_dict) + return self + @lru_cache(maxsize=1) def _get_deployment_metadata_once() -> dict: identifier_prefix = CONFIG.IDENTIFIER_PREFIX metadata = ( - CONFIG.DEPLOYMENT_METADATA.dict(exclude_none=True) if CONFIG.DEPLOYMENT_METADATA else {} + CONFIG.DEPLOYMENT_METADATA.model_dump(exclude_none=True) + if CONFIG.DEPLOYMENT_METADATA + else {} + ) + metadata.update( + { + "identifier_prefix": identifier_prefix, + "max_upload_bytes": CONFIG.MAX_CONTENT_LENGTH, + "features": FEATURE_FLAGS, + } ) - metadata.update({"identifier_prefix": identifier_prefix}) - metadata.update({"max_upload_bytes": CONFIG.MAX_CONTENT_LENGTH}) - return metadata @@ -95,18 +117,15 @@ def get_info(): versions, features and so on. """ - metadata = _get_deployment_metadata_once() + + response_data = JSONAPIResponse( + data=Data(id="/", type="info", attributes=Info(**_get_deployment_metadata_once())), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), + links=Links(self=request.url), + ) return ( - jsonify( - json.loads( - JSONAPIResponse( - data=Data(id="/", type="info", attributes=Info(**metadata)), - meta=Meta(query=request.query_string), - links=Links(self=request.url), - ).json() - ) - ), + jsonify(json.loads(response_data.model_dump_json())), 200, ) @@ -149,30 +168,11 @@ def list_block_types(): for block_type, block in BLOCK_TYPES.items() ], meta=Meta(query=request.query_string), - ).json() + ).model_dump_json() ) ) -def get_all_items_models(): - return Item.__subclasses__() - - -def generate_schemas(): - schemas: dict[str, dict] = {} - - for model_class in get_all_items_models() + [Collection]: - model_type = model_class.schema()["properties"]["type"]["default"] - - schemas[model_type] = model_class.schema(by_alias=False) - - return schemas - - -# Generate once on import -SCHEMAS = generate_schemas() - - @INFO.route("/info/types", methods=["GET"]) def list_supported_types(): """Returns a list of supported schemas.""" @@ -190,10 +190,10 @@ def list_supported_types(): "schema": schema, }, ) - for item_type, schema in SCHEMAS.items() + for item_type, schema in ITEM_SCHEMAS.items() ], meta=Meta(query=request.query_string), - ).json() + ).model_dump_json() ) ) @@ -201,7 +201,7 @@ def list_supported_types(): @INFO.route("/info/types/", methods=["GET"]) def get_schema_type(item_type): """Returns the schema of the given type.""" - if item_type not in SCHEMAS: + if item_type not in ITEM_SCHEMAS: return jsonify( {"status": "error", "detail": f"Item type {item_type} not found for this deployment"} ), 404 @@ -215,11 +215,11 @@ def get_schema_type(item_type): attributes={ "version": __version__, "api_version": __api_version__, - "schema": SCHEMAS[item_type], + "schema": ITEM_SCHEMAS[item_type], }, ), meta=Meta(query=request.query_string), - ).json() + ).model_dump_json() ) ) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 6f8fc8bc7..47a693dd9 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -506,8 +506,8 @@ def search_items(): Returns: response list of dictionaries containing the matching items in order of descending match score. - """ + """ query = request.args.get("query", type=str) nresults = request.args.get("nresults", default=100, type=int) types = request.args.get("types", default=None) @@ -662,7 +662,7 @@ def _create_sample( elif CONFIG.TESTING and not current_user.is_authenticated: # Set fake ID to ObjectId("000000000000000000000000") so a dummy user can be created # locally for testing creator UI elements - new_sample["creator_ids"] = [PUBLIC_USER_ID] + new_sample["creator_ids"] = [str(PUBLIC_USER_ID)] new_sample["creators"] = [ { "display_name": "Public testing user", @@ -703,7 +703,6 @@ def _create_sample( try: # Check on relationship fields and prefill new_sample = entry_reference_lookup(new_sample) - data_model: Item = model(**new_sample) except ValidationError as error: @@ -717,7 +716,9 @@ def _create_sample( # the `Entry` model. try: result = flask_mongo.db.items.insert_one( - data_model.dict(exclude={"creators", "collections", "groups"}) + data_model.model_dump( + exclude={"creators", "collections", "groups"}, exclude_none=True, by_alias=True + ) ) except DuplicateKeyError as error: raise Conflict(f"Duplicate key error: {str(error)}.") @@ -752,7 +753,7 @@ def _create_sample( data = { "status": "success", "item_id": data_model.item_id, - "sample_list_entry": data_model.dict(), + "sample_list_entry": data_model.model_dump(), } return (data, 201) # 201 Created @@ -1123,7 +1124,6 @@ def get_item_data( elif refcode: if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" - match = {"refcode": refcode} else: raise BadRequest("No item_id or refcode provided.") @@ -1148,6 +1148,7 @@ def get_item_data( try: doc = list(cursor)[0] + except IndexError: doc = None @@ -1162,7 +1163,6 @@ def get_item_data( 404, ) - # determine the item type and validate according to the appropriate schema try: ItemModel = ITEM_MODELS[doc["type"]] except KeyError: @@ -1219,7 +1219,7 @@ def get_item_data( ) # loop over and collect all 'outer' relationships presented by other items - incoming_relationships: dict[RelationshipType, set[str]] = {} + incoming_relationships: dict[RelationshipType, set] = {} for d in relationships_query_results: for k in d["relationships"]: if k["relation"] not in incoming_relationships: @@ -1229,7 +1229,7 @@ def get_item_data( ) # loop over and aggregate all 'inner' relationships presented by this item - inlined_relationships: dict[RelationshipType, set[str]] = {} + inlined_relationships: dict[RelationshipType, set] = {} if doc.relationships is not None: inlined_relationships = { relation: { @@ -1249,7 +1249,7 @@ def get_item_data( ) # Must be exported to JSON first to apply the custom pydantic JSON encoders - return_dict = json.loads(doc.json(exclude_unset=True)) + return_dict = doc.model_dump(mode="json", exclude_unset=True) if item_id is None: item_id = return_dict["item_id"] @@ -1557,7 +1557,7 @@ def restore_version(refcode): # Insert validated data flask_mongo.db.item_versions.insert_one( - validated_restored_version.dict(by_alias=True, exclude_none=True) + validated_restored_version.model_dump(by_alias=True, exclude_none=True) ) return jsonify( @@ -1637,6 +1637,7 @@ def save_item(): # These keys should not be updated here and cannot be modified by the user through this endpoint for k in ( "_id", + "immutable_id", "file_ObjectIds", "files", "creators", @@ -1745,6 +1746,42 @@ def save_item(): ] updated_data["collections"] = inaccessible + existing_item = flask_mongo.db.items.find_one({"item_id": item_id}) + if existing_item: + existing_relationships = existing_item.get("relationships", []) + non_collection_relationships = [ + rel for rel in existing_relationships if rel.get("type") != "collections" + ] + + collection_relationships = [] + for coll in updated_data.get("collections", []): + immutable_id = coll.get("immutable_id") + collection_id = coll.get("collection_id") + + if immutable_id: + if isinstance(immutable_id, str): + from bson import ObjectId + + immutable_id = ObjectId(immutable_id) + elif collection_id: + collection_doc = flask_mongo.db.collections.find_one( + {"collection_id": collection_id} + ) + if collection_doc: + immutable_id = collection_doc["_id"] + + if immutable_id: + collection_relationships.append( + { + "relation": None, + "immutable_id": immutable_id, + "type": "collections", + "description": "Is a member of", + } + ) + + updated_data["relationships"] = non_collection_relationships + collection_relationships + item_type = item["type"] preserve_relationships = "collections" not in updated_data @@ -1754,7 +1791,13 @@ def save_item(): try: item = entry_reference_lookup(item) - item = ITEM_MODELS[item_type](**item).dict() + item = ITEM_MODELS[item_type](**item).model_dump( + exclude_none=True, + exclude_unset=True, + by_alias=True, + exclude={"collections", "creators", "immutable_id"}, + ) + except ValidationError as exc: return ( jsonify( @@ -1769,8 +1812,10 @@ def save_item(): item["relationships"] = original_relationships # remove collections and creators and any other reference fields - item.pop("collections") - item.pop("creators") + item.pop("collections", None) + item.pop("creators", None) + item.pop("immutable_id", None) + item.pop("files", None) # Update the item FIRST (transaction safety: item update before version save) result = flask_mongo.db.items.update_one( diff --git a/pydatalab/src/pydatalab/routes/v0_1/remotes.py b/pydatalab/src/pydatalab/routes/v0_1/remotes.py index be50502f9..4e69952a2 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/remotes.py +++ b/pydatalab/src/pydatalab/routes/v0_1/remotes.py @@ -76,7 +76,9 @@ def list_remote_directories(): response = {} response["meta"] = {} - response["meta"]["remotes"] = [json.loads(d.json()) for d in CONFIG.REMOTE_FILESYSTEMS] + response["meta"]["remotes"] = [ + json.loads(d.model_dump_json()) for d in CONFIG.REMOTE_FILESYSTEMS + ] if all_directory_structures: oldest_update = min(d["last_updated"] for d in all_directory_structures) response["meta"]["oldest_cache_update"] = oldest_update.isoformat() @@ -139,7 +141,7 @@ def get_remote_directory(remote_id: str): response: dict[str, Any] = {} response["meta"] = {} - response["meta"]["remote"] = json.loads(d.json()) + response["meta"]["remote"] = json.loads(d.model_dump_json()) response["data"] = directory_structure return jsonify(response), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/users.py b/pydatalab/src/pydatalab/routes/v0_1/users.py index b59aca49d..f790a12bf 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/users.py +++ b/pydatalab/src/pydatalab/routes/v0_1/users.py @@ -6,11 +6,12 @@ from bson import ObjectId from flask import Blueprint, jsonify, request from flask_login import current_user +from pydantic import ValidationError from werkzeug.exceptions import BadRequest, Forbidden, Unauthorized from pydatalab.config import CONFIG from pydatalab.logger import LOGGER -from pydatalab.models.people import AccountStatus, DisplayName, EmailStr, Person +from pydatalab.models.people import Person from pydatalab.mongo import ( USERS_FTS_FIELDS, build_search_pipeline, @@ -49,22 +50,22 @@ def save_user(user_id): update = {} - try: - if display_name: - update["display_name"] = DisplayName(display_name) + if display_name: + update["display_name"] = display_name - except ValueError: - raise BadRequest(f"Invalid display name {display_name!r} was passed") + if contact_email or contact_email in (None, ""): + if contact_email in ("", None): + update["contact_email"] = None + else: + update["contact_email"] = contact_email - try: - if contact_email or contact_email in (None, ""): - if contact_email in ("", None): - update["contact_email"] = None - else: - update["contact_email"] = EmailStr(contact_email) + if account_status: + update["account_status"] = account_status - except ValueError: - raise BadRequest(f"Invalid email address {contact_email!r} was passed") + try: + _ = Person(**update) + except ValidationError as e: + raise BadRequest(f"Invalid user data: {e.errors()}") from e if "contact_email" in update or "display_name" in update: existing = ( @@ -128,12 +129,6 @@ def save_user(user_id): trigger_email_verification = False LOGGER.critical("Unable to send verification email on this deployment: %s", e) - try: - if account_status: - update["account_status"] = AccountStatus(account_status) - except ValueError: - raise BadRequest(f"Invalid account status {account_status!r} was passed") - if not update: return jsonify({"status": "success", "message": "No update to perform."}), 200 From b76e1155fe40448851b6fad4253d0eed69cdcd44 Mon Sep 17 00:00:00 2001 From: Benjamin Charmes Date: Tue, 11 Nov 2025 01:14:10 +0000 Subject: [PATCH 09/24] Update various blocks for pydantic v2 Chatblock patch; set default values in block More updates for blocks --- pydatalab/src/pydatalab/apps/chat/blocks.py | 15 ++++++++------- pydatalab/src/pydatalab/apps/chat/models.py | 2 +- pydatalab/src/pydatalab/apps/nmr/models.py | 5 ++--- pydatalab/src/pydatalab/apps/xrd/blocks.py | 8 +++++--- pydatalab/src/pydatalab/blocks/base.py | 6 +++--- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pydatalab/src/pydatalab/apps/chat/blocks.py b/pydatalab/src/pydatalab/apps/chat/blocks.py index 67371cc29..b952fe8ca 100644 --- a/pydatalab/src/pydatalab/apps/chat/blocks.py +++ b/pydatalab/src/pydatalab/apps/chat/blocks.py @@ -2,7 +2,7 @@ import warnings from langchain_core.messages import AIMessage, HumanMessage, SystemMessage -from pydantic import Field, validator +from pydantic import Field, field_validator from pydatalab.blocks.base import DataBlock from pydatalab.models import ITEM_MODELS @@ -16,15 +16,16 @@ class ChatBlockResponse(DataBlockResponse): messages: list[dict] = Field(default_factory=list) - prompt: str | None + prompt: str | None = None model: str available_models: dict[str, ModelCard] | None = Field( - datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, validate_default=True, datalab_exclude_from_db=True, datalab_exclude_from_load=True ) - token_count: int | None + token_count: int | None = None temperature: float - @validator("available_models", pre=True, always=True) + @field_validator("available_models", mode="before") + @classmethod def set_available_models(cls, _): return AVAILABLE_MODELS @@ -206,8 +207,8 @@ def _prepare_item_json_for_chat(self, item_id: str, item_data: dict | None = Non item_model.blocks_obj = { k: block for k, block in item_model.blocks_obj.items() if block.blocktype != "chat" } - item_data = item_model.dict(exclude_none=True, exclude_unset=True) - item_data["type"] = item_model.type + item_info = item_model.model_dump(exclude_none=True, exclude_unset=True) + item_info["type"] = item_model.type # strip irrelevant or large fields item_filenames = { diff --git a/pydatalab/src/pydatalab/apps/chat/models.py b/pydatalab/src/pydatalab/apps/chat/models.py index 03efc7246..756ca1970 100644 --- a/pydatalab/src/pydatalab/apps/chat/models.py +++ b/pydatalab/src/pydatalab/apps/chat/models.py @@ -9,7 +9,7 @@ class ModelCard(BaseModel): context_window: int input_cost_usd_per_MTok: float output_cost_usd_per_MTok: float - chat_client: type[BaseChatModel] | None = Field(exclude=True) + chat_client: type[BaseChatModel] | None = Field(None, exclude=True) __all__ = ("AVAILABLE_MODELS", "ModelCard") diff --git a/pydatalab/src/pydatalab/apps/nmr/models.py b/pydatalab/src/pydatalab/apps/nmr/models.py index 269684c26..73a981b4a 100644 --- a/pydatalab/src/pydatalab/apps/nmr/models.py +++ b/pydatalab/src/pydatalab/apps/nmr/models.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from pydatalab.models.blocks import DataBlockResponse @@ -46,8 +46,7 @@ class NMRMetadata(BaseModel): title: str | None = None """Title or description of the NMR experiment.""" - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") class NMRModel(DataBlockResponse): diff --git a/pydatalab/src/pydatalab/apps/xrd/blocks.py b/pydatalab/src/pydatalab/apps/xrd/blocks.py index 0524904b9..aceb86caf 100644 --- a/pydatalab/src/pydatalab/apps/xrd/blocks.py +++ b/pydatalab/src/pydatalab/apps/xrd/blocks.py @@ -273,7 +273,7 @@ def generate_xrd_plot(self, filenames: list[str | Path] | None = None) -> None: except Exception as exc: warnings.warn(f"Could not parse file {f['location']} as XRD data. Error: {exc}") continue - peak_information[str(f["immutable_id"])] = PeakInformation(**peak_data).dict() + peak_information[str(f["immutable_id"])] = PeakInformation(**peak_data).model_dump() pattern_df["normalized intensity (staggered)"] += ind pattern_dfs.append(pattern_df) @@ -302,7 +302,9 @@ def generate_xrd_plot(self, filenames: list[str | Path] | None = None) -> None: peak_model = PeakInformation(**peak_data) if self.data.get("computed") is None: self.data["computed"] = {"peak_data": {}} - self.data["computed"]["peak_data"][str(file_info["immutable_id"])] = peak_model.dict() + self.data["computed"]["peak_data"][str(file_info["immutable_id"])] = ( + peak_model.model_dump() + ) pattern_dfs = [pattern_df] else: @@ -321,7 +323,7 @@ def generate_xrd_plot(self, filenames: list[str | Path] | None = None) -> None: peak_model = PeakInformation(**peak_data) if self.data.get("computed") is None: self.data["computed"] = {"peak_data": {}} - self.data["computed"]["peak_data"][f] = peak_model.dict() + self.data["computed"]["peak_data"][f] = peak_model.model_dump() pattern_dfs = [pattern_df] if pattern_dfs: diff --git a/pydatalab/src/pydatalab/blocks/base.py b/pydatalab/src/pydatalab/blocks/base.py index 258afd826..374f0d443 100644 --- a/pydatalab/src/pydatalab/blocks/base.py +++ b/pydatalab/src/pydatalab/blocks/base.py @@ -209,7 +209,7 @@ def to_db(self) -> dict: for (f, s) in self.block_db_model.schema()["properties"].items() if s.get("datalab_exclude_from_db") } - return self.block_db_model(**self.data).dict( + return self.block_db_model(**self.data).model_dump( exclude=exclude_fields, exclude_unset=True, exclude_none=True, @@ -261,7 +261,7 @@ def to_web(self) -> dict[str, Any]: else: self.data.pop("warnings", None) - return self.block_db_model(**self.data).dict(exclude_unset=True, exclude_none=True) + return self.block_db_model(**self.data).model_dump(exclude_unset=True, exclude_none=True) def process_events(self, events: list[dict] | dict): """Handle any supported events passed to the block.""" @@ -357,5 +357,5 @@ def update_from_web(self, data: dict): if s.get("datalab_exclude_from_load") } [data.pop(f, None) for f in exclude_fields] - self.data.update(self.block_db_model(**data).dict()) + self.data.update(self.block_db_model(**data).model_dump(exclude_unset=True)) return self From b44a3874b178abf9c36d384951d202a2227ddfa0 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 20:34:24 +0100 Subject: [PATCH 10/24] More model updates More updates for models Fixes for models Simplify item versioning model Remove unecessary by_alias=True --- pydatalab/schemas/cell.json | 46 +++++++++++----- pydatalab/schemas/equipment.json | 50 ++++++++++++----- pydatalab/schemas/sample.json | 46 +++++++++++----- pydatalab/schemas/startingmaterial.json | 54 +++++++++++++------ pydatalab/src/pydatalab/models/blocks.py | 6 ++- pydatalab/src/pydatalab/models/equipment.py | 2 +- pydatalab/src/pydatalab/models/export_task.py | 5 +- pydatalab/src/pydatalab/models/files.py | 4 +- .../pydatalab/models/starting_materials.py | 6 +-- pydatalab/src/pydatalab/models/tasks.py | 12 ++--- pydatalab/src/pydatalab/models/traits.py | 4 +- pydatalab/src/pydatalab/models/versions.py | 52 +++++------------- pydatalab/src/pydatalab/mongo.py | 2 +- pydatalab/src/pydatalab/routes/v0_1/items.py | 7 +-- pydatalab/src/pydatalab/versioning.py | 4 +- 15 files changed, 179 insertions(+), 121 deletions(-) diff --git a/pydatalab/schemas/cell.json b/pydatalab/schemas/cell.json index 771d40fb0..c66e9dcb3 100644 --- a/pydatalab/schemas/cell.json +++ b/pydatalab/schemas/cell.json @@ -84,14 +84,15 @@ "additionalProperties": true, "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "collection_id": { + "immutable_id": { "description": "The immutable ID of the collection.", - "title": "Collection Id", + "format": "objectid", + "title": "Immutable Id", "type": "string" } }, "required": [ - "collection_id" + "immutable_id" ], "title": "CollectionReference", "type": "object" @@ -239,6 +240,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", "title": "B64 Encoded Image" }, @@ -254,6 +256,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "A JSON-encoded string containing the Bokeh plot data, if any.", "title": "Bokeh Plot Data" }, @@ -272,6 +275,20 @@ "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", "title": "Computed" }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, "metadata": { "anyOf": [ { @@ -290,9 +307,7 @@ }, "required": [ "blocktype", - "block_id", - "b64_encoded_image", - "bokeh_plot_data" + "block_id" ], "title": "DataBlockResponse", "type": "object" @@ -609,10 +624,18 @@ "title": "Source" }, "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The timestamp for the original file upload.", - "format": "datetime", - "title": "Time Added", - "type": "string" + "title": "Time Added" }, "metadata": { "anyOf": [ @@ -678,15 +701,14 @@ "type": "null" } ], + "default": null, "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ "name", "extension", - "time_added", - "is_live", - "checksums" + "is_live" ], "title": "File", "type": "object" diff --git a/pydatalab/schemas/equipment.json b/pydatalab/schemas/equipment.json index 5482268c6..e0312f30a 100644 --- a/pydatalab/schemas/equipment.json +++ b/pydatalab/schemas/equipment.json @@ -14,14 +14,15 @@ "additionalProperties": true, "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "collection_id": { + "immutable_id": { "description": "The immutable ID of the collection.", - "title": "Collection Id", + "format": "objectid", + "title": "Immutable Id", "type": "string" } }, "required": [ - "collection_id" + "immutable_id" ], "title": "CollectionReference", "type": "object" @@ -169,6 +170,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", "title": "B64 Encoded Image" }, @@ -184,6 +186,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "A JSON-encoded string containing the Bokeh plot data, if any.", "title": "Bokeh Plot Data" }, @@ -202,6 +205,20 @@ "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", "title": "Computed" }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, "metadata": { "anyOf": [ { @@ -220,9 +237,7 @@ }, "required": [ "blocktype", - "block_id", - "b64_encoded_image", - "bokeh_plot_data" + "block_id" ], "title": "DataBlockResponse", "type": "object" @@ -470,10 +485,18 @@ "title": "Source" }, "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The timestamp for the original file upload.", - "format": "datetime", - "title": "Time Added", - "type": "string" + "title": "Time Added" }, "metadata": { "anyOf": [ @@ -539,15 +562,14 @@ "type": "null" } ], + "default": null, "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ "name", "extension", - "time_added", - "is_live", - "checksums" + "is_live" ], "title": "File", "type": "object" @@ -1306,13 +1328,13 @@ "type": "null" } ], + "default": null, "description": "Contact information for equipment (e.g., email address or phone number).", "title": "Contact" } }, "required": [ - "item_id", - "contact" + "item_id" ], "title": "Equipment", "type": "object" diff --git a/pydatalab/schemas/sample.json b/pydatalab/schemas/sample.json index 57b0fb8ce..2d85b667b 100644 --- a/pydatalab/schemas/sample.json +++ b/pydatalab/schemas/sample.json @@ -14,14 +14,15 @@ "additionalProperties": true, "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "collection_id": { + "immutable_id": { "description": "The immutable ID of the collection.", - "title": "Collection Id", + "format": "objectid", + "title": "Immutable Id", "type": "string" } }, "required": [ - "collection_id" + "immutable_id" ], "title": "CollectionReference", "type": "object" @@ -211,6 +212,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", "title": "B64 Encoded Image" }, @@ -226,6 +228,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "A JSON-encoded string containing the Bokeh plot data, if any.", "title": "Bokeh Plot Data" }, @@ -244,6 +247,20 @@ "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", "title": "Computed" }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, "metadata": { "anyOf": [ { @@ -262,9 +279,7 @@ }, "required": [ "blocktype", - "block_id", - "b64_encoded_image", - "bokeh_plot_data" + "block_id" ], "title": "DataBlockResponse", "type": "object" @@ -581,10 +596,18 @@ "title": "Source" }, "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The timestamp for the original file upload.", - "format": "datetime", - "title": "Time Added", - "type": "string" + "title": "Time Added" }, "metadata": { "anyOf": [ @@ -650,15 +673,14 @@ "type": "null" } ], + "default": null, "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ "name", "extension", - "time_added", - "is_live", - "checksums" + "is_live" ], "title": "File", "type": "object" diff --git a/pydatalab/schemas/startingmaterial.json b/pydatalab/schemas/startingmaterial.json index 40ea9e4c1..a096ffb55 100644 --- a/pydatalab/schemas/startingmaterial.json +++ b/pydatalab/schemas/startingmaterial.json @@ -14,14 +14,15 @@ "additionalProperties": true, "description": "A reference to a collection, used for inlining collection info within other models.", "properties": { - "collection_id": { + "immutable_id": { "description": "The immutable ID of the collection.", - "title": "Collection Id", + "format": "objectid", + "title": "Immutable Id", "type": "string" } }, "required": [ - "collection_id" + "immutable_id" ], "title": "CollectionReference", "type": "object" @@ -211,6 +212,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "Any base64-encoded image data associated with the block, keyed by `file_id`.", "title": "B64 Encoded Image" }, @@ -226,6 +228,7 @@ ], "datalab_exclude_from_db": true, "datalab_exclude_from_load": true, + "default": null, "description": "A JSON-encoded string containing the Bokeh plot data, if any.", "title": "Bokeh Plot Data" }, @@ -244,6 +247,20 @@ "description": "Any processed or computed data associated with the block, small enough to store and filter directly in the database,\ni.e., strings or a few hundred numbers not exceeding 16KB in size.\nExamples could include peak positions, and widths, but not the full spectrum.", "title": "Computed" }, + "processed": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "datalab_exclude_from_load": true, + "default": null, + "title": "Processed" + }, "metadata": { "anyOf": [ { @@ -262,9 +279,7 @@ }, "required": [ "blocktype", - "block_id", - "b64_encoded_image", - "bokeh_plot_data" + "block_id" ], "title": "DataBlockResponse", "type": "object" @@ -581,10 +596,18 @@ "title": "Source" }, "time_added": { + "anyOf": [ + { + "format": "datetime", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, "description": "The timestamp for the original file upload.", - "format": "datetime", - "title": "Time Added", - "type": "string" + "title": "Time Added" }, "metadata": { "anyOf": [ @@ -650,15 +673,14 @@ "type": "null" } ], + "default": null, "description": "Content checksums (MD5 and SHA-256) of the file." } }, "required": [ "name", "extension", - "time_added", - "is_live", - "checksums" + "is_live" ], "title": "File", "type": "object" @@ -1498,6 +1520,7 @@ "type": "null" } ], + "default": null, "description": "The name of the substance in the container.", "title": "Name" }, @@ -1567,6 +1590,7 @@ "type": "null" } ], + "default": null, "description": "The chemical purity of this container with regards to the defined substance.", "title": "Chemical Purity" }, @@ -1618,6 +1642,7 @@ "type": "null" } ], + "default": null, "description": "Supplier or manufacturer of the chemical.", "title": "Supplier" }, @@ -1649,10 +1674,7 @@ } }, "required": [ - "item_id", - "name", - "chemical_purity", - "supplier" + "item_id" ], "title": "StartingMaterial", "type": "object" diff --git a/pydatalab/src/pydatalab/models/blocks.py b/pydatalab/src/pydatalab/models/blocks.py index a89af0ffc..f1e02a4d4 100644 --- a/pydatalab/src/pydatalab/models/blocks.py +++ b/pydatalab/src/pydatalab/models/blocks.py @@ -44,12 +44,12 @@ class DataBlockResponse(BaseModel): """Any warnings that occurred during block processing.""" b64_encoded_image: dict[str, str] | None = Field( - datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, datalab_exclude_from_db=True, datalab_exclude_from_load=True ) """Any base64-encoded image data associated with the block, keyed by `file_id`.""" bokeh_plot_data: dict | None = Field( - datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, datalab_exclude_from_db=True, datalab_exclude_from_load=True ) """A JSON-encoded string containing the Bokeh plot data, if any.""" @@ -59,6 +59,8 @@ class DataBlockResponse(BaseModel): Examples could include peak positions, and widths, but not the full spectrum. """ + processed: dict | None = Field(default=None, datalab_exclude_from_load=True) + metadata: dict | None = Field(default=None, datalab_exclude_from_load=True) """Any structured metadata associated with the block, for example, experimental acquisition parameters.""" diff --git a/pydatalab/src/pydatalab/models/equipment.py b/pydatalab/src/pydatalab/models/equipment.py index 820961905..fb2dbd9d5 100644 --- a/pydatalab/src/pydatalab/models/equipment.py +++ b/pydatalab/src/pydatalab/models/equipment.py @@ -22,7 +22,7 @@ class Equipment(Item): location: str | None = None """Place where the equipment is located""" - contact: str | None + contact: str | None = None """Contact information for equipment (e.g., email address or phone number).""" status: EquipmentStatus = Field(default=EquipmentStatus.WORKING) diff --git a/pydatalab/src/pydatalab/models/export_task.py b/pydatalab/src/pydatalab/models/export_task.py index d2085ffde..e2728888d 100644 --- a/pydatalab/src/pydatalab/models/export_task.py +++ b/pydatalab/src/pydatalab/models/export_task.py @@ -1,7 +1,7 @@ from datetime import datetime, timezone from enum import Enum -from pydantic import Field +from pydantic import ConfigDict, Field from pydatalab.models.utils import BaseModel, PyObjectId @@ -48,5 +48,4 @@ class ExportTask(BaseModel): error_message: str | None = None """Error message if status is ERROR""" - class Config: - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) diff --git a/pydatalab/src/pydatalab/models/files.py b/pydatalab/src/pydatalab/models/files.py index ca17eadd5..898b5f986 100644 --- a/pydatalab/src/pydatalab/models/files.py +++ b/pydatalab/src/pydatalab/models/files.py @@ -52,7 +52,7 @@ class File(Entry, HasOwner, HasRevisionControl): source: str | None = None """The source of the file, e.g. 'remote' or 'uploaded'.""" - time_added: IsoformatDateTime + time_added: IsoformatDateTime | None = None """The timestamp for the original file upload.""" metadata: dict[Any, Any] | None = None @@ -69,5 +69,5 @@ class File(Entry, HasOwner, HasRevisionControl): is_live: bool """Whether or not the file should be watched for future updates.""" - checksums: FileChecksums | None + checksums: FileChecksums | None = None """Content checksums (MD5 and SHA-256) of the file.""" diff --git a/pydatalab/src/pydatalab/models/starting_materials.py b/pydatalab/src/pydatalab/models/starting_materials.py index 845bd6e5a..8bbdad870 100644 --- a/pydatalab/src/pydatalab/models/starting_materials.py +++ b/pydatalab/src/pydatalab/models/starting_materials.py @@ -24,13 +24,13 @@ class StartingMaterial(Item, HasSynthesisInfo, HasSubstanceInfo): date_opened: IsoformatDateTime | None = Field(None, alias="Date opened") """The date the item was opened""" - chemical_purity: str | None = Field(alias="Chemical purity") + chemical_purity: str | None = Field(None, alias="Chemical purity") """The chemical purity of this container with regards to the defined substance.""" full_percent: str | None = Field(None, alias="Full %") """The amount of the defined substance remaining in the container, expressed as a percentage.""" - name: str | None = Field(alias="Container Name") + name: str | None = Field(None, alias="Container Name") """The name of the substance in the container.""" size: str | None = Field(None, alias="Container Size") @@ -39,7 +39,7 @@ class StartingMaterial(Item, HasSynthesisInfo, HasSubstanceInfo): size_unit: str | None = Field(None, alias="Unit") """Units for the 'size' field.""" - supplier: str | None = Field(alias="Supplier") + supplier: str | None = Field(None, alias="Supplier") """Supplier or manufacturer of the chemical.""" location: str | None = Field(None, alias="Location") diff --git a/pydatalab/src/pydatalab/models/tasks.py b/pydatalab/src/pydatalab/models/tasks.py index 14728df26..ab21d39c8 100644 --- a/pydatalab/src/pydatalab/models/tasks.py +++ b/pydatalab/src/pydatalab/models/tasks.py @@ -2,7 +2,7 @@ from enum import Enum from typing import Literal -from pydantic import Field, validator +from pydantic import ConfigDict, Field, field_validator from pydatalab.models.utils import BaseModel, PyObjectId @@ -90,9 +90,10 @@ class Task(BaseModel): spec: ExportTaskSpec | BlockProcessingTaskSpec """Task-specific data""" - @validator("spec", pre=True, always=True) - def validate_spec_type(cls, v, values): - task_type = values.get("type") + @field_validator("spec", mode="before") + @classmethod + def validate_spec_type(cls, v, info): + task_type = info.data.get("type") if task_type == TaskType.EXPORT: if not isinstance(v, ExportTaskSpec): @@ -103,5 +104,4 @@ def validate_spec_type(cls, v, values): return v - class Config: - use_enum_values = True + model_config = ConfigDict(use_enum_values=True) diff --git a/pydatalab/src/pydatalab/models/traits.py b/pydatalab/src/pydatalab/models/traits.py index 8cc0f9deb..b7805c984 100644 --- a/pydatalab/src/pydatalab/models/traits.py +++ b/pydatalab/src/pydatalab/models/traits.py @@ -52,9 +52,9 @@ class HasBlocks(BaseModel): class CollectionReference(BaseModel): """A reference to a collection, used for inlining collection info within other models.""" - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", populate_by_name=True) - collection_id: str + immutable_id: PyObjectId = Field(alias="_id") """The immutable ID of the collection.""" diff --git a/pydatalab/src/pydatalab/models/versions.py b/pydatalab/src/pydatalab/models/versions.py index 6efce3a2c..4063ebdc7 100644 --- a/pydatalab/src/pydatalab/models/versions.py +++ b/pydatalab/src/pydatalab/models/versions.py @@ -3,7 +3,7 @@ from datetime import datetime from enum import Enum -from pydantic import Field, validator +from pydantic import ConfigDict, Field, model_validator from pydatalab.models.utils import BaseModel, PyObjectId, Refcode @@ -56,17 +56,16 @@ class ItemVersion(BaseModel): user_agent: str | None = None """User agent string of the client that triggered this version. Will only be stored if it matches a known value from the datalab ecosystem.""" - @validator("restored_from_version") - def validate_restored_from_version(cls, v, values): + @model_validator(mode="after") + def validate_restored_from_version(self): """Ensure restored_from_version is only present when action='restored'.""" - action = values.get("action") - if action == VersionAction.RESTORED and v is None: + if self.action == VersionAction.RESTORED and self.restored_from_version is None: raise ValueError("restored_from_version must be provided when action='restored'") - if action != VersionAction.RESTORED and v is not None: + if self.action != VersionAction.RESTORED and self.restored_from_version is not None: raise ValueError( - f"restored_from_version should only be present when action='restored', got action='{action}'" + f"restored_from_version should only be present when action='restored', got action='{self.action}'" ) - return v + return self class VersionCounter(BaseModel): @@ -82,50 +81,25 @@ class VersionCounter(BaseModel): counter: int = Field(1, ge=1) """Current version counter value (1-indexed, matches version numbers)""" - class Config: - extra = "ignore" # Allow MongoDB's _id field and other internal fields + model_config = ConfigDict(extra="ignore") class RestoreVersionRequest(BaseModel): """Request body for restoring a version.""" - version_id: str + version_id: PyObjectId """ObjectId string of the version to restore to""" - @validator("version_id") - def validate_version_id_format(cls, v): - """Validate that version_id is a valid ObjectId string.""" - try: - from bson import ObjectId - - ObjectId(v) - except Exception as e: - raise ValueError(f"version_id must be a valid ObjectId string: {e}") - return v - - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") class CompareVersionsQuery(BaseModel): """Query parameters for comparing two versions.""" - v1: str + v1: PyObjectId """ObjectId string of the first version""" - v2: str + v2: PyObjectId """ObjectId string of the second version""" - @validator("v1", "v2") - def validate_version_ids(cls, v): - """Validate that version IDs are valid ObjectId strings.""" - try: - from bson import ObjectId - - ObjectId(v) - except Exception as e: - raise ValueError(f"Version ID must be a valid ObjectId string: {e}") - return v - - class Config: - extra = "forbid" + model_config = ConfigDict(extra="forbid") diff --git a/pydatalab/src/pydatalab/mongo.py b/pydatalab/src/pydatalab/mongo.py index 022b9af92..fb9bb7fa5 100644 --- a/pydatalab/src/pydatalab/mongo.py +++ b/pydatalab/src/pydatalab/mongo.py @@ -163,7 +163,7 @@ def insert_pydantic_model_fork_safe(model: BaseModel, collection: str) -> str: """Inserts a Pydantic model into chosen collection, returning the inserted ID.""" return ( get_database()[collection] - .insert_one(model.model_dump(by_alias=True, exclude_none=True)) + .insert_one(model.model_dump(by_alias=False, exclude_none=True)) .inserted_id ) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 47a693dd9..e9057b9c1 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -716,9 +716,7 @@ def _create_sample( # the `Entry` model. try: result = flask_mongo.db.items.insert_one( - data_model.model_dump( - exclude={"creators", "collections", "groups"}, exclude_none=True, by_alias=True - ) + data_model.model_dump(exclude={"creators", "collections", "groups"}, exclude_none=True) ) except DuplicateKeyError as error: raise Conflict(f"Duplicate key error: {str(error)}.") @@ -1557,7 +1555,7 @@ def restore_version(refcode): # Insert validated data flask_mongo.db.item_versions.insert_one( - validated_restored_version.model_dump(by_alias=True, exclude_none=True) + validated_restored_version.model_dump(exclude_none=True) ) return jsonify( @@ -1794,7 +1792,6 @@ def save_item(): item = ITEM_MODELS[item_type](**item).model_dump( exclude_none=True, exclude_unset=True, - by_alias=True, exclude={"collections", "creators", "immutable_id"}, ) diff --git a/pydatalab/src/pydatalab/versioning.py b/pydatalab/src/pydatalab/versioning.py index 1bdf8568d..13433bb25 100644 --- a/pydatalab/src/pydatalab/versioning.py +++ b/pydatalab/src/pydatalab/versioning.py @@ -192,9 +192,7 @@ def save_version_snapshot( ) # Insert validated data (convert to dict and exclude None values) - flask_mongo.db.item_versions.insert_one( - validated_version.dict(by_alias=True, exclude_none=True) - ) + flask_mongo.db.item_versions.insert_one(validated_version.model_dump(exclude_none=True)) return ( {"status": "success", "message": "Version saved.", "version": next_version_number}, 200, From b086e5598656003aa4a03ad28c3e0d6444ca7854 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 20:57:45 +0100 Subject: [PATCH 11/24] Minimal test updates --- pydatalab/tests/server/conftest.py | 22 +++---- pydatalab/tests/server/test_item_versions.py | 48 ++------------- pydatalab/tests/server/test_samples.py | 22 +++---- pydatalab/tests/test_models.py | 61 +++++++++++++++----- 4 files changed, 72 insertions(+), 81 deletions(-) diff --git a/pydatalab/tests/server/conftest.py b/pydatalab/tests/server/conftest.py index 4cc1c7938..b00855b89 100644 --- a/pydatalab/tests/server/conftest.py +++ b/pydatalab/tests/server/conftest.py @@ -479,7 +479,7 @@ def fixture_insert_complicated_sample_constituents(user_id): creator_ids=[user_id], refcode=generate_unique_refcode(), ) - flask_mongo.db.items.insert_one(sm.dict(exclude_unset=False)) + flask_mongo.db.items.insert_one(sm.model_dump(exclude_unset=False)) items.append(sm) yield items @@ -492,7 +492,7 @@ def fixture_insert_complicated_sample_constituents(user_id): def example_items(user_id, admin_user_id): """Create a collection of samples with mixed ownership between the user and admin.""" return [ - d.dict(exclude_unset=False) + d.model_dump(exclude_unset=False) for d in [ Sample( **{ @@ -579,23 +579,23 @@ def example_items(user_id, admin_user_id): @pytest.fixture(scope="module", name="default_sample_dict") -def fixture_default_sample_dict(default_sample): - return default_sample.dict(exclude_unset=True) +def fixture_default_sample_model_dump(default_sample): + return default_sample.model_dump(exclude_unset=True) @pytest.fixture(scope="module", name="default_cell_dict") -def fixture_default_cell_dict(default_cell): - return default_cell.dict(exclude_unset=True) +def fixture_default_cell_model_dump(default_cell): + return default_cell.model_dump(exclude_unset=True) @pytest.fixture(scope="module", name="default_starting_material_dict") -def fixture_default_starting_material_dict(default_starting_material): - return default_starting_material.dict(exclude_unset=True) +def fixture_default_starting_material_model_dump(default_starting_material): + return default_starting_material.model_dump(exclude_unset=True) @pytest.fixture(scope="module", name="default_equipment_dict") -def fixture_default_equipment_dict(default_equipment): - return default_equipment.dict(exclude_unset=True) +def fixture_default_equipment_model_dump(default_equipment): + return default_equipment.model_dump(exclude_unset=True) def _insert_and_cleanup_item_from_model(model): @@ -604,7 +604,7 @@ def _insert_and_cleanup_item_from_model(model): refcode = generate_unique_refcode() model.refcode = refcode - flask_mongo.db.items.insert_one(model.dict(exclude_unset=False)) + flask_mongo.db.items.insert_one(model.model_dump(exclude_unset=False)) yield model flask_mongo.db.items.delete_one({"refcode": model.refcode}) diff --git a/pydatalab/tests/server/test_item_versions.py b/pydatalab/tests/server/test_item_versions.py index 1c0bd6c63..2356177a5 100644 --- a/pydatalab/tests/server/test_item_versions.py +++ b/pydatalab/tests/server/test_item_versions.py @@ -88,7 +88,6 @@ def test_save_version_nonexistent_item(self, client): response = client.post("/items/nonexistent/save-version/") assert response.status_code == 404 - assert response.json["status"] == "error" class TestListVersions: @@ -198,8 +197,6 @@ def test_get_version_nonexistent(self, client, sample_with_version): response = client.get(f"/items/{refcode}/versions/{fake_id}/") assert response.status_code == 404 - assert response.json["status"] == "error" - assert "not found" in response.json["message"].lower() class TestCompareVersions: @@ -248,24 +245,12 @@ def test_compare_versions_missing_parameters(self, client, sample_with_version): # Missing v2 - request.args.get() returns "" for missing params, which fails ObjectId validation response = client.get(f"/items/{refcode}/compare-versions/?v1=some_id") assert response.status_code == 400 - assert response.json["message"] == "Invalid query parameters" - assert "errors" in response.json - errors = response.json["errors"] - # Should have error for v2 (empty string is invalid ObjectId) - v2_errors = [e for e in errors if "v2" in str(e["loc"])] - assert len(v2_errors) == 1 - assert "valid objectid" in v2_errors[0]["msg"].lower() + assert "Invalid query parameters" in response.json["message"] # Missing v1 - same behavior response = client.get(f"/items/{refcode}/compare-versions/?v2=some_id") assert response.status_code == 400 - assert response.json["message"] == "Invalid query parameters" - assert "errors" in response.json - errors = response.json["errors"] - # Should have error for v1 (empty string is invalid ObjectId) - v1_errors = [e for e in errors if "v1" in str(e["loc"])] - assert len(v1_errors) == 1 - assert "valid objectid" in v1_errors[0]["msg"].lower() + assert "Invalid query parameters" in response.json["message"] def test_compare_versions_invalid_id(self, client, sample_with_version): """Test comparing versions with invalid ID format.""" @@ -273,15 +258,7 @@ def test_compare_versions_invalid_id(self, client, sample_with_version): response = client.get(f"/items/{refcode}/compare-versions/?v1=invalid&v2=invalid") assert response.status_code == 400 - assert response.json["message"] == "Invalid query parameters" - # Check Pydantic's structured error response - assert "errors" in response.json - errors = response.json["errors"] - # Should have errors for both v1 and v2 - assert len(errors) == 2 - for error in errors: - assert error["loc"][0] in ["v1", "v2"] - assert "valid ObjectId" in error["msg"] + assert "Invalid query parameters" in response.json["message"] def test_compare_versions_detects_changes(self, client, sample_with_version): """Test that compare_versions properly detects changes using DeepDiff.""" @@ -449,13 +426,7 @@ def test_restore_version_missing_version_id(self, client, sample_with_version): response = client.post(f"/items/{refcode}/restore-version/", json={}) assert response.status_code == 400 - assert response.json["message"] == "Invalid request body" - # Check Pydantic's structured error response - assert "errors" in response.json - errors = response.json["errors"] - assert len(errors) == 1 - assert errors[0]["loc"] == ["version_id"] - assert "required" in errors[0]["msg"].lower() + assert "Invalid request body" in response.json["message"] def test_restore_version_invalid_id(self, client, sample_with_version): """Test restoring with invalid version ID.""" @@ -463,13 +434,7 @@ def test_restore_version_invalid_id(self, client, sample_with_version): response = client.post(f"/items/{refcode}/restore-version/", json={"version_id": "invalid"}) assert response.status_code == 400 - assert response.json["message"] == "Invalid request body" - # Check Pydantic's structured error response - assert "errors" in response.json - errors = response.json["errors"] - assert len(errors) == 1 - assert errors[0]["loc"] == ["version_id"] - assert "valid ObjectId" in errors[0]["msg"] + assert "Invalid request body" in response.json["message"] def test_restore_version_nonexistent(self, client, sample_with_version): """Test restoring non-existent version.""" @@ -478,7 +443,6 @@ def test_restore_version_nonexistent(self, client, sample_with_version): response = client.post(f"/items/{refcode}/restore-version/", json={"version_id": fake_id}) assert response.status_code == 404 - assert "not found" in response.json["message"].lower() def test_restore_version_increments_version_number(self, client, sample_with_version): """Test that restore increments the item version number.""" @@ -541,7 +505,6 @@ def test_delete_version_nonexistent(self, client, sample_with_version): response = client.delete(f"/items/{refcode}/versions/{fake_id}/") assert response.status_code == 404 - assert response.json["status"] == "error" def test_delete_version_invalid_id(self, client, sample_with_version): """Test deleting with invalid ID format.""" @@ -921,7 +884,6 @@ def test_restore_requires_write_permission(self, another_client, sample_with_ver # Should fail because another_client doesn't have access to this item assert response.status_code == 404 - assert "not found or insufficient permissions" in response.json["message"].lower() class TestEdgeCases: diff --git a/pydatalab/tests/server/test_samples.py b/pydatalab/tests/server/test_samples.py index 925e83dc6..f4173556f 100644 --- a/pydatalab/tests/server/test_samples.py +++ b/pydatalab/tests/server/test_samples.py @@ -540,8 +540,8 @@ def test_create_cell(client, default_cell): assert cell["electrolyte"][1]["item"]["chemform"] == "NaCl" assert ( - cell["positive_electrode"][0]["item"]["name"] - == default_cell.positive_electrode[0].item.name + cell["positive_electrode"][0]["item"]["item_id"] + == default_cell.positive_electrode[0].item.item_id ) # The anode has a "real" entry in the db, so it should be looked up properly # with dereferenced chemical formula and name, even if it was not provided at link time @@ -840,7 +840,7 @@ def test_remove_items_from_collection_success( response = client.post("/new-sample/", json=sample_dict) assert response.status_code == 201 - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_remove" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -912,7 +912,7 @@ def test_remove_items_from_collection_not_found(client): @pytest.mark.dependency() def test_remove_items_from_collection_no_items_provided(client, default_collection): """Test removing with no item IDs provided.""" - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_empty_items" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -928,7 +928,7 @@ def test_remove_items_from_collection_no_items_provided(client, default_collecti @pytest.mark.dependency() def test_remove_items_from_collection_no_matching_items(client, default_collection): """Test removing items that don't exist.""" - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_no_match" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -957,7 +957,7 @@ def test_remove_items_from_collection_partial_success( response = client.post("/new-sample/", json=sample_dict) assert response.status_code == 201 - collection_dict = default_collection.dict() + collection_dict = default_collection.model_dump() collection_dict["collection_id"] = "test_collection_partial" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1016,7 +1016,7 @@ def test_copy_sample_and_add_to_collection(client, default_sample_dict, default_ assert response.status_code == 201 assert response.json["status"] == "success" - collection_dict = default_collection.dict().copy() + collection_dict = default_collection.model_dump().copy() collection_dict["collection_id"] = "test_copy_collection" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1048,12 +1048,12 @@ def test_copy_sample_and_add_to_collection(client, default_sample_dict, default_ def test_copy_sample_from_collection_to_different_collection( client, default_sample_dict, default_collection ): - collection1_dict = default_collection.dict().copy() + collection1_dict = default_collection.model_dump().copy() collection1_dict["collection_id"] = "collection_1" response = client.put("/collections", json={"data": collection1_dict}) assert response.status_code == 201 - collection2_dict = default_collection.dict().copy() + collection2_dict = default_collection.model_dump().copy() collection2_dict["collection_id"] = "collection_2" response = client.put("/collections", json={"data": collection2_dict}) assert response.status_code == 201 @@ -1093,7 +1093,7 @@ def test_copy_sample_from_collection_to_different_collection( @pytest.mark.dependency(depends=["test_copy_sample_from_collection_to_different_collection"]) def test_copy_sample_without_copying_collections(client, default_sample_dict, default_collection): - collection_dict = default_collection.dict().copy() + collection_dict = default_collection.model_dump().copy() collection_dict["collection_id"] = "test_no_auto_copy_collection" response = client.put("/collections", json={"data": collection_dict}) assert response.status_code == 201 @@ -1133,7 +1133,7 @@ def test_collections_permissions(client, admin_client, default_sample_dict, defa assert response.json["status"] == "success" refcode = response.json["sample_list_entry"]["refcode"] - admin_collection = default_collection.dict().copy() + admin_collection = default_collection.model_dump().copy() admin_collection["collection_id"] = "admin_only_collection" response = admin_client.put("/collections", json={"data": admin_collection}) diff --git a/pydatalab/tests/test_models.py b/pydatalab/tests/test_models.py index c2ba5959b..c92dbb7ce 100644 --- a/pydatalab/tests/test_models.py +++ b/pydatalab/tests/test_models.py @@ -81,7 +81,7 @@ def test_sample_with_inlined_reference(): @pytest.mark.parametrize("model", ITEM_MODELS.values()) def test_generate_schemas(model): """Test that all item model schemas can be generated.""" - assert model.schema() + assert model.model_json_schema() def test_attribute_docstrings_in_schema(): @@ -187,6 +187,9 @@ def test_file(): def test_custom_and_inherited_items(): class TestItem(Item): type: str = "items_custom" + new_field: str + + TestItem.model_rebuild() item = TestItem( type="items_custom", @@ -195,9 +198,10 @@ class TestItem(Item): creators=None, date="2020-01-01 00:00", item_id="1234", + new_field="This is a new field", ) - item_dict = item.dict() + item_dict = item.model_dump() assert item_dict["type"] == "items_custom" assert item_dict["creator_ids"][0] == ObjectId("0123456789ab0123456789ab") assert item_dict["creator_ids"][1] == ObjectId("1023456789ab0123456789ab") @@ -207,8 +211,8 @@ class TestItem(Item): item_json = json.loads(item.json()) assert item_json["type"] == "items_custom" - assert item_json["creator_ids"][0] == str(ObjectId("0123456789ab0123456789ab")) - assert item_json["creator_ids"][1] == str(ObjectId("1023456789ab0123456789ab")) + assert item_json["creator_ids"][0] == "0123456789ab0123456789ab" + assert item_json["creator_ids"][1] == "1023456789ab0123456789ab" assert ( item_json["date"] == datetime.datetime.fromisoformat("2020-01-01 00:00") @@ -224,7 +228,7 @@ class TestItem(Item): item_id="1234", ) - sample_dict = sample.dict() + sample_dict = sample.model_dump() assert sample_dict["type"] == "samples" assert sample_dict["creator_ids"][0] == ObjectId("0123456789ab0123456789ab") assert sample_dict["creator_ids"][1] == ObjectId("1023456789ab0123456789ab") @@ -261,13 +265,17 @@ class TestItem(Item): "MP2018_TEST_COMMERCIAL", "MP2018_TEST_COMMERCIAL_4.5V_hold", "AAAAAA", - 111111111, + "111111111", ], ) def test_good_ids(id): """Test good human-readable IDs for validity.""" - assert HumanReadableIdentifier(id) + class TestModel(pydantic.BaseModel): + test_id: HumanReadableIdentifier + + model = TestModel(test_id=id) + assert model.test_id == id @pytest.mark.parametrize( @@ -277,6 +285,7 @@ def test_good_ids(id): "mp 1 2 3 4 5 6", "lithium & sodium", "me388-123456789-123456789-really-long-descriptive-identifier-that-should-be-the-name-but-is-otherwise-valid", + 111111111, 1111111111111111111111111111111111111111111111111, "_AAAA", "AAA_", @@ -288,8 +297,11 @@ def test_good_ids(id): def test_bad_ids(id): """Test bad human-readable IDs for invalidity.""" + class TestModel(pydantic.BaseModel): + test_id: HumanReadableIdentifier + with pytest.raises(pydantic.ValidationError): - HumanReadableIdentifier(id) + TestModel(test_id=id) def test_cell_with_inlined_reference(): @@ -312,7 +324,7 @@ def test_cell_with_inlined_reference(): assert cell assert len(cell.relationships) == 1 - cell = Cell(**json.loads(cell.json())) + cell = Cell(**json.loads(cell.model_dump_json())) assert cell assert len(cell.relationships) == 1 @@ -432,7 +444,7 @@ def test_cell_relationship_deduplication(): assert parthood[0].item_id == "test_cathode" # Re-validating an already-clean cell must not grow the relationships list. - cell = Cell(**json.loads(cell.json())) + cell = Cell(**json.loads(cell.model_dump_json())) parthood = [r for r in cell.relationships if r.relation == RelationshipType.PARTHOOD] assert len(parthood) == 1 assert parthood[0].refcode == "grey:ABCDEF" @@ -527,7 +539,7 @@ def test_sample_synthesis_relationship_deduplication(): assert parents[0].item_id == "sm_1" # Re-validating an already-clean sample must not grow the relationships list. - sample = Sample(**json.loads(sample.json())) + sample = Sample(**json.loads(sample.model_dump_json())) parents = [r for r in sample.relationships if r.relation == RelationshipType.PARENT] assert len(parents) == 1 assert parents[0].refcode == "grey:ABCDEF" @@ -590,8 +602,11 @@ def test_good_refcodes(refcode): def test_bad_refcodes(refcode): """Test bad refcodes for invalidity.""" + class TestModel(pydantic.BaseModel): + test_refcode: Refcode + with pytest.raises(pydantic.ValidationError): - Refcode(refcode) + TestModel(test_refcode=refcode) @pytest.mark.parametrize( @@ -606,7 +621,10 @@ def test_bad_refcodes(refcode): def test_good_display_name(display_name): """Test good display name for validity.""" - assert DisplayName(display_name) + class TestModel(pydantic.BaseModel): + name: DisplayName + + assert TestModel(name=display_name) @pytest.mark.parametrize( @@ -620,8 +638,11 @@ def test_good_display_name(display_name): def test_bad_display_name(display_name): """Test bad display_name for invalidity.""" + class TestModel(pydantic.BaseModel): + name: DisplayName + with pytest.raises(ValueError): - DisplayName(display_name) + TestModel(name=display_name) @pytest.mark.parametrize( @@ -631,7 +652,11 @@ def test_bad_display_name(display_name): ], ) def test_good_email(contact_email): - assert EmailStr(contact_email) + + class TestModel(pydantic.BaseModel): + email: EmailStr + + assert TestModel(email=contact_email) @pytest.mark.parametrize( @@ -644,5 +669,9 @@ def test_good_email(contact_email): ], ) def test_bad_email(contact_email): + + class TestModel(pydantic.BaseModel): + email: EmailStr + with pytest.raises(ValueError): - assert EmailStr(contact_email) + TestModel(email=contact_email) From eb174645f0f8fc37c721607a4d55c7aa435ab46d Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 22:30:36 +0100 Subject: [PATCH 12/24] Patch optional vs default value for plugin data blocks --- pydatalab/src/pydatalab/models/utils.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/pydatalab/src/pydatalab/models/utils.py b/pydatalab/src/pydatalab/models/utils.py index 96df03119..696ec620c 100644 --- a/pydatalab/src/pydatalab/models/utils.py +++ b/pydatalab/src/pydatalab/models/utils.py @@ -4,7 +4,7 @@ from collections.abc import Callable from enum import Enum from functools import partial -from typing import Annotated, Any, TypeAlias +from typing import Annotated, Any, TypeAlias, get_args import pint from bson import ObjectId @@ -29,6 +29,25 @@ class BaseModel(PydanticBaseModel): model_config = ConfigDict(use_attribute_docstrings=True) + @classmethod + def __pydantic_init_subclass__(cls, **kwargs): + """Restore pydantic v1's implicit `None` default for optional fields. + + In pydantic v2 a field annotated ``T | None`` with no assigned default is + *required*, whereas in v1 it implicitly defaulted to ``None``. This back-fills + a ``None`` default for any required field that allows ``None``, so that models + (including those in plugins not yet migrated to v2) keep their v1 semantics. + Fields that already declare a default are untouched. + """ + super().__pydantic_init_subclass__(**kwargs) + changed = False + for field in cls.model_fields.values(): + if field.is_required() and type(None) in get_args(field.annotation): + field.default = None + changed = True + if changed: + cls.model_rebuild(force=True) + class ItemType(str, Enum): """An enumeration of the types of items known by this implementation, should be made dynamic in the future.""" From d4bf525baf6d1904171a12f699e96d0872c43a7b Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 22:31:50 +0100 Subject: [PATCH 13/24] Simplify error handling of all item routes --- pydatalab/src/pydatalab/routes/v0_1/items.py | 206 +++++-------------- 1 file changed, 46 insertions(+), 160 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index e9057b9c1..b613c73ca 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -11,7 +11,7 @@ from flask_login import current_user from pydantic import ValidationError from pymongo.errors import DuplicateKeyError -from werkzeug.exceptions import BadRequest, Conflict, NotFound +from werkzeug.exceptions import BadRequest, Conflict, InternalServerError, NotFound from pydatalab.apps import BLOCK_TYPES from pydatalab.config import CONFIG @@ -515,7 +515,7 @@ def search_items(): types = types.split(",") if not query: - return jsonify({"status": "error", "message": "No query provided."}), 400 + raise BadRequest("No query provided.") permissions = get_default_permissions(user_only=False) pipeline = build_search_pipeline(query, get_items_fts_fields(), permissions) @@ -783,12 +783,9 @@ def create_samples(): sample_jsons = request_json["new_sample_datas"] if len(sample_jsons) > CONFIG.MAX_BATCH_CREATE_SIZE: - return jsonify( - { - "status": "error", - "message": f"Batch size limit exceeded. Maximum allowed: {CONFIG.MAX_BATCH_CREATE_SIZE}, requested: {len(sample_jsons)}", - } - ), 400 + raise BadRequest( + f"Batch size limit exceeded. Maximum allowed: {CONFIG.MAX_BATCH_CREATE_SIZE}, requested: {len(sample_jsons)}" + ) copy_from_item_ids = request_json.get("copy_from_item_ids") generate_ids_automatically = request_json.get("generate_ids_automatically") @@ -875,13 +872,7 @@ def _process_item_permissions( ] if not groups_requested and not creators_requested: - return ( - { - "status": "error", - "message": "No valid creator or group IDs found in the request.", - }, - 400, - ) + raise BadRequest("No valid creator or group IDs found in the request.") # Validate all creator IDs are present in the database if creator_ids: @@ -889,13 +880,7 @@ def _process_item_permissions( d for d in flask_mongo.db.users.find({"_id": {"$in": creator_ids}}, {"_id": 1}) ] if len(found_creator_ids) != len(creator_ids): - return ( - { - "status": "error", - "message": "One or more creator IDs not found in the database.", - }, - 400, - ) + raise BadRequest("One or more creator IDs not found in the database.") if group_ids: # Validate all group IDs are present in the database @@ -903,13 +888,7 @@ def _process_item_permissions( d for d in flask_mongo.db.groups.find({"_id": {"$in": group_ids}}, {"_id": 1}) ] if len(found_group_ids) != len(group_ids): - return ( - { - "status": "error", - "message": "One or more group IDs not found in the database.", - }, - 400, - ) + raise BadRequest("One or more group IDs not found in the database.") if append_mode: if creators_requested: @@ -971,12 +950,8 @@ def _process_item_permissions( ) if result.modified_count != 1: - return ( - { - "status": "error", - "message": "Failed to update permissions: you cannot remove yourself or the base owner as a creator.", - }, - 400, + raise BadRequest( + "Failed to update permissions: you cannot remove yourself or the base owner as a creator." ) return {"status": "success"}, 200 @@ -1015,12 +990,7 @@ def issue_physical_token(refcode: str): ) if not current_item: - return jsonify( - { - "status": "error", - "message": f"No valid item found with the given {refcode=}.", - } - ), 404 + raise NotFound token = secrets.token_urlsafe(16) access_document = { @@ -1075,15 +1045,7 @@ def delete_sample(): ) if result.deleted_count != 1: - return ( - jsonify( - { - "status": "error", - "message": f"Failed to delete item with {item_id=}.", - } - ), - 400, - ) + raise BadRequest(f"Failed to delete item with {item_id=}.") flask_mongo.db.api_keys.delete_many({"refcode": item["refcode"], "type": "access_token"}) @@ -1151,15 +1113,7 @@ def get_item_data( doc = None if not doc: - return ( - jsonify( - { - "status": "error", - "message": f"No matching items for {match=} with current authorization.", - } - ), - 404, - ) + raise NotFound try: ItemModel = ITEM_MODELS[doc["type"]] @@ -1272,9 +1226,7 @@ def list_versions(refcode): # Check if user has access to the item (read access) has_access, _ = check_version_access(refcode, user_only=False) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1326,9 +1278,7 @@ def get_version(refcode, version_id): # Check if user has access to the item (read access) has_access, _ = check_version_access(refcode, user_only=False) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1336,7 +1286,7 @@ def get_version(refcode, version_id): try: version_object_id = ObjectId(version_id) except (InvalidId, TypeError): - return jsonify({"status": "error", "message": f"Invalid version_id: {version_id}"}), 400 + raise BadRequest(f"Invalid version_id: {version_id}") version = list( flask_mongo.db.item_versions.aggregate( @@ -1363,7 +1313,7 @@ def get_version(refcode, version_id): version = version[0] if not version: - return jsonify({"status": "error", "message": "Version not found"}), 404 + raise NotFound return jsonify({"status": "success", "version": version}), 200 @@ -1377,9 +1327,7 @@ def compare_versions(refcode): # Check if user has access to the item (read access) has_access, _ = check_version_access(refcode, user_only=False) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1390,20 +1338,12 @@ def compare_versions(refcode): v1=request.args.get("v1", ""), v2=request.args.get("v2", "") ) except ValidationError as exc: - return jsonify( - {"status": "error", "message": "Invalid query parameters", "errors": exc.errors()} - ), 400 + raise BadRequest(f"Invalid query parameters: {exc}") - try: - v1_object_id = ObjectId(query_params.v1) - v2_object_id = ObjectId(query_params.v2) - except (InvalidId, TypeError) as e: - return jsonify({"status": "error", "message": f"Invalid version ID format: {str(e)}"}), 400 - - v1 = flask_mongo.db.item_versions.find_one({"_id": v1_object_id, "refcode": refcode}) - v2 = flask_mongo.db.item_versions.find_one({"_id": v2_object_id, "refcode": refcode}) + v1 = flask_mongo.db.item_versions.find_one({"_id": query_params.v1, "refcode": refcode}) + v2 = flask_mongo.db.item_versions.find_one({"_id": query_params.v2, "refcode": refcode}) if not v1 or not v2: - return jsonify({"status": "error", "message": "One or both versions not found"}), 404 + return NotFound("One or both versions not found") # Use DeepDiff for proper nested structure comparison # This handles nested dicts, lists, type changes, and provides detailed change information @@ -1448,29 +1388,23 @@ def restore_version(refcode): try: restore_request = RestoreVersionRequest(**request.get_json()) except ValidationError as exc: - return jsonify( - {"status": "error", "message": "Invalid request body", "errors": exc.errors()} - ), 400 + raise BadRequest(f"Invalid request body: {exc}") try: version_object_id = ObjectId(restore_request.version_id) except (InvalidId, TypeError): - return jsonify( - {"status": "error", "message": f"Invalid version_id: {restore_request.version_id}"} - ), 400 + raise BadRequest(f"Invalid version_id: {restore_request.version_id}") # Check permissions - user must have write access current_item = flask_mongo.db.items.find_one( {"refcode": refcode, **get_default_permissions(user_only=True)} ) if not current_item: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound version = flask_mongo.db.item_versions.find_one({"_id": version_object_id, "refcode": refcode}) if not version: - return jsonify({"status": "error", "message": "Version not found"}), 404 + raise NotFound restored_data = version["data"].copy() @@ -1479,12 +1413,9 @@ def restore_version(refcode): # Ensure type consistency if restored_data.get("type") != current_item.get("type"): - return jsonify( - { - "status": "error", - "message": f"Cannot restore version with different type. Current: {current_item.get('type')}, Version: {restored_data.get('type')}", - } - ), 400 + raise BadRequest( + f"Cannot restore version with different type. Current: {current_item.get('type')}, Version: {restored_data.get('type')}" + ) # Atomically get the next version number (used for both version in item_versions and item.version) next_version_number = get_next_version_number(refcode) @@ -1496,19 +1427,15 @@ def restore_version(refcode): # Validate restored data against the item model item_type = current_item["type"] if item_type not in ITEM_MODELS: - return jsonify({"status": "error", "message": f"Invalid item type: {item_type}"}), 400 + raise BadRequest(f"Invalid item type: {item_type}") try: # Validate using the appropriate model ITEM_MODELS[item_type](**restored_data) except ValidationError as exc: - return jsonify( - { - "status": "error", - "message": f"Restored data failed validation: {str(exc)}", - "output": str(exc), - } - ), 400 + raise BadRequest( + f"Restored data failed validation against schema for type {item_type}: {exc}" + ) # Perform the restore first flask_mongo.db.items.update_one({"refcode": refcode}, {"$set": restored_data}) @@ -1545,13 +1472,7 @@ def restore_version(refcode): refcode, str(exc), ) - return jsonify( - { - "status": "error", - "message": f"Restored version data validation failed: {str(exc)}", - "output": str(exc), - } - ), 400 + raise BadRequest(f"Restored version data validation failed: {exc}") # Insert validated data flask_mongo.db.item_versions.insert_one( @@ -1576,9 +1497,7 @@ def delete_version(refcode, version_id): # Check if user has write access to the item (write access required) has_access, _ = check_version_access(refcode, user_only=True) if not has_access: - return jsonify( - {"status": "error", "message": "Item not found or insufficient permissions"} - ), 404 + raise NotFound if len(refcode.split(":")) != 2: refcode = f"{CONFIG.IDENTIFIER_PREFIX}:{refcode}" @@ -1586,13 +1505,13 @@ def delete_version(refcode, version_id): try: version_object_id = ObjectId(version_id) except (InvalidId, TypeError): - return jsonify({"status": "error", "message": f"Invalid version_id: {version_id}"}), 400 + raise BadRequest(f"Invalid version_id: {version_id}") result = flask_mongo.db.item_versions.delete_one({"_id": version_object_id, "refcode": refcode}) if result.deleted_count == 1: return jsonify({"status": "success"}), 200 else: - return jsonify({"status": "error", "message": "Version not found"}), 404 + raise NotFound @ITEMS.route("/items//save-version/", methods=["POST"]) @@ -1665,23 +1584,13 @@ def save_item(): ) if not item: - return ( - jsonify( - status="error", - message=f"Unable to find item with appropriate permissions and {item_id=}.", - ), - 404, - ) + raise NotFound # Store refcode for version saving after successful update refcode = item.get("refcode") if not refcode: - return ( - jsonify( - status="error", - message=f"Item {item_id} does not have a refcode.", - ), - 400, + raise InternalServerError( + f"Item {item_id} does not have a refcode; please report this issue." ) user_only = item["type"] not in ("starting_materials", "equipment") @@ -1691,13 +1600,7 @@ def save_item(): ) if not item: - return ( - jsonify( - status="error", - message=f"Unable to find item with appropriate permissions and {item_id=}.", - ), - 404, - ) + raise NotFound if "collections" in updated_data: requested_collections = updated_data["collections"] @@ -1795,14 +1698,9 @@ def save_item(): exclude={"collections", "creators", "immutable_id"}, ) - except ValidationError as exc: - return ( - jsonify( - status="error", - message=f"Unable to update item {item_id=} ({item_type=}) with new data {updated_data}", - output=str(exc), - ), - 400, + except ValidationError: + raise BadRequest( + f"Unable to update item {item_id=} ({item_type=}) with new data {updated_data}" ) if preserve_relationships and original_relationships is not None: @@ -1821,14 +1719,7 @@ def save_item(): ) if result.matched_count != 1: - return ( - jsonify( - status="error", - message=f"{item_id} item update failed. no subdocument matched", - output=result.raw_result, - ), - 400, - ) + raise BadRequest(f"{item_id} item update failed. no subdocument matched") # Now save a version AFTER successful item update. # Only increment item.version when content actually changed (i.e., a snapshot was minted). @@ -1879,12 +1770,7 @@ def get_access_token_info(refcode: str): ) if not current_item: - return jsonify( - { - "status": "error", - "message": f"No valid item found with the given {refcode=}.", - } - ), 404 + raise NotFound existing_token = flask_mongo.db.api_keys.find_one( {"refcode": refcode, "active": True, "type": "access_token"}, From 1e0c8629c1a95ebd4993df02cd40974c24d0458e Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 23:02:52 +0100 Subject: [PATCH 14/24] Fix exports for pydantic 2 --- pydatalab/src/pydatalab/export.py | 6 +++--- pydatalab/src/pydatalab/routes/v0_1/export.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pydatalab/src/pydatalab/export.py b/pydatalab/src/pydatalab/export.py index 8835ba341..1a83a32b5 100644 --- a/pydatalab/src/pydatalab/export.py +++ b/pydatalab/src/pydatalab/export.py @@ -112,7 +112,7 @@ def write_eln_file( item_folder = root_folder / item[primary_key] - item_metadata = ITEM_MODELS[item.get("type")](**item).json(indent=2) + item_metadata = ITEM_MODELS[item.get("type")](**item).model_dump_json(indent=2) zipf.writestr(str(item_folder / "metadata.json"), item_metadata) for file in item.get("files", []): @@ -378,7 +378,7 @@ def create_eln_file( item_data = list(cursor)[0] ItemModel = ITEM_MODELS[item_data["type"]] - item_data = ItemModel(**item_data).dict() + item_data = ItemModel(**item_data).model_dump() except IndexError: item_data = None @@ -420,7 +420,7 @@ def create_eln_file( for ind, item in enumerate(all_items): ItemModel = ITEM_MODELS[item["type"]] - _all_items.append(ItemModel(**item).dict()) + _all_items.append(ItemModel(**item).model_dump()) all_items = _all_items diff --git a/pydatalab/src/pydatalab/routes/v0_1/export.py b/pydatalab/src/pydatalab/routes/v0_1/export.py index 9f32c95f7..b16d005de 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/export.py +++ b/pydatalab/src/pydatalab/routes/v0_1/export.py @@ -134,7 +134,7 @@ def _do_export( def add_stage(message: str, level: str = "info"): stage = TaskStage(timestamp=datetime.now(tz=timezone.utc), message=message, level=level) flask_mongo.db.tasks.update_one( - {"task_id": task_id}, {"$push": {"spec.stages": stage.dict()}} + {"task_id": task_id}, {"$push": {"spec.stages": stage.model_dump()}} ) try: @@ -221,7 +221,7 @@ def start_collection_export(collection_id: str): ), ) - flask_mongo.db.tasks.insert_one(export_task.dict(exclude_none=False)) + flask_mongo.db.tasks.insert_one(export_task.model_dump(exclude_none=False)) task_scheduler.add_job( func=_generate_export_in_background, @@ -390,7 +390,7 @@ def start_item_export(item_id: str): ), ) - flask_mongo.db.tasks.insert_one(export_task.dict(exclude_none=False)) + flask_mongo.db.tasks.insert_one(export_task.model_dump(exclude_none=False)) task_scheduler.add_job( func=_generate_export_in_background, From 5c0a8cfdabe00d41a3d5be2fcb3e0b726706de4d Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 23:22:56 +0100 Subject: [PATCH 15/24] Fix info endpoint for pydantic 2 --- pydatalab/src/pydatalab/routes/v0_1/info.py | 37 +++++++++------------ 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/info.py b/pydatalab/src/pydatalab/routes/v0_1/info.py index 40e4fc6a5..4bf7bd6e1 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/info.py +++ b/pydatalab/src/pydatalab/routes/v0_1/info.py @@ -5,6 +5,7 @@ from datetime import timedelta as td from datetime import timezone as tz from functools import lru_cache +from typing import Any, Generic, TypeVar from flask import Blueprint, jsonify, request from pydantic import ( @@ -13,7 +14,6 @@ ConfigDict, Field, field_validator, - model_validator, ) from pydatalab import __version__ @@ -47,14 +47,19 @@ class Links(BaseModel): model_config = ConfigDict(extra="allow") -class Data(BaseModel): +AttributesT = TypeVar("AttributesT") + + +class Data(BaseModel, Generic[AttributesT]): id: str type: str - attributes: Attributes + attributes: AttributesT + """The attributes payload, serialized as whatever concrete type the envelope is + parametrised with (e.g. ``Data[Info]``) so subclass fields are not stripped.""" -class JSONAPIResponse(BaseModel): - data: Data | list[Data] +class JSONAPIResponse(BaseModel, Generic[AttributesT]): + data: Data[AttributesT] | list[Data[AttributesT]] meta: Meta links: Links | None = None @@ -70,7 +75,7 @@ class Info(Attributes, Meta): homepage: AnyUrl | None = None source_repository: AnyUrl | None = None identifier_prefix: str - features: FeatureFlags = FEATURE_FLAGS + features: FeatureFlags | None = None max_upload_bytes: int @field_validator("maintainer", mode="before") @@ -80,18 +85,6 @@ def strip_maintainer_fields(cls, v): return MetaPerson(contact_email=v.contact_email, display_name=v.display_name) return v - @model_validator(mode="after") - def ensure_features_serialization(self): - """Ensure features are properly serialized for frontend consumption.""" - if hasattr(self.features, "model_dump"): - features_dict = self.features.model_dump() - else: - features_dict = self.features - - if not isinstance(self.features, FeatureFlags): - self.features = FeatureFlags(**features_dict) - return self - @lru_cache(maxsize=1) def _get_deployment_metadata_once() -> dict: @@ -118,7 +111,7 @@ def get_info(): """ - response_data = JSONAPIResponse( + response_data = JSONAPIResponse[Info]( data=Data(id="/", type="info", attributes=Info(**_get_deployment_metadata_once())), meta=Meta(query=request.query_string.decode() if request.query_string else ""), links=Links(self=request.url), @@ -150,7 +143,7 @@ def list_block_types(): """Returns a list of all blocks implemented in this server.""" return jsonify( json.loads( - JSONAPIResponse( + JSONAPIResponse[dict[str, Any]]( data=[ Data( id=block_type, @@ -179,7 +172,7 @@ def list_supported_types(): return jsonify( json.loads( - JSONAPIResponse( + JSONAPIResponse[dict[str, Any]]( data=[ Data( id=item_type, @@ -208,7 +201,7 @@ def get_schema_type(item_type): return jsonify( json.loads( - JSONAPIResponse( + JSONAPIResponse[dict[str, Any]]( data=Data( id=item_type, type="item_type", From c5e6b15c05f2431e3fbc473f776f3965e7fab33a Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 23:31:59 +0100 Subject: [PATCH 16/24] Unpick circular dependencies between export/items routes by moving lookups to mongo module --- pydatalab/src/pydatalab/export.py | 4 +- pydatalab/src/pydatalab/mongo.py | 40 +++++++++++++++++ pydatalab/src/pydatalab/routes/v0_1/items.py | 46 ++++---------------- 3 files changed, 50 insertions(+), 40 deletions(-) diff --git a/pydatalab/src/pydatalab/export.py b/pydatalab/src/pydatalab/export.py index 1a83a32b5..a06551765 100644 --- a/pydatalab/src/pydatalab/export.py +++ b/pydatalab/src/pydatalab/export.py @@ -12,10 +12,10 @@ from pydatalab.config import CONFIG from pydatalab.logger import LOGGER from pydatalab.models import ITEM_MODELS -from pydatalab.mongo import flask_mongo -from pydatalab.routes.v0_1.items import ( +from pydatalab.mongo import ( creators_lookup, files_lookup, + flask_mongo, groups_lookup, ) diff --git a/pydatalab/src/pydatalab/mongo.py b/pydatalab/src/pydatalab/mongo.py index fb9bb7fa5..f8afa05d2 100644 --- a/pydatalab/src/pydatalab/mongo.py +++ b/pydatalab/src/pydatalab/mongo.py @@ -24,6 +24,9 @@ "COLLECTIONS_FTS_FIELDS", "generate_heuristic_regex_search", "build_search_pipeline", + "creators_lookup", + "groups_lookup", + "files_lookup", ) flask_mongo = PyMongo() @@ -34,6 +37,43 @@ """ +def creators_lookup() -> dict: + return { + "from": "users", + "let": {"creator_ids": "$creator_ids"}, + "pipeline": [ + {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$creator_ids", []]}]}}}, + {"$addFields": {"__order": {"$indexOfArray": ["$$creator_ids", "$_id"]}}}, + {"$sort": {"__order": 1}}, + {"$project": {"_id": 1, "display_name": 1, "gravatar_hash": 1}}, + ], + "as": "creators", + } + + +def groups_lookup() -> dict: + return { + "from": "groups", + "let": {"group_ids": "$group_ids"}, + "pipeline": [ + {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$group_ids", []]}]}}}, + {"$addFields": {"__order": {"$indexOfArray": ["$$group_ids", "$_id"]}}}, + {"$sort": {"__order": 1}}, + {"$project": {"_id": 1, "display_name": 1, "group_id": 1}}, + ], + "as": "groups", + } + + +def files_lookup() -> dict: + return { + "from": "files", + "localField": "file_ObjectIds", + "foreignField": "_id", + "as": "files", + } + + @lru_cache(maxsize=1) def get_items_fts_fields() -> set[str]: """Get all string fields from item models for full-text search.""" diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index b613c73ca..4e5de851a 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -25,7 +25,14 @@ RestoreVersionRequest, VersionAction, ) -from pydatalab.mongo import build_search_pipeline, flask_mongo, get_items_fts_fields +from pydatalab.mongo import ( + build_search_pipeline, + creators_lookup, + files_lookup, + flask_mongo, + get_items_fts_fields, + groups_lookup, +) from pydatalab.permissions import ( PUBLIC_USER_ID, access_token_or_active_users, @@ -294,34 +301,6 @@ def get_samples_summary(match: dict | None = None, project: dict | None = None) ) -def creators_lookup() -> dict: - return { - "from": "users", - "let": {"creator_ids": "$creator_ids"}, - "pipeline": [ - {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$creator_ids", []]}]}}}, - {"$addFields": {"__order": {"$indexOfArray": ["$$creator_ids", "$_id"]}}}, - {"$sort": {"__order": 1}}, - {"$project": {"_id": 1, "display_name": 1, "gravatar_hash": 1}}, - ], - "as": "creators", - } - - -def groups_lookup() -> dict: - return { - "from": "groups", - "let": {"group_ids": "$group_ids"}, - "pipeline": [ - {"$match": {"$expr": {"$in": ["$_id", {"$ifNull": ["$$group_ids", []]}]}}}, - {"$addFields": {"__order": {"$indexOfArray": ["$$group_ids", "$_id"]}}}, - {"$sort": {"__order": 1}}, - {"$project": {"_id": 1, "display_name": 1, "group_id": 1}}, - ], - "as": "groups", - } - - def entry_reference_lookup(item_doc: dict) -> dict: """Looks up any field that contains an entry reference and resolves it to the item data.""" @@ -401,15 +380,6 @@ def entry_reference_lookup(item_doc: dict) -> dict: return item_doc -def files_lookup() -> dict: - return { - "from": "files", - "localField": "file_ObjectIds", - "foreignField": "_id", - "as": "files", - } - - def collections_lookup() -> dict: """Looks inside the relationships of the item, searches for IDs in the collections table and then projects only the collection ID and name for the response. From 15ce5135c5f026baa606ca5507876f430a49d30a Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Sun, 7 Jun 2026 23:53:20 +0100 Subject: [PATCH 17/24] Move `datalab_*` fields in schema to `json_schema_extra` and allow carve out for plugins --- pydatalab/src/pydatalab/apps/__init__.py | 11 ++++++++++- pydatalab/src/pydatalab/apps/chat/blocks.py | 4 +++- pydatalab/src/pydatalab/blocks/base.py | 4 ++-- pydatalab/src/pydatalab/models/blocks.py | 18 +++++++++++++----- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/pydatalab/src/pydatalab/apps/__init__.py b/pydatalab/src/pydatalab/apps/__init__.py index 9f49a0950..f48a6427e 100644 --- a/pydatalab/src/pydatalab/apps/__init__.py +++ b/pydatalab/src/pydatalab/apps/__init__.py @@ -3,8 +3,11 @@ """ +import warnings from typing import TYPE_CHECKING +from pydantic.warnings import PydanticDeprecatedSince20 + if TYPE_CHECKING: # This import is required to prevent circular imports for application-specific blocks from pydatalab.blocks.base import DataBlock # noqa @@ -116,7 +119,13 @@ def load_block_plugins(): block_plugins: dict[str, type[DataBlock]] = {} for entry_point in entry_points(group="pydatalab.apps.plugins"): - block = entry_point.load() + # Plugins may not yet be migrated to pydantic v2, so tolerate the v2 + # deprecation warnings their models emit at import time (e.g. extra `Field` + # kwargs like `datalab_exclude_*`) rather than letting them error out plugin + # loading. datalab's own deprecations are still surfaced as errors. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", PydanticDeprecatedSince20) + block = entry_point.load() if not issubclass(block, DataBlock): raise ValueError(f"Plugin {block} must be a subclass of DataBlock") diff --git a/pydatalab/src/pydatalab/apps/chat/blocks.py b/pydatalab/src/pydatalab/apps/chat/blocks.py index b952fe8ca..cc241e1a1 100644 --- a/pydatalab/src/pydatalab/apps/chat/blocks.py +++ b/pydatalab/src/pydatalab/apps/chat/blocks.py @@ -19,7 +19,9 @@ class ChatBlockResponse(DataBlockResponse): prompt: str | None = None model: str available_models: dict[str, ModelCard] | None = Field( - None, validate_default=True, datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, + validate_default=True, + json_schema_extra={"datalab_exclude_from_db": True, "datalab_exclude_from_load": True}, ) token_count: int | None = None temperature: float diff --git a/pydatalab/src/pydatalab/blocks/base.py b/pydatalab/src/pydatalab/blocks/base.py index 374f0d443..6da25ac06 100644 --- a/pydatalab/src/pydatalab/blocks/base.py +++ b/pydatalab/src/pydatalab/blocks/base.py @@ -206,7 +206,7 @@ def to_db(self) -> dict: LOGGER.debug("Casting block %s to database object.", self.__class__.__name__) exclude_fields: set[str] = { f - for (f, s) in self.block_db_model.schema()["properties"].items() + for (f, s) in self.block_db_model.model_json_schema()["properties"].items() if s.get("datalab_exclude_from_db") } return self.block_db_model(**self.data).model_dump( @@ -353,7 +353,7 @@ def update_from_web(self, data: dict): ) exclude_fields: set[str] = { f - for (f, s) in self.block_db_model.schema()["properties"].items() + for (f, s) in self.block_db_model.model_json_schema()["properties"].items() if s.get("datalab_exclude_from_load") } [data.pop(f, None) for f in exclude_fields] diff --git a/pydatalab/src/pydatalab/models/blocks.py b/pydatalab/src/pydatalab/models/blocks.py index f1e02a4d4..e6d9244e3 100644 --- a/pydatalab/src/pydatalab/models/blocks.py +++ b/pydatalab/src/pydatalab/models/blocks.py @@ -44,23 +44,31 @@ class DataBlockResponse(BaseModel): """Any warnings that occurred during block processing.""" b64_encoded_image: dict[str, str] | None = Field( - None, datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, + json_schema_extra={"datalab_exclude_from_db": True, "datalab_exclude_from_load": True}, ) """Any base64-encoded image data associated with the block, keyed by `file_id`.""" bokeh_plot_data: dict | None = Field( - None, datalab_exclude_from_db=True, datalab_exclude_from_load=True + None, + json_schema_extra={"datalab_exclude_from_db": True, "datalab_exclude_from_load": True}, ) """A JSON-encoded string containing the Bokeh plot data, if any.""" - computed: dict | None = Field(default=None, datalab_exclude_from_load=True) + computed: dict | None = Field( + default=None, json_schema_extra={"datalab_exclude_from_load": True} + ) """Any processed or computed data associated with the block, small enough to store and filter directly in the database, i.e., strings or a few hundred numbers not exceeding 16KB in size. Examples could include peak positions, and widths, but not the full spectrum. """ - processed: dict | None = Field(default=None, datalab_exclude_from_load=True) + processed: dict | None = Field( + default=None, json_schema_extra={"datalab_exclude_from_load": True} + ) - metadata: dict | None = Field(default=None, datalab_exclude_from_load=True) + metadata: dict | None = Field( + default=None, json_schema_extra={"datalab_exclude_from_load": True} + ) """Any structured metadata associated with the block, for example, experimental acquisition parameters.""" From e3bceabc0bbcb92af6c665dbaf2177cf06485e0d Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 00:16:59 +0100 Subject: [PATCH 18/24] Final wave of model_dump/dict changes --- pydatalab/src/pydatalab/apps/nmr/blocks.py | 6 ++-- pydatalab/src/pydatalab/routes/v0_1/admin.py | 8 ++++-- pydatalab/src/pydatalab/routes/v0_1/blocks.py | 4 +-- pydatalab/src/pydatalab/routes/v0_1/groups.py | 5 +++- pydatalab/src/pydatalab/routes/v0_1/users.py | 2 +- pydatalab/tests/server/test_async_blocks.py | 26 ++++++++--------- pydatalab/tests/server/test_export.py | 28 +++++++++---------- pydatalab/tests/server/test_graph.py | 2 +- pydatalab/tests/server/test_item_graph.py | 13 +++++---- pydatalab/tests/server/test_item_versions.py | 12 ++++---- pydatalab/tests/server/test_samples.py | 21 ++++++++------ pydatalab/tests/server/test_search.py | 2 +- .../tests/server/test_starting_materials.py | 2 +- pydatalab/tests/test_models.py | 6 ++-- 14 files changed, 73 insertions(+), 64 deletions(-) diff --git a/pydatalab/src/pydatalab/apps/nmr/blocks.py b/pydatalab/src/pydatalab/apps/nmr/blocks.py index 028b4b16d..c2975b875 100644 --- a/pydatalab/src/pydatalab/apps/nmr/blocks.py +++ b/pydatalab/src/pydatalab/apps/nmr/blocks.py @@ -130,7 +130,7 @@ def read_bruker_nmr_data( metadata["pulse_program_name"] = a_dic["acqus"]["PULPROG"] metadata["title"] = topspin_title - self.data["metadata"] = NMRMetadata(**metadata).dict() + self.data["metadata"] = NMRMetadata(**metadata).model_dump() return serialized_df, metadata @@ -209,7 +209,7 @@ def read_jcamp_nmr_data( pass serialized_df = df.to_dict() if (df is not None) else None - self.data["metadata"] = NMRMetadata(**metadata).dict() + self.data["metadata"] = NMRMetadata(**metadata).model_dump() return serialized_df, metadata @@ -293,7 +293,7 @@ def read_jeol_nmr_data( metadata["nscans"] = nscans serialized_df = df.to_dict() if (df is not None) else None - self.data["metadata"] = NMRMetadata(**metadata).dict() + self.data["metadata"] = NMRMetadata(**metadata).model_dump() return serialized_df, metadata diff --git a/pydatalab/src/pydatalab/routes/v0_1/admin.py b/pydatalab/src/pydatalab/routes/v0_1/admin.py index 55e886fae..eea5a6478 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/admin.py +++ b/pydatalab/src/pydatalab/routes/v0_1/admin.py @@ -125,7 +125,7 @@ def get_users(): elif not isinstance(user["managers"], list): user["managers"] = [] - return jsonify({"status": "success", "data": [Person(**d).dict() for d in users_list]}) + return jsonify({"status": "success", "data": [Person(**d).model_dump() for d in users_list]}) @ADMIN.route("/roles/", methods=["PATCH"]) @@ -357,7 +357,9 @@ def get_groups(): [{"$match": {}}, {"$lookup": members_lookup}, {"$lookup": managers_lookup}] ) - return jsonify({"status": "success", "data": [Group(**d).dict() for d in group_docs]}), 200 + return jsonify( + {"status": "success", "data": [Group(**d).model_dump() for d in group_docs]} + ), 200 @ADMIN.route("/groups", methods=["PUT"]) @@ -392,7 +394,7 @@ def create_group(): try: group_immutable_id = flask_mongo.db.groups.insert_one( - group.dict(exclude_unset=True) + group.model_dump(exclude_unset=True) ).inserted_id except pymongo.errors.DuplicateKeyError: return jsonify( diff --git a/pydatalab/src/pydatalab/routes/v0_1/blocks.py b/pydatalab/src/pydatalab/routes/v0_1/blocks.py index 174fccd08..158cad713 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/blocks.py +++ b/pydatalab/src/pydatalab/routes/v0_1/blocks.py @@ -54,7 +54,7 @@ def add_stage(message: str, level: str = "info", traceback: str | None = None): timestamp=datetime.now(tz=timezone.utc), message=message, level=level, detail=traceback ) flask_mongo.db.tasks.update_one( - {"task_id": task_id}, {"$push": {"spec.stages": stage.dict()}} + {"task_id": task_id}, {"$push": {"spec.stages": stage.model_dump()}} ) with app_ctx, req_ctx: @@ -429,7 +429,7 @@ def update_block(): ), ) - flask_mongo.db.tasks.insert_one(block_task.dict()) + flask_mongo.db.tasks.insert_one(block_task.model_dump()) task_scheduler.add_job( func=_process_block_async, diff --git a/pydatalab/src/pydatalab/routes/v0_1/groups.py b/pydatalab/src/pydatalab/routes/v0_1/groups.py index ddd732c98..6e36216a9 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/groups.py +++ b/pydatalab/src/pydatalab/routes/v0_1/groups.py @@ -46,5 +46,8 @@ def search_groups(): ] ) return jsonify( - {"status": "success", "data": list(json.loads(Group(**d).json()) for d in cursor)} + { + "status": "success", + "data": list(json.loads(Group(**d).model_dump_json()) for d in cursor), + } ), 200 diff --git a/pydatalab/src/pydatalab/routes/v0_1/users.py b/pydatalab/src/pydatalab/routes/v0_1/users.py index f790a12bf..ab9e737f0 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/users.py +++ b/pydatalab/src/pydatalab/routes/v0_1/users.py @@ -218,5 +218,5 @@ def search_users(): cursor = flask_mongo.db.users.aggregate(pipeline) return jsonify( - {"status": "success", "users": [json.loads(Person(**d).json()) for d in cursor]} + {"status": "success", "users": [json.loads(Person(**d).model_dump_json()) for d in cursor]} ), 200 diff --git a/pydatalab/tests/server/test_async_blocks.py b/pydatalab/tests/server/test_async_blocks.py index 7d143e14d..226ec9fff 100644 --- a/pydatalab/tests/server/test_async_blocks.py +++ b/pydatalab/tests/server/test_async_blocks.py @@ -147,7 +147,7 @@ def test_status_pending(self, client, user_id, database): block_id="test_block", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -182,7 +182,7 @@ def test_status_processing_with_stages(self, client, user_id, database): stages=stages, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -226,7 +226,7 @@ def test_status_ready_with_gridfs_data(self, client, user_id, database, app): block_id=block_id, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) # Write block data to GridFS from pydatalab.mongo import get_database @@ -276,7 +276,7 @@ def test_status_ready_no_gridfs_data(self, client, user_id, database): block_id=block_id, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -304,7 +304,7 @@ def test_status_error(self, client, user_id, database): block_id="test_block", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 200 @@ -334,7 +334,7 @@ def test_status_ignores_export_tasks(self, client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/blocks/{task_id}/status") assert response.status_code == 404 @@ -369,7 +369,7 @@ def test_status_ready_respects_permissions(self, another_client, user_id, databa block_id=block_id, ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) from pydatalab.mongo import get_database @@ -428,7 +428,7 @@ def test_successful_processing(self, app, user_id, database): status=TaskStatus.PENDING, spec=BlockProcessingTaskSpec(item_id=item_id, block_id=block_id), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) block_data = { "blocktype": "comment", @@ -470,7 +470,7 @@ def test_processing_error_handling(self, app, user_id, database): status=TaskStatus.PENDING, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) # Pass an invalid block type to trigger an error block_data = { @@ -509,7 +509,7 @@ def test_marks_timed_out_tasks_as_error(self, app, user_id, database): status=TaskStatus.PROCESSING, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - task_dict = task.dict() + task_dict = task.model_dump() task_dict["created_at"] = old_time database.tasks.insert_one(task_dict) @@ -534,7 +534,7 @@ def test_does_not_timeout_recent_tasks(self, app, user_id, database): status=TaskStatus.PROCESSING, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _cleanup_stale_tasks() @@ -558,7 +558,7 @@ def test_purges_old_completed_tasks_and_gridfs(self, app, user_id, database): completed_at=old_time, spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - task_dict = task.dict() + task_dict = task.model_dump() task_dict["created_at"] = old_time database.tasks.insert_one(task_dict) @@ -586,7 +586,7 @@ def test_does_not_purge_recent_completed_tasks(self, app, user_id, database): completed_at=datetime.now(tz=timezone.utc), spec=BlockProcessingTaskSpec(item_id="x", block_id="y"), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _cleanup_stale_tasks() diff --git a/pydatalab/tests/server/test_export.py b/pydatalab/tests/server/test_export.py index 6facb0643..b9337169b 100644 --- a/pydatalab/tests/server/test_export.py +++ b/pydatalab/tests/server/test_export.py @@ -78,7 +78,7 @@ def test_get_export_status_pending(client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/status") assert response.status_code == 200 @@ -107,7 +107,7 @@ def test_get_export_status_ready(client, user_id, database, tmp_path): file_path=str(file_path), ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/status") assert response.status_code == 200 @@ -136,7 +136,7 @@ def test_get_export_status_error(client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/status") assert response.status_code == 200 @@ -180,7 +180,7 @@ def test_download_export_success(client, user_id, database, tmp_path, monkeypatc file_path=str(file_path), ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/download") assert response.status_code == 200 @@ -203,7 +203,7 @@ def test_not_users_export_download(client, user_id, another_client, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = another_client.get(f"/exports/{task_id}/download") assert response.status_code == 404 @@ -223,7 +223,7 @@ def test_download_export_not_ready(client, user_id, database): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/download") assert response.status_code == 400 @@ -248,7 +248,7 @@ def test_download_export_file_missing(client, user_id, database): file_path="/nonexistent/path.eln", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) response = client.get(f"/exports/{task_id}/download") assert response.status_code == 404 @@ -275,7 +275,7 @@ def test_do_export_success(database, sample_collection, insert_default_sample, u export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _do_export(task_id, collection_id=collection_id, export_type="collection") @@ -299,7 +299,7 @@ def test_do_export_success(database, sample_collection, insert_default_sample, u export_type="item", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _do_export(task_id, item_id=item_id, export_type="item") @@ -326,7 +326,7 @@ def test_do_export_error_handling(database, user_id): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) _do_export(task_id, collection_id="nonexistent_collection", export_type="collection") @@ -367,7 +367,7 @@ def test_cleanup_old_exports(database, user_id, tmp_path, monkeypatch): file_path=str(old_file), ), ) - database.tasks.insert_one(old_task.dict()) + database.tasks.insert_one(old_task.model_dump()) # A recent export: created within the window, must be retained. recent_task_id = "recent-export-to-keep" @@ -385,7 +385,7 @@ def test_cleanup_old_exports(database, user_id, tmp_path, monkeypatch): file_path=str(recent_file), ), ) - database.tasks.insert_one(recent_task.dict()) + database.tasks.insert_one(recent_task.model_dump()) try: _cleanup_old_exports() @@ -422,7 +422,7 @@ def test_cleanup_old_exports_missing_file(database, user_id): file_path="/nonexistent/already-deleted.eln", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) try: _cleanup_old_exports() @@ -447,7 +447,7 @@ def test_do_export_status_transitions(database, sample_collection, user_id): export_type="collection", ), ) - database.tasks.insert_one(task.dict()) + database.tasks.insert_one(task.model_dump()) status_during_export = [] diff --git a/pydatalab/tests/server/test_graph.py b/pydatalab/tests/server/test_graph.py index 3e10a9dc9..9c1f30e3d 100644 --- a/pydatalab/tests/server/test_graph.py +++ b/pydatalab/tests/server/test_graph.py @@ -61,7 +61,7 @@ def test_simple_graph(admin_client): ) new_samples = [ - json.loads(d.json()) + json.loads(d.model_dump_json()) for d in [parent, child_1, child_2, child_3, child_4, missing_child, cell] ] diff --git a/pydatalab/tests/server/test_item_graph.py b/pydatalab/tests/server/test_item_graph.py index 08ffa7db4..f04169aa6 100644 --- a/pydatalab/tests/server/test_item_graph.py +++ b/pydatalab/tests/server/test_item_graph.py @@ -15,7 +15,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(material.json())}, + json={"new_sample_data": json.loads(material.model_dump_json())}, ) assert creation.status_code == 201 @@ -39,7 +39,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(parent.json())}, + json={"new_sample_data": json.loads(parent.model_dump_json())}, ) assert creation.status_code == 201 @@ -68,7 +68,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(child.json())}, + json={"new_sample_data": json.loads(child.model_dump_json())}, ) grandchild = Sample( @@ -80,7 +80,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(grandchild.json())}, + json={"new_sample_data": json.loads(grandchild.model_dump_json())}, ) great_grandchild = Sample( @@ -92,7 +92,7 @@ def test_single_starting_material(admin_client, client): creation = client.post( "/new-sample/", - json={"new_sample_data": json.loads(great_grandchild.json())}, + json={"new_sample_data": json.loads(great_grandchild.model_dump_json())}, ) graph = client.get("/item-graph").json @@ -114,7 +114,8 @@ def test_single_starting_material(admin_client, client): ) creation = admin_client.post( - "/new-sample/", json={"new_sample_data": json.loads(admin_great_great_grandchild.json())} + "/new-sample/", + json={"new_sample_data": json.loads(admin_great_great_grandchild.model_dump_json())}, ) graph = admin_client.get("/item-graph/great-grandchild?sudo=1").json diff --git a/pydatalab/tests/server/test_item_versions.py b/pydatalab/tests/server/test_item_versions.py index 2356177a5..bfcd529b4 100644 --- a/pydatalab/tests/server/test_item_versions.py +++ b/pydatalab/tests/server/test_item_versions.py @@ -24,7 +24,7 @@ def sample_with_version(client, user_id): "synthesis_description": "Initial synthesis", } ) - flask_mongo.db.items.insert_one(sample.dict(exclude_unset=False)) + flask_mongo.db.items.insert_one(sample.model_dump(exclude_unset=False)) yield sample @@ -525,7 +525,7 @@ def test_save_item_creates_version(self, client, sample_with_version): refcode_short = sample_with_version.refcode.split(":")[1] # Modify and save the item using save-item endpoint - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "Updated via save-item" response = client.post("/save-item/", json={"item_id": item_id, "data": item_data}) @@ -549,7 +549,7 @@ def test_save_item_increments_version(self, client, sample_with_version): refcode = sample_with_version.refcode - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "First real change" client.post("/save-item/", json={"item_id": sample_with_version.item_id, "data": item_data}) @@ -617,7 +617,7 @@ def test_save_item_endpoint_creates_manual_save_action(self, client, sample_with refcode_short = sample_with_version.refcode.split(":")[1] # Save via save-item endpoint (user clicking save button) - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "Updated via save-item" client.post("/save-item/", json={"item_id": sample_with_version.item_id, "data": item_data}) @@ -785,7 +785,7 @@ def test_save_item_identical_does_not_create_version(self, client, sample_with_v refcode = sample_with_version.refcode.split(":")[1] full_refcode = sample_with_version.refcode - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "A new description" client.post("/save-item/", json={"item_id": item_id, "data": item_data}) @@ -805,7 +805,7 @@ def test_save_item_with_change_creates_new_version(self, client, sample_with_ver refcode = sample_with_version.refcode.split(":")[1] full_refcode = sample_with_version.refcode - item_data = sample_with_version.dict(exclude_unset=False) + item_data = sample_with_version.model_dump(exclude_unset=False) item_data["description"] = "First real change" client.post("/save-item/", json={"item_id": item_id, "data": item_data}) diff --git a/pydatalab/tests/server/test_samples.py b/pydatalab/tests/server/test_samples.py index f4173556f..b25874df7 100644 --- a/pydatalab/tests/server/test_samples.py +++ b/pydatalab/tests/server/test_samples.py @@ -276,7 +276,7 @@ def test_item_regex_search( def test_new_sample_with_relationships( client, complicated_sample, insert_complicated_sample_constituents ): - complicated_sample_json = json.loads(complicated_sample.json()) + complicated_sample_json = json.loads(complicated_sample.model_dump_json()) response = client.post("/new-sample/", json=complicated_sample_json) # Test that 201: Created is emitted assert response.status_code == 201, response.json @@ -340,7 +340,7 @@ def test_new_sample_with_relationships( description="This is a new relationship", ) ) - derived_sample_json = json.loads(derived_sample.json()) + derived_sample_json = json.loads(derived_sample.model_dump_json()) response = client.post("/new-sample/", json=derived_sample_json) # Test that 201: Created is emitted @@ -444,7 +444,7 @@ def test_copy_from_sample(client, complicated_sample): """ complicated_sample.item_id = "new_complicated_sample" - complicated_sample_json = json.loads(complicated_sample.json()) + complicated_sample_json = json.loads(complicated_sample.model_dump_json()) response = client.post("/new-sample/", json=complicated_sample_json) # Test that 201: Created is emitted @@ -474,12 +474,13 @@ def test_copy_from_sample(client, complicated_sample): @pytest.mark.dependency(depends=["test_copy_from_sample"]) def test_create_multiple_samples(client, complicated_sample): - samples = [complicated_sample, complicated_sample.copy()] + samples = [complicated_sample, complicated_sample.model_copy()] samples[0].item_id = "another_new_complicated_sample" samples[1].item_id = "additional_new_complicated_sample" response = client.post( - "/new-samples/", json={"new_sample_datas": [json.loads(s.json()) for s in samples]} + "/new-samples/", + json={"new_sample_datas": [json.loads(s.model_dump_json()) for s in samples]}, ) assert response.status_code == 207, response.json assert response.json["nsuccess"] == 2, response.json @@ -493,7 +494,7 @@ def test_create_multiple_samples(client, complicated_sample): response = client.post( "/new-samples/", json={ - "new_sample_datas": [json.loads(s.json()) for s in samples], + "new_sample_datas": [json.loads(s.model_dump_json()) for s in samples], "copy_from_item_ids": [ "another_new_complicated_sample", "additional_new_complicated_sample", @@ -515,7 +516,7 @@ def test_create_multiple_samples(client, complicated_sample): @pytest.mark.dependency(depends=["test_create_multiple_samples"]) def test_create_cell(client, default_cell): - response = client.post("/new-sample/", json=json.loads(default_cell.json())) + response = client.post("/new-sample/", json=json.loads(default_cell.model_dump_json())) assert response.status_code == 201, response.json assert response.json["status"] == "success" @@ -588,7 +589,9 @@ def test_create_collections(client, default_collection, database): assert response.status_code == 200 # Create an empty collection - response = client.put("/collections", json={"data": json.loads(default_collection.json())}) + response = client.put( + "/collections", json={"data": json.loads(default_collection.model_dump_json())} + ) assert response.status_code == 201, response.json assert response.json["status"] == "success" assert response.json["data"]["collection_id"] == "test_collection" @@ -607,7 +610,7 @@ def test_create_collections(client, default_collection, database): new_collection = copy.deepcopy(default_collection) new_collection.collection_id = "test_collection_2" - data = json.loads(new_collection.json()) + data = json.loads(new_collection.model_dump_json()) data.update( { "starting_members": [ diff --git a/pydatalab/tests/server/test_search.py b/pydatalab/tests/server/test_search.py index 40e4771dd..d9b0cabe9 100644 --- a/pydatalab/tests/server/test_search.py +++ b/pydatalab/tests/server/test_search.py @@ -52,7 +52,7 @@ def test_collection_regex_search(query, should_find, real_mongo_client, client, if real_mongo_client is None: pytest.skip("Skipping search tests, not connected to real MongoDB") - test_collection = default_collection.dict() + test_collection = default_collection.model_dump() clean_query = query.replace("=", "_").replace("%23", "hash_").replace("'", "") test_collection["collection_id"] = f"test_coll_{clean_query}" test_collection["title"] = "test collection" diff --git a/pydatalab/tests/server/test_starting_materials.py b/pydatalab/tests/server/test_starting_materials.py index 3ee28080a..080559e5d 100644 --- a/pydatalab/tests/server/test_starting_materials.py +++ b/pydatalab/tests/server/test_starting_materials.py @@ -133,7 +133,7 @@ def test_starting_materials_list_includes_collections( client, default_starting_material_dict, default_collection ): """GET /starting-materials/ should include a collections array with collection_id for each item.""" - collection_data = json.loads(default_collection.json()) + collection_data = json.loads(default_collection.model_dump_json()) collection_data["starting_members"] = [{"item_id": default_starting_material_dict["item_id"]}] response = client.put("/collections", json={"data": collection_data}) assert response.status_code == 201, response.json diff --git a/pydatalab/tests/test_models.py b/pydatalab/tests/test_models.py index c92dbb7ce..625ae5d08 100644 --- a/pydatalab/tests/test_models.py +++ b/pydatalab/tests/test_models.py @@ -52,7 +52,7 @@ def test_sample_with_inlined_reference(): assert parents[0].item_id == a.item_id assert parents[0].refcode == a.refcode - b_both = Sample(**json.loads(b_both.json())) + b_both = Sample(**json.loads(b_both.model_dump_json())) parents = [r for r in b_both.relationships if r.relation == RelationshipType.PARENT] assert len(parents) == 1 @@ -209,7 +209,7 @@ class TestItem(Item): tzinfo=datetime.timezone.utc ) - item_json = json.loads(item.json()) + item_json = json.loads(item.model_dump_json()) assert item_json["type"] == "items_custom" assert item_json["creator_ids"][0] == "0123456789ab0123456789ab" assert item_json["creator_ids"][1] == "1023456789ab0123456789ab" @@ -239,7 +239,7 @@ class TestItem(Item): "2020-01-01 00:00" ).replace(tzinfo=datetime.timezone.utc) - sample_json = json.loads(sample.json()) + sample_json = json.loads(sample.model_dump_json()) assert sample_json["type"] == "samples" assert sample_json["creator_ids"][0] == str(ObjectId("0123456789ab0123456789ab")) assert sample_json["creator_ids"][1] == str(ObjectId("1023456789ab0123456789ab")) From d8a52ea60f99510216731457c4184d40d162e499 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 00:19:53 +0100 Subject: [PATCH 19/24] Minor tweaks suggested by copilot --- pydatalab/src/pydatalab/models/people.py | 4 ---- pydatalab/src/pydatalab/routes/v0_1/info.py | 6 +++--- pydatalab/src/pydatalab/routes/v0_1/items.py | 8 ++++++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pydatalab/src/pydatalab/models/people.py b/pydatalab/src/pydatalab/models/people.py index ec3485ade..5a3897419 100644 --- a/pydatalab/src/pydatalab/models/people.py +++ b/pydatalab/src/pydatalab/models/people.py @@ -190,10 +190,6 @@ def add_missing_type(cls, v): v = "people" return v - @classmethod - def set_default_type(cls, _): - return "people" - @staticmethod def new_user_from_identity( identity: Identity, diff --git a/pydatalab/src/pydatalab/routes/v0_1/info.py b/pydatalab/src/pydatalab/routes/v0_1/info.py index 4bf7bd6e1..dd74ebb90 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/info.py +++ b/pydatalab/src/pydatalab/routes/v0_1/info.py @@ -160,7 +160,7 @@ def list_block_types(): ) for block_type, block in BLOCK_TYPES.items() ], - meta=Meta(query=request.query_string), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), ).model_dump_json() ) ) @@ -185,7 +185,7 @@ def list_supported_types(): ) for item_type, schema in ITEM_SCHEMAS.items() ], - meta=Meta(query=request.query_string), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), ).model_dump_json() ) ) @@ -211,7 +211,7 @@ def get_schema_type(item_type): "schema": ITEM_SCHEMAS[item_type], }, ), - meta=Meta(query=request.query_string), + meta=Meta(query=request.query_string.decode() if request.query_string else ""), ).model_dump_json() ) ) diff --git a/pydatalab/src/pydatalab/routes/v0_1/items.py b/pydatalab/src/pydatalab/routes/v0_1/items.py index 4e5de851a..6b96249b6 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/items.py +++ b/pydatalab/src/pydatalab/routes/v0_1/items.py @@ -695,9 +695,13 @@ def _create_sample( raise BadRequest(f"Failed to add new item {new_sample['item_id']!r} to database.") # Save initial version snapshot after successful item creation + refcode = data_model.refcode + if not isinstance(refcode, str): + raise InternalServerError(f"Invalid refcode generated for new item: {refcode!r}") try: version_resp, version_status = save_version_snapshot( - data_model.refcode, action=VersionAction.CREATED + refcode, + action=VersionAction.CREATED, # type: ignore ) if version_status != 200: LOGGER.error( @@ -1313,7 +1317,7 @@ def compare_versions(refcode): v1 = flask_mongo.db.item_versions.find_one({"_id": query_params.v1, "refcode": refcode}) v2 = flask_mongo.db.item_versions.find_one({"_id": query_params.v2, "refcode": refcode}) if not v1 or not v2: - return NotFound("One or both versions not found") + raise NotFound("One or both versions not found") # Use DeepDiff for proper nested structure comparison # This handles nested dicts, lists, type changes, and provides detailed change information From 45183c532d420f8bb76adff94b9b84470e679450 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 00:28:35 +0100 Subject: [PATCH 20/24] Fix docker build by reordering tasks imports --- pydatalab/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index dad1da782..7b747714b 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -9,8 +9,6 @@ import tomlkit from invoke import Collection, task -from pydatalab.models.utils import UserRole - ns = Collection() dev = Collection("dev") admin = Collection("admin") @@ -263,13 +261,15 @@ def create_mongo_indices(_): @task -def change_user_role(_, display_name: str, role: "UserRole"): +def change_user_role(_, display_name: str, role: str): """This task takes a user's name and gives them the desired role.""" from bson import ObjectId from pydatalab.models.utils import UserRole from pydatalab.mongo import _get_active_mongo_client + role = UserRole(role.upper()) + try: role = getattr(UserRole, role.upper()) except AttributeError: From 1e89b2560c61bb84cd96a2ba8c93a4588233468e Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 10:05:05 +0100 Subject: [PATCH 21/24] Fix for auth routes --- pydatalab/src/pydatalab/login.py | 2 +- pydatalab/src/pydatalab/routes/v0_1/auth.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pydatalab/src/pydatalab/login.py b/pydatalab/src/pydatalab/login.py index 16682d0d7..b7521bc5f 100644 --- a/pydatalab/src/pydatalab/login.py +++ b/pydatalab/src/pydatalab/login.py @@ -105,7 +105,7 @@ def groups_lookup() -> dict: } -def get_by_id(user_id: str) -> LoginUser | None: +def get_by_id(user_id: str | ObjectId) -> LoginUser | None: """Lookup the user database ID and create a new `LoginUser` with the relevant metadata. diff --git a/pydatalab/src/pydatalab/routes/v0_1/auth.py b/pydatalab/src/pydatalab/routes/v0_1/auth.py index fe814ae18..eec258ccd 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/auth.py +++ b/pydatalab/src/pydatalab/routes/v0_1/auth.py @@ -617,6 +617,7 @@ def attach_identity_to_user( if user is None: raise RuntimeError("Failed to insert user into database") + wrapped_login_user(user) # Send email notification to admins _send_admin_email_notification(user) @@ -625,7 +626,8 @@ def attach_identity_to_user( # Log the user into the session with this identity if user is not None: - wrapped_login_user(user) + login_user = get_by_id(user.immutable_id) + wrapped_login_user(login_user) def _validate_magic_link_request(email: str, referrer: str) -> None: From 4ccfa2d0ec8b2aeb83832057456c0956875e88d3 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 10:05:22 +0100 Subject: [PATCH 22/24] Update plugin config validator for pydantic 2 --- pydatalab/tasks.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pydatalab/tasks.py b/pydatalab/tasks.py index 7b747714b..c3c9204ad 100644 --- a/pydatalab/tasks.py +++ b/pydatalab/tasks.py @@ -64,17 +64,16 @@ class PluginConfigModel(BaseModel): dependencies: list[str] = [] tool: ToolSection = ToolSection() - @model_validator(mode="before") - @classmethod - def _sources_must_match_dependencies(cls, values): - deps = {d.split("[")[0].strip() for d in values.get("dependencies", [])} - sources = values.get("tool", ToolSection()).uv.sources + @model_validator(mode="after") + def _sources_must_match_dependencies(self): + deps = {d.split("[")[0].strip() for d in self.dependencies} + sources = self.tool.uv.sources orphans = sorted(set(sources) - deps) if orphans: raise ValueError( f"[tool.uv.sources] entries have no matching dependency: {orphans}" ) - return values + return self return PluginConfigModel From 4265c794855c4ba75e6b3e80e077ca8eebb1c732 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Mon, 8 Jun 2026 11:25:57 +0100 Subject: [PATCH 23/24] More auth route fixes --- pydatalab/src/pydatalab/routes/v0_1/auth.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/auth.py b/pydatalab/src/pydatalab/routes/v0_1/auth.py index eec258ccd..a4f9a1fcb 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/auth.py +++ b/pydatalab/src/pydatalab/routes/v0_1/auth.py @@ -613,11 +613,12 @@ def attach_identity_to_user( ) LOGGER.debug("Inserting new user model %s into database", user) inserted_id = insert_pydantic_model_fork_safe(user, "users") - user = get_by_id(inserted_id) - if user is None: + login_user = get_by_id(inserted_id) + if login_user is None: raise RuntimeError("Failed to insert user into database") - wrapped_login_user(user) + user = login_user.person + # Send email notification to admins _send_admin_email_notification(user) From 338772e1306d3c9854733dbaf1cab2fb9f7098e0 Mon Sep 17 00:00:00 2001 From: BenSmithGreyGroup Date: Mon, 22 Jun 2026 18:18:57 +0100 Subject: [PATCH 24/24] Initial commit for refinement block --- pydatalab/src/pydatalab/apps/__init__.py | 7 + .../pydatalab/apps/xrd_refinement/__init__.py | 3 + .../pydatalab/apps/xrd_refinement/blocks.py | 284 ++++++++++++++++++ 3 files changed, 294 insertions(+) create mode 100644 pydatalab/src/pydatalab/apps/xrd_refinement/__init__.py create mode 100644 pydatalab/src/pydatalab/apps/xrd_refinement/blocks.py diff --git a/pydatalab/src/pydatalab/apps/__init__.py b/pydatalab/src/pydatalab/apps/__init__.py index f48a6427e..d34960efd 100644 --- a/pydatalab/src/pydatalab/apps/__init__.py +++ b/pydatalab/src/pydatalab/apps/__init__.py @@ -102,6 +102,13 @@ def load_app_blocks(): except ImportError as e: _check_error(e) + try: + from pydatalab.apps.xrd_refinement import XRDRefinementBlock + + app_blocks.append(XRDRefinementBlock) + except ImportError as e: + _check_error(e) + return app_blocks diff --git a/pydatalab/src/pydatalab/apps/xrd_refinement/__init__.py b/pydatalab/src/pydatalab/apps/xrd_refinement/__init__.py new file mode 100644 index 000000000..4c348448b --- /dev/null +++ b/pydatalab/src/pydatalab/apps/xrd_refinement/__init__.py @@ -0,0 +1,3 @@ +from .blocks import XRDRefinementBlock + +__all__ = ["XRDRefinementBlock"] diff --git a/pydatalab/src/pydatalab/apps/xrd_refinement/blocks.py b/pydatalab/src/pydatalab/apps/xrd_refinement/blocks.py new file mode 100644 index 000000000..f1b28bdde --- /dev/null +++ b/pydatalab/src/pydatalab/apps/xrd_refinement/blocks.py @@ -0,0 +1,284 @@ +import tempfile +import warnings +import zipfile +from pathlib import Path + +import bokeh +import numpy as np +import pandas as pd +from bokeh.events import DoubleTap +from bokeh.models import ( + ColumnDataSource, + CustomJS, + HoverTool, + Legend, + LegendItem, + Segment, + Span, +) +from bokeh.plotting import figure + +from pydatalab.blocks.base import DataBlock +from pydatalab.bokeh_plots import DATALAB_BOKEH_THEME +from pydatalab.file_utils import get_file_info_by_id +from pydatalab.logger import LOGGER + +TOOLS = "box_zoom, reset, crosshair, save" + + +def _parse_output_file(path: Path) -> pd.DataFrame | None: + """Parse a TOPAS-style 3-column refinement output file (2θ, Yobs, Ycalc).""" + try: + df = pd.read_csv(path, sep=r"\s+", header=None, dtype=float) + if df.shape[1] >= 3: + df = df.iloc[:, :3] + df.columns = ["two_theta", "yobs", "ycalc"] + return df + except Exception: + LOGGER.debug("Could not parse %s as 3-column output file", path, exc_info=True) + return None + + +def _parse_hkl_file(path: Path) -> pd.DataFrame | None: + """Parse a 7-column HKL reflection file (h k l M d 2θ I).""" + try: + df = pd.read_csv(path, sep=r"\s+", header=None, comment="#", dtype=float) + if df.shape[1] >= 7: + df = df.iloc[:, :7] + df.columns = ["h", "k", "l", "M", "d", "two_theta", "intensity"] + return df + except Exception: + LOGGER.debug("Could not parse %s as HKL reflection file", path, exc_info=True) + return None + + +def _detect_files( + extract_dir: Path, +) -> tuple[pd.DataFrame | None, pd.DataFrame | None]: + """Find and parse the output and HKL files inside an extracted zip directory.""" + output_df = None + hkl_df = None + + all_files = sorted(extract_dir.rglob("*")) + text_files = [f for f in all_files if f.is_file() and not f.name.startswith(".")] + + # Prefer files named output.txt / hkl.txt, then fall back to column-count sniffing + named_output = next((f for f in text_files if f.name.lower() == "output.txt"), None) + named_hkl = next((f for f in text_files if f.name.lower() == "hkl.txt"), None) + + if named_output: + output_df = _parse_output_file(named_output) + if named_hkl: + hkl_df = _parse_hkl_file(named_hkl) + + # Fall back: try every remaining file and guess by column count + for f in text_files: + if output_df is not None and hkl_df is not None: + break + try: + df = pd.read_csv(f, sep=r"\s+", header=None, comment="#", dtype=float, nrows=5) + except Exception: + LOGGER.debug("Skipping file during format sniffing: %s", f, exc_info=True) + continue + + ncols = df.shape[1] + if output_df is None and ncols == 3 and f != named_output: + candidate = _parse_output_file(f) + if candidate is not None: + output_df = candidate + elif hkl_df is None and ncols >= 7 and f != named_hkl: + candidate = _parse_hkl_file(f) + if candidate is not None: + hkl_df = candidate + + return output_df, hkl_df + + +def _build_refinement_plot(output_df: pd.DataFrame, hkl_df: pd.DataFrame | None): + """Build a Bokeh figure showing observed, calculated, residual, and (optionally) HKL ticks.""" + + two_theta = output_df["two_theta"].to_numpy() + yobs = output_df["yobs"].to_numpy() + ycalc = output_df["ycalc"].to_numpy() + residual_raw = yobs - ycalc + + ymax = max(yobs.max(), ycalc.max()) + ymin = yobs.min() + yrange = ymax - ymin + + # Place residual below the pattern with a gap + residual_offset = ymin - 0.20 * yrange + residual = residual_raw + residual_offset + + p = figure( + sizing_mode="scale_width", + aspect_ratio=2.5, + x_axis_label="2θ (°)", + y_axis_label="Intensity", + tools=TOOLS, + ) + p.toolbar.logo = "grey" + p.xaxis.ticker.desired_num_ticks = 10 + p.yaxis.ticker.desired_num_ticks = 5 + + main_source = ColumnDataSource( + {"two_theta": two_theta, "yobs": yobs, "ycalc": ycalc, "residual": residual} + ) + + r_obs = p.circle( + x="two_theta", + y="yobs", + source=main_source, + size=3, + color="black", + legend_label="Observed", + ) + r_calc = p.line( + x="two_theta", + y="ycalc", + source=main_source, + color="red", + line_width=1.5, + legend_label="Calculated", + ) + r_resid = p.line( + x="two_theta", + y="residual", + source=main_source, + color="green", + line_width=1, + legend_label="Residual", + ) + + zero_line = Span( + location=residual_offset, + dimension="width", + line_color="gray", + line_dash="dashed", + line_width=1, + ) + p.add_layout(zero_line) + + p.add_tools( + HoverTool( + renderers=[r_obs], + tooltips=[("2θ", "@two_theta{0.000}°"), ("Obs", "@yobs{0.0}"), ("Calc", "@ycalc{0.0}")], + mode="vline", + ) + ) + + legend_items = [ + LegendItem(label="Observed", renderers=[r_obs]), + LegendItem(label="Calculated", renderers=[r_calc]), + LegendItem(label="Residual", renderers=[r_resid]), + ] + + # HKL tick marks + if hkl_df is not None: + hkl_sorted = hkl_df.sort_values("two_theta") + norm_intensity = hkl_sorted["intensity"].to_numpy() + max_i = norm_intensity.max() + if max_i > 0: + norm_intensity = norm_intensity / max_i + + tick_base = residual_offset - 0.08 * yrange + tick_top = tick_base - norm_intensity * 0.10 * yrange + + hkl_source = ColumnDataSource( + { + "x0": hkl_sorted["two_theta"].to_numpy(), + "x1": hkl_sorted["two_theta"].to_numpy(), + "y0": np.full(len(hkl_sorted), tick_base), + "y1": tick_top, + "h": hkl_sorted["h"].astype(int).astype(str).to_numpy(), + "k": hkl_sorted["k"].astype(int).astype(str).to_numpy(), + "l": hkl_sorted["l"].astype(int).astype(str).to_numpy(), + } + ) + + r_hkl = p.add_glyph( + hkl_source, Segment(x0="x0", y0="y0", x1="x1", y1="y1", line_color="blue", line_width=1) + ) + r_hkl.hover_glyph = None + + p.add_tools( + HoverTool( + renderers=[r_hkl], + tooltips=[("hkl", "(@h @k @l)"), ("2θ", "@x0{0.000}°")], + ) + ) + + legend_items.append(LegendItem(label="Reflections", renderers=[r_hkl])) + + # Move legend outside the plot, with click_policy="hide" for toggling + p.legend.visible = False + external_legend = Legend( + items=legend_items, + click_policy="hide", + background_fill_alpha=0.8, + spacing=2, + margin=2, + ) + p.add_layout(external_legend, "right") + + p.js_on_event(DoubleTap, CustomJS(args=dict(p=p), code="p.reset.emit()")) + + return p + + +class XRDRefinementBlock(DataBlock): + blocktype = "xrd_refinement" + name = "XRD Refinement" + description = "Visualize Rietveld refinement results from a TOPAS output zip file." + accepted_file_extensions = (".zip",) + + defaults: dict = {} + + @property + def plot_functions(self): + return (self.generate_refinement_plot,) + + def generate_refinement_plot(self) -> None: + if not self.data.get("file_id"): + return + + try: + file_info = get_file_info_by_id(self.data["file_id"], update_if_live=True) + except Exception as exc: + warnings.warn(f"XRDRefinementBlock: could not retrieve file: {exc}") + return + + location = file_info.get("location") + if not location or not zipfile.is_zipfile(location): + warnings.warn(f"XRDRefinementBlock: file is not a valid zip archive: {location}") + return + + with tempfile.TemporaryDirectory() as tmpdir: + extract_dir = Path(tmpdir) + try: + with zipfile.ZipFile(location) as zf: + zf.extractall(extract_dir) + except Exception as exc: + warnings.warn(f"XRDRefinementBlock: failed to extract zip: {exc}") + return + + output_df, hkl_df = _detect_files(extract_dir) + + if output_df is None: + warnings.warn("XRDRefinementBlock: no 3-column output file found in zip") + return + + if hkl_df is None: + LOGGER.info("XRDRefinementBlock: no HKL file found; plotting without reflections") + + p = _build_refinement_plot(output_df, hkl_df) + + self.data["computed"] = { + "n_points": len(output_df), + "two_theta_range": [ + float(output_df["two_theta"].min()), + float(output_df["two_theta"].max()), + ], + "n_reflections": len(hkl_df) if hkl_df is not None else 0, + } + self.data["bokeh_plot_data"] = bokeh.embed.json_item(p, theme=DATALAB_BOKEH_THEME)