fix: batch release of #406, #545 and #546 (TMbed payloads, N/A styles, shared constants)

.github/workflows/bundle-contract.yml

@@ -27,6 +27,11 @@ name: Bundle format contract (Python ↔ TypeScript)
 #
 # Cost: ~1 minute, about half of it the Python generator that writes the fixture
 # bundles by invoking the real `protspace bundle` CLI.
+#
+# It also checks the generated shared-constants modules against
+# contracts/shared-constants.json, for the same reason: a hand edit to only the
+# Python module would skip ci.yml, and one to only the TypeScript module would skip
+# protspace-ci.yml.
 
 on:
   push:
@@ -85,5 +90,8 @@ jobs:
       - name: Install Node dependencies
         run: pnpm install --frozen-lockfile
 
+      - name: Check generated shared constants
+        run: pnpm shared-constants:check
+
       - name: Run bundle format contract
         run: pnpm test:contract

.github/workflows/protspace-ci.yml

@@ -20,24 +20,23 @@ name: protspace CI (Python)
 #
 # The same holds for every other file outside apps/protspace that the tests read
 # (protspace cannot import TypeScript or protspace_prep, so its copies are checked
-# against their source): the web's palette ids, prep's
-# Biocentral outage patterns, the web-side v3 fixtures the decoder is checked
-# against, the served demo bundle, the web catalog whose defaultView each showcase
-# example opens on, and the perf manifest stage_perf.py writes. A change to only one
-# of them would otherwise never run the test that reads it. A checkout holding only
-# apps/protspace, the uv workspace files and the files listed below passes the fast
-# suite with no skips; a new test reading outside it belongs in the list. The web's
-# missing-value tokens are not here: bundle-contract.yml, which runs on every pull
-# request, checks them against the Python ones.
+# against their source): prep's Biocentral outage patterns, the web-side v3 fixtures
+# the decoder is checked against, the served demo bundle, the web catalog whose
+# defaultView each showcase example opens on, and the perf manifest stage_perf.py
+# writes. A change to only one of them would otherwise never run the test that reads
+# it. A checkout holding only apps/protspace, the uv workspace files and the files
+# listed below passes the fast suite with no skips; a new test reading outside it
+# belongs in the list. Constants shared with the web (missing-value tokens, palette
+# ids, ...) are not here: both sides are generated from
+# contracts/shared-constants.json, and bundle-contract.yml, which runs on every pull
+# request, checks the generated modules and the contract.
 
 on:
   push:
     branches: [main]
     paths:
       - 'apps/protspace/**'
       - 'docs/guide/python-cli.md'
-      - 'packages/utils/src/visualization/color-scheme.ts'
-      - 'packages/utils/src/visualization/numeric-binning.ts'
       - 'apps/prep/src/protspace_prep/pipeline.py'
       - 'packages/core/src/components/data-loader/utils/__fixtures__/v3-sample.parquetbundle'
       - 'apps/web/tests/fixtures/venom_eat_stats_811_v3.parquetbundle'
@@ -50,8 +49,6 @@ on:
     paths:
       - 'apps/protspace/**'
       - 'docs/guide/python-cli.md'
-      - 'packages/utils/src/visualization/color-scheme.ts'
-      - 'packages/utils/src/visualization/numeric-binning.ts'
       - 'apps/prep/src/protspace_prep/pipeline.py'
       - 'packages/core/src/components/data-loader/utils/__fixtures__/v3-sample.parquetbundle'
       - 'apps/web/tests/fixtures/venom_eat_stats_811_v3.parquetbundle'

AGENTS.md

@@ -37,10 +37,13 @@ One-time CLI setup is in [CONTRIBUTING.md](CONTRIBUTING.md#openspec-one-time-per
 ## Before committing
 
 Always run `pnpm precommit` before any git commit. It is
-`lint-staged && quality && docs:annotations:check && docs:examples:check && docs:build`:
+`lint-staged && quality && shared-constants:check && docs:annotations:check &&
+docs:examples:check && docs:build`:
 
 - ESLint `--fix` and Prettier `--write`, on staged files only (lint-staged)
 - TypeScript typecheck, Knip, and Knip dependency validation (`pnpm quality`)
+- `shared-constants:check` — the generated TypeScript and Python modules must match
+  `contracts/shared-constants.json` (see below)
 - `docs:annotations:check` — the generated annotation reference must match its source
 - `docs:examples:check` — the generated Example datasets page must match the example catalog,
   the bundle manifest and its prose (`docs/scripts/example-details.ts`)
@@ -69,6 +72,14 @@ Move these in the same PR — `pnpm precommit` covers none, and all three have s
 Pin a fact that has to live in two places with a test, not a comment asking the next reader to
 keep them in step — see `apps/protspace/tests/test_docs_extras_sync.py`.
 
+A **value** both the Python package and the web app hold (a token list, a palette id, a colour)
+goes in `contracts/shared-constants.json` instead: `pnpm shared-constants` generates it into
+`packages/utils/src/generated/` and `apps/protspace/src/protspace/_generated/`, both committed,
+and each side re-exports it under the name its code already uses. Never edit the generated
+modules. A **rule** both sides implement (how a cell is matched, which column holds the ids) is
+code, and is pinned by running both on the same inputs in the bundle contract suite
+(`tests/contract`). [`contracts/README.md`](contracts/README.md) has the details.
+
 ## End-to-end tests (Playwright)
 
 `e2e.yml` alone drives the real app in a browser; the unit suites run in jsdom, which has no

CONTRIBUTING.md

@@ -198,6 +198,14 @@ pnpm test:ci      # Run the CI test suite locally
 pnpm docs:build   # Verify the documentation build
 ```
 
+**Constants shared with the Python package** (missing-value tokens, the N/A key and colour,
+Kelly's colours, palette ids, point shapes, boolean labels) live in
+`contracts/shared-constants.json`. Edit them there and run `pnpm shared-constants`, which
+regenerates `packages/utils/src/generated/shared-constants.ts` and
+`apps/protspace/src/protspace/_generated/shared_constants.py`; commit all three.
+`pnpm shared-constants:check` (in `pnpm precommit` and `bundle-contract.yml`) fails while they
+disagree. See [`contracts/README.md`](contracts/README.md) for what belongs there.
+
 ### Code Quality Standards
 
 **Automated CI checks** (all PRs must pass):
@@ -209,7 +217,8 @@ pnpm docs:build   # Verify the documentation build
 5. Test suite execution
 6. Documentation build
 7. Bundle format contract (`bundle-contract.yml`), `pnpm test:contract` against a real Python-produced
-   bundle. Deliberately **unfiltered**: it runs on every PR, including PRs that touch no code it covers.
+   bundle, plus `pnpm shared-constants:check`. Deliberately **unfiltered**: it runs on every PR,
+   including PRs that touch no code it covers.
 8. E2E, Playwright (`e2e.yml`), on PRs touching the web app, the packages it builds on, or the root
    files that decide what those resolve to — `e2e.yml` owns the exact list — plus nightly on `main`.
    Outside those paths, dispatch it: `gh workflow run e2e.yml --ref <branch>`.

apps/prep/tests/test_pipeline.py

@@ -339,7 +339,7 @@ async def test_pipeline_uses_normalized_fasta_for_embed_and_annotate(ctx):
     normalized = ctx.output_dir / "input.normalized.fasta"
     assert seen_inputs["embed"] == str(normalized)
     assert seen_inputs["annotate"] == str(normalized)
-    assert ">P12345" in normalized.read_text()
+    assert normalized.read_text() == ">P12345\nMAAAAAA\n"
 
 
 async def test_embed_failure_with_connection_refused_is_classified_as_biocentral_unavailable(

apps/protspace/CLAUDE.md

@@ -122,6 +122,8 @@ Model shortcuts are defined in `MODEL_SHORT_KEYS` (the models the client lists i
 
 ```
 src/protspace/
+├── _generated/
+│   └── shared_constants.py     # Values shared with the web app, generated from contracts/shared-constants.json (`pnpm shared-constants`); never edit, re-exported under their old names
 ├── cli/
 │   ├── app.py                  # Typer app root, shared utilities, setup_logging()
 │   ├── common_options.py       # Shared Typer options
@@ -315,7 +317,8 @@ For a live count run `uv run pytest tests/ --collect-only -q`.
 | `test_biocentral_retriever.py` | Biocentral prediction retriever (TMbed parsing, per-sequence), batches of at most `_BATCH_SIZE`, a failed batch keeps the others and warns clear of `_BIOCENTRAL_DOWN_PATTERNS`, long sequences still sent, no usable server fails the source and the warning says why, a model the client no longer lists fails the source (blank columns would be cached as complete) and every model the retriever names is one the client lists |
 | `test_taxonomy_annotation_retriever.py` | Taxonomy via UniProt Taxonomy API (mocked + integration) |
 | `test_config_validation.py` | DimensionReductionConfig parameter validation |
-| `test_style_warnings.py` | `protspace style` warnings: numeric-column detection (tsenoner/protspace-legacy#67) + `selectedPaletteId` validation (categorical vs gradient palette, per column type) + pinned palette-catalog contract |
+| `test_style_warnings.py` | `protspace style` warnings: numeric-column detection (tsenoner/protspace-legacy#67) + `selectedPaletteId` validation (categorical vs gradient palette, per column type); `kellys` is a categorical palette |
+| `test_shared_constants.py` | Every name the package exports for a constant shared with the web app (`MISSING_VALUE_TOKENS`, `NA_INTERNAL`, `NA_PINNED_COLOR`, `KELLYS_COLORS`, the palette ids, `ALLOWED_SHAPES`, `ARROW_BOOLEAN_LABELS`) holds the value generated from `contracts/shared-constants.json`; the generated module is plain data (no imports) |
 | `test_h5_parse_identifier.py` | HDF5 key parsing, identifier extraction |
 | `test_base_data_processor.py` | BaseProcessor: reduction (the MDS input actually fed: a similarity matrix converted with `sqrt(max(s) - s)`, a distance matrix passed through), output creation, save (incl. settings in unbundled output) |
 | `test_ted_retriever.py` | TED domain retriever (mocked AlphaFold API, CATH names), final retry pass for failed lookups (10-in-a-row cut-off, 404 never retried), parallel lookups equal to one at a time under jitter, one session, CATH names loaded once, a slow first-pass lookup not holding up the others, a tripped final-pass breaker stopping lookups still retrying |
@@ -326,7 +329,7 @@ For a live count run `uv run pytest tests/ --collect-only -q`.
 | `test_transfer_cli.py` | Transfer orchestration core (k>1 confidence, cosine metric, self-transfer, a cell spelled missing such as `N/A` filled and never copied) and the `transfer` CLI: bad input (no matching embedding, no query match, both id columns) as a usage error with its message, `--help` rendering, `protein_id` and legacy v1 bundles end to end |
 | `test_predictions_overlay.py` | Building the per-cell prediction overlay columns |
 | `test_display_decode.py` | Display-side decoding of encoded values, multi-hit rendering, gated-off passthrough |
-| `test_missing_values.py` | One definition of a missing cell (`is_missing_value`, the web's `normalizeMissingValue`; the bundle contract suite pins the two): the Dash plot and `stats` read cased, padded and `__NA__` spellings and pandas nulls as missing; every fixed producer vocabulary (TMbed, signal peptide, PDB, fragment, reviewed, boolean labels) stays a category, hit by hit; no string literal in `data/annotations/` or `data/parsers/` is spelled like a missing value |
+| `test_missing_values.py` | One definition of a missing cell (`is_missing_value`, the web's `normalizeMissingValue`; the bundle contract suite pins the two): the Dash plot and `stats` read cased, padded and `__NA__` spellings and pandas nulls as missing; `protspace style` stores an N/A colour, shape, pin or hide named by any spelling under `__NA__`, once, a restyle replacing the stored one, and `--generate-template` lists N/A once; every fixed producer vocabulary (TMbed, signal peptide, PDB, fragment, reviewed, boolean labels) stays a category, hit by hit; no string literal in `data/annotations/` or `data/parsers/` is spelled like a missing value |
 | `test_toxprot_demo.py` | Signal-peptide bound parsing, mature-FASTA stripping, bundle post-processing (column filter/reorder) |
 | `test_build_showcase.py` | `scripts/generate_examples/build_showcase.py`, the showcase bundle build: bundle reading and rebuilding (legacy and v3), projection selection, column order from the web catalog's `defaultView`, styles filtering, mature chains and the three-finger-toxins hold-out and EAT gates, provenance stamping read back by `write_manifest.py`, step markers and reruns, fetch release checks, `stage-release` never giving a published name new bytes; one offline end-to-end EAT build (three-finger toxins), run in CI |
 | `test_write_manifest.py` | `scripts/generate_examples/write_manifest.py`: every record value read from the file (legacy and v3 containers), provenance and the build command's machine paths redacted, a bundle stamped for another example refused, retained files kept one release and never under a current file's name, the Zenodo DOI kept while the bytes are unchanged, `--check` |

apps/protspace/scripts/generate_toxprot_demo.py

@@ -5,10 +5,11 @@
 the mature peptides with ProtT5 and ESM2-650M, then runs DR + the
 sequence-independent annotation sources via `protspace prepare`. The
 sequence-based sources (InterPro, Biocentral) run separately through
-`protspace annotate` on the FULL-LENGTH sequences: local FASTA sequences
-take priority over UniProt's, so feeding them the mature peptides made
-InterPro miss by MD5 (73 % empty Pfam) and made TMbed predict no signal
-peptide for secreted toxins.
+`protspace annotate` on the FULL-LENGTH sequences: Biocentral predicts the
+local FASTA sequence over UniProt's, so feeding it the mature peptides made
+TMbed predict no signal peptide for secreted toxins. (InterPro once missed by
+MD5 the same way, 73 % empty Pfam; it now prefers UniProt's canonical
+sequence, falling back to the FASTA's only without one.)
 
 Finally the bundle keeps every annotation column (protein_families first),
 replaces `length` with the mature length, and patches the settings JSON:

apps/protspace/src/protspace/_generated/__init__.py

@@ -0,0 +1,6 @@
+"""Modules generated from sources outside the package; never edit them by hand.
+
+``shared_constants`` comes from ``contracts/shared-constants.json`` at the repository
+root (``pnpm shared-constants``). It is committed so the wheel needs nothing outside
+``apps/protspace``.
+"""

apps/protspace/src/protspace/_generated/shared_constants.py

@@ -0,0 +1,130 @@
+"""Constants the protspace package shares with the ProtSpace web app.
+
+Generated by scripts/shared-constants/generate-shared-constants.mts from
+contracts/shared-constants.json. Do not edit: change the JSON, run
+``pnpm shared-constants`` and commit both generated modules.
+``pnpm shared-constants:check`` fails while they disagree.
+"""
+
+# Strings that mean "this annotation value is missing". A cell is missing
+# when it is null, a non-finite number, blank, or, trimmed and lower-cased,
+# one of these: `normalizeMissingValue` in the web, `is_missing_value` in
+# Python. That rule is code on each side, so the bundle contract suite runs
+# both on the same probe spellings.
+#
+# `__na__` reserves the spelling of NA_VALUE: the bundle writer drops that
+# token unconditionally (it is the sentinel a missing cell carries), so a
+# genuine category literally spelled `__NA__` would be silently deleted on
+# export. Reading it as missing at ingestion makes the two agree.
+MISSING_VALUE_TOKENS = frozenset(
+    {
+        "na",
+        "n/a",
+        "nan",
+        "null",
+        "none",
+        "__na__",
+    }
+)
+
+# Internal key of the N/A category: the legend value a missing cell gets,
+# and the key its colour and shape are saved under in a bundle's settings.
+# Never displayed; the web shows it as `N/A`.
+NA_VALUE = "__NA__"
+
+# Default swatch colour of the N/A category, which a saved category colour
+# overrides (categorical annotations only). Python gives a pinned N/A this
+# colour, and `protspace style --generate-template` pre-fills it for N/A, so
+# a CLI-styled legend matches the web's default.
+NA_DEFAULT_COLOR = "#DDDDDD"
+
+# Kelly's 21 colours of maximum contrast, in the order categories take them
+# (the category in slot i gets colour i). Medium gray (#848482) is left out,
+# too close to the greys of Other (#999999) and N/A. Very light and very
+# dark gray come last, for the least frequent categories. In order: vivid
+# yellow, strong purple, vivid orange, very light blue, vivid red, grayish
+# yellow, vivid green, strong purplish pink, strong blue, strong yellowish
+# pink, strong violet, vivid orange yellow, strong purplish red, vivid
+# greenish yellow, strong reddish brown, vivid yellowish green, deep
+# yellowish brown, vivid reddish orange, dark olive green, very light gray,
+# very dark gray.
+#
+# Reference: Kelly, Kenneth L. Twenty-two colors of maximum contrast.
+# Color Engineering 3.26-27 (1965).
+KELLYS_COLORS = (
+    "#F3C300",
+    "#875692",
+    "#F38400",
+    "#A1CAF1",
+    "#BE0032",
+    "#C2B280",
+    "#008856",
+    "#E68FAC",
+    "#0067A5",
+    "#F99379",
+    "#604E97",
+    "#F6A600",
+    "#B3446C",
+    "#DCD300",
+    "#882D17",
+    "#8DB600",
+    "#654522",
+    "#E25822",
+    "#2B3D26",
+    "#F2F3F4",
+    "#222222",
+)
+
+# Palettes for categorical annotations: the only ids a bundle's
+# `selectedPaletteId` may name. `protspace style` warns on any other id, and
+# the web draws a gradient or unknown id with DEFAULT_CATEGORICAL_PALETTE_ID.
+# The web's `COLOR_SCHEMES` holds the colours and must list exactly these ids
+# plus GRADIENT_PALETTE_IDS (the type check fails otherwise). A new palette
+# also goes in docs/guide/styling.md (Color palettes).
+CATEGORICAL_PALETTE_IDS = (
+    "kellys",
+    "okabeIto",
+    "tolBright",
+    "set2",
+    "dark2",
+    "tableau10",
+)
+
+# Sequential palettes for numeric annotations, chosen in the web's numeric
+# legend settings rather than through `selectedPaletteId`.
+GRADIENT_PALETTE_IDS = (
+    "viridis",
+    "cividis",
+    "inferno",
+    "batlow",
+    "plasma",
+)
+
+# The palette a categorical annotation starts on, and the one the web falls
+# back to; one of CATEGORICAL_PALETTE_IDS.
+DEFAULT_CATEGORICAL_PALETTE_ID = "kellys"
+
+# The gradient a numeric annotation starts on; one of GRADIENT_PALETTE_IDS.
+DEFAULT_NUMERIC_PALETTE_ID = "batlow"
+
+# Shapes a category can be drawn with: the `shape` values a bundle's
+# settings may hold. The web's `PointShape` type is exactly these.
+POINT_SHAPES = (
+    "circle",
+    "square",
+    "diamond",
+    "triangle-up",
+    "triangle-down",
+    "plus",
+)
+
+# Labels of an Arrow `bool` annotation column, false then true. Lower case,
+# as the browser has always shown one (`String(true)`), so legend colours
+# saved against a v2 bundle keep matching. Both v3 writers store a bool
+# column under these labels, and the web's writer keeps `sourceType: bool`
+# only while every label is one of them. Not Python's CANONICAL_BOOLEANS
+# (`False`/`True`), which are string cells the pipeline writes itself.
+ARROW_BOOLEAN_LABELS = (
+    "false",
+    "true",
+)

apps/protspace/src/protspace/core/constants.py

@@ -3,14 +3,13 @@
 
 import pandas as pd
 
+# The spellings the web app reads as a missing annotation value, compared against
+# the trimmed, lower-cased cell. Both languages generate them from
+# contracts/shared-constants.json; the bundle contract suite (tests/contract) checks
+# that the two matching rules agree.
+from protspace._generated.shared_constants import MISSING_VALUE_TOKENS
 from protspace.utils.arrow_reader import ArrowReader
 
-# The spellings the web app reads as a missing annotation value: its
-# ``MISSING_VALUE_TOKENS`` (packages/utils/src/visualization/missing-values.ts),
-# which it compares against the trimmed, lower-cased cell. The bundle contract
-# suite (tests/contract) fails when the two sets differ.
-MISSING_VALUE_TOKENS = frozenset({"na", "n/a", "nan", "null", "none", "__na__"})
-
 
 def is_missing_value(value) -> bool:
     """Whether the web app reads this annotation cell as missing (N/A).