Tomviz 3.1

.github/workflows/build_and_test.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.11, 3.12, 3.13]
+        python-version: [3.11, 3.12, 3.13, 3.14]
         config:
         - {
             name: Linux,
@@ -77,6 +77,10 @@ jobs:
     - name: Install Test Requirements
       run: |
         conda install -y --override-channels -c conda-forge --file tomviz/tests/python/requirements-dev.txt
+        # The pipeline engine is the standalone tomviz-pipeline package
+        # from PyPI; the compatible range comes from the one place it is
+        # defined, and its dependencies come from runtime_requirements.txt.
+        pip install --no-deps -U "$(python tomviz/tomviz/python/tomviz/_pipeline_requirement.py)"
         pip install --no-build-isolation --no-deps -U tomviz/tomviz/python
         pip install --no-build-isolation --no-deps -U tomviz/acquisition
         # These do not have to be in the same environment as Tomviz, but the dependencies

.github/workflows/runtime_requirements.txt

@@ -9,4 +9,5 @@ pygments
 pystackreg
 h5py
 click
+tqdm
 scikit-image

.gitignore

@@ -3,3 +3,4 @@
 .cache
 *.egg-info
 __pycache__
+tomviz/python/build

BUILDING.md

@@ -43,15 +43,35 @@ Tomviz python library itself.
 
 ```bash
 conda install -y --override-channels -c conda-forge --file tomviz/.github/workflows/runtime_requirements.txt
+pip install --no-deps -U "$(python tomviz/tomviz/python/tomviz/_pipeline_requirement.py)"
 pip install --no-build-isolation --no-deps -U tomviz/tomviz/python
 ```
 
+The pipeline engine is the separately released
+[tomviz-pipeline](https://pypi.org/project/tomviz-pipeline/) package;
+`tomviz/python` is the application-side layer on top of it. The
+compatible tomviz-pipeline range is defined once, in
+`tomviz/python/tomviz/_pipeline_requirement.py`, which prints it when
+run as a script — hence the `$(python ...)` above. Both installs use
+`--no-deps` so pip never replaces the conda-managed packages. If your
+environment predates that split it still has the old in-tree package
+installed under the same name and the two will shadow each other — run
+`pip uninstall tomviz-pipeline tomviz-app` and delete
+`tomviz/tomviz/python/build/` before the two `pip install` lines above.
+
 Tomviz may then be ran as follows:
 
 ```bash
 ./tomviz-build/bin/tomviz
 ```
 
+To run the test suite (`ctest` in the build directory), also install the
+test requirements:
+
+```bash
+conda install -y --override-channels -c conda-forge --file tomviz/tests/python/requirements-dev.txt
+```
+
 All of the build instructions above put into a single script are as follows:
 
 ```bash
@@ -67,5 +87,6 @@ bash tomviz/.github/workflows/scripts/build_tomviz.sh
 
 # Install runtime dependencies
 conda install -y --override-channels -c conda-forge --file tomviz/.github/workflows/runtime_requirements.txt
+pip install --no-deps -U "$(python tomviz/tomviz/python/tomviz/_pipeline_requirement.py)"
 pip install --no-build-isolation --no-deps -U tomviz/tomviz/python
 ```

CMakeLists.txt

@@ -106,10 +106,10 @@ include(Git)
 determine_version("${tomviz_SOURCE_DIR}" "${GIT_EXECUTABLE}" "tomviz")
 # The version being developed here. Authoritative, because "git
 # describe" keeps reporting the previous release until the matching tag
-# is created, and a 3.0 build must not label itself 2.3.1. Also what
+# is created, and a 3.1 build must not label itself 3.0.0. Also what
 # source tarballs (no git at all) fall back on.
 set(tomviz_version_major 3)
-set(tomviz_version_minor 0)
+set(tomviz_version_minor 1)
 set(tomviz_version_patch 0)
 set(tomviz_version_extra)
 set(tomviz_version
@@ -118,7 +118,7 @@ set(tomviz_version
 # git describe refines that in two ways: it supplies the commit-count
 # and sha suffix that distinguishes a development build from the release
 # (empty when sitting exactly on the tag), and once a tag newer than the
-# version above exists it takes over the number too, so tagging 3.0.1
+# version above exists it takes over the number too, so tagging 3.1.1
 # needs no edit here. Packaging that builds outside a git checkout can
 # pass -DTOMVIZ_GIT_DESCRIBE=<version> to drive both.
 if(tomviz_VERSION_FULL)
@@ -166,6 +166,21 @@ find_package(ITK 4.9)
 set(PYBIND11_CPP_STANDARD "-std=c++17" CACHE STRING "")
 add_subdirectory(${PROJECT_SOURCE_DIR}/thirdparty/pybind11)
 
+# Route pybind11's GIL management through CPython's PyGILState_Ensure /
+# PyGILState_Release (see pybind11/gil_simple.h) instead of its own
+# thread-state caching. Tomviz embeds Python alongside VTK/ParaView,
+# which pervasively use PyGILState_Ensure (e.g. vtkSmartPyObject's
+# destructor). Mixing the two GIL bookkeeping schemes across threads —
+# a pipeline worker running a Python transform via pybind while the
+# main thread tears down a Python-bearing widget via VTK — corrupts the
+# worker's thread state (PyThreadState with a NULL interpreter), a
+# SIGSEGV in the next Python allocation. Defined here as an INTERFACE
+# definition so every translation unit that includes pybind11/gil.h
+# agrees on the gil_scoped_* type (it changes a type alias; a per-target
+# definition would be an ODR violation).
+target_compile_definitions(pybind11::pybind11
+  INTERFACE PYBIND11_SIMPLE_GIL_MANAGEMENT)
+
 option(tomviz_ENABLE_LOOKING_GLASS "Enable the looking glass plugin" OFF)
 if (tomviz_ENABLE_LOOKING_GLASS)
   if (NOT DEFINED tomviz_LOOKING_GLASS_PLUGIN_PATH)

docker/tomviz-builder/Dockerfile

@@ -15,6 +15,7 @@ RUN echo "#!/bin/bash" > build.sh && \
   echo "pip2 install -q -r ../tomviz/tests/python/requirements-dev.txt" >> build.sh && \
   echo "pip3 install -q -r ../tomviz/tests/python/requirements-dev.txt" >> build.sh && \
   echo "pip2 install -q -r ../tomviz/acquisition/requirements-dev.txt" >> build.sh  && \
+  echo "pip3 install -q \"tomviz-pipeline>=3.1.3\"" >> build.sh && \
   echo "pip3 install -q ../tomviz/tomviz/python/" >> build.sh && \
   echo "ctest -VV -S /tomviz/cmake/CircleContinuous.cmake" >> build.sh && \
   chmod u+x build.sh

docker/tomviz-pipeline/Dockerfile

@@ -1,13 +1,13 @@
-FROM library/python:3.7-slim
+FROM library/python:3.12-slim
 MAINTAINER Chris Harris <chris.harris@kitware.com>
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 ENV PYTHONUNBUFFERED=ON
 
-COPY tomviz/python/ /tmp/python/
-
-RUN pip install /tmp/python/ && \
-  rm -rf /tmp/python/
+# The pipeline runtime is the standalone tomviz-pipeline package. scipy
+# is not a dependency of the engine but is widely used by operator
+# scripts.
+RUN pip install "tomviz-pipeline>=3.1.3" scipy
 
 ENTRYPOINT ["tomviz-pipeline"]
 

docker/tomviz-tomopy-pipeline/Dockerfile

@@ -6,10 +6,10 @@ ENV LANG=C.UTF-8
 # Install tomopy
 RUN conda install -c conda-forge tomopy
 
-COPY tomviz/python/ /tmp/python/
-
-RUN pip install /tmp/python/ && \
-  rm -rf /tmp/python/
+# The pipeline runtime is the standalone tomviz-pipeline package. scipy
+# is not a dependency of the engine but is widely used by operator
+# scripts.
+RUN pip install "tomviz-pipeline>=3.1.3" scipy
 
 ENTRYPOINT ["tomviz-pipeline"]
 

examples/batch/README.md

@@ -0,0 +1,50 @@
+# Batch processing example
+
+One pipeline, many datasets, no GUI. This directory builds a small
+reproducible case for the `tomviz-pipeline` command line tool:
+
+ * `make_example.py` writes four volumes with a different number of
+   spheres in each (`example/inputs/particles_0N.emd`) and a pipeline
+   `example/particles.tvsm` that blurs, thresholds and labels the spheres.
+   The pipeline is assembled from the operator scripts in `tomviz/python`,
+   the same thing you get by saving it from the tomviz GUI.
+ * `run.sh` runs that pipeline over every input in one call.
+
+```sh
+pip install "tomviz-pipeline>=3.1.7"   # 3.1.7 added the label map writer
+./run.sh
+```
+
+`tomviz-pipeline` runs the state file once per matched input and writes
+each run's leaf outputs (here, the label map from Connected Components)
+to its own directory:
+
+```
+example/results/
+  run_0/4_Connected_Components__volume.emd
+  run_0/state.tvsm
+  run_1/4_Connected_Components__volume.emd
+  run_1/state.tvsm
+  run_2/...
+  run_3/...
+```
+
+Each `state.tvsm` is the pipeline as it ran, with that run's input pinned;
+it can be opened in tomviz or passed back to `tomviz-pipeline -s`.
+
+Open a label map in tomviz, or read it from Python:
+
+```python
+from tomviz_pipeline.io import load_dataset
+
+labels = load_dataset('example/results/run_2/4_Connected_Components__volume.emd')
+print(labels.active_scalars.max(), 'spheres')   # 4
+```
+
+To apply your own pipeline the same way, save it from the GUI
+(`File > Save State`) and point `tomviz-pipeline -s` at it; the `--input`
+override works with any state file whose only source is a file reader.
+The full set of options, including multi-source pipelines and running in
+an external Python environment, is in the
+[External Pipelines](https://tomviz.readthedocs.io/en/latest/pipelines.html)
+documentation.

examples/batch/make_example.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+"""Build the inputs and the pipeline for the batch-processing example.
+
+    python make_example.py [DIRECTORY] [--count N] [--size S]
+
+writes DIRECTORY/inputs/particles_00.emd ... particles_0N.emd, each a
+noisy volume with a different number of spheres in it, and
+DIRECTORY/particles.tvsm, a pipeline that blurs, thresholds and labels
+the spheres. The pipeline is assembled from the operator scripts shipped
+in tomviz/python, so it is the same thing saving it from the GUI gives
+you, and the example never drifts from the operators.
+
+Needs the tomviz-pipeline package (pip install tomviz-pipeline).
+"""
+import argparse
+import json
+from pathlib import Path
+
+import numpy as np
+
+from tomviz_pipeline import Dataset
+from tomviz_pipeline.writers.emd import write_emd
+
+OPERATORS = Path(__file__).resolve().parents[2] / 'tomviz' / 'python'
+
+
+def make_volume(index, size, rng):
+    """A volume with ``2 + index`` non-overlapping spheres in noise.
+
+    The spheres sit on a jittered 2 x 2 x 2 lattice, so they never touch
+    and Connected Components finds exactly as many as were placed."""
+    spheres = 2 + index
+    volume = rng.normal(0.0, 0.15, (size, size, size)).astype(np.float32)
+    lattice = [(i, j, k) for i in (0.25, 0.75) for j in (0.25, 0.75)
+               for k in (0.25, 0.75)]
+    rng.shuffle(lattice)
+    x, y, z = np.mgrid[:size, :size, :size]
+    for site in lattice[:spheres]:
+        center = np.array(site) * size + rng.uniform(-size / 16, size / 16, 3)
+        radius = rng.uniform(size / 12, size / 8)
+        inside = ((x - center[0])**2 + (y - center[1])**2 +
+                  (z - center[2])**2) <= radius**2
+        volume[inside] = 1.0
+    dataset = Dataset({'Scalars': np.asfortranarray(volume)}, 'Scalars')
+    dataset.spacing = (1.0, 1.0, 1.0)
+    return dataset, spheres
+
+
+def operator_node(node_id, name, label, **arguments):
+    """A Python operator node the way a state file stores it: the
+    description and the script travel inside the file."""
+    return {
+        'id': node_id,
+        'type': 'transform.legacyPython',
+        'label': label,
+        'description': (OPERATORS / f'{name}.json').read_text(),
+        'script': (OPERATORS / f'{name}.py').read_text(),
+        'arguments': arguments,
+    }
+
+
+def make_state(first_input):
+    nodes = [
+        {'id': 1, 'type': 'source.reader', 'label': 'Reader',
+         'fileNames': [str(first_input)]},
+        operator_node(2, 'GaussianFilter', 'Gaussian Blur', sigma=1.0),
+        operator_node(3, 'BinaryThreshold', 'Binary Threshold',
+                      lower_threshold=0.5, upper_threshold=10.0),
+        operator_node(4, 'ConnectedComponents', 'Connected Components',
+                      background_value=0, connectivity=3, min_size=10),
+    ]
+    # Binary Threshold delivers its label map on a port named after its
+    # child dataset, which is where Connected Components reads from.
+    links = [
+        {'from': {'node': 1, 'port': 'volume'},
+         'to': {'node': 2, 'port': 'volume'}},
+        {'from': {'node': 2, 'port': 'volume'},
+         'to': {'node': 3, 'port': 'volume'}},
+        {'from': {'node': 3, 'port': 'thresholded_segmentation'},
+         'to': {'node': 4, 'port': 'volume'}},
+    ]
+    return {
+        'schemaVersion': 2,
+        'pipeline': {'nextNodeId': 5, 'nodes': nodes, 'links': links},
+    }
+
+
+def build(directory, count=4, size=64, seed=0):
+    """Write the inputs and the state file. Returns the state file path,
+    the input paths and how many spheres each input holds."""
+    directory = Path(directory)
+    inputs_dir = directory / 'inputs'
+    inputs_dir.mkdir(parents=True, exist_ok=True)
+    rng = np.random.default_rng(seed)
+
+    inputs, spheres = [], []
+    for index in range(count):
+        dataset, n = make_volume(index, size, rng)
+        path = inputs_dir / f'particles_{index:02d}.emd'
+        write_emd(dataset, path)
+        inputs.append(path)
+        spheres.append(n)
+
+    state_path = directory / 'particles.tvsm'
+    state_path.write_text(json.dumps(make_state(inputs[0]), indent=2))
+    return state_path, inputs, spheres
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__.split('\n')[0])
+    parser.add_argument('directory', nargs='?', default='example')
+    parser.add_argument('--count', type=int, default=4,
+                        help='how many input volumes to write')
+    parser.add_argument('--size', type=int, default=64,
+                        help='edge length of each volume in voxels')
+    args = parser.parse_args()
+
+    state_path, inputs, spheres = build(args.directory, args.count,
+                                        args.size)
+    for path, n in zip(inputs, spheres):
+        print(f'{path}: {n} spheres')
+    print(f'{state_path}: Gaussian Blur -> Binary Threshold -> '
+          'Connected Components')
+
+
+if __name__ == '__main__':
+    main()

examples/batch/run.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+# Run the batch-processing example end to end: make the inputs and the
+# pipeline, then process every input with one tomviz-pipeline call.
+set -e
+cd "$(dirname "$0")"
+
+python make_example.py example
+tomviz-pipeline -s example/particles.tvsm -o example/results \
+    --input 'example/inputs/*.emd'
+
+echo
+echo "Results:"
+find example/results -type f | sort