GTAO test

README.md

@@ -103,6 +103,8 @@ The "frontend" UI thread handles user actions and rendering, while the "backend"
 - [Compressed segmentation format](src/sliceview/compressed_segmentation/README.md)
 - [Data chunk management](src/chunk_manager/)
 - [On-GPU hashing](src/gpu_hash/)
+- [Volume rendering](src/volume_rendering/)
+- [Screen-space ambient occlusion (SSAO) rendering](src/ssao/)
 
 # Building
 

docs/user-guide/index.rst

@@ -12,10 +12,13 @@ Topics
   to segmentation layers.
 - :doc:`annotation_shaders` — writing custom GLSL shader code to drive
   the visual appearance of annotations from their properties.
+- :doc:`rendering` — perspective-view rendering modes, including
+  screen-space ambient occlusion.
 
 .. toctree::
    :hidden:
 
    navigation
    annotations
    annotation_shaders
+   rendering

docs/user-guide/rendering.rst

@@ -0,0 +1,39 @@
+Rendering
+=========
+
+Neuroglancer offers several rendering modes for displaying data in the
+perspective (3-D) view.
+
+Volume rendering
+----------------
+
+TODO
+
+.. _ssao:
+
+Screen-space ambient occlusion (SSAO)
+-------------------------------------
+
+SSAO simulates shadows on 3-D mesh surfaces by darkening crevices and
+concavities where ambient light would be occluded. It adds depth cues that help
+you perceive shapes, and makes the display more appealing.
+
+.. image:: https://storage.googleapis.com/neuroglancer-doc-videos/0/user-guide/ssao/ssao-off.png
+   :alt: A mesh rendered without SSAO.
+   :width: 45%
+
+.. image:: https://storage.googleapis.com/neuroglancer-doc-videos/0/user-guide/ssao/ssao-on.png
+   :alt: The same mesh rendered with SSAO, showing shading in crevices.
+   :width: 45%
+
+Use the :kbd:`q` key to toggle SSAO on and off. The settings panel has three
+controls for SSAO:
+
+- **Enable SSAO (shadows)**: the :kbd:`q` toggle.
+- **SSAO intensity**: a slider; higher values give darker shadow effects.
+- **SSAO radius**: a slider; higher values give broader, softer shadows.
+
+SSAO applies only to mesh surfaces (segmentation layers with object surface
+meshes). It does not apply to other opaque geometry like skeletons and
+annotations. SSAO is disabled in any perspective view that contains a
+volume-rendering layer.

python/neuroglancer/__init__.py

@@ -118,5 +118,6 @@
     row_layout,  # noqa: F401
     column_layout,  # noqa: F401
     LayerGroupViewer,  # noqa: F401
+    Ssao,  # noqa: F401
     ViewerState,  # noqa: F401
 )

python/neuroglancer/viewer_state.py

@@ -1879,6 +1879,16 @@ class DataSelectionState(SidePanelLocation):
     layers = wrapped_property("layers", LayerSelectedValues)
 
 
+@export
+class Ssao(JsonObjectWrapper):
+    __slots__ = ()
+    supports_validation = True
+
+    enabled = wrapped_property("enabled", optional(bool, False))
+    intensity = wrapped_property("intensity", optional(float, 1.8))
+    radius = wrapped_property("radius", optional(float, 0.05))
+
+
 @export
 class ViewerState(JsonObjectWrapper):
     """Complete Neuroglancer viewer state.
@@ -1928,6 +1938,7 @@ class ViewerState(JsonObjectWrapper):
         "showAxisLines", optional(bool, True)
     )
     wire_frame = wireFrame = wrapped_property("wireFrame", optional(bool, False))
+    ssao = wrapped_property("ssao", Ssao)
     enable_adaptive_downsampling = enableAdaptiveDownsampling = wrapped_property(
         "enableAdaptiveDownsampling", optional(bool, True)
     )

src/data_panel_layout.css

@@ -49,3 +49,28 @@
 .neuroglancer-data-panel-layout-controls > button:hover {
   color: #6ff;
 }
+
+.neuroglancer-data-panel-status {
+  position: absolute;
+  top: 2px;
+  left: 2px;
+  display: flex;
+  flex-direction: column;
+  align-items: flex-start;
+  gap: 2px;
+  pointer-events: none;
+}
+
+.neuroglancer-data-panel-status > .neuroglancer-display-dimensions-widget {
+  position: static;
+  pointer-events: auto;
+}
+
+.neuroglancer-frame-time-indicator {
+  padding: 2px 4px;
+  background-color: rgba(0, 0, 0, 0.3);
+  color: white;
+  font-family: monospace;
+  font-size: 0.75rem;
+  white-space: nowrap;
+}

src/data_panel_layout.ts

@@ -42,6 +42,7 @@ import type { RenderLayerRole } from "#src/renderlayer.js";
 import { SliceView } from "#src/sliceview/frontend.js";
 import type { SliceViewerState } from "#src/sliceview/panel.js";
 import { SliceViewPanel } from "#src/sliceview/panel.js";
+import type { TrackableSSAO } from "#src/ssao/trackable_ssao_params.js";
 import { TrackableBoolean } from "#src/trackable_boolean.js";
 import type {
   WatchableSet,
@@ -94,13 +95,15 @@ export interface ViewerUIState
   wireFrame: TrackableBoolean;
   enableAdaptiveDownsampling: TrackableBoolean;
   showScaleBar: TrackableBoolean;
+  showFrameTime: TrackableBoolean;
   scaleBarOptions: TrackableValue<ScaleBarOptions>;
   visibleLayerRoles: WatchableSet<RenderLayerRole>;
   selectedLayer: SelectedLayerState;
   inputEventBindings: InputEventBindings;
   crossSectionBackgroundColor: TrackableRGB;
   perspectiveViewBackgroundColor: TrackableRGB;
   hideCrossSectionBackground3D: TrackableBoolean;
+  ssao: TrackableSSAO;
   pickRadius: TrackableValue<number>;
 }
 
@@ -184,6 +187,7 @@ export function getCommonViewerState(viewer: ViewerUIState) {
     visibility: viewer.visibility,
     scaleBarOptions: viewer.scaleBarOptions,
     hideCrossSectionBackground3D: viewer.hideCrossSectionBackground3D,
+    ssao: viewer.ssao,
     pickRadius: viewer.pickRadius,
   };
 }
@@ -213,19 +217,39 @@ function addDisplayDimensionsWidget(
   panel: RenderedDataPanel,
 ) {
   const { navigationState } = panel;
-  panel.element.appendChild(
-    layout.registerDisposer(
-      new DisplayDimensionsWidget(
-        navigationState.pose.displayDimensionRenderInfo.addRef(),
-        navigationState.zoomFactor,
-        navigationState.depthRange.addRef(),
-        navigationState.pose.orientation.addRef(),
-        panel.boundsUpdated,
-        panel.renderViewport,
-        panel instanceof SliceViewPanel,
-      ),
-    ).element,
+  const statusContainer = document.createElement("div");
+  statusContainer.className = "neuroglancer-data-panel-status";
+  const displayDimensionsWidget = layout.registerDisposer(
+    new DisplayDimensionsWidget(
+      navigationState.pose.displayDimensionRenderInfo.addRef(),
+      navigationState.zoomFactor,
+      navigationState.depthRange.addRef(),
+      navigationState.pose.orientation.addRef(),
+      panel.boundsUpdated,
+      panel.renderViewport,
+      panel instanceof SliceViewPanel,
+    ),
   );
+  statusContainer.appendChild(displayDimensionsWidget.element);
+
+  const frameTimeElement = document.createElement("div");
+  frameTimeElement.className = "neuroglancer-frame-time-indicator";
+  statusContainer.appendChild(frameTimeElement);
+  panel.element.appendChild(statusContainer);
+
+  const { display, showFrameTime } = layout.container.viewer;
+  const updateFrameTime = () => {
+    const frameTime = display.getLastFrameTimesInMs(1)[0];
+    frameTimeElement.textContent =
+      frameTime === undefined ? "-- ms" : `${frameTime.toFixed(1)} ms`;
+  };
+  const updateFrameTimeVisibility = () => {
+    frameTimeElement.style.display = showFrameTime.value ? "block" : "none";
+    if (showFrameTime.value) updateFrameTime();
+  };
+  layout.registerDisposer(display.frameTimeUpdated.add(updateFrameTime));
+  layout.registerDisposer(showFrameTime.changed.add(updateFrameTimeVisibility));
+  updateFrameTimeVisibility();
 }
 
 function registerRelatedLayouts(

src/display_context.ts

@@ -433,6 +433,7 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter {
   gl: GL;
   updateStarted = new NullarySignal();
   updateFinished = new NullarySignal();
+  frameTimeUpdated = new NullarySignal();
   continuousCameraMotionStarted = new NullarySignal();
   continuousCameraMotionFinished = new NullarySignal();
   multiChannelSetupFinished = new NullarySignal();
@@ -684,6 +685,7 @@ export class DisplayContext extends RefCounted implements FrameNumberCounter {
     this.updateFinished.dispatch();
     this.framerateMonitor.endLastTimeQuery(gl, ext);
     this.framerateMonitor.grabAnyFinishedQueryResults(gl);
+    this.frameTimeUpdated.dispatch();
   }
 
   getDepthArray(): Float32Array<ArrayBuffer> {

src/layer_group_viewer.ts

@@ -51,6 +51,7 @@ import {
   WatchableDisplayDimensionRenderInfo,
 } from "#src/navigation_state.js";
 import type { RenderLayerRole } from "#src/renderlayer.js";
+import type { TrackableSSAO } from "#src/ssao/trackable_ssao_params.js";
 import { TrackableBoolean } from "#src/trackable_boolean.js";
 import type {
   TrackableValue,
@@ -94,6 +95,7 @@ export interface LayerGroupViewerState {
   wireFrame: TrackableBoolean;
   enableAdaptiveDownsampling: TrackableBoolean;
   showScaleBar: TrackableBoolean;
+  showFrameTime: TrackableBoolean;
   scaleBarOptions: TrackableScaleBarOptions;
   showPerspectiveSliceViews: TrackableBoolean;
   layerSpecification: Owned<LayerListSpecification>;
@@ -104,6 +106,7 @@ export interface LayerGroupViewerState {
   crossSectionBackgroundColor: TrackableRGB;
   perspectiveViewBackgroundColor: TrackableRGB;
   hideCrossSectionBackground3D: TrackableBoolean;
+  ssao: TrackableSSAO;
   pickRadius: TrackableValue<number>;
 }
 
@@ -367,6 +370,9 @@ export class LayerGroupViewer extends RefCounted {
   get showScaleBar() {
     return this.viewerState.showScaleBar;
   }
+  get showFrameTime() {
+    return this.viewerState.showFrameTime;
+  }
   get showPerspectiveSliceViews() {
     return this.viewerState.showPerspectiveSliceViews;
   }
@@ -391,6 +397,9 @@ export class LayerGroupViewer extends RefCounted {
   get scaleBarOptions() {
     return this.viewerState.scaleBarOptions;
   }
+  get ssao() {
+    return this.viewerState.ssao;
+  }
   layerPanel: LayerBar | undefined;
   layout: DataPanelLayoutContainer;
   toolBinder: LocalToolBinder<this>;

src/layer_groups_layout.ts

@@ -409,6 +409,7 @@ function getCommonViewerState(viewer: Viewer) {
     wireFrame: viewer.wireFrame,
     enableAdaptiveDownsampling: viewer.enableAdaptiveDownsampling,
     showScaleBar: viewer.showScaleBar,
+    showFrameTime: viewer.showFrameTime,
     scaleBarOptions: viewer.scaleBarOptions,
     showPerspectiveSliceViews: viewer.showPerspectiveSliceViews,
     inputEventBindings: viewer.inputEventBindings,
@@ -421,6 +422,7 @@ function getCommonViewerState(viewer: Viewer) {
     crossSectionBackgroundColor: viewer.crossSectionBackgroundColor,
     perspectiveViewBackgroundColor: viewer.perspectiveViewBackgroundColor,
     hideCrossSectionBackground3D: viewer.hideCrossSectionBackground3D,
+    ssao: viewer.ssao,
     pickRadius: viewer.uiConfiguration.pickRadius,
   };
 }

src/mesh/frontend.ts

@@ -36,6 +36,7 @@ import {
   validateOctree,
 } from "#src/mesh/multiscale.js";
 import type { PerspectivePanel } from "#src/perspective_view/panel.js";
+import { perspectivePanelEmitWithNormals } from "#src/perspective_view/panel.js";
 import type {
   PerspectiveViewReadyRenderContext,
   PerspectiveViewRenderContext,
@@ -294,6 +295,16 @@ export class MeshShaderManager {
     mat3.invert(tempMat3, tempMat3);
     mat3.transpose(tempMat3, tempMat3);
     gl.uniformMatrix3fv(shader.uniform("uNormalMatrix"), false, tempMat3);
+    if (renderContext.emitter === perspectivePanelEmitWithNormals) {
+      // Combined model→view normal transform for SSAO. Uses `modelMat`
+      // directly, bypassing `uNormalMatrix`'s `canonicalVoxelFactors` scaling,
+      // which gives wrong oblique normals on anisotropic data.
+      mat4.multiply(tempMat4, projectionParameters.viewMatrix, modelMat);
+      mat4.invert(tempMat4, tempMat4);
+      mat3FromMat4(tempMat3, tempMat4);
+      mat3.transpose(tempMat3, tempMat3);
+      gl.uniformMatrix3fv(shader.uniform("uViewNormalMatrix"), false, tempMat3);
+    }
   }
 
   drawFragmentHelper(
@@ -358,7 +369,8 @@ export class MeshShaderManager {
     return parameterizedEmitterDependentShaderGetter(layer, layer.gl, {
       memoizeKey: `mesh/MeshShaderManager/${this.fragmentRelativeVertices}/${this.vertexPositionFormat}`,
       parameters: silhouetteRenderingEnabled,
-      defineShader: (builder, silhouetteRenderingEnabled) => {
+      defineShader: (builder, silhouetteRenderingEnabled, _, emitter) => {
+        const emitNormals = emitter === perspectivePanelEmitWithNormals;
         this.vertexPositionHandler.defineShader(builder);
         builder.addAttribute("highp vec2", "aVertexNormal");
         builder.addVarying("highp vec4", "vColor");
@@ -367,6 +379,12 @@ export class MeshShaderManager {
         builder.addUniform("highp mat3", "uNormalMatrix");
         builder.addUniform("highp mat4", "uModelViewProjection");
         builder.addUniform("highp uint", "uPickID");
+        if (emitNormals) {
+          builder.addVarying("highp vec3", "vViewNormal");
+          builder.addUniform("highp mat3", "uViewNormalMatrix");
+          // Hover-highlighted segments opt out of AO by writing the zero-RGB sentinel to `vViewNormal`; see below.
+          builder.addUniform("highp float", "uHighlighted");
+        }
         if (silhouetteRenderingEnabled) {
           builder.addUniform("highp float", "uSilhouettePower");
         }
@@ -395,13 +413,26 @@ float absCosAngle = abs(dot(normal, uLightDirection.xyz));
 float lightingFactor = absCosAngle + uLightDirection.w;
 vColor = vec4(lightingFactor * uColor.rgb, uColor.a);
 `;
+        if (emitNormals) {
+          // Bypass `normal` (post-uNormalMatrix); `uViewNormalMatrix`
+          // already encodes the full model→view transform. Multiply by
+          // (1 - uHighlighted) so hovered segments emit the zero sentinel.
+          vertexMain += `
+vViewNormal = (1.0 - uHighlighted) *
+              normalize(uViewNormalMatrix * (normalMultiplier * origNormal));
+`;
+        }
         if (silhouetteRenderingEnabled) {
           vertexMain += `
 vColor *= pow(1.0 - absCosAngle, uSilhouettePower);
 `;
         }
         builder.setVertexMain(vertexMain);
-        builder.setFragmentMain("emit(vColor, uPickID);");
+        builder.setFragmentMain(
+          emitNormals
+            ? "emit(vColor, uPickID, gl_FrontFacing ? vViewNormal : -vViewNormal);"
+            : "emit(vColor, uPickID);",
+        );
       },
     });
   }
@@ -516,6 +547,15 @@ export class MeshLayer extends PerspectiveViewRenderLayer<ThreeDimensionalRender
         ++presentChunks;
         if (renderContext.emitColor) {
           meshShaderManager.setColor(gl, shader, color!);
+          if (renderContext.emitter === perspectivePanelEmitWithNormals) {
+            const isHighlighted =
+              displayState.hoverHighlight.value &&
+              displayState.segmentSelectionState.isSelected(objectId);
+            gl.uniform1f(
+              shader.uniform("uHighlighted"),
+              isHighlighted ? 1.0 : 0.0,
+            );
+          }
         }
         if (renderContext.emitPickID) {
           meshShaderManager.setPickID(gl, shader, pickIndex!);
@@ -896,6 +936,15 @@ export class MultiscaleMeshLayer extends PerspectiveViewRenderLayer<ThreeDimensi
         }
         if (renderContext.emitColor) {
           meshShaderManager.setColor(gl, shader, color!);
+          if (renderContext.emitter === perspectivePanelEmitWithNormals) {
+            const isHighlighted =
+              displayState.hoverHighlight.value &&
+              displayState.segmentSelectionState.isSelected(objectId);
+            gl.uniform1f(
+              shader.uniform("uHighlighted"),
+              isHighlighted ? 1.0 : 0.0,
+            );
+          }
         }
         if (renderContext.emitPickID) {
           meshShaderManager.setPickID(gl, shader, pickIndex!);