feat: expose datashader categorical blending on hvPlot views

lumen/ai/agents/hvplot.py

@@ -6,6 +6,7 @@
 from pydantic.fields import FieldInfo
 
 from ...views import hvPlotUIView
+from ...views.base import GRIDDED_KINDS, VALUE_AGGREGATORS
 from ..config import PROMPTS_DIR
 from ..context import TContext
 from ..translate import param_to_pydantic
@@ -56,12 +57,45 @@ def _get_model(self, prompt_name: str, schema: dict[str, Any]) -> type[BaseModel
             extra_fields={
                 "chain_of_thought": (str, FieldInfo(description="Your thought process behind the plot.")),
             },
+            # Only this one view is being described. Expanding subclasses is for
+            # callers that want a union over a taxonomy, and here it would walk
+            # every subclass of every base, down into Panel and Bokeh objects
+            # that have no JSON schema and nothing to do with a plot.
+            process_subclasses=False,
         )
         return model[self.view_type.__name__]
 
+    @staticmethod
+    def _drop_conflicting_axes(spec: dict[str, Any]) -> None:
+        """Remove axis assignments that contradict each other.
+
+        The prompt asks for x, y, by and groupby to name distinct columns, and
+        the model does not always oblige. A groupby repeating x or y raises
+        while the plot is built, and one repeating by is worse than that: it
+        pages each category into its own frame, so a datashaded plot renders
+        without complaint and blends nothing.
+        """
+        taken = {spec.get("x"), spec.get("y"), *(spec.get("by") or [])}
+        groupby = [col for col in (spec.get("groupby") or []) if col not in taken]
+        if groupby:
+            spec["groupby"] = groupby
+        else:
+            spec.pop("groupby", None)
+        # z belongs to the gridded kinds, plus heatmap, which takes the same
+        # column as C. Elsewhere hvPlot only warns that it is unused, which is
+        # a warning nobody reads.
+        kind = spec.get("kind")
+        if kind not in GRIDDED_KINDS and kind != "heatmap":
+            spec.pop("z", None)
+        # Reducing a value column needs one named, and the spec has no field for
+        # it, so an aggregator asking for that has nothing to work on.
+        if spec.get("aggregator") in VALUE_AGGREGATORS:
+            spec.pop("aggregator", None)
+
     async def _extract_spec(self, context: TContext, spec: dict[str, Any]):
         pipeline = context["pipeline"]
         spec = {key: val for key, val in spec.items() if val is not None}
+        self._drop_conflicting_axes(spec)
         spec["type"] = "hvplot_ui"
         self.view_type.validate(spec)
         spec.pop("type", None)
@@ -70,6 +104,11 @@ async def _extract_spec(self, context: TContext, spec: dict[str, Any]):
         spec["responsive"] = True
         data = await get_data(pipeline)
         if len(data) > 20000 and spec["kind"] in ("line", "scatter", "points"):
-            spec["rasterize"] = True
-            spec["cnorm"] = "log"
+            if spec.get("by"):
+                # rasterize reduces to a single number per pixel, which throws
+                # away the category; datashade blends the ones sharing a pixel.
+                spec["datashade"] = True
+            else:
+                spec["rasterize"] = True
+                spec["cnorm"] = "log"
         return spec

lumen/ai/prompts/hvPlotAgent/main.jinja2

@@ -6,6 +6,12 @@ no repeated columns allowed. Do not arbitrarily set `groupby` and `by` fields un
 histogram requested, use `y` instead of `x`. If x categorical or strings, prefer barh over bar, and use `y` for
 values. If column has `format: geometry`, set `kind` from its `geometry_type`: `polygons` for Polygon or
 MultiPolygon, `paths` for LineString or MultiLineString, otherwise `points`.
+
+To color a dense scatter or point cloud by a category, set `by` to that column and `datashade=True`; this
+aggregates per pixel and blends the categories sharing one, instead of drawing points on top of each other.
+Add `dynspread=True` when the points are sparse enough to disappear. Set `color_key` to an explicit
+{category: hex} mapping only when the user names colors or asks for a specific palette; otherwise leave it
+unset and a categorical palette is chosen. Do not set `datashade` for a plot with no `by`.
 {%- if gridded is defined and gridded %}
 
 For gridded xarray data (dimensions {{ gridded.dims | join(', ') }}, variables {{ gridded.data_vars | join(', ') }}), pick framing request calls for:

lumen/ai/translate.py

@@ -38,6 +38,7 @@
     param.CalendarDate: DATE_TYPE,
     param.CalendarDateRange: tuple[DATE_TYPE],
     param.Parameter: object,
+    param.Path: str,
     param.Color: Color,
     param.Callable: Callable,
 }
@@ -279,7 +280,17 @@ def parameter_to_field(parameter: param.Parameter, created_models: dict[str, typ
     elif parameter.name == "margin":
         type_ = float | tuple[float, float] | tuple[float, float, float, float]
     else:
-        raise NotImplementedError(f"Parameter {parameter.name!r} of {param_type.__name__!r} not supported")
+        # Every branch above dispatches on the exact class, so a Parameter
+        # subclass that is not listed lands here: Magnitude is a Number,
+        # Filename is a Path, and any future subclass is the same story.
+        # Resolving through the MRO keeps those usable rather than aborting the
+        # whole model; param.Parameter is mapped, so the walk always terminates.
+        type_ = next(
+            (PARAM_TYPE_MAPPING[base] for base in param_type.__mro__[1:] if base in PARAM_TYPE_MAPPING),
+            object,
+        )
+        if parameter.default is not None and parameter.default is not PydanticUndefined:
+            field_kwargs["default"] = parameter.default
 
     if hasattr(parameter, "bounds") and parameter.bounds and type_ in [int, float]:
         try:
@@ -329,6 +340,14 @@ def param_to_pydantic(
     if parameterized_name in created_models:
         return created_models
 
+    current_excluded: list[str]
+    if isinstance(excluded, str) and hasattr(parameterized, excluded):
+        current_excluded = getattr(parameterized, excluded, [])
+    elif isinstance(excluded, list):  # If excluded is already a list
+        current_excluded = excluded
+    else:  # Fallback: excluded is a string but not an attribute, or other unexpected type
+        current_excluded = []
+
     pydantic_model_bases = []
     # Iterate over direct base classes of `parameterized`
     for param_parent_cls in parameterized.__bases__:
@@ -344,9 +363,15 @@ def param_to_pydantic(
                     param_parent_cls,
                     base_model=base_model,
                     created_models=created_models,
-                    # schema, excluded, extra_fields are generally not propagated
-                    # to parents unless explicitly needed, as they are often
-                    # specific to the current class conversion.
+                    # `excluded` has to reach the parent: the generated models
+                    # inherit, so a name dropped only here still arrives via the
+                    # parent's model. Leaving it out also made the caller's
+                    # exclusions no protection against walking into whatever
+                    # those params point at, which for a View is every other
+                    # component in Lumen.
+                    excluded=current_excluded,
+                    # schema and extra_fields stay local: they describe this
+                    # class's own fields.
                     process_subclasses=process_subclasses,  # PROPAGATE
                 )
 
@@ -368,14 +393,6 @@ def param_to_pydantic(
         elif base_model not in pydantic_model_bases:  # Should be redundant if list is empty
             pydantic_model_bases.append(base_model)
 
-    current_excluded: list[str]
-    if isinstance(excluded, str) and hasattr(parameterized, excluded):
-        current_excluded = getattr(parameterized, excluded, [])
-    elif isinstance(excluded, list):  # If excluded is already a list
-        current_excluded = excluded
-    else:  # Fallback: excluded is a string but not an attribute, or other unexpected type
-        current_excluded = []
-
     field_params = list(getattr(parameterized, "_field_params", []))
 
     fields = {}

lumen/tests/ai/test_hvplot_datashade.py

@@ -0,0 +1,175 @@
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pandas as pd
+import param
+import pytest
+
+from pydantic import BaseModel
+
+from lumen.ai.agents.hvplot import hvPlotAgent
+from lumen.ai.config import PROMPTS_DIR
+from lumen.ai.translate import param_to_pydantic
+from lumen.views.base import hvPlotBaseView
+
+
+async def extract(spec, n_rows):
+    """Run _extract_spec against a frame of the given size."""
+    data = pd.DataFrame({"x": range(n_rows), "y": range(n_rows), "c": ["a"] * n_rows})
+    with patch("lumen.ai.agents.hvplot.get_data", return_value=data):
+        return await hvPlotAgent()._extract_spec(
+            {"pipeline": SimpleNamespace(table="t")}, dict(spec)
+        )
+
+
+# ---- The LLM has to be able to express the spec ----
+
+def test_datashade_params_reach_the_pydantic_schema():
+    """The agent derives the LLM's schema from the view's params, so a param
+    that param_to_pydantic cannot map is a field the model can never fill."""
+    class Probe(param.Parameterized):
+        by = hvPlotBaseView.param.by
+        color_key = hvPlotBaseView.param.color_key
+        datashade = hvPlotBaseView.param.datashade
+        dynspread = hvPlotBaseView.param.dynspread
+
+    models = param_to_pydantic(Probe, base_model=BaseModel, process_subclasses=False)
+    properties = models["Probe"].model_json_schema()["properties"]
+
+    assert {"datashade", "dynspread", "color_key"} <= set(properties)
+
+
+# ---- The large-frame default has to preserve the category ----
+
+async def test_large_categorical_frame_uses_datashade():
+    """rasterize reduces each pixel to one number, which discards the category."""
+    spec = await extract({"kind": "points", "x": "x", "y": "y", "by": ["c"]}, 20_001)
+
+    assert spec["datashade"] is True
+    assert "rasterize" not in spec
+    assert "cnorm" not in spec
+
+
+async def test_large_frame_without_by_still_rasterizes():
+    spec = await extract({"kind": "points", "x": "x", "y": "y"}, 20_001)
+
+    assert spec["rasterize"] is True
+    assert spec["cnorm"] == "log"
+    assert "datashade" not in spec
+
+
+async def test_small_categorical_frame_is_left_alone():
+    """Under the threshold every point is drawn, so neither operation applies."""
+    spec = await extract({"kind": "points", "x": "x", "y": "y", "by": ["c"]}, 10)
+
+    assert "datashade" not in spec
+    assert "rasterize" not in spec
+
+
+@pytest.mark.parametrize("kind", ["bar", "heatmap", "hist"])
+async def test_non_point_kinds_are_untouched(kind):
+    spec = await extract({"kind": kind, "x": "x", "y": "y", "by": ["c"]}, 20_001)
+
+    assert "datashade" not in spec
+    assert "rasterize" not in spec
+
+
+# ---- The prompt has to mention it, or the model never tries ----
+
+def test_prompt_documents_categorical_datashading():
+    prompt = (PROMPTS_DIR / "hvPlotAgent" / "main.jinja2").read_text()
+
+    assert "datashade" in prompt
+    assert "color_key" in prompt
+
+
+# ---- The schema the LLM is actually handed ----
+
+def test_get_model_builds_the_view_schema():
+    """_get_model walked the whole component tree until it hit a param type it
+    could not map, so no hvPlot view schema could be built at all."""
+    schema = {
+        "lon": {"type": "number"},
+        "lat": {"type": "number"},
+        "family": {"type": "string", "enum": ["Irish", "Italian"]},
+    }
+
+    model = hvPlotAgent()._get_model("main", schema)
+    properties = model.model_json_schema()["properties"]
+
+    assert {"kind", "x", "y", "by", "datashade", "dynspread", "color_key"} <= set(properties)
+
+
+def test_get_model_omits_excluded_names():
+    """The generated models inherit, so anything excluded has to stay out of the
+    parent models too."""
+    model = hvPlotAgent()._get_model("main", {"lon": {"type": "number"}})
+    properties = model.model_json_schema()["properties"]
+
+    excluded = {"pipeline", "source", "transforms", "download", "controls",
+                "field", "selection_group"}
+
+    assert not (excluded & set(properties))
+
+
+# ---- The model does not always honour the "distinct columns" instruction ----
+
+async def test_groupby_repeating_by_is_dropped():
+    """A groupby equal to by pages each category into its own frame, so the
+    plot renders and blends nothing."""
+    spec = await extract(
+        {"kind": "points", "x": "x", "y": "y", "by": ["c"], "groupby": ["c"]}, 20_001
+    )
+
+    assert "groupby" not in spec
+    assert spec["datashade"] is True
+
+
+async def test_groupby_repeating_the_axes_is_dropped():
+    """A groupby naming x or y raises while the plot is being built."""
+    spec = await extract(
+        {"kind": "points", "x": "x", "y": "y", "by": ["c"], "groupby": ["x", "y", "c"]}, 20_001
+    )
+
+    assert "groupby" not in spec
+
+
+async def test_a_distinct_groupby_survives():
+    spec = await extract(
+        {"kind": "line", "x": "x", "y": "y", "by": ["c"], "groupby": ["other"]}, 10
+    )
+
+    assert spec["groupby"] == ["other"]
+
+
+async def test_z_is_dropped_for_non_gridded_kinds():
+    spec = await extract({"kind": "points", "x": "x", "y": "y", "z": "c"}, 10)
+
+    assert "z" not in spec
+
+
+@pytest.mark.parametrize("kind", ["image", "quadmesh", "heatmap"])
+async def test_z_is_kept_for_gridded_kinds(kind):
+    spec = await extract({"kind": kind, "x": "x", "y": "y", "z": "c"}, 10)
+
+    assert spec["z"] == "c"
+
+
+# ---- Aggregator the spec cannot support ----
+
+async def test_value_aggregator_is_dropped():
+    """The spec has no field naming a value column, so mean has nothing to
+    reduce and hvPlot would raise from inside the datashader operation."""
+    spec = await extract(
+        {"kind": "points", "x": "x", "y": "y", "rasterize": True, "aggregator": "mean"}, 10
+    )
+
+    assert "aggregator" not in spec
+
+
+async def test_row_counting_aggregator_survives():
+    spec = await extract(
+        {"kind": "points", "x": "x", "y": "y", "rasterize": True, "aggregator": "count"}, 10
+    )
+
+    assert spec["aggregator"] == "count"

lumen/tests/ai/test_translate.py

@@ -1,3 +1,5 @@
+from typing import get_args
+
 import param
 import pytest
 
@@ -573,3 +575,46 @@ class PlotConfig(param.Parameterized):
     instance = PydanticPlotConfig(xlim=(1.0, 5.0), ylim=(-1.0, 1.0))
     assert instance.xlim == (1.0, 5.0)
     assert instance.ylim == (-1.0, 1.0)
+
+
+class _UnmappedTypes(param.Parameterized):
+    """Parameter subclasses that no branch of parameter_to_field names."""
+
+    alpha = param.Magnitude(default=1.0)
+
+    yaml_file = param.Filename(default=None, check_exists=False)
+
+
+def test_param_subclasses_resolve_through_the_mro():
+    """Dispatch is by exact class, so an unlisted subclass used to abort the
+    whole model rather than the one field."""
+    models = param_to_pydantic(_UnmappedTypes, base_model=BaseModel, process_subclasses=False)
+    fields = models["_UnmappedTypes"].model_fields
+
+    assert fields["alpha"].annotation is float  # Magnitude is a Number
+    # Filename is a Path, and allows None, so it arrives as Optional[str].
+    assert str in get_args(fields["yaml_file"].annotation)
+
+
+class _Parent(param.Parameterized):
+
+    secret = param.String(default="hidden")
+
+    kept = param.String(default="visible")
+
+
+class _Child(_Parent):
+
+    own = param.String(default="mine")
+
+
+def test_excluded_names_do_not_return_via_the_parent():
+    """The generated models inherit, so an exclusion that stops at the child
+    still arrives through the parent's model."""
+    models = param_to_pydantic(
+        _Child, base_model=BaseModel, excluded=["secret"], process_subclasses=False
+    )
+    properties = models["_Child"].model_json_schema()["properties"]
+
+    assert "secret" not in properties
+    assert {"kept", "own"} <= set(properties)