RFC, feat: Add (basic) support for pandas SparseDtype

docs/concepts/sparse.md

@@ -0,0 +1,55 @@
+# Sparse columns
+
+pandas can store a column as a [`SparseArray`](https://pandas.pydata.org/docs/user_guide/sparse.html),
+which compresses repeated occurrences of a single *fill value* (typically `0` or `NaN`) to save memory.
+
+No other Narwhals backend has an equivalent concept.
+
+Narwhals treats sparsity as a **storage detail, not a logical dtype**: a sparse column has the same Narwhals dtype as
+its dense equivalent. Beyond that, keeping columns sparse is a **best-effort, pandas-only convenience**.
+
+## Reading the schema
+
+A `Sparse[<subtype>]` column reports the Narwhals dtype of its dense `<subtype>` (e.g. `Sparse[int64, 0]` maps to `Int64`),
+rather than `Unknown`:
+
+```python exec="yes" source="above" session="sparse"
+import narwhals as nw
+import pandas as pd
+
+frame = pd.DataFrame(
+    {
+        "a": pd.arrays.SparseArray([0, 1, 0, 2]),
+        "b": pd.arrays.SparseArray([0.0, 1.5, 0.0, 2.5]),
+    }
+)
+```
+
+```python exec="yes" source="material-block" result="python" session="sparse"
+print("native dtypes: ", frame.dtypes.to_dict())
+print("narwhals schema:", nw.from_native(frame).schema)
+```
+
+## Casting
+
+`cast` keeps a column sparse when it can do so *without altering the data*, and densifies otherwise.
+A sparse column stays sparse when the target is a numeric or temporal NumPy dtype and the source fill
+value is representable in it; it densifies when:
+
+* the target is not a NumPy dtype (nullable, PyArrow-backed) as pandas' `SparseDtype` can only wrap NumPy dtypes;
+* the target is a string / object dtype: casting a sparse column to these is a no-op that would leave the stored values unconverted;
+* the source fill value cannot be represented in the target (e.g. a `NaN` fill cast to an integer subtype), since no
+    fill would preserve the compressed entries.
+
+```python exec="yes" source="material-block" result="python" session="sparse"
+s = nw.from_native(frame["a"], series_only=True)
+
+print("cast to Float64:", nw.to_native(s.cast(nw.Float64)).dtype)
+print("cast to String:", nw.to_native(s.cast(nw.String)).dtype)
+```
+
+!!! note
+    Sparse *input* is accepted and preserved on a best-effort basis, for pandas only.
+
+    There is no sparse output on any other backend, and Narwhals only explicitly preserves sparsity in `cast`.
+    Other operations delegate to pandas, which may or may not keep the column sparse.

src/narwhals/_pandas_like/dataframe.py

@@ -13,6 +13,7 @@
     broadcast_series_to_index,
     get_dtype_backend,
     import_array_module,
+    is_object_native_dtype,
     iter_dtype_backends,
     narwhals_to_native_dtype,
     native_to_narwhals_dtype,
@@ -424,13 +425,13 @@ def iter_rows(
     def schema(self) -> dict[str, DType]:
         native_dtypes = self.native.dtypes
         return {
-            col: native_to_narwhals_dtype(
-                native_dtypes[col], self._version, self._implementation
-            )
-            if native_dtypes[col] != "object"
-            else object_native_to_narwhals_dtype(
+            col: object_native_to_narwhals_dtype(
                 self.native[col], self._version, self._implementation
             )
+            if is_object_native_dtype(native_dtypes[col])
+            else native_to_narwhals_dtype(
+                native_dtypes[col], self._version, self._implementation
+            )
             for col in self.native.columns
         }
 

src/narwhals/_pandas_like/series.py

@@ -17,6 +17,8 @@
     broadcast_series_to_index,
     get_dtype_backend,
     import_array_module,
+    is_object_native_dtype,
+    keep_sparse_dtype,
     narwhals_to_native_dtype,
     native_to_narwhals_dtype,
     object_native_to_narwhals_dtype,
@@ -238,11 +240,13 @@ def name(self) -> str:
     def dtype(self) -> DType:
         native_dtype = self.native.dtype
         return (
-            native_to_narwhals_dtype(native_dtype, self._version, self._implementation)
-            if native_dtype != "object"
-            else object_native_to_narwhals_dtype(
+            object_native_to_narwhals_dtype(
                 self.native, self._version, self._implementation
             )
+            if is_object_native_dtype(native_dtype)
+            else native_to_narwhals_dtype(
+                native_dtype, self._version, self._implementation
+            )
         )
 
     @property
@@ -313,9 +317,10 @@ def scatter(
         return None if in_place else self._with_native(series)
 
     def cast(self, dtype: IntoDType) -> Self:
-        if self.dtype == dtype and self.native.dtype != "object":
+        if self.dtype == dtype and not is_object_native_dtype(self.native.dtype):
             # Avoid dealing with pandas' type-system if we can. Note that it's only
-            # safe to do this if we're not starting with object dtype, see tests/expr_and_series/cast_test.py::test_cast_object_pandas
+            # safe to do this if we're not starting with object dtype (incl. a sparse
+            # `object` subtype), see tests/expr_and_series/cast_test.py::test_cast_object_pandas
             # for an example of why.
             return self._with_native(self.native, preserve_broadcast=True)
         pd_dtype = narwhals_to_native_dtype(
@@ -324,6 +329,8 @@ def cast(self, dtype: IntoDType) -> Self:
             implementation=self._implementation,
             version=self._version,
         )
+        # Keep a sparse column sparse (when the target subtype allows it).
+        pd_dtype = keep_sparse_dtype(pd_dtype, self.native.dtype)
         return self._with_native(self.native.astype(pd_dtype), preserve_broadcast=True)
 
     def item(self, index: int | None = None) -> Any:

src/narwhals/_pandas_like/utils.py

@@ -262,9 +262,18 @@ def object_native_to_narwhals_dtype(
         # objects, so we can just return String.
         return dtypes.String()
 
-    infer = pd.api.types.infer_dtype
-    # Arbitrary limit of 100 elements to use to sniff dtype.
-    inferred_dtype = "empty" if series is None else infer(series.head(100), skipna=True)
+    if series is None:  # pragma: no cover
+        # Only reachable via the `allow_object` path in `native_to_narwhals_dtype`.
+        inferred_dtype = "empty"
+    else:
+        # Arbitrary limit of 100 elements to use to sniff dtype.
+        sample: Any = series.head(100)
+        if is_dtype_sparse(sample.dtype):
+            # Densify the (small) sample so a sparse `object` column is sniffed the
+            # same way as its dense equivalent.
+            sample = sample.astype(sample.dtype.subtype)
+        infer = pd.api.types.infer_dtype
+        inferred_dtype = infer(sample, skipna=True)
     if inferred_dtype == "string":
         return dtypes.String()
     if inferred_dtype == "empty" and version is not Version.V1:
@@ -314,6 +323,11 @@ def native_to_narwhals_dtype(
     *,
     allow_object: bool = False,
 ) -> DType:
+    if is_dtype_sparse(native_dtype):
+        # A pandas sparse column has the same logical dtype as its dense `subtype`
+        # (e.g. `Sparse[int64, 0]` -> `Int64`). We don't densify here: that's a
+        # metadata-only unwrap, so reading a schema stays cheap.
+        native_dtype = native_dtype.subtype
     str_dtype = str(native_dtype)
 
     if is_dtype_pyarrow(native_dtype) or str_dtype.startswith(CUDF_BASE_DTYPE_PREFIX):
@@ -381,6 +395,70 @@ def is_dtype_pyarrow(dtype: Any) -> TypeIs[pd.ArrowDtype]:
     return hasattr(pd, "ArrowDtype") and isinstance(dtype, pd.ArrowDtype)
 
 
+# NOTE: Returns `bool` rather than `TypeIs[pd.SparseDtype]` on purpose: the
+# `pandas-stubs` `SparseDtype` doesn't expose `.subtype`, so narrowing would only
+# get in the way of reading it.
+@functools.lru_cache(maxsize=16)
+def is_dtype_sparse(dtype: Any) -> bool:
+    """Return `True` if `dtype` is a pandas `SparseDtype`."""
+    return isinstance(dtype, pd.SparseDtype)
+
+
+def is_object_native_dtype(native_dtype: Any) -> bool:
+    """Return `True` for the `object` dtype, including a sparse `object` subtype.
+
+    These need dtype sniffing (`object_native_to_narwhals_dtype`) rather than the
+    string-based mapping in `native_to_narwhals_dtype`.
+    """
+    if is_dtype_sparse(native_dtype):
+        native_dtype = native_dtype.subtype
+    return native_dtype == "object"
+
+
+def keep_sparse_dtype(
+    target: str | PandasDtype, source_native_dtype: Any
+) -> str | PandasDtype:
+    """Re-wrap a dense cast `target` so a sparse column stays sparse, when safe.
+
+    A `SparseDtype` source stays sparse only when `target` resolves to a numeric or
+    temporal numpy subtype (`dtype.kind` in `iufbMm`: (un)signed int, float, bool,
+    datetime, timedelta); we then return `SparseDtype(subtype, fill_value)`, carrying
+    the source fill across. Every other `target` densifies (returns `target` as-is),
+    because for those there is no sparse result that preserves the data:
+
+    - Non-numpy targets (nullable / pyarrow / categorical): `pd.SparseDtype` only
+      accepts numpy subtypes and raises `TypeError` on anything else.
+    - Object / string subtypes: these *are* numpy dtypes, but casting to them is a
+      no-op passthrough. `pd.SparseDtype(str)` collapses to `Sparse[object]`, and an
+      `object` `astype` never applies `str()`, so `.cast(String)` would leave the
+      values un-stringified. Only densifying routes through a real `astype(str)`.
+    - Numeric/temporal targets whose source fill is unrepresentable (e.g. a `NaN` fill
+      cast to an integer subtype): no fill keeps the compressed entries intact.
+
+    Carrying the fill is mandatory: `astype(SparseDtype(subtype))` keeps the sparse
+    index but resets the fill to the subtype default, silently rewriting the compressed
+    entries (`Sparse[int64, 0]` -> `Sparse[float64, nan]` turns every stored `0` into `NaN`).
+
+    References:
+    - `numpy.dtype.kind`: https://numpy.org/doc/stable/reference/generated/numpy.dtype.kind.html
+    - `pandas.SparseDtype`: https://pandas.pydata.org/docs/reference/api/pandas.SparseDtype.html
+    - pandas sparse guide: https://pandas.pydata.org/docs/user_guide/sparse.html
+    """
+    if not is_dtype_sparse(source_native_dtype):
+        return target
+    try:
+        sparse_dtype: Any = pd.SparseDtype(target)  # `Any`: stubs hide `.subtype`
+        subtype = sparse_dtype.subtype
+        if subtype.kind not in "iufbMm":
+            return target
+        # `np.asarray(..., dtype=subtype)` (not `subtype.type(...)`) raises on an
+        # unrepresentable fill instead of silently coercing.
+        fill_value: Any = np.asarray(source_native_dtype.fill_value, dtype=subtype)[()]
+    except (TypeError, ValueError):
+        return target
+    return pd.SparseDtype(subtype, fill_value)
+
+
 dtypes = Version.MAIN.dtypes
 NW_TO_PD_DTYPES_INVARIANT: Mapping[type[DType], str] = {
     # TODO(Unassigned): is there no pyarrow-backed categorical?

tests/dtypes/pandas_extension_dtypes_test.py

@@ -5,19 +5,29 @@
 
 import pytest
 
+pytest.importorskip("numpy")
 pytest.importorskip("pandas")
 
-
+import numpy as np
 import pandas as pd
 
 import narwhals as nw
+from tests.utils import PANDAS_VERSION
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
     from typing_extensions import Self
 
 
+# `cast`-preserving sparse support is best-effort.
+# pandas<2.0 has different sparse `astype` behaviour.
+# Schema *reading* of numeric/bool subtypes works everywhere.
+REQUIRES_PANDAS_V2 = pytest.mark.skipif(
+    PANDAS_VERSION < (2,), reason="sparse cast support targets pandas>=2.0"
+)
+
+
 class CustomInt16Dtype(pd.api.extensions.ExtensionDtype):  # pragma: no cover
     name = "custom_int_16"
     type = int
@@ -101,3 +111,115 @@ def test_schema_with_ext() -> None:
     assert nw_schema == nw.Schema(
         {"non-hash-int16": nw.Unknown(), "hash-int-32": nw.Unknown()}
     )
+
+
+def test_sparse_schema() -> None:
+    # See https://github.com/narwhals-dev/narwhals/issues/3722.
+    # Sparse columns map to the narwhals dtype of their dense subtype (not `Unknown`).
+    # The string/object subtype needs sample densification (which pre-2.0 pandas doesn't
+    # do), so it's covered separately by `test_sparse_object_cast_string_densifies`.
+    df = pd.DataFrame(
+        {
+            "a": pd.arrays.SparseArray([0, 1, 0, 2]),
+            "b": pd.arrays.SparseArray([0.0, 1.5, 0.0, 2.5]),
+            "c": pd.arrays.SparseArray([True, False, False, True]),
+        }
+    )
+    assert nw.from_native(df).schema == nw.Schema(
+        {"a": nw.Int64(), "b": nw.Float64(), "c": nw.Boolean()}
+    )
+    assert nw.from_native(df["a"], series_only=True).dtype == nw.Int64
+
+
+@REQUIRES_PANDAS_V2
+def test_sparse_cast_preserves_sparsity() -> None:
+    # Casting a sparse column to a numpy-subtype target keeps it sparse *and* must not
+    # corrupt the data: a naive `astype(SparseDtype("float64"))` keeps the sparse index
+    # but resets the fill to `NaN`, turning every compressed `0` into `NaN`.
+    s = nw.from_native(
+        pd.Series(pd.arrays.SparseArray([0, 0, 1, 0, 2])), series_only=True
+    )
+    result = s.cast(nw.Float64)
+    native = nw.to_native(result)
+    assert isinstance(native.dtype, pd.SparseDtype)
+    # `pandas-stubs`' `SparseDtype` doesn't expose `.subtype`, so assert the densified
+    # dtype instead (which also confirms the values round-trip correctly).
+    arr = native.to_numpy()
+    assert arr.dtype == "float64"
+    assert arr.tolist() == [0.0, 0.0, 1.0, 0.0, 2.0]
+
+    # Same logical dtype: stays sparse (no densification).
+    assert isinstance(nw.to_native(s.cast(nw.Int64)).dtype, pd.SparseDtype)
+
+
+@REQUIRES_PANDAS_V2
+@pytest.mark.parametrize(
+    ("data", "target"),
+    [
+        # NaN fill can't be represented as an integer subtype -> densify.
+        (pd.arrays.SparseArray([0.0, 1.0, 2.0]), nw.Int64),
+        # String / object subtypes densify.
+        (pd.arrays.SparseArray([0, 1, 2]), nw.String),
+        # `SparseDtype` can only wrap numpy dtypes, so non-numpy targets densify.
+        (pd.arrays.SparseArray([0, 1, 2]), nw.Categorical),
+    ],
+)
+def test_sparse_cast_densifies_when_unsupported(
+    data: Any, target: nw.dtypes.DType
+) -> None:
+    s = nw.from_native(pd.Series(data), series_only=True)
+    native = nw.to_native(s.cast(target))
+    assert not isinstance(native.dtype, pd.SparseDtype)
+
+
+@REQUIRES_PANDAS_V2
+@pytest.mark.parametrize(
+    ("data", "target"),
+    [
+        # Non-zero fill carried across to a wider numeric subtype: the fill (7) must
+        # survive as 7.0, not reset to the float default (NaN).
+        (pd.arrays.SparseArray([7, 7, 1, 7, 2], fill_value=7), nw.Float64),
+        # Bool subtype (`kind == "b"`) stays sparse.
+        (pd.arrays.SparseArray([True, False, False, True]), nw.Int64),
+        # Datetime subtype (`kind == "M"`); source unit differs from target to force
+        # the slow path through `keep_sparse_dtype`.
+        (
+            pd.arrays.SparseArray(
+                ["2020-01-01", "2020-01-01", "2020-01-02"], dtype="datetime64[ms]"
+            ),
+            nw.Datetime("us"),
+        ),
+        # Timedelta subtype (`kind == "m"`); source unit differs from target.
+        (pd.arrays.SparseArray([0, 0, 5], dtype="timedelta64[us]"), nw.Duration("ns")),
+    ],
+)
+def test_sparse_cast_stays_sparse(data: Any, target: nw.dtypes.DType) -> None:
+    # A sparse cast must equal the dense cast, only kept sparse. Comparing the two
+    # densified results proves the fill was carried (a reset fill would corrupt the
+    # compressed entries) without hard-coding per-dtype expected values.
+    sparse_native = pd.Series(data)
+    dense_native = pd.Series(np.asarray(data))
+
+    sparse_result = nw.to_native(
+        nw.from_native(sparse_native, series_only=True).cast(target)
+    )
+    dense_result = nw.to_native(
+        nw.from_native(dense_native, series_only=True).cast(target)
+    )
+
+    assert isinstance(sparse_result.dtype, pd.SparseDtype)
+    assert not isinstance(dense_result.dtype, pd.SparseDtype)
+    np.testing.assert_array_equal(sparse_result.to_numpy(), dense_result.to_numpy())
+
+
+@REQUIRES_PANDAS_V2
+def test_sparse_object_cast_string_densifies() -> None:
+    # A sparse `object` column sniffs as `String`, so casting to `String` is a no-op at
+    # the narwhals level. The fast path in `cast` must NOT keep it sparse: an
+    # `astype(str)` on a sparse `object` column does not actually stringify the values.
+    s = nw.from_native(
+        pd.Series(pd.arrays.SparseArray(["x", "y", "x", "z"])), series_only=True
+    )
+    assert s.dtype == nw.String
+    native = nw.to_native(s.cast(nw.String))
+    assert not isinstance(native.dtype, pd.SparseDtype)

zensical.toml

@@ -24,6 +24,7 @@ nav = [
         "concepts/column_names.md",
         "concepts/boolean.md",
         "concepts/null_handling.md",
+        "concepts/sparse.md",
     ]},
     {"Overhead" = "overhead.md"},
     {"Perfect backwards compatibility policy" = "backcompat.md"},