feat: Extend plugin support for IO and from_* methods/functions

docs/extending.md

@@ -59,7 +59,40 @@ handle plugins. For this integration to work, any plugin architecture must conta
 
     Take a look at the `Plugin` protocol in `narwhals/plugins.py` for the
     signatures.
-
+
+### Supporting `backend=...` in Narwhals functions
+
+Functions and constructors which accept a `backend` argument can also dispatch to a
+plugin. Users can pass:
+
+- the plugin's entry point name (e.g. `backend="narwhals-grizzlies"`),
+- the plugin's module name (e.g. `backend="narwhals_grizzlies"`),
+- or the plugin's module itself (e.g. `backend=narwhals_grizzlies`).
+
+There are two mechanisms, depending on the function:
+
+1. **IO functions** (`read_csv`, `scan_csv`, `read_parquet`, `scan_parquet`): the plugin
+   should expose a same-named function at the top level of its namespace which returns a
+   **native** object (which is then passed through `narwhals.from_native`):
+
+    - `read_csv(source, separator=",", **kwargs)`, `scan_csv(source, separator=",", **kwargs)`
+    - `read_parquet(source, **kwargs)`, `scan_parquet(source, **kwargs)`
+
+    If the required hook is missing, an informative `AttributeError` is raised.
+
+2. **Eager constructors** (`from_dict`, `from_dicts`, `from_numpy`, `from_arrow`,
+   `new_series`, as well as the `DataFrame.from_*` and `Series.from_*` classmethods):
+   these are eager-only, and go through the compliant namespace returned by the plugin's
+   `__narwhals_namespace__`. If that namespace implements the `EagerNamespace` protocol
+   (in particular the `_dataframe` and `_series` properties, and the `from_dict`,
+   `from_dicts`, `from_numpy`, `from_arrow` and `from_iterable` constructors on the
+   respective compliant classes), these functions work with no extra plugin code.
+   Lazy-only plugins get an informative `ValueError` instead.
+
+Methods which internally construct Series (for example `Series.scatter`, or
+`DataFrame.filter` with a list of booleans) use the compliant namespace of the object
+they are called on, so they also work for eager plugins.
+
 ## Can I see an example?
 
 Yes! For a reference plugin, please check out [narwhals-daft](https://github.com/narwhals-dev/narwhals-daft).

packages/test-plugin/src/test_plugin/__init__.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
     from typing_extensions import TypeIs
@@ -21,3 +21,30 @@ def is_native(native_object: object) -> TypeIs[DictFrame]:
 
 
 NATIVE_PACKAGE = "builtins"
+
+
+# The functions below are the IO extension hooks used by `narwhals.functions`
+# when `backend` resolves to this plugin (https://github.com/narwhals-dev/narwhals/issues/3713).
+# Eager constructors (`from_dict`, `new_series`, `Series.from_iterable`, ...) instead go
+# through `DictNamespace._dataframe` / `DictNamespace._series`.
+
+
+def scan_csv(source: str, separator: str = ",", **kwargs: Any) -> DictFrame:  # noqa: ARG001
+    import csv
+    from pathlib import Path
+
+    with Path(source).open(newline="", encoding="utf-8") as file:
+        header, *rows = list(csv.reader(file, delimiter=separator))
+    return {name: [row[index] for row in rows] for index, name in enumerate(header)}
+
+
+def scan_parquet(source: str, **kwargs: Any) -> DictFrame:
+    import pyarrow.parquet as pq
+
+    result: DictFrame = pq.read_table(source, **kwargs).to_pydict()
+    return result
+
+
+# `read_*` returns the same native object; narwhals then enforces eagerness downstream.
+read_csv = scan_csv
+read_parquet = scan_parquet

packages/test-plugin/src/test_plugin/dataframe.py

@@ -11,15 +11,91 @@
 from narwhals.typing import CompliantLazyFrame
 
 if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
     from typing import TypeAlias
 
     from typing_extensions import Self
 
-    from narwhals import LazyFrame  # noqa: F401
+    from narwhals import DataFrame, LazyFrame  # noqa: F401
+    from narwhals._utils import _LimitedContext
 
 DictFrame: TypeAlias = dict[str, list[Any]]
 
 
+class DictDataFrame:
+    """Minimal eager frame, kept to the smallest surface exercised in narwhals' tests."""
+
+    _implementation = Implementation.UNKNOWN
+
+    def __init__(self, native_dataframe: DictFrame, *, version: Version) -> None:
+        self._native_frame: DictFrame = native_dataframe
+        self._version = version
+
+    @classmethod
+    def from_dict(
+        cls,
+        data: Mapping[str, Any],
+        /,
+        *,
+        context: _LimitedContext,
+        schema: Any = None,  # noqa: ARG003
+    ) -> Self:
+        return cls(
+            {name: list(values) for name, values in data.items()},
+            version=context._version,
+        )
+
+    @classmethod
+    def from_dicts(
+        cls,
+        data: Sequence[Mapping[str, Any]],
+        /,
+        *,
+        context: _LimitedContext,
+        schema: Any = None,  # noqa: ARG003
+    ) -> Self:
+        columns: list[str] = list(data[0]) if data else []
+        return cls(
+            {name: [row[name] for row in data] for name in columns},
+            version=context._version,
+        )
+
+    @classmethod
+    def from_numpy(
+        cls, data: Any, /, *, context: _LimitedContext, schema: Any = None
+    ) -> Self:
+        names = (
+            list(schema)
+            if schema is not None
+            else [str(index) for index in range(data.shape[1])]
+        )
+        return cls(
+            {name: data[:, index].tolist() for index, name in enumerate(names)},
+            version=context._version,
+        )
+
+    @classmethod
+    def from_arrow(cls, data: Any, /, *, context: _LimitedContext) -> Self:
+        import pyarrow as pa
+
+        return cls(pa.table(data).to_pydict(), version=context._version)
+
+    def __narwhals_dataframe__(self) -> Self:
+        return self
+
+    def __narwhals_namespace__(self) -> Any:
+        from test_plugin.namespace import DictNamespace
+
+        return DictNamespace(version=self._version)
+
+    @property
+    def native(self) -> DictFrame:
+        return self._native_frame
+
+    def to_narwhals(self) -> DataFrame[Any]:
+        return self._version.dataframe(self, level="full")
+
+
 class DictLazyFrame(
     CompliantLazyFrame[Any, "DictFrame", "LazyFrame[DictFrame]"],  # type: ignore[type-var]
     ValidateBackendVersion,

packages/test-plugin/src/test_plugin/namespace.py

@@ -3,11 +3,12 @@
 from typing import TYPE_CHECKING, Any
 
 from narwhals._compliant import CompliantNamespace
-from narwhals._utils import not_implemented
-from test_plugin.dataframe import DictFrame, DictLazyFrame
+from narwhals._utils import Implementation, not_implemented
+from test_plugin.dataframe import DictDataFrame, DictFrame, DictLazyFrame
 
 if TYPE_CHECKING:
     from narwhals.utils import Version
+    from test_plugin.series import DictSeries
 
 
 class DictNamespace(CompliantNamespace[DictLazyFrame, Any]):
@@ -17,9 +18,22 @@ def __init__(self, *, version: Version) -> None:
     def from_native(self, native_object: DictFrame) -> DictLazyFrame:
         return DictLazyFrame(native_object, version=self._version)
 
+    @property
+    def _dataframe(self) -> type[DictDataFrame]:
+        return DictDataFrame
+
+    @property
+    def _series(self) -> type[DictSeries]:
+        from test_plugin.series import DictSeries
+
+        return DictSeries
+
+    # NOTE: `not_implemented.__get__` reads `instance._implementation` to build its
+    # error message, so `_implementation` itself must be a real value.
+    _implementation = Implementation.UNKNOWN
+
     is_native: Any = not_implemented()
     _expr: Any = not_implemented()
-    _implementation: Any = not_implemented()
     corr: Any = not_implemented()
     cov: Any = not_implemented()
     len: Any = not_implemented()

packages/test-plugin/src/test_plugin/series.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from narwhals._utils import Implementation
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from typing_extensions import Self
+
+    from narwhals._utils import Version, _LimitedContext
+    from narwhals.series import Series
+
+
+class DictSeries:
+    """Minimal eager series, kept to the smallest surface exercised in narwhals' tests."""
+
+    _implementation = Implementation.UNKNOWN
+
+    def __init__(
+        self, values: Iterable[Any], *, name: str = "", version: Version
+    ) -> None:
+        self._values: list[Any] = list(values)
+        self._name = name
+        self._version = version
+
+    @classmethod
+    def from_iterable(
+        cls,
+        data: Iterable[Any],
+        /,
+        *,
+        context: _LimitedContext,
+        name: str = "",
+        dtype: Any = None,  # noqa: ARG003
+    ) -> Self:
+        return cls(data, name=name, version=context._version)
+
+    @classmethod
+    def from_numpy(cls, data: Any, /, *, context: _LimitedContext) -> Self:
+        return cls(data.tolist(), version=context._version)
+
+    def __narwhals_series__(self) -> Self:
+        return self
+
+    def __narwhals_namespace__(self) -> Any:
+        from test_plugin.namespace import DictNamespace
+
+        return DictNamespace(version=self._version)
+
+    @property
+    def native(self) -> list[Any]:
+        return self._values
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    def alias(self, name: str) -> Self:
+        return self.__class__(self._values, name=name, version=self._version)
+
+    def is_empty(self) -> bool:
+        return not self._values
+
+    def scatter(self, indices: Self, values: Self) -> Self:
+        data = list(self._values)
+        for index, value in zip(indices.native, values.native, strict=True):
+            data[index] = value
+        return self.__class__(data, name=self._name, version=self._version)
+
+    def to_narwhals(self) -> Series[Any]:
+        return self._version.series(self, level="full")

src/narwhals/_utils.py

@@ -77,6 +77,7 @@
     from narwhals._compliant.any_namespace import NamespaceAccessor
     from narwhals._compliant.typing import (
         Accessor,
+        EagerNamespaceAny,
         EvalNames,
         NativeDataFrameT,
         NativeLazyFrameT,
@@ -1625,6 +1626,82 @@ def is_eager_allowed(impl: Implementation, /) -> TypeIs[_EagerAllowedImpl]:
     }
 
 
+def _ensure_eager_capable(
+    namespace: Any, /, *, source: str, function_name: str
+) -> EagerNamespaceAny:
+    """Duck-check that `namespace` implements the `EagerNamespace` protocol.
+
+    `_series` and `_dataframe` may be regular properties, missing entirely, or
+    `not_implemented` descriptors (which raise on instance access).
+    """
+    try:
+        eager_capable = namespace._series is not None and namespace._dataframe is not None
+    except (AttributeError, NotImplementedError):
+        eager_capable = False
+    if not eager_capable:
+        msg = (
+            f"Plugin backend {source!r} does not provide eager support (its "
+            "compliant namespace does not implement the `EagerNamespace` protocol), "
+            f"but `{function_name}` is an eager-only function."
+        )
+        raise ValueError(msg)
+    return cast("EagerNamespaceAny", namespace)
+
+
+def eager_namespace(
+    backend: IntoBackend[Backend],
+    /,
+    *,
+    version: Version,
+    function_name: str,
+    hint_example: str,
+) -> EagerNamespaceAny:
+    """Resolve `backend` to an eager-capable compliant namespace.
+
+    Built-in eager backends resolve directly. Anything unknown to `Implementation` is
+    resolved via the plugin entry-point registry, in which case the plugin's
+    `__narwhals_namespace__` must return a namespace implementing the `EagerNamespace`
+    protocol (in particular, the `_series` and `_dataframe` properties).
+    Built-in lazy-only backends raise an informative `ValueError`, suggesting
+    `hint_example` followed by a `.lazy(...)` call.
+    """
+    implementation = Implementation.from_backend(backend)
+    if is_eager_allowed(implementation):
+        # NOTE: `cast` is required as the overload returns a union of concrete
+        # namespaces, whose invariant type parameters don't unify with `*Any` aliases.
+        return cast(
+            "EagerNamespaceAny", version.namespace.from_backend(implementation).compliant
+        )
+    if implementation is not Implementation.UNKNOWN:
+        msg = (
+            f"{implementation} support in Narwhals is lazy-only, but `{function_name}` is an eager-only function.\n\n"
+            "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n"
+            f"    {hint_example}.lazy('{implementation}')"
+        )
+        raise ValueError(msg)
+    from narwhals.plugins import _backend_namespace, _plugin_hook
+
+    module = _backend_namespace(backend)
+    namespace = _plugin_hook(module, "__narwhals_namespace__")(version=version)
+    return _ensure_eager_capable(
+        namespace, source=module.__name__, function_name=function_name
+    )
+
+
+def eager_namespace_from_compliant(
+    compliant_object: Any, /, *, function_name: str
+) -> EagerNamespaceAny:
+    """Resolve the eager namespace of a compliant object originating from a plugin.
+
+    `Implementation.UNKNOWN` cannot be resolved back to a plugin, so methods which
+    internally construct series use the namespace of the compliant object itself.
+    """
+    namespace = compliant_object.__narwhals_namespace__()
+    return _ensure_eager_capable(
+        namespace, source=type(namespace).__name__, function_name=function_name
+    )
+
+
 def can_lazyframe_collect(impl: Implementation, /) -> TypeIs[_LazyFrameCollectImpl]:
     """Return True if `LazyFrame.collect(impl)` is allowed."""
     return impl in {Implementation.PANDAS, Implementation.POLARS, Implementation.PYARROW}