diff --git a/docs/extending.md b/docs/extending.md index e3a44334fa..a7eb001f65 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -60,6 +60,27 @@ handle plugins. For this integration to work, any plugin architecture must conta Take a look at the `Plugin` protocol in `narwhals/plugins.py` for the signatures. + 3. an `_implementation` attribute set to `narwhals.Implementation.UNKNOWN`, on every + compliant class the plugin exposes (namespace, dataframe, lazyframe, series): + + ```py + from narwhals import Implementation + + + class GrizzliesNamespace: + _implementation = Implementation.UNKNOWN + ... + + + class GrizzliesDataFrame: + _implementation = Implementation.UNKNOWN + ... + ``` + + A plugin's backend is, by definition, not one of Narwhals' own `Implementation` + members, and this attribute is what says so, telling plugin objects apart from + built-in ones. + ## Supporting `backend=...` in Narwhals functions Functions and constructors which accept a `backend` argument can also dispatch to a @@ -69,8 +90,8 @@ plugin. Users can pass: - the plugin's module name (e.g. `backend="narwhals_grizzlies"`), - or the plugin's module itself (e.g. `backend=narwhals_grizzlies`). -In all cases, dispatch goes through the compliant namespace returned by the plugin's -`__narwhals_namespace__`: +All three spellings are resolved the same way built-in backends are, and dispatch goes +through the compliant namespace returned by the plugin's `__narwhals_namespace__`: 1. **IO functions** (`read_csv`, `scan_csv`, `read_parquet`, `scan_parquet`): these call same-named methods on the compliant namespace, following the diff --git a/src/narwhals/_namespace.py b/src/narwhals/_namespace.py index bfe00d0026..84d225a789 100644 --- a/src/narwhals/_namespace.py +++ b/src/narwhals/_namespace.py @@ -34,7 +34,7 @@ is_native_spark_like, is_native_sqlframe, ) -from narwhals._utils import Implementation, Version +from narwhals._utils import Implementation, Version, is_into_plugin if TYPE_CHECKING: from typing import TypeAlias @@ -57,6 +57,7 @@ Ibis, IntoBackend, PandasLike, + PluginName, Polars, SparkLike, ) @@ -135,12 +136,12 @@ def from_backend(cls, backend: EagerAllowed, /) -> EagerAllowedNamespace: ... @overload @classmethod def from_backend( - cls, backend: IntoBackend[Backend], / + cls, backend: IntoBackend[Backend | PluginName], / ) -> Namespace[CompliantNamespaceAny]: ... @classmethod def from_backend( - cls: type[Namespace[Any]], backend: IntoBackend[Backend], / + cls: type[Namespace[Any]], backend: IntoBackend[Backend | PluginName], / ) -> Namespace[Any]: """Instantiate from native namespace module, string, or Implementation. @@ -184,8 +185,18 @@ def from_backend( from narwhals._ibis.namespace import IbisNamespace ns = IbisNamespace(version=version) - else: - msg = "Not supported Implementation" # pragma: no cover + elif is_into_plugin(backend, impl): + # NOTE: Anything unknown to `Implementation` is resolved as a plugin, + # either by entry point name, by module name, or as the plugin module itself. + from narwhals.plugins import _plugin_namespace, _resolve_plugin + + plugin = _resolve_plugin(backend) + ns = _plugin_namespace(plugin, version=version) + else: # pragma: no cover + # NOTE: Unreachable and defensive only check; `UNKNOWN` is handled + # above and every other member of `Implementation` is matched by one + # of the branches. + msg = "Not supported Implementation" raise AssertionError(msg) return cls(ns) diff --git a/src/narwhals/_utils.py b/src/narwhals/_utils.py index e84cb1b5fc..6324bc73b7 100644 --- a/src/narwhals/_utils.py +++ b/src/narwhals/_utils.py @@ -67,7 +67,7 @@ if TYPE_CHECKING: from collections.abc import Set # noqa: PYI025 from types import ModuleType - from typing import Concatenate, TypeAlias + from typing import Concatenate, TypeAlias, TypeGuard import pandas as pd import polars as pl @@ -1654,6 +1654,24 @@ def is_eager_allowed(impl: Implementation, /) -> TypeIs[_EagerAllowedImpl]: } +# NOTE: Keep `TypeGuard`, not `TypeIs`, as the two narrow by different rules. +def is_into_plugin( + backend: IntoBackend[Backend | PluginName], # noqa: ARG001 + impl: Implementation, + /, +) -> TypeGuard[IntoBackend[PluginName]]: + """Return True if `backend` names a plugin, rather than a built-in backend. + + `Implementation.UNKNOWN` means exactly "not one of Narwhals' own backends", so a + `backend` which resolves to it can only be a plugin's name or a plugin module. + + Arguments: + backend: Backend spelling, as given by the user. + impl: Result of `Implementation.from_backend(backend)`. + """ + return impl is Implementation.UNKNOWN + + # TODO(Unassigned): Generalize _hasattr_static? # See https://github.com/narwhals-dev/narwhals/pull/3753#discussion_r3653098839 def _is_eager_namespace(obj: object, /) -> TypeIs[EagerNamespaceAny]: @@ -1725,30 +1743,29 @@ def eager_namespace( ) -> EagerNamespaceAny | EagerNamespaceKnown: """Resolve `backend` to an eager-allowed compliant namespace. - Built-in eager backends resolve directly. Anything unknown to `Implementation` is - resolved via the plugin entry-point registry, in which case the plugin's - `__narwhals_namespace__` must return a namespace implementing the `EagerNamespace` - protocol (in particular, the `_series` and `_dataframe` properties). + Built-in eager backends and plugins resolve through the same + `Namespace.from_backend`; for a plugin, the namespace returned by + `__narwhals_namespace__` must implement the `EagerNamespace` protocol (in + particular, the `_series` and `_dataframe` properties). Built-in lazy-only backends raise an informative `ValueError`, suggesting the `EAGER_HINT_EXAMPLES` entry for `function_name` followed by a `.lazy(...)` call. """ implementation = Implementation.from_backend(backend) if is_eager_allowed(implementation): return version.namespace.from_backend(implementation).compliant - if implementation is not Implementation.UNKNOWN: - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `{function_name}` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" {EAGER_HINT_EXAMPLES[function_name]}.lazy('{implementation}')" - ) - raise ValueError(msg) - from narwhals.plugins import _backend_namespace, _plugin_namespace + if is_into_plugin(backend, implementation): + from narwhals.plugins import _backend_name - plugin = _backend_namespace(backend) - namespace = _plugin_namespace(plugin, version=version) - return _ensure_eager_allowed( - namespace, source=plugin.__name__, function_name=function_name + namespace = version.namespace.from_backend(backend).compliant + return _ensure_eager_allowed( + namespace, source=_backend_name(backend), function_name=function_name + ) + msg = ( + f"{implementation} support in Narwhals is lazy-only, but `{function_name}` is an eager-only function.\n\n" + "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" + f" {EAGER_HINT_EXAMPLES[function_name]}.lazy('{implementation}')" ) + raise ValueError(msg) def eager_namespace_from_compliant( diff --git a/src/narwhals/functions.py b/src/narwhals/functions.py index 014f5046a0..e64484a7fb 100644 --- a/src/narwhals/functions.py +++ b/src/narwhals/functions.py @@ -3,7 +3,7 @@ import platform import sys from collections.abc import Iterable, Mapping, Sequence -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Concatenate, Literal, overload from narwhals._expression_parsing import ( ExprKind, @@ -19,6 +19,7 @@ eager_namespace, flatten, is_eager_allowed, + is_into_plugin, is_nested_literal, is_sequence_of, normalize_path, @@ -33,7 +34,7 @@ ) from narwhals.exceptions import InvalidOperationError from narwhals.expr import Expr -from narwhals.plugins import plugin_io_method +from narwhals.plugins import _backend_name, _ensure_io_method from narwhals.schema import Schema from narwhals.translate import to_native @@ -44,7 +45,7 @@ from typing_extensions import Self, TypeIs - from narwhals._compliant.typing import CompliantFrameAny + from narwhals._compliant.typing import CompliantDataFrameAny, CompliantFrameAny from narwhals._translate import IntoArrowTable from narwhals._typing import Backend, EagerAllowed, IntoBackend, PluginName from narwhals.dataframe import DataFrame, LazyFrame @@ -57,7 +58,9 @@ IntoDType, IntoExpr, IntoSchema, + IOMethodName, NonNestedLiteral, + NormalizedPath, PythonLiteral, _2DArray, ) @@ -549,6 +552,47 @@ def show_versions() -> None: print(f"{k:>13}: {stat}") # noqa: T201 +@overload +def _io_method( + backend: IntoBackend[Backend | PluginName], + method_name: Literal["read_csv", "read_parquet"], + /, + *, + version: Version, +) -> Callable[Concatenate[NormalizedPath, ...], CompliantDataFrameAny]: ... + + +@overload +def _io_method( + backend: IntoBackend[Backend | PluginName], + method_name: Literal["scan_csv", "scan_parquet"], + /, + *, + version: Version, +) -> Callable[Concatenate[NormalizedPath, ...], CompliantFrameAny]: ... + + +def _io_method( + backend: IntoBackend[Backend | PluginName], + method_name: IOMethodName, + /, + *, + version: Version, +) -> Callable[Concatenate[NormalizedPath, ...], CompliantFrameAny]: + """Bind the `method_name` method of `backend`'s compliant namespace. + + Built-in backends and plugins share one resolution path (`Namespace.from_backend`) + and one dispatch mechanism (a same-named method on the compliant namespace), but only + a plugin may be missing the method, so only a plugin is checked for it. + """ + impl = Implementation.from_backend(backend) + namespace = version.namespace.from_backend(backend).compliant + if is_into_plugin(backend, impl): + _ensure_io_method(namespace, method_name, source=_backend_name(backend)) + method: Callable[..., CompliantFrameAny] = getattr(namespace, method_name) + return method + + def read_csv( source: FileSource, *, @@ -584,20 +628,16 @@ def read_csv( └──────────────────┘ """ impl = Implementation.from_backend(backend) - if is_eager_allowed(impl): - ns = Version.MAIN.namespace.from_backend(impl).compliant - frame = ns.read_csv(normalize_path(source), separator=separator, **kwargs) - return frame.to_narwhals() - if impl is Implementation.UNKNOWN: - read = plugin_io_method(backend, "read_csv", version=Version.MAIN) - frame = read(normalize_path(source), separator=separator, **kwargs) - result: DataFrame[Any] = frame.to_narwhals() - return result - msg = ( - f"Expected eager backend, found {impl}.\n\n" - f"Hint: use nw.scan_csv(source={source}, backend={backend})" - ) - raise ValueError(msg) + if not is_eager_allowed(impl) and impl is not Implementation.UNKNOWN: + msg = ( + f"Expected eager backend, found {impl}.\n\n" + f"Hint: use nw.scan_csv(source={source}, backend={backend})" + ) + raise ValueError(msg) + read = _io_method(backend, "read_csv", version=Version.MAIN) + frame = read(normalize_path(source), separator=separator, **kwargs) + result: DataFrame[Any] = frame.to_narwhals() + return result def scan_csv( @@ -640,12 +680,7 @@ def scan_csv( │ z │ 3 │ └─────────┴───────┘ """ - impl = Implementation.from_backend(backend) - scan: Callable[..., CompliantFrameAny] - if impl is Implementation.UNKNOWN: - scan = plugin_io_method(backend, "scan_csv", version=Version.MAIN) - else: - scan = Version.MAIN.namespace.from_backend(impl).compliant.scan_csv + scan = _io_method(backend, "scan_csv", version=Version.MAIN) frame = scan(normalize_path(source), separator=separator, **kwargs) result: LazyFrame[Any] = frame.to_narwhals().lazy() return result @@ -686,20 +721,16 @@ def read_parquet( └──────────────────┘ """ impl = Implementation.from_backend(backend) - if is_eager_allowed(impl): - ns = Version.MAIN.namespace.from_backend(impl).compliant - frame = ns.read_parquet(normalize_path(source), **kwargs) - return frame.to_narwhals() - if impl is Implementation.UNKNOWN: - read = plugin_io_method(backend, "read_parquet", version=Version.MAIN) - frame = read(normalize_path(source), **kwargs) - result: DataFrame[Any] = frame.to_narwhals() - return result - msg = ( - f"Expected eager backend, found {impl}.\n\n" - f"Hint: use nw.scan_parquet(source={source}, backend={backend})" - ) - raise ValueError(msg) + if not is_eager_allowed(impl) and impl is not Implementation.UNKNOWN: + msg = ( + f"Expected eager backend, found {impl}.\n\n" + f"Hint: use nw.scan_parquet(source={source}, backend={backend})" + ) + raise ValueError(msg) + read = _io_method(backend, "read_parquet", version=Version.MAIN) + frame = read(normalize_path(source), **kwargs) + result: DataFrame[Any] = frame.to_narwhals() + return result def scan_parquet( @@ -764,12 +795,7 @@ def scan_parquet( | b: [[4,5]] | └──────────────────┘ """ - impl = Implementation.from_backend(backend) - scan: Callable[..., CompliantFrameAny] - if impl is Implementation.UNKNOWN: - scan = plugin_io_method(backend, "scan_parquet", version=Version.MAIN) - else: - scan = Version.MAIN.namespace.from_backend(impl).compliant.scan_parquet + scan = _io_method(backend, "scan_parquet", version=Version.MAIN) frame = scan(normalize_path(source), **kwargs) result: LazyFrame[Any] = frame.to_narwhals().lazy() return result diff --git a/src/narwhals/plugins.py b/src/narwhals/plugins.py index 00ad585573..f94f9b0314 100644 --- a/src/narwhals/plugins.py +++ b/src/narwhals/plugins.py @@ -16,7 +16,7 @@ import sys from functools import cache from types import ModuleType -from typing import TYPE_CHECKING, Any, Literal, Protocol, cast, overload +from typing import TYPE_CHECKING, Any, Protocol, cast from narwhals._compliant import CompliantNamespace from narwhals._typing import PluginName @@ -24,7 +24,7 @@ from narwhals.exceptions import PluginError if TYPE_CHECKING: - from collections.abc import Callable, Iterator + from collections.abc import Iterator from importlib.metadata import EntryPoints from typing import TypeAlias @@ -34,16 +34,12 @@ CompliantDataFrameAny, CompliantFrameAny, CompliantLazyFrameAny, + CompliantNamespaceAny, CompliantSeriesAny, ) - from narwhals._typing import Backend, IntoBackend + from narwhals.typing import IntoBackend, IOMethodName from narwhals.utils import Version - IOMethodName: TypeAlias = Literal[ - "read_csv", "read_parquet", "scan_csv", "scan_parquet" - ] - """Name of a Narwhals IO function, dispatched to a same-named namespace method.""" - __all__ = ["Plugin", "PluginName", "from_native"] @@ -95,7 +91,12 @@ def _find_plugin(backend_name: str, /) -> Plugin | None: return None -def _backend_namespace(backend: IntoBackend[Backend | PluginName], /) -> Plugin: +def _backend_name(backend: IntoBackend[PluginName], /) -> str: + """Spelling of a plugin `backend`, used to identify it in error messages.""" + return backend.__name__ if isinstance(backend, ModuleType) else backend + + +def _resolve_plugin(backend: IntoBackend[PluginName], /) -> Plugin: """Resolve a backend which is not a Narwhals `Implementation` to a plugin. The plugin is expected to implement the `Plugin` protocol, in particular the @@ -105,7 +106,7 @@ def _backend_namespace(backend: IntoBackend[Backend | PluginName], /) -> Plugin: # NOTE: A user-provided module is only *claimed* to be a `Plugin`; the runtime # guard is `_plugin_namespace`, which raises `PluginError` if it is not one. return cast("Plugin", backend) - if isinstance(backend, str) and (plugin := _find_plugin(backend)) is not None: + if (plugin := _find_plugin(backend)) is not None: return plugin installed = ", ".join(_plugin_names()) or "" msg = ( @@ -127,38 +128,16 @@ def _plugin_namespace(plugin: Plugin, /, *, version: Version) -> PluginNamespace return namespace -@overload -def plugin_io_method( - backend: IntoBackend[Backend | PluginName], - method_name: Literal["read_csv", "read_parquet"], - /, - *, - version: Version, -) -> Callable[..., CompliantDataFrameAny]: ... - - -@overload -def plugin_io_method( - backend: IntoBackend[Backend | PluginName], - method_name: Literal["scan_csv", "scan_parquet"], - /, - *, - version: Version, -) -> Callable[..., CompliantFrameAny]: ... - - -def plugin_io_method( - backend: IntoBackend[Backend | PluginName], - method_name: IOMethodName, - /, - *, - version: Version, -) -> Callable[..., CompliantFrameAny]: - """Resolve `backend` to the `method_name` method of a plugin's compliant namespace. +def _ensure_io_method( + namespace: CompliantNamespaceAny, method_name: IOMethodName, /, *, source: str +) -> None: + """Raise unless a plugin's compliant namespace implements `method_name`. IO functions share a single dispatch mechanism with built-in backends: they call same-named methods on the compliant namespace (see the "IO functions" section of the [extension docs](../extending.md/#io-functions-the-namespace-contract)). + Built-in namespaces always implement the methods for their kind, so only plugins + need checking. Note: `PluginNamespace` deliberately does not declare the IO methods: they are an @@ -168,17 +147,13 @@ def plugin_io_method( from narwhals._utils import not_implemented - plugin = _backend_namespace(backend) - namespace = _plugin_namespace(plugin, version=version) method = getattr_static(namespace, method_name, None) if method is None or isinstance(method, not_implemented): msg = ( - f"Plugin backend {plugin.__name__!r} is expected to implement " + f"Plugin backend {source!r} is expected to implement " f"`{method_name}` on its compliant namespace to support `narwhals.{method_name}`." ) raise PluginError(msg) - bound_method: Callable[..., CompliantFrameAny] = getattr(namespace, method_name) - return bound_method class PluginNamespace(CompliantNamespace[FrameT, Any], Protocol[FrameT, FromNativeR_co]): diff --git a/src/narwhals/typing.py b/src/narwhals/typing.py index 018ab589ac..e1d79a0d34 100644 --- a/src/narwhals/typing.py +++ b/src/narwhals/typing.py @@ -405,6 +405,8 @@ def Binary(self) -> type[dtypes.Binary]: ... "MultiIndexSelector[_T] | MultiNameSelector[_T] | SizedMultiBoolSelector[_T]" ) +IOMethodName: TypeAlias = Literal["read_csv", "read_parquet", "scan_csv", "scan_parquet"] +"""Name of a Narwhals IO function, dispatched to a same-named namespace method.""" __all__ = [ "Backend", diff --git a/tests/namespace_test.py b/tests/namespace_test.py index 97b5670b7b..11506ad1ed 100644 --- a/tests/namespace_test.py +++ b/tests/namespace_test.py @@ -3,6 +3,7 @@ import re from collections import deque from collections.abc import Callable, Iterable, Sequence +from types import ModuleType from typing import TYPE_CHECKING, Any, TypeVar, cast import pytest @@ -10,6 +11,8 @@ import narwhals as nw from narwhals._namespace import Namespace from narwhals._utils import Version +from narwhals.exceptions import PluginError +from narwhals.plugins import PluginName if TYPE_CHECKING: from typing import TypeAlias @@ -72,6 +75,34 @@ def test_namespace_from_backend_name(backend: BackendName) -> None: assert namespace.version is Version.MAIN +@pytest.mark.parametrize( + "backend", + [PluginName("test-plugin"), PluginName("test_plugin")], + ids=["entry-point-name", "module-name"], +) +def test_namespace_from_backend_plugin(backend: PluginName) -> None: + pytest.importorskip("test_plugin") + namespace = Namespace.from_backend(backend) + if TYPE_CHECKING: + assert_type(namespace, "Namespace[CompliantNamespace[Any, Any]]") + assert repr(namespace) == "Namespace[DictNamespace]" + assert namespace.implementation is nw.Implementation.UNKNOWN + assert namespace.version is Version.MAIN + + +def test_namespace_from_backend_plugin_not_installed() -> None: + with pytest.raises(ValueError, match="Unsupported backend: 'not-a-backend'"): + Namespace.from_backend(PluginName("not-a-backend")) + + +def test_namespace_from_backend_plugin_invalid() -> None: + not_a_plugin = ModuleType("empty_plugin") + with pytest.raises( + PluginError, match="expected to implement `__narwhals_namespace__`" + ): + Namespace.from_backend(not_a_plugin) + + def test_namespace_from_native_object(constructor: Constructor) -> None: data = {"a": [1, 2, 3], "b": [4, 5, 6]} frame = constructor(data)