diff --git a/docs/api-reference/exceptions.md b/docs/api-reference/exceptions.md index 066ecf2af3..c76ae45845 100644 --- a/docs/api-reference/exceptions.md +++ b/docs/api-reference/exceptions.md @@ -11,6 +11,7 @@ - InvalidOperationError - MultiOutputExpressionError - NarwhalsUnstableWarning + - PluginError - ShapeError - UnsupportedDTypeError show_source: false diff --git a/docs/api-reference/index.md b/docs/api-reference/index.md index 535e309b8e..304ee3213c 100644 --- a/docs/api-reference/index.md +++ b/docs/api-reference/index.md @@ -26,3 +26,4 @@ - [narwhals.selectors](selectors.md) - [narwhals.typing](typing.md) - [narwhals.utils](utils.md) +- [narwhals.plugins](plugins.md) diff --git a/docs/api-reference/plugins.md b/docs/api-reference/plugins.md new file mode 100644 index 0000000000..b004d5705a --- /dev/null +++ b/docs/api-reference/plugins.md @@ -0,0 +1,13 @@ +# `narwhals.plugins` + +For an overview of how to write a plugin, see [extensions and plugins](../extending.md). + +::: narwhals.plugins + handler: python + options: + members: + - Plugin + - PluginName + - from_native + show_source: false + show_bases: false diff --git a/docs/api-reference/typing.md b/docs/api-reference/typing.md index f8cdea5b66..7fb7578476 100644 --- a/docs/api-reference/typing.md +++ b/docs/api-reference/typing.md @@ -21,6 +21,8 @@ Narwhals comes fully statically typed. In addition to `nw.DataFrame`, `nw.Expr`, - Backend - EagerAllowed - LazyAllowed + - FileSource + - NormalizedPath - IntoDType - IntoSchema - SizeUnit diff --git a/docs/extending.md b/docs/extending.md index eef38b1a80..e3a44334fa 100644 --- a/docs/extending.md +++ b/docs/extending.md @@ -60,11 +60,54 @@ handle plugins. For this integration to work, any plugin architecture must conta Take a look at the `Plugin` protocol in `narwhals/plugins.py` for the signatures. -## IO functions: the namespace contract +## Supporting `backend=...` in Narwhals functions + +Functions and constructors which accept a `backend` argument can also dispatch to a +plugin. Users can pass: + +- the plugin's entry point name (e.g. `backend="narwhals-grizzlies"`), +- the plugin's module name (e.g. `backend="narwhals_grizzlies"`), +- or the plugin's module itself (e.g. `backend=narwhals_grizzlies`). + +In all cases, dispatch goes through the compliant namespace returned by the plugin's +`__narwhals_namespace__`: + +1. **IO functions** (`read_csv`, `scan_csv`, `read_parquet`, `scan_parquet`): these + call same-named methods on the compliant namespace, following the + [namespace contract](#io-functions-the-namespace-contract) below. If the plugin's + namespace does not implement the required method, an informative + [`PluginError`](api-reference/exceptions.md) is raised. + +2. **Eager constructors** (`from_dict`, `from_dicts`, `from_numpy`, `from_arrow`, + `new_series`, as well as the `DataFrame.from_*` and `Series.from_*` classmethods): + these are eager-only. If the plugin's compliant namespace implements the + `EagerNamespace` protocol (importable from `narwhals.compliant`, alongside + `EagerDataFrame`, `EagerSeries` and `EagerExpr`; in particular the + `_dataframe` and `_series` properties, + and the `from_dict`, `from_dicts`, `from_numpy`, `from_arrow` and `from_iterable` + constructors on the respective compliant classes), these functions work with no + extra plugin code. Lazy-only plugins get an informative + [`PluginError`](api-reference/exceptions.md) instead. + +Methods which internally construct Series (for example `Series.scatter`, or +`DataFrame.filter` with a list of booleans) use the compliant namespace of the object +they are called on, so they also work for eager plugins. + +!!! tip "Type checking" + + The `backend` parameters of these functions are typed with + [`PluginName`](api-reference/plugins.md), a `str` + [`NewType`](https://docs.python.org/3/library/typing.html#newtype): plugin names + are only known at runtime, so an opaque string does not type check, but an + explicitly wrapped one does, e.g. + `nw.from_dict(data, backend=PluginName("narwhals-grizzlies"))`. + Passing the plain string works at runtime all the same. + +### IO functions: the namespace contract The Narwhals IO functions (`read_csv`, `scan_csv`, `read_parquet`, `scan_parquet`) dispatch to same-named methods on the compliant namespace. This is a single mechanism, -shared by built-in backends and extensions alike: to support these functions, a +shared by built-in backends and plugins alike: to support these functions, a compliant namespace implements (a subset of): ```py diff --git a/packages/test-plugin/src/test_plugin/dataframe.py b/packages/test-plugin/src/test_plugin/dataframe.py index 56bc928cf7..7770756c73 100644 --- a/packages/test-plugin/src/test_plugin/dataframe.py +++ b/packages/test-plugin/src/test_plugin/dataframe.py @@ -8,18 +8,94 @@ Version, not_implemented, ) -from narwhals.typing import CompliantLazyFrame +from narwhals.compliant import CompliantLazyFrame if TYPE_CHECKING: + from collections.abc import Mapping, Sequence from typing import TypeAlias from typing_extensions import Self - from narwhals import LazyFrame # noqa: F401 + from narwhals import DataFrame, LazyFrame + from narwhals._utils import _LimitedContext DictFrame: TypeAlias = dict[str, list[Any]] +class DictDataFrame: + """Minimal eager frame, kept to the smallest surface exercised in narwhals' tests.""" + + _implementation = Implementation.UNKNOWN + + def __init__(self, native_dataframe: DictFrame, *, version: Version) -> None: + self._native_frame: DictFrame = native_dataframe + self._version = version + + @classmethod + def from_dict( + cls, + data: Mapping[str, Any], + /, + *, + context: _LimitedContext, + schema: Any = None, # noqa: ARG003 + ) -> Self: + return cls( + {name: list(values) for name, values in data.items()}, + version=context._version, + ) + + @classmethod + def from_dicts( + cls, + data: Sequence[Mapping[str, Any]], + /, + *, + context: _LimitedContext, + schema: Any = None, # noqa: ARG003 + ) -> Self: + columns: list[str] = list(data[0]) if data else [] + return cls( + {name: [row[name] for row in data] for name in columns}, + version=context._version, + ) + + @classmethod + def from_numpy( + cls, data: Any, /, *, context: _LimitedContext, schema: Any = None + ) -> Self: + names = ( + list(schema) + if schema is not None + else [str(index) for index in range(data.shape[1])] + ) + return cls( + {name: data[:, index].tolist() for index, name in enumerate(names)}, + version=context._version, + ) + + @classmethod + def from_arrow(cls, data: Any, /, *, context: _LimitedContext) -> Self: + import pyarrow as pa + + return cls(pa.table(data).to_pydict(), version=context._version) + + def __narwhals_dataframe__(self) -> Self: + return self + + def __narwhals_namespace__(self) -> Any: + from test_plugin.namespace import DictNamespace + + return DictNamespace(version=self._version) + + @property + def native(self) -> DictFrame: + return self._native_frame + + def to_narwhals(self) -> DataFrame[Any]: + return self._version.dataframe(self, level="full") + + class DictLazyFrame( CompliantLazyFrame[Any, "DictFrame", "LazyFrame[DictFrame]"], # type: ignore[type-var] ValidateBackendVersion, @@ -33,6 +109,9 @@ def __init__(self, native_dataframe: DictFrame, *, version: Version) -> None: def __narwhals_lazyframe__(self) -> Self: return self + def to_narwhals(self) -> LazyFrame[Any]: + return self._version.lazyframe(self, level="lazy") + @property def columns(self) -> list[str]: # pragma: no cover return list(self._native_frame.keys()) @@ -74,7 +153,6 @@ def _with_version(self, version: Version) -> Self: sink_parquet = not_implemented() sort = not_implemented() tail = not_implemented() - to_narwhals = not_implemented() unique = not_implemented() unpivot = not_implemented() with_columns = not_implemented() diff --git a/packages/test-plugin/src/test_plugin/namespace.py b/packages/test-plugin/src/test_plugin/namespace.py index 46419516a6..ded5768d41 100644 --- a/packages/test-plugin/src/test_plugin/namespace.py +++ b/packages/test-plugin/src/test_plugin/namespace.py @@ -2,12 +2,14 @@ from typing import TYPE_CHECKING, Any -from narwhals._compliant import CompliantNamespace -from narwhals._utils import not_implemented -from test_plugin.dataframe import DictFrame, DictLazyFrame +from narwhals._utils import Implementation, not_implemented +from narwhals.compliant import CompliantNamespace +from test_plugin.dataframe import DictDataFrame, DictFrame, DictLazyFrame if TYPE_CHECKING: + from narwhals.typing import NormalizedPath from narwhals.utils import Version + from test_plugin.series import DictSeries class DictNamespace(CompliantNamespace[DictLazyFrame, Any]): @@ -17,9 +19,53 @@ def __init__(self, *, version: Version) -> None: def from_native(self, native_object: DictFrame) -> DictLazyFrame: return DictLazyFrame(native_object, version=self._version) + @property + def _dataframe(self) -> type[DictDataFrame]: + return DictDataFrame + + @property + def _series(self) -> type[DictSeries]: + from test_plugin.series import DictSeries + + return DictSeries + + # IO methods below follow the namespace contract used by `narwhals.functions` + # (see "IO functions: the namespace contract" in `docs/extending.md`). + # `scan_*` delegate to `read_*`, mirroring the defaults `EagerNamespace` provides. + + def read_csv( + self, source: NormalizedPath, *, separator: str = ",", **kwds: Any + ) -> DictDataFrame: + import csv + from pathlib import Path + + with Path(source).open(newline="", encoding="utf-8") as file: + header, *rows = list(csv.reader(file, delimiter=separator)) + data = {name: [row[index] for row in rows] for index, name in enumerate(header)} + return DictDataFrame(data, version=self._version) + + def read_parquet(self, source: NormalizedPath, **kwds: Any) -> DictDataFrame: + import pyarrow.parquet as pq + + data: DictFrame = pq.read_table(source, **kwds).to_pydict() + return DictDataFrame(data, version=self._version) + + def scan_csv( + self, source: NormalizedPath, *, separator: str = ",", **kwds: Any + ) -> DictLazyFrame: + data = self.read_csv(source, separator=separator, **kwds).native + return DictLazyFrame(data, version=self._version) + + def scan_parquet(self, source: NormalizedPath, **kwds: Any) -> DictLazyFrame: + data = self.read_parquet(source, **kwds).native + return DictLazyFrame(data, version=self._version) + + # NOTE: `not_implemented.__get__` reads `instance._implementation` to build its + # error message, so `_implementation` itself must be a real value. + _implementation = Implementation.UNKNOWN + is_native: Any = not_implemented() _expr: Any = not_implemented() - _implementation: Any = not_implemented() corr: Any = not_implemented() cov: Any = not_implemented() len: Any = not_implemented() @@ -37,5 +83,3 @@ def from_native(self, native_object: DictFrame) -> DictLazyFrame: selectors: Any = not_implemented() coalesce: Any = not_implemented() struct: Any = not_implemented() - scan_csv: Any = not_implemented() - scan_parquet: Any = not_implemented() diff --git a/packages/test-plugin/src/test_plugin/series.py b/packages/test-plugin/src/test_plugin/series.py new file mode 100644 index 0000000000..dc6c8cf84b --- /dev/null +++ b/packages/test-plugin/src/test_plugin/series.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from narwhals._utils import Implementation + +if TYPE_CHECKING: + from collections.abc import Iterable + + from typing_extensions import Self + + from narwhals._utils import _LimitedContext + from narwhals.series import Series + from narwhals.utils import Version + + +class DictSeries: + """Minimal eager series, kept to the smallest surface exercised in narwhals' tests.""" + + _implementation = Implementation.UNKNOWN + + def __init__( + self, values: Iterable[Any], *, name: str = "", version: Version + ) -> None: + self._values: list[Any] = list(values) + self._name = name + self._version = version + + @classmethod + def from_iterable( + cls, + data: Iterable[Any], + /, + *, + context: _LimitedContext, + name: str = "", + dtype: Any = None, # noqa: ARG003 + ) -> Self: + return cls(data, name=name, version=context._version) + + @classmethod + def from_numpy(cls, data: Any, /, *, context: _LimitedContext) -> Self: + return cls(data.tolist(), version=context._version) + + def __narwhals_series__(self) -> Self: + return self + + def __narwhals_namespace__(self) -> Any: + from test_plugin.namespace import DictNamespace + + return DictNamespace(version=self._version) + + @property + def native(self) -> list[Any]: + return self._values + + @property + def name(self) -> str: + return self._name + + def alias(self, name: str) -> Self: + return self.__class__(self._values, name=name, version=self._version) + + def is_empty(self) -> bool: + return not self._values + + def scatter(self, indices: Self, values: Self) -> Self: + data = list(self._values) + for index, value in zip(indices.native, values.native, strict=True): + data[index] = value + return self.__class__(data, name=self._name, version=self._version) + + def to_narwhals(self) -> Series[Any]: + return self._version.series(self, level="full") diff --git a/src/narwhals/_namespace.py b/src/narwhals/_namespace.py index 87b2f2b359..bfe00d0026 100644 --- a/src/narwhals/_namespace.py +++ b/src/narwhals/_namespace.py @@ -61,6 +61,9 @@ SparkLike, ) + EagerNamespaceKnown: TypeAlias = ( + PandasLikeNamespace | ArrowNamespace | PolarsNamespace + ) EagerAllowedNamespace: TypeAlias = "Namespace[PandasLikeNamespace] | Namespace[ArrowNamespace] | Namespace[PolarsNamespace]" __all__ = ["Namespace"] diff --git a/src/narwhals/_utils.py b/src/narwhals/_utils.py index 90e893e2e8..e84cb1b5fc 100644 --- a/src/narwhals/_utils.py +++ b/src/narwhals/_utils.py @@ -60,6 +60,7 @@ ColumnNotFoundError, DuplicateError, InvalidOperationError, + PluginError, ShapeError, ) @@ -77,11 +78,14 @@ from narwhals._compliant.any_namespace import NamespaceAccessor from narwhals._compliant.typing import ( Accessor, + CompliantDataFrameAny, + CompliantSeriesAny, + EagerNamespaceAny, EvalNames, NativeDataFrameT, NativeLazyFrameT, ) - from narwhals._namespace import Namespace + from narwhals._namespace import EagerNamespaceKnown, Namespace from narwhals._native import ( NativeArrow, NativeCuDF, @@ -1650,6 +1654,117 @@ def is_eager_allowed(impl: Implementation, /) -> TypeIs[_EagerAllowedImpl]: } +# TODO(Unassigned): Generalize _hasattr_static? +# See https://github.com/narwhals-dev/narwhals/pull/3753#discussion_r3653098839 +def _is_eager_namespace(obj: object, /) -> TypeIs[EagerNamespaceAny]: + """Duck-check that `obj` implements the `EagerNamespace` protocol. + + Note: + `_hasattr_static` alone is not enough: `_series` and `_dataframe` may be + `not_implemented` descriptors, which exist statically but raise on instance + access, so the statically-retrieved attribute is checked against `not_implemented`. + """ + return all( + (attr := getattr_static(obj, name, None)) is not None + and not isinstance(attr, not_implemented) + for name in ("_series", "_dataframe") + ) + + +def _ensure_eager_allowed( + namespace: object, /, *, source: str, function_name: str +) -> EagerNamespaceAny: + """Raise unless `namespace` implements the `EagerNamespace` protocol.""" + if not _is_eager_namespace(namespace): + msg = ( + f"Plugin backend {source!r} does not provide eager support (its " + "compliant namespace does not implement the `EagerNamespace` protocol), " + f"but `{function_name}` is an eager-only function." + ) + raise PluginError(msg) + return namespace + + +EagerFunctionName: TypeAlias = Literal[ + "new_series", + "from_dict", + "from_dicts", + "from_numpy", + "from_arrow", + "DataFrame.from_arrow", + "DataFrame.from_dict", + "DataFrame.from_dicts", + "DataFrame.from_numpy", + "Series.from_iterable", + "Series.from_numpy", +] +"""Name of an eager-only Narwhals function or constructor which accepts `backend`.""" + +EAGER_HINT_EXAMPLES: Mapping[EagerFunctionName, str] = { + "new_series": "nw.new_series('a', [1,2,3], backend='pyarrow').to_frame()", + "from_dict": "nw.from_dict({'a': [1, 2]}, backend='pyarrow')", + "from_dicts": "nw.from_dicts([{'a': 1}, {'a': 2}], backend='pyarrow')", + "from_numpy": "nw.from_numpy(arr, backend='pyarrow')", + "from_arrow": "nw.from_arrow(df, backend='pyarrow')", + "DataFrame.from_arrow": "nw.DataFrame.from_arrow(df, backend='pyarrow')", + "DataFrame.from_dict": "nw.DataFrame.from_dict({'a': [1, 2]}, backend='pyarrow')", + "DataFrame.from_dicts": "nw.DataFrame.from_dicts([{'a': 1}, {'a': 2}], backend='pyarrow')", + "DataFrame.from_numpy": "nw.DataFrame.from_numpy(arr, backend='pyarrow')", + "Series.from_iterable": "nw.Series.from_iterable('a', [1,2,3], backend='pyarrow').to_frame()", + "Series.from_numpy": "nw.Series.from_numpy(arr, backend='pyarrow').to_frame()", +} +"""Per-function `.lazy(...)` hint, shown when an eager-only function is given a lazy backend.""" + + +def eager_namespace( + backend: IntoBackend[Backend | PluginName], + /, + *, + version: Version, + function_name: EagerFunctionName, +) -> EagerNamespaceAny | EagerNamespaceKnown: + """Resolve `backend` to an eager-allowed compliant namespace. + + Built-in eager backends resolve directly. Anything unknown to `Implementation` is + resolved via the plugin entry-point registry, in which case the plugin's + `__narwhals_namespace__` must return a namespace implementing the `EagerNamespace` + protocol (in particular, the `_series` and `_dataframe` properties). + Built-in lazy-only backends raise an informative `ValueError`, suggesting the + `EAGER_HINT_EXAMPLES` entry for `function_name` followed by a `.lazy(...)` call. + """ + implementation = Implementation.from_backend(backend) + if is_eager_allowed(implementation): + return version.namespace.from_backend(implementation).compliant + if implementation is not Implementation.UNKNOWN: + msg = ( + f"{implementation} support in Narwhals is lazy-only, but `{function_name}` is an eager-only function.\n\n" + "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" + f" {EAGER_HINT_EXAMPLES[function_name]}.lazy('{implementation}')" + ) + raise ValueError(msg) + from narwhals.plugins import _backend_namespace, _plugin_namespace + + plugin = _backend_namespace(backend) + namespace = _plugin_namespace(plugin, version=version) + return _ensure_eager_allowed( + namespace, source=plugin.__name__, function_name=function_name + ) + + +def eager_namespace_from_compliant( + compliant_object: CompliantDataFrameAny | CompliantSeriesAny, /, *, function_name: str +) -> EagerNamespaceAny: + """Resolve the eager namespace of a compliant object originating from a plugin. + + `Implementation.UNKNOWN` cannot be resolved back to a plugin, so methods which + internally construct series use the namespace of the compliant object itself. + """ + namespace = compliant_object.__narwhals_namespace__() + return _ensure_eager_allowed( + namespace, source=type(namespace).__name__, function_name=function_name + ) + + def can_lazyframe_collect(impl: Implementation, /) -> TypeIs[_LazyFrameCollectImpl]: """Return True if `LazyFrame.collect(impl)` is allowed.""" return impl in {Implementation.PANDAS, Implementation.POLARS, Implementation.PYARROW} diff --git a/src/narwhals/compliant.py b/src/narwhals/compliant.py index a97ce01c2d..c0bab8289a 100644 --- a/src/narwhals/compliant.py +++ b/src/narwhals/compliant.py @@ -9,6 +9,10 @@ CompliantLazyFrame, CompliantNamespace, CompliantSeries, + EagerDataFrame, + EagerExpr, + EagerNamespace, + EagerSeries, ) from narwhals._compliant.any_namespace import ( CatNamespace, @@ -32,6 +36,10 @@ "CompliantSelectorNamespace", "CompliantSeries", "DateTimeNamespace", + "EagerDataFrame", + "EagerExpr", + "EagerNamespace", + "EagerSeries", "ListNamespace", "StringNamespace", "StructNamespace", diff --git a/src/narwhals/dataframe.py b/src/narwhals/dataframe.py index e18bb409f9..0bbcb3ca1a 100644 --- a/src/narwhals/dataframe.py +++ b/src/narwhals/dataframe.py @@ -29,11 +29,12 @@ _resolve_sample_size, can_lazyframe_collect, check_columns_exist, + eager_namespace, + eager_namespace_from_compliant, flatten, generate_repr, is_compliant_dataframe, is_compliant_lazyframe, - is_eager_allowed, is_index_selector, is_iterator, is_lazy_allowed, @@ -548,17 +549,11 @@ def from_arrow( if not (supports_arrow_c_stream(native_frame) or is_pyarrow_table(native_frame)): msg = f"Given object of type {type(native_frame)} does not support PyCapsule interface" raise TypeError(msg) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = cls._version.namespace.from_backend(implementation).compliant - compliant = ns._dataframe.from_arrow(native_frame, context=ns) - return cls(compliant, level="full") - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `DataFrame.from_arrow` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.DataFrame.from_arrow(df, backend='pyarrow').lazy('{implementation}')" + ns = eager_namespace( + backend, version=cls._version, function_name="DataFrame.from_arrow" ) - raise ValueError(msg) + compliant = ns._dataframe.from_arrow(native_frame, context=ns) + return cls(compliant, level="full") @classmethod def from_dict( @@ -610,18 +605,11 @@ def from_dict( if backend is None: data, backend = _from_dict_no_backend(data) schema = dict(schema) if schema is not None else None - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = cls._version.namespace.from_backend(implementation).compliant - compliant = ns._dataframe.from_dict(data, schema=schema, context=ns) - return cls(compliant, level="full") - # NOTE: (#2786) needs resolving for extensions - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `DataFrame.from_dict` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.DataFrame.from_dict({{'a': [1, 2]}}, backend='pyarrow').lazy('{implementation}')" + ns = eager_namespace( + backend, version=cls._version, function_name="DataFrame.from_dict" ) - raise ValueError(msg) + compliant = ns._dataframe.from_dict(data, schema=schema, context=ns) + return cls(compliant, level="full") @classmethod def from_dicts( @@ -684,18 +672,11 @@ def from_dicts( └──────────────────────────┘ """ schema = dict(schema) if schema is not None else None - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = cls._version.namespace.from_backend(implementation).compliant - compliant = ns._dataframe.from_dicts(data, schema=schema, context=ns) - return cls(compliant, level="full") - # NOTE: (#2786) needs resolving for extensions - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `DataFrame.from_dicts` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.DataFrame.from_dicts([{{'a': 1}}, {{'a': 2}}], backend='pyarrow').lazy('{implementation}')" + ns = eager_namespace( + backend, version=cls._version, function_name="DataFrame.from_dicts" ) - raise ValueError(msg) + compliant = ns._dataframe.from_dicts(data, schema=schema, context=ns) + return cls(compliant, level="full") @classmethod def from_numpy( @@ -760,16 +741,11 @@ def from_numpy( raise TypeError(msg) if not (schema is None or is_sequence_of(schema, str)): schema = Schema(schema) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = cls._version.namespace.from_backend(implementation).compliant - return cls(ns.from_numpy(data, schema), level="full") - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `DataFrame.from_numpy` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.DataFrame.from_numpy(arr, backend='pyarrow').lazy('{implementation}')" + ns = eager_namespace( + backend, version=cls._version, function_name="DataFrame.from_numpy" ) - raise ValueError(msg) + compliant = ns._dataframe.from_numpy(data, schema=schema, context=ns) + return cls(compliant, level="full") def __len__(self) -> int: return self._compliant_frame.__len__() @@ -1727,9 +1703,19 @@ def filter( 1 2 7 b """ impl = self.implementation + + def into_series(values: list[bool]) -> Series[Any]: + if impl is Implementation.UNKNOWN: # type: ignore[comparison-overlap] + ns = eager_namespace_from_compliant( + self._compliant_frame, function_name="DataFrame.filter(list[bool])" + ) + return self._series( + ns._series.from_iterable(values, context=ns, name=""), level="full" + ) + return self._series.from_iterable("", values, backend=impl) + parsed_predicates = ( - self._series.from_iterable("", p, backend=impl) if is_list_of(p, bool) else p - for p in predicates + into_series(p) if is_list_of(p, bool) else p for p in predicates ) return super().filter(*parsed_predicates, **constraints) diff --git a/src/narwhals/exceptions.py b/src/narwhals/exceptions.py index 447ff87a1d..c5f57ef1c5 100644 --- a/src/narwhals/exceptions.py +++ b/src/narwhals/exceptions.py @@ -103,6 +103,14 @@ def from_invalid_type(cls: type, invalid_type: type) -> InvalidIntoExprError: return InvalidIntoExprError(message) +class PluginError(NarwhalsError): + """Exception raised when a plugin backend does not support the requested operation. + + This includes missing extension hooks (e.g. `read_csv`, `__narwhals_namespace__`) + and lazy-only plugins used with eager-only functions. + """ + + class UnsupportedDTypeError(NarwhalsError): """Exception raised when trying to convert to a DType which is not supported by the given backend.""" diff --git a/src/narwhals/functions.py b/src/narwhals/functions.py index fb661554b3..014f5046a0 100644 --- a/src/narwhals/functions.py +++ b/src/narwhals/functions.py @@ -16,6 +16,7 @@ Implementation, Version, deprecate_native_namespace, + eager_namespace, flatten, is_eager_allowed, is_nested_literal, @@ -32,16 +33,18 @@ ) from narwhals.exceptions import InvalidOperationError from narwhals.expr import Expr +from narwhals.plugins import plugin_io_method from narwhals.schema import Schema -from narwhals.translate import from_native, to_native +from narwhals.translate import to_native if TYPE_CHECKING: + from collections.abc import Callable from types import ModuleType from typing import TypeAlias from typing_extensions import Self, TypeIs - from narwhals._native import NativeDataFrame, NativeLazyFrame, NativeSeries + from narwhals._compliant.typing import CompliantFrameAny from narwhals._translate import IntoArrowTable from narwhals._typing import Backend, EagerAllowed, IntoBackend, PluginName from narwhals.dataframe import DataFrame, LazyFrame @@ -213,27 +216,9 @@ def _new_series_impl( *, backend: IntoBackend[EagerAllowed | PluginName], ) -> Series[Any]: - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = Version.MAIN.namespace.from_backend(implementation).compliant - series = ns._series.from_iterable(values, name=name, context=ns, dtype=dtype) - return series.to_narwhals() - if implementation is Implementation.UNKNOWN: # pragma: no cover - _native_namespace = implementation.to_native_namespace() - try: - native_series: NativeSeries = _native_namespace.new_series( - name, values, dtype - ) - return from_native(native_series, series_only=True).alias(name) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `new_series` constructor." - raise AttributeError(msg) from e - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `new_series` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.new_series('a', [1,2,3], backend='pyarrow').to_frame().lazy('{implementation}')" - ) - raise ValueError(msg) + ns = eager_namespace(backend, version=Version.MAIN, function_name="new_series") + series = ns._series.from_iterable(values, name=name, context=ns, dtype=dtype) + return series.to_narwhals() @deprecate_native_namespace(warn_version="1.26.0") @@ -290,28 +275,8 @@ def from_dict( if schema and data and (diff := set(schema.keys()).symmetric_difference(data.keys())): msg = f"Keys in `schema` and `data` are expected to match, found unmatched keys: {diff}" raise InvalidOperationError(msg) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = Version.MAIN.namespace.from_backend(implementation).compliant - return ns._dataframe.from_dict(data, schema=schema, context=ns).to_narwhals() - if implementation is Implementation.UNKNOWN: # pragma: no cover - _native_namespace = implementation.to_native_namespace() - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement `from_dict` function in the top-level namespace. - native_frame: NativeDataFrame = _native_namespace.from_dict( - data, schema=schema - ) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `from_dict` function." - raise AttributeError(msg) from e - return from_native(native_frame, eager_only=True) - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `from_dict` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.from_dict({{'a': [1, 2]}}, backend='pyarrow').lazy('{implementation}')" - ) - raise ValueError(msg) + ns = eager_namespace(backend, version=Version.MAIN, function_name="from_dict") + return ns._dataframe.from_dict(data, schema=schema, context=ns).to_narwhals() def _from_dict_no_backend( @@ -449,28 +414,8 @@ def from_numpy( raise TypeError(msg) if not (schema is None or is_sequence_of(schema, str)): schema = Schema(schema) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = Version.MAIN.namespace.from_backend(implementation).compliant - return ns.from_numpy(data, schema).to_narwhals() - if implementation is Implementation.UNKNOWN: # pragma: no cover - _native_namespace = implementation.to_native_namespace() - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement `from_numpy` function in the top-level namespace. - native_frame: NativeDataFrame = _native_namespace.from_numpy( - data, schema=schema - ) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `from_numpy` function." - raise AttributeError(msg) from e - return from_native(native_frame, eager_only=True) - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `from_numpy` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.from_numpy(arr, backend='pyarrow').lazy('{implementation}')" - ) - raise ValueError(msg) + ns = eager_namespace(backend, version=Version.MAIN, function_name="from_numpy") + return ns._dataframe.from_numpy(data, schema=schema, context=ns).to_narwhals() def _is_into_schema(obj: Any) -> TypeIs[_IntoSchema]: @@ -521,26 +466,8 @@ def from_arrow( if not (supports_arrow_c_stream(native_frame) or is_pyarrow_table(native_frame)): msg = f"Given object of type {type(native_frame)} does not support PyCapsule interface" raise TypeError(msg) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = Version.MAIN.namespace.from_backend(implementation).compliant - return ns._dataframe.from_arrow(native_frame, context=ns).to_narwhals() - if implementation is Implementation.UNKNOWN: # pragma: no cover - _native_namespace = implementation.to_native_namespace() - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement PyCapsule support - native: NativeDataFrame = _native_namespace.DataFrame(native_frame) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `DataFrame` class which accepts object which supports PyCapsule Interface." - raise AttributeError(msg) from e - return from_native(native, eager_only=True) - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `from_arrow` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.from_arrow(df, backend='pyarrow').lazy('{implementation}')" - ) - raise ValueError(msg) + ns = eager_namespace(backend, version=Version.MAIN, function_name="from_arrow") + return ns._dataframe.from_arrow(native_frame, context=ns).to_narwhals() def _get_sys_info() -> dict[str, str]: @@ -661,18 +588,11 @@ def read_csv( ns = Version.MAIN.namespace.from_backend(impl).compliant frame = ns.read_csv(normalize_path(source), separator=separator, **kwargs) return frame.to_narwhals() - if impl is Implementation.UNKNOWN: # pragma: no cover - native_namespace = impl.to_native_namespace() - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement `read_csv` function in the top-level namespace. - native_frame: NativeDataFrame = native_namespace.read_csv( - source=source, **kwargs - ) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `read_csv` function." - raise AttributeError(msg) from e - return from_native(native_frame, eager_only=True) + if impl is Implementation.UNKNOWN: + read = plugin_io_method(backend, "read_csv", version=Version.MAIN) + frame = read(normalize_path(source), separator=separator, **kwargs) + result: DataFrame[Any] = frame.to_narwhals() + return result msg = ( f"Expected eager backend, found {impl}.\n\n" f"Hint: use nw.scan_csv(source={source}, backend={backend})" @@ -721,20 +641,12 @@ def scan_csv( └─────────┴───────┘ """ impl = Implementation.from_backend(backend) - if impl is Implementation.UNKNOWN: # pragma: no cover - native_namespace = impl.to_native_namespace() - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement `scan_csv` function in the top-level namespace. - native_frame: NativeDataFrame | NativeLazyFrame = native_namespace.scan_csv( - source=source, **kwargs - ) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `scan_csv` function." - raise AttributeError(msg) from e - return from_native(native_frame).lazy() - ns = Version.MAIN.namespace.from_backend(impl).compliant - frame = ns.scan_csv(normalize_path(source), separator=separator, **kwargs) + scan: Callable[..., CompliantFrameAny] + if impl is Implementation.UNKNOWN: + scan = plugin_io_method(backend, "scan_csv", version=Version.MAIN) + else: + scan = Version.MAIN.namespace.from_backend(impl).compliant.scan_csv + frame = scan(normalize_path(source), separator=separator, **kwargs) result: LazyFrame[Any] = frame.to_narwhals().lazy() return result @@ -778,18 +690,11 @@ def read_parquet( ns = Version.MAIN.namespace.from_backend(impl).compliant frame = ns.read_parquet(normalize_path(source), **kwargs) return frame.to_narwhals() - if impl is Implementation.UNKNOWN: # pragma: no cover - native_namespace = impl.to_native_namespace() - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement `read_parquet` function in the top-level namespace. - native_frame: NativeDataFrame = native_namespace.read_parquet( - source=source, **kwargs - ) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `read_parquet` function." - raise AttributeError(msg) from e - return from_native(native_frame, eager_only=True) + if impl is Implementation.UNKNOWN: + read = plugin_io_method(backend, "read_parquet", version=Version.MAIN) + frame = read(normalize_path(source), **kwargs) + result: DataFrame[Any] = frame.to_narwhals() + return result msg = ( f"Expected eager backend, found {impl}.\n\n" f"Hint: use nw.scan_parquet(source={source}, backend={backend})" @@ -860,20 +765,12 @@ def scan_parquet( └──────────────────┘ """ impl = Implementation.from_backend(backend) - if impl is Implementation.UNKNOWN: # pragma: no cover - native_namespace = impl.to_native_namespace() - try: - # implementation is UNKNOWN, Narwhals extension using this feature should - # implement `scan_parquet` function in the top-level namespace. - native_frame: NativeDataFrame | NativeLazyFrame = ( - native_namespace.scan_parquet(source=source, **kwargs) - ) - except AttributeError as e: - msg = "Unknown namespace is expected to implement `scan_parquet` function." - raise AttributeError(msg) from e - return from_native(native_frame).lazy() - ns = Version.MAIN.namespace.from_backend(impl).compliant - frame = ns.scan_parquet(normalize_path(source), **kwargs) + scan: Callable[..., CompliantFrameAny] + if impl is Implementation.UNKNOWN: + scan = plugin_io_method(backend, "scan_parquet", version=Version.MAIN) + else: + scan = Version.MAIN.namespace.from_backend(impl).compliant.scan_parquet + frame = scan(normalize_path(source), **kwargs) result: LazyFrame[Any] = frame.to_narwhals().lazy() return result diff --git a/src/narwhals/plugins.py b/src/narwhals/plugins.py index ae823eec59..00ad585573 100644 --- a/src/narwhals/plugins.py +++ b/src/narwhals/plugins.py @@ -15,14 +15,16 @@ import sys from functools import cache -from typing import TYPE_CHECKING, Any, Protocol +from types import ModuleType +from typing import TYPE_CHECKING, Any, Literal, Protocol, cast, overload from narwhals._compliant import CompliantNamespace from narwhals._typing import PluginName from narwhals._typing_compat import TypeVar +from narwhals.exceptions import PluginError if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import Callable, Iterator from importlib.metadata import EntryPoints from typing import TypeAlias @@ -34,8 +36,14 @@ CompliantLazyFrameAny, CompliantSeriesAny, ) + from narwhals._typing import Backend, IntoBackend from narwhals.utils import Version + IOMethodName: TypeAlias = Literal[ + "read_csv", "read_parquet", "scan_csv", "scan_parquet" + ] + """Name of a Narwhals IO function, dispatched to a same-named namespace method.""" + __all__ = ["Plugin", "PluginName", "from_native"] @@ -62,18 +70,179 @@ def _discover_entrypoints() -> EntryPoints: return eps(group=group) +def _plugin_names() -> tuple[str, ...]: + """Entry point names of all installed plugins.""" + return tuple(entry_point.name for entry_point in _discover_entrypoints()) + + +def _find_plugin(backend_name: str, /) -> Plugin | None: + """Return the first installed plugin matching `backend_name`. + + `backend_name` is matched against both the entry point name and its module, + e.g. both `"my-plugin"` and `"my_plugin"` for a plugin registered as: + + [project.entry-points.'narwhals.plugins'] + my-plugin = 'my_plugin' + + Note: + The parameter is a plain `str`, not a `PluginName`: the module spelling is a + valid input and is *not* an entry point name. + """ + for entry_point in _discover_entrypoints(): + if backend_name in {entry_point.name, entry_point.module}: + plugin: Plugin = entry_point.load() + return plugin + return None + + +def _backend_namespace(backend: IntoBackend[Backend | PluginName], /) -> Plugin: + """Resolve a backend which is not a Narwhals `Implementation` to a plugin. + + The plugin is expected to implement the `Plugin` protocol, in particular the + `__narwhals_namespace__` function returning a compliant namespace. + """ + if isinstance(backend, ModuleType): + # NOTE: A user-provided module is only *claimed* to be a `Plugin`; the runtime + # guard is `_plugin_namespace`, which raises `PluginError` if it is not one. + return cast("Plugin", backend) + if isinstance(backend, str) and (plugin := _find_plugin(backend)) is not None: + return plugin + installed = ", ".join(_plugin_names()) or "" + msg = ( + f"Unsupported backend: {backend!r}.\n\n" + "Expected one of Narwhals' built-in backends (e.g. 'pandas', 'polars', " + "'pyarrow'), a native namespace module, or the name of an installed " + f"Narwhals plugin (installed plugins: {installed})." + ) + raise ValueError(msg) + + +def _plugin_namespace(plugin: Plugin, /, *, version: Version) -> PluginNamespace: + """Get `plugin`'s compliant namespace, raising if `__narwhals_namespace__` is missing.""" + name = "__narwhals_namespace__" + if (hook := getattr(plugin, name, None)) is None: + msg = f"Plugin backend {plugin.__name__!r} is expected to implement `{name}` function." + raise PluginError(msg) + namespace: PluginNamespace = hook(version=version) + return namespace + + +@overload +def plugin_io_method( + backend: IntoBackend[Backend | PluginName], + method_name: Literal["read_csv", "read_parquet"], + /, + *, + version: Version, +) -> Callable[..., CompliantDataFrameAny]: ... + + +@overload +def plugin_io_method( + backend: IntoBackend[Backend | PluginName], + method_name: Literal["scan_csv", "scan_parquet"], + /, + *, + version: Version, +) -> Callable[..., CompliantFrameAny]: ... + + +def plugin_io_method( + backend: IntoBackend[Backend | PluginName], + method_name: IOMethodName, + /, + *, + version: Version, +) -> Callable[..., CompliantFrameAny]: + """Resolve `backend` to the `method_name` method of a plugin's compliant namespace. + + IO functions share a single dispatch mechanism with built-in backends: they call + same-named methods on the compliant namespace (see the "IO functions" section of + the [extension docs](../extending.md/#io-functions-the-namespace-contract)). + + Note: + `PluginNamespace` deliberately does not declare the IO methods: they are an + optional subset of a plugin (e.g. a lazy-only plugin implements `scan_*` only). + """ + from inspect import getattr_static + + from narwhals._utils import not_implemented + + plugin = _backend_namespace(backend) + namespace = _plugin_namespace(plugin, version=version) + method = getattr_static(namespace, method_name, None) + if method is None or isinstance(method, not_implemented): + msg = ( + f"Plugin backend {plugin.__name__!r} is expected to implement " + f"`{method_name}` on its compliant namespace to support `narwhals.{method_name}`." + ) + raise PluginError(msg) + bound_method: Callable[..., CompliantFrameAny] = getattr(namespace, method_name) + return bound_method + + class PluginNamespace(CompliantNamespace[FrameT, Any], Protocol[FrameT, FromNativeR_co]): - def from_native(self, data: Any, /) -> FromNativeR_co: ... + """A `CompliantNamespace` which can also wrap native objects via `from_native`.""" + + def from_native(self, data: Any, /) -> FromNativeR_co: + """Wrap a native object into a compliant DataFrame, LazyFrame, or Series.""" + ... class Plugin(Protocol[FrameT, FromNativeR_co]): + """Top-level interface a plugin module is expected to implement. + + A plugin is a module registered in the `narwhals.plugins` + [entry point](https://packaging.python.org/en/latest/specifications/entry-points/) + group: + + ```toml + [project.entry-points.'narwhals.plugins'] + narwhals-grizzlies = 'narwhals_grizzlies' + ``` + + Narwhals discovers installed plugins at runtime and uses this interface to + recognise their native objects (`NATIVE_PACKAGE`, `is_native`) and to obtain + a compliant namespace (`__narwhals_namespace__`), through which all further + dispatch happens. + + See [extensions and plugins](../extending.md) for a complete walk-through. + """ + @property - def NATIVE_PACKAGE(self) -> LiteralString: ... # noqa: N802 + def __name__(self) -> str: + """Name of the plugin module, used to identify the backend in error messages. + + Automatically provided: a plugin *is* a module, and every module has a `__name__`. + """ + ... + + @property + def NATIVE_PACKAGE(self) -> LiteralString: # noqa: N802 + """Name of the package providing the plugin's native objects, e.g. `"grizzlies"`. + + Used as a cheap pre-check when converting native objects: the plugin is only + consulted if this package is already imported and the inspected object's class + might originate from it. + """ + ... def __narwhals_namespace__( self, version: Version - ) -> PluginNamespace[FrameT, FromNativeR_co]: ... - def is_native(self, native_object: object, /) -> bool: ... + ) -> PluginNamespace[FrameT, FromNativeR_co]: + """Return a compliant namespace for the given Narwhals API version. + + The returned namespace is the plugin's dispatch hub: its `from_native` method + wraps native objects, IO functions call its `scan_*`/`read_*` methods, and + eager constructors use its `_dataframe`/`_series` classes (see the + [`backend=...` section](../extending.md/#supporting-backend-in-narwhals-functions) + of the extension docs). + """ + ... + + def is_native(self, native_object: object, /) -> bool: + """Return whether `native_object` is a native object of the plugin's library.""" + ... @cache @@ -110,16 +279,18 @@ def from_native(native_object: Any, version: Version) -> CompliantAny | None: Returns: If the following conditions are met + - at least 1 plugin is installed - at least 1 installed plugin supports `type(native_object)` - Then for the **first matching plugin**, the result of the call below. - This *should* be an object accepted by a Narwhals Dataframe, Lazyframe, or Series: + Then for the **first matching plugin**, the result of the call below. + + This *should* be an object accepted by a Narwhals Dataframe, Lazyframe, or Series: - plugin: Plugin - plugin.__narwhals_namespace__(version).from_native(native_object) + plugin: Plugin + plugin.__narwhals_namespace__(version).from_native(native_object) - In all other cases, `None` is returned instead. + In all other cases, `None` is returned instead. """ return next(_iter_from_native(native_object, version), None) diff --git a/src/narwhals/series.py b/src/narwhals/series.py index d0688cba41..1bc268bd34 100644 --- a/src/narwhals/series.py +++ b/src/narwhals/series.py @@ -2,7 +2,6 @@ import math from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence -from functools import partial from typing import TYPE_CHECKING, Any, ClassVar, Generic, Literal, cast, overload from narwhals._expression_parsing import ExprKind, ExprNode @@ -13,10 +12,11 @@ _Implementation, _resolve_sample_size, _validate_rolling_arguments, + eager_namespace, + eager_namespace_from_compliant, ensure_type, generate_repr, is_compliant_series, - is_eager_allowed, is_index_selector, qualified_type_name, supports_arrow_c_stream, @@ -165,19 +165,13 @@ def from_numpy( raise ValueError(msg) if dtype: _validate_into_dtype(dtype) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = cls._version.namespace.from_backend(implementation).compliant - compliant = ns.from_numpy(values).alias(name) - if dtype: - return cls(compliant.cast(dtype), level="full") - return cls(compliant, level="full") - msg = ( # pragma: no cover - f"{implementation} support in Narwhals is lazy-only, but `Series.from_numpy` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.Series.from_numpy(arr, backend='pyarrow').to_frame().lazy('{implementation}')" + ns = eager_namespace( + backend, version=cls._version, function_name="Series.from_numpy" ) - raise ValueError(msg) # pragma: no cover + compliant = ns._series.from_numpy(values, context=ns).alias(name) + if dtype: + return cls(compliant.cast(dtype), level="full") + return cls(compliant, level="full") @classmethod def from_iterable( @@ -227,19 +221,11 @@ def from_iterable( if not isinstance(values, Iterable): msg = f"Expected values to be an iterable, got: {qualified_type_name(values)!r}." raise TypeError(msg) - implementation = Implementation.from_backend(backend) - if is_eager_allowed(implementation): - ns = cls._version.namespace.from_backend(implementation).compliant - compliant = ns._series.from_iterable( - values, context=ns, name=name, dtype=dtype - ) - return cls(compliant, level="full") - msg = ( - f"{implementation} support in Narwhals is lazy-only, but `Series.from_iterable` is an eager-only function.\n\n" - "Hint: you may want to use an eager backend and then call `.lazy`, e.g.:\n\n" - f" nw.Series.from_iterable('a', [1,2,3], backend='pyarrow').to_frame().lazy('{implementation}')" + ns = eager_namespace( + backend, version=cls._version, function_name="Series.from_iterable" ) - raise ValueError(msg) + compliant = ns._series.from_iterable(values, context=ns, name=name, dtype=dtype) + return cls(compliant, level="full") implementation: _Implementation = _Implementation() """Return [`narwhals.Implementation`][] of native Series. @@ -433,9 +419,18 @@ def scatter( a: [[999,888,3]] b: [[4,5,6]] """ - into_series = partial( - type(self).from_iterable, name="", backend=self.implementation - ) + impl = self.implementation + + def into_series(values: Any, dtype: IntoDType | None = None) -> Series[Any]: + if impl is Implementation.UNKNOWN: # type: ignore[comparison-overlap] + ns = eager_namespace_from_compliant( + self._compliant_series, function_name="Series.scatter" + ) + compliant = ns._series.from_iterable( + values, context=ns, name="", dtype=dtype + ) + return self._with_compliant(compliant) + return type(self).from_iterable("", values, dtype, backend=impl) if not isinstance(indices, Series): if not isinstance(indices, Iterable): diff --git a/src/narwhals/testing/asserts/series.py b/src/narwhals/testing/asserts/series.py index 5bdcbb2ea7..80ad2b673f 100644 --- a/src/narwhals/testing/asserts/series.py +++ b/src/narwhals/testing/asserts/series.py @@ -5,20 +5,28 @@ from narwhals._utils import qualified_type_name from narwhals.dependencies import is_narwhals_series -from narwhals.dtypes import Array, Boolean, Categorical, List, String, Struct -from narwhals.functions import new_series +from narwhals.dtypes import Array, Categorical, List, String, Struct from narwhals.testing.asserts.utils import raise_series_assertion_error if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Iterable from typing import TypeAlias from narwhals.series import Series - from narwhals.typing import IntoSeriesT, SeriesT + from narwhals.typing import IntoDType, IntoSeriesT, SeriesT CheckFn: TypeAlias = Callable[[Series[Any], Series[Any]], None] +def _series_like( + reference: SeriesT, values: Iterable[Any], dtype: IntoDType | None +) -> SeriesT: + """Build a Series from `values` using `reference`'s own backend.""" + source = reference._compliant_series + compliant = type(source).from_iterable(values, name="", context=source, dtype=dtype) + return reference._with_compliant(compliant) + + def assert_series_equal( left: Series[IntoSeriesT], right: Series[IntoSeriesT], @@ -161,7 +169,6 @@ def _check_exact_values( categorical_as_str: bool, ) -> None: """Check exact value equality for various data types.""" - left_impl = left.implementation left_dtype, right_dtype = left.dtype, right.dtype is_not_equal_mask: Series[Any] @@ -182,9 +189,9 @@ def _check_exact_values( abs_tol=abs_tol, categorical_as_str=categorical_as_str, ) + # `_check_list_like` raises on the first mismatch; reaching here means equal. _check_list_like(left, right, left_dtype, right_dtype, check_fn=check_fn) - # If `_check_list_like` didn't raise, then every nested element is equal - is_not_equal_mask = new_series("", [False], dtype=Boolean(), backend=left_impl) + return elif isinstance(left_dtype, Struct) and isinstance(right_dtype, Struct): check_fn = partial( assert_series_equal, @@ -196,18 +203,15 @@ def _check_exact_values( abs_tol=abs_tol, categorical_as_str=categorical_as_str, ) + # `_check_struct` raises on the first mismatch; reaching here means equal. _check_struct(left, right, left_dtype, right_dtype, check_fn=check_fn) - # If `_check_struct` didn't raise, then every nested element is equal - is_not_equal_mask = new_series("", [False], dtype=Boolean(), backend=left_impl) + return elif isinstance(left_dtype, Categorical) and isinstance(right_dtype, Categorical): - # If `_check_categorical` didn't raise, then the categories sources/encodings are - # the same, and we can use equality - _not_equal = _check_categorical( - left, right, categorical_as_str=categorical_as_str - ) - is_not_equal_mask = new_series( - "", [_not_equal], dtype=Boolean(), backend=left_impl - ) + # A raise from `_check_categorical` means the categories sources/encodings + # differ; otherwise it returns whether any element differs. + if _check_categorical(left, right, categorical_as_str=categorical_as_str): + raise_series_assertion_error("exact value mismatch", left, right) + return else: is_not_equal_mask = left != right @@ -241,12 +245,11 @@ def _check_list_like( # Check row by row after transforming each array/list into a new series. # Notice that order within the array/list must be the same, regardless of # `check_order` value at the top level. - impl = left_vals.implementation try: - for left_val, right_val in zip(left_vals, right_vals, strict=True): + for lv, rv in zip(left_vals, right_vals, strict=True): check_fn( - new_series("", values=left_val, dtype=left_dtype.inner, backend=impl), - new_series("", values=right_val, dtype=right_dtype.inner, backend=impl), + _series_like(reference=left_vals, values=lv, dtype=left_dtype.inner), + _series_like(reference=right_vals, values=rv, dtype=right_dtype.inner), ) except AssertionError: raise_series_assertion_error("nested value mismatch", left_vals, right_vals) diff --git a/src/narwhals/typing.py b/src/narwhals/typing.py index 9173c0b612..018ab589ac 100644 --- a/src/narwhals/typing.py +++ b/src/narwhals/typing.py @@ -413,6 +413,7 @@ def Binary(self) -> type[dtypes.Binary]: ... "CompliantSeries", "DataFrameT", "EagerAllowed", + "FileSource", "Frame", "FrameT", "IntoBackend", @@ -426,4 +427,5 @@ def Binary(self) -> type[dtypes.Binary]: ... "IntoSeries", "IntoSeriesT", "LazyAllowed", + "NormalizedPath", ] diff --git a/tests/plugins_test.py b/tests/plugins_test.py index 11bcd813e3..6b986d9fec 100644 --- a/tests/plugins_test.py +++ b/tests/plugins_test.py @@ -1,6 +1,9 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +import re +import types +from functools import partial +from typing import TYPE_CHECKING, Any, Protocol, cast, get_args import pytest @@ -8,17 +11,36 @@ import narwhals.stable.v1.dependencies as nw_v1_dependencies import narwhals.stable.v2.dependencies as nw_v2_dependencies from narwhals import dependencies as nw_dependencies +from narwhals._compliant import CompliantNamespace +from narwhals._utils import EAGER_HINT_EXAMPLES, EagerFunctionName, not_implemented +from narwhals.exceptions import PluginError from narwhals.plugins import PluginName +from tests.utils import PYARROW_VERSION if TYPE_CHECKING: + from collections.abc import Callable, Mapping, Sequence + from pathlib import Path + from types import ModuleType + from typing import TypeAlias + + import pyarrow as pa from typing_extensions import Self - from narwhals._typing import EagerAllowed, IntoBackend + from narwhals._typing import Backend, EagerAllowed, IntoBackend from narwhals.plugins import Plugin + from narwhals.typing import _1DArray, _2DArray from narwhals.utils import Version + _ConstructorData: TypeAlias = "Mapping[str, Any] | Sequence[Mapping[str, Any]] | Callable[[], _2DArray | pa.Table]" + +plugin_module = pytest.importorskip("test_plugin") + DEPENDENCIES_MODULES = (nw_dependencies, nw_v1_dependencies, nw_v2_dependencies) +BACKEND = PluginName("test-plugin") +DATA: dict[str, Any] = {"a": [1, 1, 2], "b": [4, 5, 6]} +ROWS = [{"a": 1, "b": 4}, {"a": 1, "b": 5}, {"a": 2, "b": 6}] + class FakeNative: """Native object of an imaginary plugin-backed library.""" @@ -70,27 +92,292 @@ def load(self) -> FakePlugin: return self._plugin -def test_plugin() -> None: - pytest.importorskip("test_plugin") - df_native = {"a": [1, 1, 2], "b": [4, 5, 6]} - lf = nw.from_native(df_native) # type: ignore[call-overload] +class PluginModule(types.ModuleType): + """An ad-hoc, *deliberately incomplete* plugin module. + + Real plugins live in `packages/`, but the error paths below need namespaces which + violate the contract on purpose, so they cannot be packaged. + """ + + _factory: Callable[[], object] + + def __init__(self, name: str, factory: Callable[[], object]) -> None: + super().__init__(name) + self._factory = factory + + def __narwhals_namespace__(self, version: Version) -> object: + return self._factory() + + +class BackendFn(Protocol): + """A narwhals function whose remaining argument is `backend`.""" + + def __call__( + self, *, backend: IntoBackend[Backend | PluginName] + ) -> nw.DataFrame[Any] | nw.LazyFrame[Any]: ... + + +class NotImplementedNamespace(CompliantNamespace[Any, Any]): + """A namespace which declares, but does not provide, the optional plugin methods. + + `not_implemented` descriptors exist statically yet raise on instance access, so they + must not be mistaken for support. + """ + + scan_csv = not_implemented() + read_csv = not_implemented() + scan_parquet = not_implemented() + read_parquet = not_implemented() + _series = not_implemented() + _dataframe = not_implemented() + + +def _np_2d_array() -> _2DArray: + pytest.importorskip("numpy") + import numpy as np + + return cast("_2DArray", np.array([[1, 4], [1, 5], [2, 6]])) + + +def _np_1d_array() -> _1DArray: + pytest.importorskip("numpy") + import numpy as np + + return cast("_1DArray", np.array([1, 2, 3])) + + +def _arrow_table() -> pa.Table: + pytest.importorskip("pyarrow") + import pyarrow as pa + + return pa.table(DATA) + + +@pytest.fixture +def csv_path(tmp_path: Path) -> str: + path = tmp_path / "file.csv" + path.write_text("a,b\n1,4\n1,5\n2,6\n", encoding="utf-8") + return str(path) + + +@pytest.fixture +def parquet_path(tmp_path: Path) -> str: + pq = pytest.importorskip("pyarrow.parquet") + path = str(tmp_path / "file.parquet") + pq.write_table(_arrow_table(), path) + return path + + +def test_plugin_is_lazy() -> None: + lf = nw.from_native(DATA) # type: ignore[call-overload] assert isinstance(lf, nw.LazyFrame) assert lf.columns == ["a", "b"] def test_not_implemented() -> None: - pytest.importorskip("test_plugin") - df_native = {"a": [1, 1, 2], "b": [4, 5, 6]} - lf = nw.from_native(df_native) # type: ignore[call-overload] + lf = nw.from_native(DATA) # type: ignore[call-overload] with pytest.raises( NotImplementedError, match="is not implemented for: 'DictLazyFrame'" ): lf.select(nw.col("a").ewm_mean()) +@pytest.mark.parametrize( + "backend", + ["test-plugin", "test_plugin", plugin_module], + ids=["entry-point-name", "module-name", "module"], +) +@pytest.mark.parametrize( + ("scan_function", "path_fixture"), + [(nw.scan_csv, "csv_path"), (nw.scan_parquet, "parquet_path")], + ids=["scan_csv", "scan_parquet"], +) +def test_scan_plugin( + request: pytest.FixtureRequest, + scan_function: Callable[..., nw.LazyFrame[Any]], + path_fixture: str, + backend: str | ModuleType, +) -> None: + """`backend` resolves via the entry point name, its module name, or the module itself.""" + lf = scan_function(request.getfixturevalue(path_fixture), backend=backend) + assert isinstance(lf, nw.LazyFrame) + assert lf.columns == ["a", "b"] + + +@pytest.mark.parametrize( + ("read_function", "path_fixture", "expected"), + [ + (nw.read_csv, "csv_path", {"a": ["1", "1", "2"], "b": ["4", "5", "6"]}), + (nw.read_parquet, "parquet_path", DATA), + ], + ids=["read_csv", "read_parquet"], +) +def test_read_plugin( + request: pytest.FixtureRequest, + read_function: Callable[..., nw.DataFrame[Any]], + path_fixture: str, + expected: dict[str, Any], +) -> None: + """`read_*` dispatch to the namespace's eager half of the IO contract.""" + df = read_function(request.getfixturevalue(path_fixture), backend=BACKEND) + assert isinstance(df, nw.DataFrame) + assert df.to_native() == expected + + +@pytest.mark.parametrize( + ("dataframe_constructor", "data", "kwargs"), + [ + (nw.from_dict, DATA, {}), + (nw.from_dicts, ROWS, {}), + (nw.from_numpy, _np_2d_array, {"schema": ["a", "b"]}), + pytest.param( + nw.from_arrow, + _arrow_table, + {}, + marks=pytest.mark.skipif(PYARROW_VERSION < (14,), reason="too old"), + ), + (nw.DataFrame.from_dict, DATA, {}), + (nw.DataFrame.from_dicts, ROWS, {}), + (nw.DataFrame.from_numpy, _np_2d_array, {"schema": ["a", "b"]}), + pytest.param( + nw.DataFrame.from_arrow, + _arrow_table, + {}, + marks=pytest.mark.skipif(PYARROW_VERSION < (14,), reason="too old"), + ), + ], +) +def test_eager_dataframe_constructors_plugin( + dataframe_constructor: Callable[..., nw.DataFrame[Any]], + data: _ConstructorData, + kwargs: dict[str, Any], +) -> None: + """Eager constructors dispatch to the plugin's `EagerNamespace`-compliant namespace.""" + # Factories are deferred, so that `importorskip` runs at test time. + native_data = data() if callable(data) else data + df = dataframe_constructor(native_data, backend=BACKEND, **kwargs) + assert isinstance(df, nw.DataFrame) + assert df.to_native() == DATA + + +@pytest.mark.parametrize( + ("series_constructor", "values"), + [ + (nw.new_series, [1, 2, 3]), + (nw.Series.from_iterable, [1, 2, 3]), + (nw.Series.from_numpy, _np_1d_array), + ], +) +def test_eager_series_constructors_plugin( + series_constructor: Callable[..., nw.Series[Any]], + values: list[int] | Callable[[], _1DArray], +) -> None: + """Eager constructors dispatch to the plugin's `EagerNamespace`-compliant namespace.""" + # The factory is deferred, so that `importorskip` runs at test time. + native_values = values() if callable(values) else values + s = series_constructor("a", native_values, backend=BACKEND) + assert isinstance(s, nw.Series) + assert s.name == "a" + assert s.to_native() == [1, 2, 3] + + +def test_series_scatter_plugin() -> None: + """`scatter` constructs indices/values via the plugin's own namespace.""" + s = nw.Series.from_iterable("a", [1, 2, 3], backend=BACKEND) + assert s.scatter([0, 2], [99, 77]).to_native() == [99, 2, 77] + assert s.scatter(1, 50).to_native() == [1, 50, 3] + # Original Series is unchanged, and empty indices are a no-op. + assert s.to_native() == [1, 2, 3] + assert s.scatter([], []).to_native() == [1, 2, 3] + + +def test_dataframe_filter_mask_plugin() -> None: + """`filter(list[bool])` builds the mask series via the plugin's own namespace.""" + df = nw.from_dict(DATA, backend=BACKEND) + with pytest.raises(NotImplementedError, match="'all_horizontal' is not implemented"): + df.filter([True, False, True]) + + +@pytest.mark.parametrize( + "function", + [ + partial(nw.scan_csv, "x.csv"), + partial(nw.read_csv, "x.csv"), + partial(nw.scan_parquet, "x.parquet"), + partial(nw.read_parquet, "x.parquet"), + partial(nw.from_dict, DATA), + ], +) +def test_plugin_missing_narwhals_namespace(function: BackendFn) -> None: + """IO and eager functions require the plugin to implement `__narwhals_namespace__`.""" + empty_namespace = types.ModuleType("empty_plugin") + with pytest.raises( + PluginError, match="expected to implement `__narwhals_namespace__`" + ): + function(backend=empty_namespace) + + +@pytest.mark.parametrize("make_namespace", [object, NotImplementedNamespace]) +@pytest.mark.parametrize( + ("io_function", "source"), + [ + (nw.scan_csv, "x.csv"), + (nw.read_csv, "x.csv"), + (nw.scan_parquet, "x.parquet"), + (nw.read_parquet, "x.parquet"), + ], +) +def test_plugin_missing_io_method( + io_function: Callable[..., nw.DataFrame[Any] | nw.LazyFrame[Any]], + source: str, + make_namespace: Callable[[], object], +) -> None: + """A plugin whose compliant namespace lacks the IO method raises an informative PluginError. + + Both a plainly absent method and a `not_implemented` placeholder count as missing. + """ + minimal_plugin = PluginModule("minimal_plugin", make_namespace) + with pytest.raises( + PluginError, match=f"expected to implement `{io_function.__name__}`" + ): + io_function(source, backend=minimal_plugin) + + +@pytest.mark.parametrize("make_namespace", [object, NotImplementedNamespace]) +@pytest.mark.parametrize( + "function", + [ + partial(nw.from_dict, DATA), + partial(nw.from_dicts, ROWS), + partial(nw.new_series, "a", [1]), + partial(nw.Series.from_iterable, "a", [1]), + partial(nw.DataFrame.from_dict, DATA), + ], +) +def test_plugin_not_eager_allowed( + function: Callable[..., nw.DataFrame[Any] | nw.Series[Any]], + make_namespace: Callable[[], object], +) -> None: + """Eager functions require an `EagerNamespace`-compliant plugin namespace.""" + lazy_plugin = PluginModule("lazy_plugin", make_namespace) + with pytest.raises(PluginError, match="does not provide eager support"): + function(backend=lazy_plugin) + + +def test_unknown_backend_raises() -> None: + """A string matching neither a built-in backend nor an installed plugin.""" + with pytest.raises(ValueError, match="Unsupported backend: 'not-a-backend'"): + nw.scan_csv("x.csv", backend="not-a-backend") # type: ignore[arg-type] + + +def test_from_native_unsupported_object() -> None: + """An object no installed plugin recognises falls through to the unsupported-type error.""" + with pytest.raises(TypeError, match="Unsupported dataframe type"): + nw.from_native(object()) # type: ignore[call-overload] + + def test_is_into_lazyframe() -> None: # https://github.com/narwhals-dev/narwhals/issues/3714 - pytest.importorskip("test_plugin") df_native = {"a": [1, 1, 2], "b": [4, 5, 6]} for dependencies in DEPENDENCIES_MODULES: assert dependencies.is_into_lazyframe(df_native) @@ -98,7 +385,6 @@ def test_is_into_lazyframe() -> None: def test_is_into_dataframe() -> None: # `test_plugin` converts to a LazyFrame, so `is_into_dataframe` should not match. - pytest.importorskip("test_plugin") df_native = {"a": [1, 1, 2], "b": [4, 5, 6]} for dependencies in DEPENDENCIES_MODULES: assert not dependencies.is_into_dataframe(df_native) @@ -117,11 +403,10 @@ def test_is_into_mocked_plugin( ) -> None: from narwhals import plugins - monkeypatch.setattr( - plugins, - "_discover_entrypoints", - lambda: (FakeEntryPoint(FakePlugin(compliant_cls)),), - ) + def fake_entrypoints() -> tuple[FakeEntryPoint, ...]: + return (FakeEntryPoint(FakePlugin(compliant_cls)),) + + monkeypatch.setattr(plugins, "_discover_entrypoints", fake_entrypoints) native = FakeNative() for dependencies in DEPENDENCIES_MODULES: assert dependencies.is_into_dataframe(native) is (expected_kind == "dataframe") @@ -130,12 +415,32 @@ def test_is_into_mocked_plugin( def test_typing() -> None: - pytest.importorskip("test_plugin") import test_plugin _plugin: Plugin = test_plugin +def test_eager_hint_examples_exhaustive() -> None: + assert set(get_args(EagerFunctionName)) == set(EAGER_HINT_EXAMPLES) + + +@pytest.mark.parametrize( + ("function", "function_name"), + [ + (partial(nw.from_dict, DATA), "from_dict"), + (partial(nw.new_series, "a", [1]), "new_series"), + (partial(nw.DataFrame.from_dicts, ROWS), "DataFrame.from_dicts"), + ], +) +def test_eager_only_lazy_backend_hint( + function: Callable[..., Any], function_name: str +) -> None: + """A lazy-only *built-in* backend gets the per-function hint, keyed by `function_name`.""" + hint = EAGER_HINT_EXAMPLES[function_name] # type: ignore[index] + with pytest.raises(ValueError, match=re.escape(f" {hint}.lazy(")): + function(backend="duckdb") + + def test_plugin_name_runtime() -> None: # `PluginName` is a `NewType`: identity at runtime, nominal for type checkers. name = PluginName("some-plugin") diff --git a/tests/testing/assert_series_equal_test.py b/tests/testing/assert_series_equal_test.py index 0517fa8aaf..c73c631e10 100644 --- a/tests/testing/assert_series_equal_test.py +++ b/tests/testing/assert_series_equal_test.py @@ -380,23 +380,17 @@ def test_categorical_as_str( categorical_as_str: bool, context: AbstractContextManager[Any], ) -> None: - if ( - "polars" in str(constructor_eager) - and POLARS_VERSION >= (1, 32) - and not categorical_as_str - ): + name = str(constructor_eager) + if "polars" in name and POLARS_VERSION >= (1, 32) and not categorical_as_str: # https://github.com/pola-rs/polars/pull/23016 removed StringCache, it still # exists but it does nothing in python. request.applymarker(pytest.mark.xfail) - if "pyarrow_table" in str(constructor_eager) and not categorical_as_str: + if "pyarrow_table" in name and not categorical_as_str: # pyarrow dictionary dtype compares values, not the encoding. request.applymarker(pytest.mark.xfail) - if "pyarrow_table" in str(constructor_eager) and PYARROW_VERSION < ( - 15, - 0, - ): # pragma: no cover + if "pyarrow_table" in name and PYARROW_VERSION < (15, 0): # pragma: no cover reason = ( "pyarrow.lib.ArrowNotImplementedError: Unsupported cast from string to " "dictionary using function cast_dictionary" @@ -415,3 +409,21 @@ def test_categorical_as_str( assert_series_equal( left, right, check_names=False, categorical_as_str=categorical_as_str ) + + +def test_categorical_as_str_value_mismatch(constructor_eager: ConstructorEager) -> None: + name = str(constructor_eager) + if "pyarrow_table" in name and PYARROW_VERSION < (15, 0): # pragma: no cover + reason = ( + "pyarrow.lib.ArrowNotImplementedError: Unsupported cast from string to " + "dictionary using function cast_dictionary" + ) + pytest.skip(reason=reason) + + data = {"a": ["beluga", "orca", "orca", "beluga"]} + frame = nw.from_native(constructor_eager(data), eager_only=True) + s = frame["a"].cast(nw.Categorical()) + left, right = s[:2], s[2:] + + with _assertion_error("exact value mismatch"): + assert_series_equal(left, right) diff --git a/zensical.toml b/zensical.toml index 31ac3aaa7f..36dd9e48d0 100644 --- a/zensical.toml +++ b/zensical.toml @@ -81,6 +81,7 @@ nav = [ "api-reference/testing.md", "api-reference/typing.md", "api-reference/utils.md", + "api-reference/plugins.md", ]}, {"This" = "this.md"} ]