diff --git a/src/narwhals/_arrow/series.py b/src/narwhals/_arrow/series.py index 74fbac0df1..336e7e7e8d 100644 --- a/src/narwhals/_arrow/series.py +++ b/src/narwhals/_arrow/series.py @@ -34,6 +34,7 @@ not_implemented, ) from narwhals.dependencies import is_numpy_array_1d +from narwhals.dtypes import _validate_cast_temporal_to_numeric from narwhals.exceptions import InvalidOperationError, ShapeError if TYPE_CHECKING: @@ -547,6 +548,7 @@ def is_nan(self) -> Self: return self._with_native(pc.is_nan(self.native), preserve_broadcast=True) def cast(self, dtype: IntoDType) -> Self: + _validate_cast_temporal_to_numeric(source=self.dtype, target=dtype) data_type = narwhals_to_native_dtype(dtype, self._version) return self._with_native(pc.cast(self.native, data_type), preserve_broadcast=True) diff --git a/src/narwhals/_compliant/expr.py b/src/narwhals/_compliant/expr.py index b893de01da..1c672d4a7b 100644 --- a/src/narwhals/_compliant/expr.py +++ b/src/narwhals/_compliant/expr.py @@ -33,6 +33,7 @@ qualified_type_name, ) from narwhals.dependencies import is_numpy_array, is_numpy_scalar +from narwhals.dtypes import _validate_cast_temporal_to_numeric from narwhals.exceptions import MultiOutputExpressionError if TYPE_CHECKING: @@ -937,6 +938,30 @@ def fn(names: Sequence[str]) -> Sequence[str]: def name(self) -> LazyExprNameNamespace[Self]: return LazyExprNameNamespace(self) + def _validate_temporal_to_numeric_cast( + self, lf: CompliantLazyFrameT, dtype: IntoDType + ) -> None: + """Guard against casting a temporal expression to a numeric dtype. + + Unlike the eager backends, lazy backends have no materialized dtype for an + arbitrary expression, so we resolve the true pre-cast dtype from the schema. + """ + if not dtype.is_numeric(): + return + try: + if (md := self._opt_metadata) is not None and md.is_pure_selection: + frame_schema = lf.collect_schema() + sources = [frame_schema[name] for name in self._evaluate_output_names(lf)] + else: + sources = list(lf.select(self).collect_schema().values()) + except Exception: # noqa: BLE001 + # NOTE: The guard is best-effort: some lazy backends (e.g. `sqlframe`) fail + # to resolve schemas for certain expressions (such as all-null columns). + # Without the source dtype we cannot validate, so we let the cast proceed. + return + for source in sources: + _validate_cast_temporal_to_numeric(source=source, target=dtype) + ewm_mean = not_implemented() # type: ignore[misc] map_batches = not_implemented() # type: ignore[misc] cat: not_implemented = not_implemented() # type: ignore[assignment] diff --git a/src/narwhals/_dask/expr.py b/src/narwhals/_dask/expr.py index 3961b5ee7e..fc375c7345 100644 --- a/src/narwhals/_dask/expr.py +++ b/src/narwhals/_dask/expr.py @@ -618,11 +618,17 @@ def func(df: DaskLazyFrame) -> Sequence[dx.Series]: ) def cast(self, dtype: IntoDType) -> Self: - def func(expr: dx.Series) -> dx.Series: + def func(df: DaskLazyFrame) -> list[dx.Series]: + self._validate_temporal_to_numeric_cast(df, dtype) native_dtype = narwhals_to_native_dtype(dtype, self._version) - return expr.astype(native_dtype) + return [expr.astype(native_dtype) for expr in self._call(df)] - return self._with_callable(func) + return self.__class__( + func, + evaluate_output_names=self._evaluate_output_names, + alias_output_names=self._alias_output_names, + version=self._version, + ) def is_finite(self) -> Self: import dask.array as da diff --git a/src/narwhals/_duckdb/expr.py b/src/narwhals/_duckdb/expr.py index dd9bfbdced..4504ffb3e9 100644 --- a/src/narwhals/_duckdb/expr.py +++ b/src/narwhals/_duckdb/expr.py @@ -38,6 +38,7 @@ ) from narwhals._duckdb.dataframe import DuckDBLazyFrame from narwhals._duckdb.namespace import DuckDBNamespace + from narwhals._duckdb.utils import duckdb_dtypes from narwhals._typing import NoDefault from narwhals._utils import _LimitedContext from narwhals.typing import FillNullStrategy, IntoDType, RollingInterpolationMethod @@ -270,15 +271,18 @@ def _fill_constant(expr: Expression, value: Any) -> Expression: return self._with_elementwise(_fill_constant, value=value) def cast(self, dtype: IntoDType) -> Self: - def func(df: DuckDBLazyFrame) -> list[Expression]: + def native_dtype(df: DuckDBLazyFrame) -> duckdb_dtypes.DuckDBPyType: + self._validate_temporal_to_numeric_cast(df, dtype) tz = DeferredTimeZone(df.native) - native_dtype = narwhals_to_native_dtype(dtype, self._version, tz) - return [expr.cast(native_dtype) for expr in self(df)] + return narwhals_to_native_dtype(dtype, self._version, tz) + + def func(df: DuckDBLazyFrame) -> list[Expression]: + dtype_ = native_dtype(df) + return [expr.cast(dtype_) for expr in self(df)] def window_f(df: DuckDBLazyFrame, inputs: DuckDBWindowInputs) -> list[Expression]: - tz = DeferredTimeZone(df.native) - native_dtype = narwhals_to_native_dtype(dtype, self._version, tz) - return [expr.cast(native_dtype) for expr in self.window_function(df, inputs)] + dtype_ = native_dtype(df) + return [expr.cast(dtype_) for expr in self.window_function(df, inputs)] return self.__class__( func, diff --git a/src/narwhals/_expression_parsing.py b/src/narwhals/_expression_parsing.py index 69ad8e85f7..c12557cfbe 100644 --- a/src/narwhals/_expression_parsing.py +++ b/src/narwhals/_expression_parsing.py @@ -180,6 +180,18 @@ def is_scalar_like(self) -> bool: ExprKind.ORDERABLE_AGGREGATION, } + @property + def is_selection(self) -> bool: + # Pure column selection: preserves dtype and maps each output directly + # into one or multiple columns, e.g. `nw.col('a')`, `nw.nth(0)`, `nw.all()`. + return self in { + ExprKind.ALL, + ExprKind.COL, + ExprKind.EXCLUDE, + ExprKind.NTH, + ExprKind.SELECTOR, + } + def is_scalar_like(obj: CompliantExprAny) -> bool: return obj._metadata.is_scalar_like @@ -414,6 +426,11 @@ def iter_nodes_reversed(self) -> Iterator[ExprNode]: yield current.current_node current = current.prev + @property + def is_pure_selection(self) -> bool: + """Whether or not this is a bare column selection with no dtype-altering operation.""" + return self.prev is None and self.current_node.kind.is_selection + @classmethod def from_node( cls, node: ExprNode, *compliant_exprs: CompliantExprAny diff --git a/src/narwhals/_ibis/expr.py b/src/narwhals/_ibis/expr.py index d0d496b33c..1e07952985 100644 --- a/src/narwhals/_ibis/expr.py +++ b/src/narwhals/_ibis/expr.py @@ -283,12 +283,17 @@ def _fill_null(expr: ir.Value, value: ir.Scalar) -> ir.Value: return self._with_callable(_fill_null, value=value) def cast(self, dtype: IntoDType) -> Self: - def _func(expr: ir.Column) -> ir.Value: + def func(df: IbisLazyFrame) -> list[ir.Value]: + self._validate_temporal_to_numeric_cast(df, dtype) native_dtype = narwhals_to_native_dtype(dtype, self._version) - # ibis `cast` overloads do not include DataType, only literals - return expr.cast(native_dtype) # type: ignore[unused-ignore] + return [expr.cast(native_dtype) for expr in self(df)] # pyright: ignore[reportArgumentType, reportCallIssue] - return self._with_callable(_func) + return self.__class__( + func, + evaluate_output_names=self._evaluate_output_names, + alias_output_names=self._alias_output_names, + version=self._version, + ) def is_unique(self) -> Self: return self._with_callable( diff --git a/src/narwhals/_pandas_like/series.py b/src/narwhals/_pandas_like/series.py index 7941402316..ea73051f97 100644 --- a/src/narwhals/_pandas_like/series.py +++ b/src/narwhals/_pandas_like/series.py @@ -27,7 +27,7 @@ from narwhals._typing_compat import assert_never from narwhals._utils import NO_DEFAULT, Implementation, is_list_of from narwhals.dependencies import is_numpy_array_1d, is_pandas_like_series -from narwhals.dtypes import String +from narwhals.dtypes import String, _validate_cast_temporal_to_numeric from narwhals.exceptions import InvalidOperationError if TYPE_CHECKING: @@ -313,6 +313,7 @@ def scatter( return None if in_place else self._with_native(series) def cast(self, dtype: IntoDType) -> Self: + _validate_cast_temporal_to_numeric(source=self.dtype, target=dtype) if self.dtype == dtype and self.native.dtype != "object": # Avoid dealing with pandas' type-system if we can. Note that it's only # safe to do this if we're not starting with object dtype, see tests/expr_and_series/cast_test.py::test_cast_object_pandas diff --git a/src/narwhals/_polars/series.py b/src/narwhals/_polars/series.py index b82a911f61..34e763e7fb 100644 --- a/src/narwhals/_polars/series.py +++ b/src/narwhals/_polars/series.py @@ -22,6 +22,7 @@ ) from narwhals._utils import NO_DEFAULT, Implementation, requires from narwhals.dependencies import is_numpy_array_1d, is_pandas_index +from narwhals.dtypes import _validate_cast_temporal_to_numeric if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping, Sequence @@ -289,7 +290,8 @@ def __getitem__(self, item: MultiIndexSelector[Self]) -> Any | Self: return self._from_native_object(self.native.__getitem__(item)) def cast(self, dtype: IntoDType) -> Self: - dtype_pl = narwhals_to_native_dtype(dtype, self._version) + _validate_cast_temporal_to_numeric(source=self.dtype, target=dtype) + dtype_pl = narwhals_to_native_dtype(dtype, version=self._version) return self._with_native(self.native.cast(dtype_pl)) def clip(self, lower_bound: PolarsSeries, upper_bound: PolarsSeries) -> Self: diff --git a/src/narwhals/_spark_like/expr.py b/src/narwhals/_spark_like/expr.py index 7bc8cf3b9b..3b4248b8ed 100644 --- a/src/narwhals/_spark_like/expr.py +++ b/src/narwhals/_spark_like/expr.py @@ -34,6 +34,7 @@ ) from narwhals._spark_like.dataframe import SparkLikeLazyFrame from narwhals._spark_like.namespace import SparkLikeNamespace + from narwhals._spark_like.utils import _NativeDType from narwhals._typing import NoDefault from narwhals._utils import _LimitedContext from narwhals.typing import ( @@ -249,19 +250,21 @@ def __neg__(self) -> Self: return self._with_elementwise(neg) def cast(self, dtype: IntoDType) -> Self: - def func(df: SparkLikeLazyFrame) -> Sequence[Column]: - spark_dtype = narwhals_to_native_dtype( + def native_dtype(df: SparkLikeLazyFrame) -> _NativeDType: + self._validate_temporal_to_numeric_cast(df, dtype) + return narwhals_to_native_dtype( dtype, self._version, self._native_dtypes, df.native.sparkSession ) - return [expr.cast(spark_dtype) for expr in self(df)] + + def func(df: SparkLikeLazyFrame) -> Sequence[Column]: + dtype_ = native_dtype(df) + return [expr.cast(dtype_) for expr in self(df)] def window_f( df: SparkLikeLazyFrame, inputs: SparkWindowInputs ) -> Sequence[Column]: - spark_dtype = narwhals_to_native_dtype( - dtype, self._version, self._native_dtypes, df.native.sparkSession - ) - return [expr.cast(spark_dtype) for expr in self.window_function(df, inputs)] + dtype_ = native_dtype(df) + return [expr.cast(dtype_) for expr in self.window_function(df, inputs)] return self.__class__( func, diff --git a/src/narwhals/dtypes.py b/src/narwhals/dtypes.py index cadf0f5683..1a41f7871a 100644 --- a/src/narwhals/dtypes.py +++ b/src/narwhals/dtypes.py @@ -59,6 +59,29 @@ def _validate_into_dtype(dtype: Any) -> None: raise TypeError(msg) +def _validate_cast_temporal_to_numeric( + source: DType | type[DType], target: IntoDType +) -> None: + """Validate that we're not casting from temporal to numeric types. + + Arguments: + source: The source data type. + target: The target data type to cast to. + + Raises: + InvalidOperationError: If attempting to cast from temporal to numeric. + """ + if source.is_temporal() and target.is_numeric(): + msg = ( + "Casting from temporal type to numeric is not supported.\n\n" + "Hint: Use `.dt` accessor methods instead, such as:\n" + " - `.dt.timestamp()` for Unix timestamp.\n" + " - `.dt.year()`, `.dt.month()`, `.dt.day()`, ..., for date components.\n" + " - `.dt.total_seconds()`, `.dt.total_milliseconds()`, ..., for duration total time." + ) + raise InvalidOperationError(msg) + + class DTypeClass(type): """Metaclass for DType classes. diff --git a/src/narwhals/expr.py b/src/narwhals/expr.py index 590d1bb3e4..9a0381e692 100644 --- a/src/narwhals/expr.py +++ b/src/narwhals/expr.py @@ -172,6 +172,15 @@ def cast(self, dtype: IntoDType) -> Self: Arguments: dtype: Data type that the object will be cast into. + Note: + Unlike polars, we don't allow to cast from a temporal to a numeric data type. + + Use `.dt` accessor methods instead, such as: + + * `.dt.timestamp()` for Unix timestamp. + * `.dt.year()`, `.dt.month()`, `.dt.day()`, ..., for date components. + * `.dt.total_seconds()`, `.dt.total_milliseconds()`, ..., for duration total time. + Examples: >>> import pandas as pd >>> import narwhals as nw diff --git a/src/narwhals/series.py b/src/narwhals/series.py index b4c03ecf90..e8953c9c6f 100644 --- a/src/narwhals/series.py +++ b/src/narwhals/series.py @@ -620,6 +620,15 @@ def cast(self, dtype: IntoDType) -> Self: Arguments: dtype: Data type that the object will be cast into. + Note: + Unlike polars, we don't allow to cast from a temporal to a numeric data type. + + Use `.dt` accessor methods instead, such as: + + * `.dt.timestamp()` for Unix timestamp. + * `.dt.year()`, `.dt.month()`, `.dt.day()`, ..., for date components. + * `.dt.total_seconds()`, `.dt.total_milliseconds()`, ..., for duration total time. + Examples: >>> import pyarrow as pa >>> import narwhals as nw diff --git a/tests/expr_and_series/cast_test.py b/tests/expr_and_series/cast_test.py index d008d95c98..c5d0bbe230 100644 --- a/tests/expr_and_series/cast_test.py +++ b/tests/expr_and_series/cast_test.py @@ -6,7 +6,7 @@ import pytest import narwhals as nw -from narwhals.exceptions import UnsupportedDTypeError +from narwhals.exceptions import InvalidOperationError, UnsupportedDTypeError from tests.utils import ( PANDAS_VERSION, POLARS_VERSION, @@ -452,3 +452,86 @@ def test_cast_object_pandas() -> None: s = nw.from_native(pd.DataFrame({"a": [2, 3, None]}, dtype=object))["a"] assert s[0] == 2 assert s.cast(nw.String)[0] == "2" + + +NUMERIC_DTYPES = [ + nw.Int8, + nw.Int16, + nw.Int32, + nw.Int64, + nw.Float32, + nw.Float64, + nw.UInt32, + nw.UInt64, +] + + +@pytest.mark.parametrize( + "values", [[datetime(2000, 1, 1, 12, 0), None], [timedelta(365, 59), None]] +) +@pytest.mark.parametrize(("target_dtype"), NUMERIC_DTYPES) +def test_cast_temporal_to_numeric_raises_expr( + constructor: Constructor, + request: pytest.FixtureRequest, + values: list[datetime] | list[timedelta], + target_dtype: nw.dtypes.DType, +) -> None: + if "polars" in str(constructor): + reason = "Polars expressions wrap native expressions" + request.applymarker(pytest.mark.xfail(reason=reason)) + + if isinstance(values[0], timedelta) and "spark" in str(constructor): + reason = "interval not implemented" + request.applymarker(pytest.mark.xfail(reason=reason)) + + df = nw.from_native(constructor({"a": values})).lazy() + msg = "Casting from temporal type to numeric" + with pytest.raises(InvalidOperationError, match=msg): + df.select(nw.col("a").cast(target_dtype)).collect() + + +@pytest.mark.parametrize( + "values", + [ + [datetime(2000, 1, 1, 12, 0), datetime(2000, 1, 2, 12, 0), None], + [timedelta(2, 59), timedelta(1, 59), None], + ], +) +@pytest.mark.parametrize(("target_dtype"), NUMERIC_DTYPES) +def test_cast_temporal_to_numeric_raises_series( + constructor_eager: ConstructorEager, + values: list[datetime] | list[timedelta], + target_dtype: nw.dtypes.DType, +) -> None: + df = nw.from_native(constructor_eager({"a": values}), eager_only=True) + series = df["a"] + msg = "Casting from temporal type to numeric" + with pytest.raises(InvalidOperationError, match=msg): + series.cast(target_dtype) + + +@pytest.mark.parametrize("target_dtype", [nw.Int64, nw.Float64]) +def test_cast_derived_temporal_to_numeric_allowed( + constructor: Constructor, target_dtype: nw.dtypes.DType +) -> None: + data = {"a": [datetime(2000, 1, 1, 12, 0), datetime(2001, 1, 1, 12, 0)]} + df = nw.from_native(constructor(data)) + result = df.select(nw.col("a").dt.year().cast(target_dtype)) + assert_equal_data(result, {"a": [2000, 2001]}) + + +@pytest.mark.parametrize("target_dtype", [nw.Int64, nw.Float64]) +def test_cast_chained_temporal_to_numeric_raises( + constructor: Constructor, + request: pytest.FixtureRequest, + target_dtype: nw.dtypes.DType, +) -> None: + if "polars" in str(constructor): + reason = "Polars expressions wrap native expressions" + request.applymarker(pytest.mark.xfail(reason=reason)) + + df = nw.from_native(constructor({"a": ["2020-01-01T12:34:56"]})).lazy() + temporal = nw.col("a").str.to_datetime(format="%Y-%m-%dT%H:%M:%S") + msg = "Casting from temporal type to numeric" + with pytest.raises(InvalidOperationError, match=msg): + df.select(temporal.cast(target_dtype)).collect()