Skip to content
Open
2 changes: 2 additions & 0 deletions src/narwhals/_arrow/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
not_implemented,
)
from narwhals.dependencies import is_numpy_array_1d
from narwhals.dtypes import _validate_cast_temporal_to_numeric
from narwhals.exceptions import InvalidOperationError, ShapeError

if TYPE_CHECKING:
Expand Down Expand Up @@ -547,6 +548,7 @@ def is_nan(self) -> Self:
return self._with_native(pc.is_nan(self.native), preserve_broadcast=True)

def cast(self, dtype: IntoDType) -> Self:
_validate_cast_temporal_to_numeric(source=self.dtype, target=dtype)
data_type = narwhals_to_native_dtype(dtype, self._version)
return self._with_native(pc.cast(self.native, data_type), preserve_broadcast=True)

Expand Down
25 changes: 25 additions & 0 deletions src/narwhals/_compliant/expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
qualified_type_name,
)
from narwhals.dependencies import is_numpy_array, is_numpy_scalar
from narwhals.dtypes import _validate_cast_temporal_to_numeric
from narwhals.exceptions import MultiOutputExpressionError

if TYPE_CHECKING:
Expand Down Expand Up @@ -937,6 +938,30 @@ def fn(names: Sequence[str]) -> Sequence[str]:
def name(self) -> LazyExprNameNamespace[Self]:
return LazyExprNameNamespace(self)

def _validate_temporal_to_numeric_cast(
self, lf: CompliantLazyFrameT, dtype: IntoDType
) -> None:
"""Guard against casting a temporal expression to a numeric dtype.

Unlike the eager backends, lazy backends have no materialized dtype for an
arbitrary expression, so we resolve the true pre-cast dtype from the schema.
"""
if not dtype.is_numeric():
return
try:
if (md := self._opt_metadata) is not None and md.is_pure_selection:
frame_schema = lf.collect_schema()
sources = [frame_schema[name] for name in self._evaluate_output_names(lf)]
else:
sources = list(lf.select(self).collect_schema().values())
except Exception: # noqa: BLE001
# NOTE: The guard is best-effort: some lazy backends (e.g. `sqlframe`) fail
# to resolve schemas for certain expressions (such as all-null columns).
# Without the source dtype we cannot validate, so we let the cast proceed.
return
for source in sources:
_validate_cast_temporal_to_numeric(source=source, target=dtype)

ewm_mean = not_implemented() # type: ignore[misc]
map_batches = not_implemented() # type: ignore[misc]
cat: not_implemented = not_implemented() # type: ignore[assignment]
Expand Down
12 changes: 9 additions & 3 deletions src/narwhals/_dask/expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -618,11 +618,17 @@ def func(df: DaskLazyFrame) -> Sequence[dx.Series]:
)

def cast(self, dtype: IntoDType) -> Self:
def func(expr: dx.Series) -> dx.Series:
def func(df: DaskLazyFrame) -> list[dx.Series]:
self._validate_temporal_to_numeric_cast(df, dtype)
native_dtype = narwhals_to_native_dtype(dtype, self._version)
return expr.astype(native_dtype)
return [expr.astype(native_dtype) for expr in self._call(df)]

return self._with_callable(func)
return self.__class__(
func,
evaluate_output_names=self._evaluate_output_names,
alias_output_names=self._alias_output_names,
version=self._version,
)

def is_finite(self) -> Self:
import dask.array as da
Expand Down
16 changes: 10 additions & 6 deletions src/narwhals/_duckdb/expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
)
from narwhals._duckdb.dataframe import DuckDBLazyFrame
from narwhals._duckdb.namespace import DuckDBNamespace
from narwhals._duckdb.utils import duckdb_dtypes
from narwhals._typing import NoDefault
from narwhals._utils import _LimitedContext
from narwhals.typing import FillNullStrategy, IntoDType, RollingInterpolationMethod
Expand Down Expand Up @@ -270,15 +271,18 @@ def _fill_constant(expr: Expression, value: Any) -> Expression:
return self._with_elementwise(_fill_constant, value=value)

def cast(self, dtype: IntoDType) -> Self:
def func(df: DuckDBLazyFrame) -> list[Expression]:
def native_dtype(df: DuckDBLazyFrame) -> duckdb_dtypes.DuckDBPyType:
self._validate_temporal_to_numeric_cast(df, dtype)
tz = DeferredTimeZone(df.native)
native_dtype = narwhals_to_native_dtype(dtype, self._version, tz)
return [expr.cast(native_dtype) for expr in self(df)]
return narwhals_to_native_dtype(dtype, self._version, tz)

def func(df: DuckDBLazyFrame) -> list[Expression]:
dtype_ = native_dtype(df)
return [expr.cast(dtype_) for expr in self(df)]

def window_f(df: DuckDBLazyFrame, inputs: DuckDBWindowInputs) -> list[Expression]:
tz = DeferredTimeZone(df.native)
native_dtype = narwhals_to_native_dtype(dtype, self._version, tz)
return [expr.cast(native_dtype) for expr in self.window_function(df, inputs)]
dtype_ = native_dtype(df)
return [expr.cast(dtype_) for expr in self.window_function(df, inputs)]

return self.__class__(
func,
Expand Down
17 changes: 17 additions & 0 deletions src/narwhals/_expression_parsing.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,18 @@ def is_scalar_like(self) -> bool:
ExprKind.ORDERABLE_AGGREGATION,
}

@property
def is_selection(self) -> bool:
# Pure column selection: preserves dtype and maps each output directly
# into one or multiple columns, e.g. `nw.col('a')`, `nw.nth(0)`, `nw.all()`.
return self in {
ExprKind.ALL,
ExprKind.COL,
ExprKind.EXCLUDE,
ExprKind.NTH,
ExprKind.SELECTOR,
}


def is_scalar_like(obj: CompliantExprAny) -> bool:
return obj._metadata.is_scalar_like
Expand Down Expand Up @@ -414,6 +426,11 @@ def iter_nodes_reversed(self) -> Iterator[ExprNode]:
yield current.current_node
current = current.prev

@property
def is_pure_selection(self) -> bool:
"""Whether or not this is a bare column selection with no dtype-altering operation."""
return self.prev is None and self.current_node.kind.is_selection

@classmethod
def from_node(
cls, node: ExprNode, *compliant_exprs: CompliantExprAny
Expand Down
13 changes: 9 additions & 4 deletions src/narwhals/_ibis/expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,12 +283,17 @@ def _fill_null(expr: ir.Value, value: ir.Scalar) -> ir.Value:
return self._with_callable(_fill_null, value=value)

def cast(self, dtype: IntoDType) -> Self:
def _func(expr: ir.Column) -> ir.Value:
def func(df: IbisLazyFrame) -> list[ir.Value]:
self._validate_temporal_to_numeric_cast(df, dtype)
native_dtype = narwhals_to_native_dtype(dtype, self._version)
# ibis `cast` overloads do not include DataType, only literals
return expr.cast(native_dtype) # type: ignore[unused-ignore]
return [expr.cast(native_dtype) for expr in self(df)] # pyright: ignore[reportArgumentType, reportCallIssue]

return self._with_callable(_func)
return self.__class__(
func,
evaluate_output_names=self._evaluate_output_names,
alias_output_names=self._alias_output_names,
version=self._version,
)

def is_unique(self) -> Self:
return self._with_callable(
Expand Down
3 changes: 2 additions & 1 deletion src/narwhals/_pandas_like/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
from narwhals._typing_compat import assert_never
from narwhals._utils import NO_DEFAULT, Implementation, is_list_of
from narwhals.dependencies import is_numpy_array_1d, is_pandas_like_series
from narwhals.dtypes import String
from narwhals.dtypes import String, _validate_cast_temporal_to_numeric
from narwhals.exceptions import InvalidOperationError

if TYPE_CHECKING:
Expand Down Expand Up @@ -313,6 +313,7 @@ def scatter(
return None if in_place else self._with_native(series)

def cast(self, dtype: IntoDType) -> Self:
_validate_cast_temporal_to_numeric(source=self.dtype, target=dtype)
if self.dtype == dtype and self.native.dtype != "object":
# Avoid dealing with pandas' type-system if we can. Note that it's only
# safe to do this if we're not starting with object dtype, see tests/expr_and_series/cast_test.py::test_cast_object_pandas
Expand Down
4 changes: 3 additions & 1 deletion src/narwhals/_polars/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
)
from narwhals._utils import NO_DEFAULT, Implementation, requires
from narwhals.dependencies import is_numpy_array_1d, is_pandas_index
from narwhals.dtypes import _validate_cast_temporal_to_numeric

if TYPE_CHECKING:
from collections.abc import Iterable, Iterator, Mapping, Sequence
Expand Down Expand Up @@ -289,7 +290,8 @@ def __getitem__(self, item: MultiIndexSelector[Self]) -> Any | Self:
return self._from_native_object(self.native.__getitem__(item))

def cast(self, dtype: IntoDType) -> Self:
dtype_pl = narwhals_to_native_dtype(dtype, self._version)
_validate_cast_temporal_to_numeric(source=self.dtype, target=dtype)
dtype_pl = narwhals_to_native_dtype(dtype, version=self._version)
return self._with_native(self.native.cast(dtype_pl))

def clip(self, lower_bound: PolarsSeries, upper_bound: PolarsSeries) -> Self:
Expand Down
17 changes: 10 additions & 7 deletions src/narwhals/_spark_like/expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
)
from narwhals._spark_like.dataframe import SparkLikeLazyFrame
from narwhals._spark_like.namespace import SparkLikeNamespace
from narwhals._spark_like.utils import _NativeDType
from narwhals._typing import NoDefault
from narwhals._utils import _LimitedContext
from narwhals.typing import (
Expand Down Expand Up @@ -249,19 +250,21 @@ def __neg__(self) -> Self:
return self._with_elementwise(neg)

def cast(self, dtype: IntoDType) -> Self:
def func(df: SparkLikeLazyFrame) -> Sequence[Column]:
spark_dtype = narwhals_to_native_dtype(
def native_dtype(df: SparkLikeLazyFrame) -> _NativeDType:
self._validate_temporal_to_numeric_cast(df, dtype)
return narwhals_to_native_dtype(
dtype, self._version, self._native_dtypes, df.native.sparkSession
)
return [expr.cast(spark_dtype) for expr in self(df)]

def func(df: SparkLikeLazyFrame) -> Sequence[Column]:
dtype_ = native_dtype(df)
return [expr.cast(dtype_) for expr in self(df)]

def window_f(
df: SparkLikeLazyFrame, inputs: SparkWindowInputs
) -> Sequence[Column]:
spark_dtype = narwhals_to_native_dtype(
dtype, self._version, self._native_dtypes, df.native.sparkSession
)
return [expr.cast(spark_dtype) for expr in self.window_function(df, inputs)]
dtype_ = native_dtype(df)
return [expr.cast(dtype_) for expr in self.window_function(df, inputs)]

return self.__class__(
func,
Expand Down
23 changes: 23 additions & 0 deletions src/narwhals/dtypes.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,29 @@ def _validate_into_dtype(dtype: Any) -> None:
raise TypeError(msg)


def _validate_cast_temporal_to_numeric(
source: DType | type[DType], target: IntoDType
) -> None:
"""Validate that we're not casting from temporal to numeric types.

Arguments:
source: The source data type.
target: The target data type to cast to.

Raises:
InvalidOperationError: If attempting to cast from temporal to numeric.
"""
if source.is_temporal() and target.is_numeric():
msg = (
"Casting from temporal type to numeric is not supported.\n\n"
"Hint: Use `.dt` accessor methods instead, such as:\n"
" - `.dt.timestamp()` for Unix timestamp.\n"
" - `.dt.year()`, `.dt.month()`, `.dt.day()`, ..., for date components.\n"
" - `.dt.total_seconds()`, `.dt.total_milliseconds()`, ..., for duration total time."
)
raise InvalidOperationError(msg)


class DTypeClass(type):
"""Metaclass for DType classes.

Expand Down
9 changes: 9 additions & 0 deletions src/narwhals/expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,15 @@ def cast(self, dtype: IntoDType) -> Self:
Arguments:
dtype: Data type that the object will be cast into.

Note:
Unlike polars, we don't allow to cast from a temporal to a numeric data type.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TIL: polars allows also casting to Float, not only to Integer


Use `.dt` accessor methods instead, such as:

* `.dt.timestamp()` for Unix timestamp.
* `.dt.year()`, `.dt.month()`, `.dt.day()`, ..., for date components.
* `.dt.total_seconds()`, `.dt.total_milliseconds()`, ..., for duration total time.

Examples:
>>> import pandas as pd
>>> import narwhals as nw
Expand Down
9 changes: 9 additions & 0 deletions src/narwhals/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -620,6 +620,15 @@ def cast(self, dtype: IntoDType) -> Self:
Arguments:
dtype: Data type that the object will be cast into.

Note:
Unlike polars, we don't allow to cast from a temporal to a numeric data type.

Use `.dt` accessor methods instead, such as:

* `.dt.timestamp()` for Unix timestamp.
* `.dt.year()`, `.dt.month()`, `.dt.day()`, ..., for date components.
* `.dt.total_seconds()`, `.dt.total_milliseconds()`, ..., for duration total time.

Examples:
>>> import pyarrow as pa
>>> import narwhals as nw
Expand Down
85 changes: 84 additions & 1 deletion tests/expr_and_series/cast_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import pytest

import narwhals as nw
from narwhals.exceptions import UnsupportedDTypeError
from narwhals.exceptions import InvalidOperationError, UnsupportedDTypeError
from tests.utils import (
PANDAS_VERSION,
POLARS_VERSION,
Expand Down Expand Up @@ -452,3 +452,86 @@ def test_cast_object_pandas() -> None:
s = nw.from_native(pd.DataFrame({"a": [2, 3, None]}, dtype=object))["a"]
assert s[0] == 2
assert s.cast(nw.String)[0] == "2"


NUMERIC_DTYPES = [
nw.Int8,
nw.Int16,
nw.Int32,
nw.Int64,
nw.Float32,
nw.Float64,
nw.UInt32,
nw.UInt64,
]


@pytest.mark.parametrize(
"values", [[datetime(2000, 1, 1, 12, 0), None], [timedelta(365, 59), None]]
)
@pytest.mark.parametrize(("target_dtype"), NUMERIC_DTYPES)
def test_cast_temporal_to_numeric_raises_expr(
constructor: Constructor,
request: pytest.FixtureRequest,
values: list[datetime] | list[timedelta],
target_dtype: nw.dtypes.DType,
) -> None:
if "polars" in str(constructor):
reason = "Polars expressions wrap native expressions"
request.applymarker(pytest.mark.xfail(reason=reason))

if isinstance(values[0], timedelta) and "spark" in str(constructor):
reason = "interval not implemented"
request.applymarker(pytest.mark.xfail(reason=reason))

df = nw.from_native(constructor({"a": values})).lazy()
msg = "Casting from temporal type to numeric"
with pytest.raises(InvalidOperationError, match=msg):
df.select(nw.col("a").cast(target_dtype)).collect()


@pytest.mark.parametrize(
"values",
[
[datetime(2000, 1, 1, 12, 0), datetime(2000, 1, 2, 12, 0), None],
[timedelta(2, 59), timedelta(1, 59), None],
],
)
@pytest.mark.parametrize(("target_dtype"), NUMERIC_DTYPES)
def test_cast_temporal_to_numeric_raises_series(
constructor_eager: ConstructorEager,
values: list[datetime] | list[timedelta],
target_dtype: nw.dtypes.DType,
) -> None:
df = nw.from_native(constructor_eager({"a": values}), eager_only=True)
series = df["a"]
msg = "Casting from temporal type to numeric"
with pytest.raises(InvalidOperationError, match=msg):
series.cast(target_dtype)


@pytest.mark.parametrize("target_dtype", [nw.Int64, nw.Float64])
def test_cast_derived_temporal_to_numeric_allowed(
constructor: Constructor, target_dtype: nw.dtypes.DType
) -> None:
data = {"a": [datetime(2000, 1, 1, 12, 0), datetime(2001, 1, 1, 12, 0)]}
df = nw.from_native(constructor(data))
result = df.select(nw.col("a").dt.year().cast(target_dtype))
assert_equal_data(result, {"a": [2000, 2001]})


@pytest.mark.parametrize("target_dtype", [nw.Int64, nw.Float64])
def test_cast_chained_temporal_to_numeric_raises(
constructor: Constructor,
request: pytest.FixtureRequest,
target_dtype: nw.dtypes.DType,
) -> None:
if "polars" in str(constructor):
reason = "Polars expressions wrap native expressions"
request.applymarker(pytest.mark.xfail(reason=reason))

df = nw.from_native(constructor({"a": ["2020-01-01T12:34:56"]})).lazy()
temporal = nw.col("a").str.to_datetime(format="%Y-%m-%dT%H:%M:%S")
msg = "Casting from temporal type to numeric"
with pytest.raises(InvalidOperationError, match=msg):
df.select(temporal.cast(target_dtype)).collect()
Loading