diff --git a/src/narwhals/_arrow/series.py b/src/narwhals/_arrow/series.py index 74fbac0df1..e542cfcc86 100644 --- a/src/narwhals/_arrow/series.py +++ b/src/narwhals/_arrow/series.py @@ -123,6 +123,22 @@ def maybe_extract_py_scalar(value: Any, return_py_scalar: bool) -> Any: # noqa: return value +def _cast_scalar_non_strict(value: ScalarAny, data_type: pa.DataType) -> ScalarAny: + try: + return pc.cast(value, data_type) + except pa.ArrowInvalid: + return pa.scalar(None, type=data_type) + + +def _cast_chunk_non_strict(chunk: ArrayAny, data_type: pa.DataType) -> ArrayAny: + try: + return pc.cast(chunk, data_type) + except pa.ArrowInvalid: + return pa.array( + [_cast_scalar_non_strict(value, data_type) for value in chunk], type=data_type + ) + + class ArrowSeries(EagerSeries["ChunkedArrayAny"]): _implementation = Implementation.PYARROW @@ -546,9 +562,25 @@ def is_null(self) -> Self: def is_nan(self) -> Self: return self._with_native(pc.is_nan(self.native), preserve_broadcast=True) - def cast(self, dtype: IntoDType) -> Self: + def cast(self, dtype: IntoDType, *, strict: bool = True) -> Self: data_type = narwhals_to_native_dtype(dtype, self._version) - return self._with_native(pc.cast(self.native, data_type), preserve_broadcast=True) + if strict: + native = pc.cast(self.native, data_type) + else: + dtypes = self._version.dtypes + base_type = dtype.base_type() + is_fallible_cast = base_type.is_numeric() or issubclass( + base_type, (dtypes.Date, dtypes.Datetime, dtypes.Duration) + ) + if not is_fallible_cast: + native = pc.cast(self.native, data_type) + else: + chunks = [ + _cast_chunk_non_strict(chunk, data_type) + for chunk in self.native.chunks + ] + native = pa.chunked_array(chunks, type=data_type) + return self._with_native(native, preserve_broadcast=True) def null_count(self, *, _return_py_scalar: bool = True) -> int: return maybe_extract_py_scalar(self.native.null_count, _return_py_scalar) diff --git a/src/narwhals/_compliant/column.py b/src/narwhals/_compliant/column.py index 31275efc07..c9ca4280bf 100644 --- a/src/narwhals/_compliant/column.py +++ b/src/narwhals/_compliant/column.py @@ -60,7 +60,7 @@ def __narwhals_namespace__(self) -> CompliantNamespace[Any, Any]: ... def abs(self) -> Self: ... def alias(self, name: str) -> Self: ... - def cast(self, dtype: IntoDType) -> Self: ... + def cast(self, dtype: IntoDType, *, strict: bool = True) -> Self: ... def clip(self, lower_bound: Self, upper_bound: Self) -> Self: ... def clip_lower(self, lower_bound: Self) -> Self: ... def clip_upper(self, upper_bound: Self) -> Self: ... diff --git a/src/narwhals/_compliant/expr.py b/src/narwhals/_compliant/expr.py index 0fd0ced0f8..98d1067689 100644 --- a/src/narwhals/_compliant/expr.py +++ b/src/narwhals/_compliant/expr.py @@ -449,8 +449,8 @@ def func(df: EagerDataFrameT) -> list[EagerSeriesT]: version=self._version, ) - def cast(self, dtype: IntoDType) -> Self: - return self._reuse_series("cast", dtype=dtype) + def cast(self, dtype: IntoDType, *, strict: bool = True) -> Self: + return self._reuse_series("cast", dtype=dtype, strict=strict) def _with_binary(self, operator: str, other: Self, /) -> Self: return self._reuse_series(operator, other=other) diff --git a/src/narwhals/_dask/expr.py b/src/narwhals/_dask/expr.py index d2a34ac5b4..6b34943d60 100644 --- a/src/narwhals/_dask/expr.py +++ b/src/narwhals/_dask/expr.py @@ -618,7 +618,11 @@ def func(df: DaskLazyFrame) -> Sequence[dx.Series]: version=self._version, ) - def cast(self, dtype: IntoDType) -> Self: + def cast(self, dtype: IntoDType, *, strict: bool = True) -> Self: + if not strict: + msg = "`cast(..., strict=False)` is not yet implemented for the dask backend." + raise NotImplementedError(msg) + def func(expr: dx.Series) -> dx.Series: native_dtype = narwhals_to_native_dtype(dtype, self._version) return expr.astype(native_dtype) diff --git a/src/narwhals/_duckdb/expr.py b/src/narwhals/_duckdb/expr.py index d3cd18aafd..2e76fa77f6 100644 --- a/src/narwhals/_duckdb/expr.py +++ b/src/narwhals/_duckdb/expr.py @@ -114,6 +114,12 @@ def _any_value(self, expr: Expression, *, ignore_nulls: bool) -> Expression: else self._function("first", expr) ) + @staticmethod + def _cast_non_strict(expr: Expression, native_dtype: Any) -> Expression: + # DuckDB's relational API has `Expression.cast`, but does not expose a + # TRY_CAST expression constructor. + return sql_expression(f"TRY_CAST({expr} AS {native_dtype})") + def __narwhals_namespace__(self) -> DuckDBNamespace: # pragma: no cover from narwhals._duckdb.namespace import DuckDBNamespace @@ -269,16 +275,23 @@ def _fill_constant(expr: Expression, value: Any) -> Expression: assert value is not None # noqa: S101 return self._with_elementwise(_fill_constant, expression_args={"value": value}) - def cast(self, dtype: IntoDType) -> Self: + def cast(self, dtype: IntoDType, *, strict: bool = True) -> Self: + def _cast(expr: Expression, native_dtype: Any) -> Expression: + if strict: + return expr.cast(native_dtype) + return self._cast_non_strict(expr, native_dtype) + def func(df: DuckDBLazyFrame) -> list[Expression]: tz = DeferredTimeZone(df.native) native_dtype = narwhals_to_native_dtype(dtype, self._version, tz) - return [expr.cast(native_dtype) for expr in self(df)] + return [_cast(expr, native_dtype) for expr in self(df)] def window_f(df: DuckDBLazyFrame, inputs: DuckDBWindowInputs) -> list[Expression]: tz = DeferredTimeZone(df.native) native_dtype = narwhals_to_native_dtype(dtype, self._version, tz) - return [expr.cast(native_dtype) for expr in self.window_function(df, inputs)] + return [ + _cast(expr, native_dtype) for expr in self.window_function(df, inputs) + ] return self.__class__( func, diff --git a/src/narwhals/_ibis/expr.py b/src/narwhals/_ibis/expr.py index 56f634799a..488c545d44 100644 --- a/src/narwhals/_ibis/expr.py +++ b/src/narwhals/_ibis/expr.py @@ -285,11 +285,13 @@ def _fill_null(expr: ir.Value, value: ir.Scalar) -> ir.Value: assert value is not None # noqa: S101 return self._with_callable(_fill_null, expression_args={"value": value}) - def cast(self, dtype: IntoDType) -> Self: + def cast(self, dtype: IntoDType, *, strict: bool = True) -> Self: def _func(expr: ir.Column) -> ir.Value: native_dtype = narwhals_to_native_dtype(dtype, self._version) - # ibis `cast` overloads do not include DataType, only literals - return expr.cast(native_dtype) # type: ignore[unused-ignore] + # ibis `cast`/`try_cast` overloads do not include DataType, only literals + if strict: + return expr.cast(native_dtype) # type: ignore[unused-ignore] + return expr.try_cast(native_dtype) # type: ignore[unused-ignore] return self._with_callable(_func) diff --git a/src/narwhals/_integer_bounds.py b/src/narwhals/_integer_bounds.py new file mode 100644 index 0000000000..75e0c8779e --- /dev/null +++ b/src/narwhals/_integer_bounds.py @@ -0,0 +1,31 @@ +"""Min/max representable values per integer dtype, shared across backends. + +Used by non-strict (`strict=False`) casting to null-out (rather than silently wrap +or raise on) out-of-range values before handing off to a backend's native cast. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from narwhals._utils import Version + +if TYPE_CHECKING: + from collections.abc import Mapping + + from narwhals.dtypes import DType + +__all__ = ["INTEGER_BOUNDS"] + +_dtypes = Version.MAIN.dtypes + +INTEGER_BOUNDS: Mapping[type[DType], tuple[int, int]] = { + _dtypes.Int8: (-128, 127), + _dtypes.Int16: (-32_768, 32_767), + _dtypes.Int32: (-2_147_483_648, 2_147_483_647), + _dtypes.Int64: (-(2**63), 2**63 - 1), + _dtypes.UInt8: (0, 255), + _dtypes.UInt16: (0, 65_535), + _dtypes.UInt32: (0, 2**32 - 1), + _dtypes.UInt64: (0, 2**64 - 1), +} diff --git a/src/narwhals/_pandas_like/series.py b/src/narwhals/_pandas_like/series.py index 48ee244c22..8fafb1f01b 100644 --- a/src/narwhals/_pandas_like/series.py +++ b/src/narwhals/_pandas_like/series.py @@ -15,6 +15,7 @@ align_and_extract_native, binary_string_sum_fallback, broadcast_series_to_index, + cast_non_strict, get_dtype_backend, import_array_module, narwhals_to_native_dtype, @@ -312,19 +313,31 @@ def scatter( return None if in_place else self._with_native(series) - def cast(self, dtype: IntoDType) -> Self: + def cast(self, dtype: IntoDType, *, strict: bool = True) -> Self: if self.dtype == dtype and self.native.dtype != "object": # Avoid dealing with pandas' type-system if we can. Note that it's only # safe to do this if we're not starting with object dtype, see tests/expr_and_series/cast_test.py::test_cast_object_pandas # for an example of why. return self._with_native(self.native, preserve_broadcast=True) + dtype_backend = get_dtype_backend(self.native.dtype, self._implementation) pd_dtype = narwhals_to_native_dtype( dtype, - dtype_backend=get_dtype_backend(self.native.dtype, self._implementation), + dtype_backend=dtype_backend, implementation=self._implementation, version=self._version, ) - return self._with_native(self.native.astype(pd_dtype), preserve_broadcast=True) + if strict: + native = self.native.astype(pd_dtype) + else: + native = cast_non_strict( + self.native, + dtype, + pd_dtype=pd_dtype, + dtype_backend=dtype_backend, + implementation=self._implementation, + version=self._version, + ) + return self._with_native(native, preserve_broadcast=True) def item(self, index: int | None = None) -> Any: # cuDF doesn't have Series.item(). diff --git a/src/narwhals/_pandas_like/utils.py b/src/narwhals/_pandas_like/utils.py index 273b341c1f..59854baaae 100644 --- a/src/narwhals/_pandas_like/utils.py +++ b/src/narwhals/_pandas_like/utils.py @@ -18,6 +18,7 @@ US_PER_SECOND, ) from narwhals._exceptions import issue_warning +from narwhals._integer_bounds import INTEGER_BOUNDS from narwhals._utils import ( Implementation, Version, @@ -560,6 +561,97 @@ def narwhals_to_native_arrow_dtype( raise NotImplementedError(msg) +def _cast_integer_non_strict( + native: pd.Series[Any], + dtype: IntoDType, + *, + pd_dtype: Any, + dtype_backend: DTypeBackend, + implementation: Implementation, + version: Version, +) -> pd.Series[Any]: + base_type = dtype.base_type() + source_is_numeric = pd.api.types.is_numeric_dtype(native.dtype) + numeric = pd.to_numeric(native, errors="coerce") + if ( + not source_is_numeric + and pd.api.types.is_float_dtype(numeric.dtype) + and numeric.abs().gt(2**53).any() + ): + msg = ( + "Pandas converted integer-like input outside the exact Float64 range " + "while performing a non-strict cast. This conversion is not supported " + "because it may silently lose integer precision." + ) + raise NotImplementedError(msg) + + truncated = np.trunc(numeric) + if source_is_numeric: + # Match Pandas' native numeric-to-integer cast, which truncates fractions. + numeric = truncated + else: + # Pandas does not consider fractional text to be valid integer text. + numeric = numeric.where(numeric.isna() | numeric.eq(truncated)) + + lo, hi = INTEGER_BOUNDS[base_type] + numeric = numeric.where(numeric.isna() | numeric.between(lo, hi)) + target: Any = pd_dtype + if ( + dtype_backend is None + and implementation is not Implementation.CUDF + and numeric.isna().any() + ): + # A plain NumPy integer dtype can't hold a null, so use Pandas' + # nullable representation when coercion introduces or preserves one. + target = narwhals_to_native_dtype( + dtype, "numpy_nullable", implementation, version + ) + return numeric.astype(target) + + +def cast_non_strict( + native: pd.Series[Any], + dtype: IntoDType, + *, + pd_dtype: Any, + dtype_backend: DTypeBackend, + implementation: Implementation, + version: Version, +) -> pd.Series[Any]: + """Non-strict (`strict=False`) counterpart to `.astype(pd_dtype)`-based casting. + + Dispatches on the target dtype family and uses pandas' own vectorized "coerce" + tools, so that values which can't be cast become null instead of raising. + """ + base_type = dtype.base_type() + if base_type.is_integer(): + return _cast_integer_non_strict( + native, + dtype, + pd_dtype=pd_dtype, + dtype_backend=dtype_backend, + implementation=implementation, + version=version, + ) + if base_type.is_float(): + numeric = pd.to_numeric(native, errors="coerce") + return numeric.astype(pd_dtype) + if issubclass(base_type, dtypes.Datetime): + converted = pd.to_datetime(native, errors="coerce") + # `to_datetime` parses at nanosecond resolution. Normalize before casting + # to an Arrow-backed coarser unit, which otherwise rejects lossy conversion. + converted = converted.dt.as_unit(dtype.time_unit) + return converted.astype(pd_dtype) + if issubclass(base_type, dtypes.Date): + return pd.to_datetime(native, errors="coerce").astype(pd_dtype) + if issubclass(base_type, dtypes.Duration): + return pd.to_timedelta(native, errors="coerce").astype(pd_dtype) + # Pandas has no coercing conversion API for the remaining target families. + # Delegate to its regular cast rather than inventing cross-backend semantics; + # unsupported conversions may therefore still raise with `strict=False`. + return native.astype(pd_dtype) + + def int_dtype_mapper(dtype: Any) -> str: if "pyarrow" in str(dtype): return "Int64[pyarrow]" diff --git a/src/narwhals/_polars/expr.py b/src/narwhals/_polars/expr.py index a52dc45b35..53a6f6c917 100644 --- a/src/narwhals/_polars/expr.py +++ b/src/narwhals/_polars/expr.py @@ -103,9 +103,9 @@ def _renamed_min_periods(self, min_samples: int, /) -> dict[str, Any]: name = "min_periods" if self._backend_version < (1, 21, 0) else "min_samples" return {name: min_samples} - def cast(self, dtype: IntoDType) -> Self: + def cast(self, dtype: IntoDType, *, strict: bool = True) -> Self: dtype_pl = narwhals_to_native_dtype(dtype, self._version) - return self._with_native(self.native.cast(dtype_pl)) + return self._with_native(self.native.cast(dtype_pl, strict=strict)) def clip_lower(self, lower_bound: PolarsExpr) -> Self: lower_native = extract_native(lower_bound) diff --git a/src/narwhals/_polars/series.py b/src/narwhals/_polars/series.py index b82a911f61..5b82885d19 100644 --- a/src/narwhals/_polars/series.py +++ b/src/narwhals/_polars/series.py @@ -288,9 +288,9 @@ def __getitem__(self, item: MultiIndexSelector[Self]) -> Any | Self: return self._from_native_object(self.native.__getitem__(item.native)) return self._from_native_object(self.native.__getitem__(item)) - def cast(self, dtype: IntoDType) -> Self: + def cast(self, dtype: IntoDType, *, strict: bool = True) -> Self: dtype_pl = narwhals_to_native_dtype(dtype, self._version) - return self._with_native(self.native.cast(dtype_pl)) + return self._with_native(self.native.cast(dtype_pl, strict=strict)) def clip(self, lower_bound: PolarsSeries, upper_bound: PolarsSeries) -> Self: return self._with_native( diff --git a/src/narwhals/_spark_like/expr.py b/src/narwhals/_spark_like/expr.py index d3e26f684e..7ecb281edc 100644 --- a/src/narwhals/_spark_like/expr.py +++ b/src/narwhals/_spark_like/expr.py @@ -248,7 +248,11 @@ def __neg__(self) -> Self: neg = cast("Callable[..., Column]", operator.neg) return self._with_elementwise(neg) - def cast(self, dtype: IntoDType) -> Self: + def cast(self, dtype: IntoDType, *, strict: bool = True) -> Self: + if not strict: + msg = "`cast(..., strict=False)` is not yet implemented for spark-like backends." + raise NotImplementedError(msg) + def func(df: SparkLikeLazyFrame) -> Sequence[Column]: spark_dtype = narwhals_to_native_dtype( dtype, self._version, self._native_dtypes, df.native.sparkSession diff --git a/src/narwhals/expr.py b/src/narwhals/expr.py index 8772c4bf0a..1acea64922 100644 --- a/src/narwhals/expr.py +++ b/src/narwhals/expr.py @@ -166,11 +166,13 @@ def pipe( """ return function(self, *args, **kwargs) - def cast(self, dtype: IntoDType) -> Self: + def cast(self, dtype: IntoDType, *, strict: bool = True) -> Self: """Redefine an object's data type. Arguments: dtype: Data type that the object will be cast into. + strict: If `True` (default), raise an error if a value can't be cast into the + target type. If `False`, replace it with `null` instead. Examples: >>> import pandas as pd @@ -188,7 +190,9 @@ def cast(self, dtype: IntoDType) -> Self: └──────────────────┘ """ _validate_dtype(dtype) - return self._append_node(ExprNode(ExprKind.ELEMENTWISE, "cast", dtype=dtype)) + return self._append_node( + ExprNode(ExprKind.ELEMENTWISE, "cast", dtype=dtype, strict=strict) + ) # --- binary --- def _with_binary(self, attr: str, other: Self | Any) -> Self: diff --git a/src/narwhals/series.py b/src/narwhals/series.py index d0688cba41..d6471c4121 100644 --- a/src/narwhals/series.py +++ b/src/narwhals/series.py @@ -614,11 +614,13 @@ def ewm_mean( ) ) - def cast(self, dtype: IntoDType) -> Self: + def cast(self, dtype: IntoDType, *, strict: bool = True) -> Self: """Cast between data types. Arguments: dtype: Data type that the object will be cast into. + strict: If `True` (default), raise an error if a value can't be cast into the + target type. If `False`, replace it with `null` instead. Examples: >>> import pyarrow as pa @@ -636,7 +638,7 @@ def cast(self, dtype: IntoDType) -> Self: ] """ _validate_dtype(dtype) - return self._with_compliant(self._compliant_series.cast(dtype)) + return self._with_compliant(self._compliant_series.cast(dtype, strict=strict)) def to_frame(self) -> DataFrame[Any]: """Convert to dataframe. diff --git a/tests/expr_and_series/cast_test.py b/tests/expr_and_series/cast_test.py index d008d95c98..142e59cd0f 100644 --- a/tests/expr_and_series/cast_test.py +++ b/tests/expr_and_series/cast_test.py @@ -1,6 +1,6 @@ from __future__ import annotations -from datetime import datetime, time, timedelta, timezone +from datetime import date, datetime, time, timedelta, timezone from typing import TYPE_CHECKING import pytest @@ -191,6 +191,379 @@ def test_cast_to_float16(constructor: Constructor) -> None: assert_equal_data(result, data) +# Backends for which `cast(..., strict=False)` isn't (yet) implemented. +CAST_STRICT_FALSE_UNSUPPORTED = ("dask", "pyspark", "sqlframe") + +# The Ibis test constructor uses DuckDB, so Ibis `try_cast` compiles to DuckDB's +# TRY_CAST and consequently has the same value-conversion semantics. +DUCKDB_TRY_CAST_CONSTRUCTORS = ("duckdb", "ibis") +DATETIME_PARSING_CONSTRUCTORS = (*DUCKDB_TRY_CAST_CONSTRUCTORS, "pandas") + +INTEGER_CAST_CASES = [ + pytest.param(nw.Int8, -128, 127, id="int8"), + pytest.param(nw.Int16, -32_768, 32_767, id="int16"), + pytest.param(nw.Int32, -2_147_483_648, 2_147_483_647, id="int32"), + pytest.param(nw.Int64, -(2**63), 2**63 - 1, id="int64"), + pytest.param(nw.UInt8, 0, 255, id="uint8"), + pytest.param(nw.UInt16, 0, 65_535, id="uint16"), + pytest.param(nw.UInt32, 0, 2**32 - 1, id="uint32"), + pytest.param(nw.UInt64, 0, 2**64 - 1, id="uint64"), +] + + +def test_cast_strict_false_string_to_numeric(constructor: Constructor) -> None: + data = {"a": ["1", "2", "-1.5", "abc", None]} + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(nw.Int64, strict=False)).lazy().collect() + return + + result = df.select(nw.col("a").cast(nw.Int64, strict=False)) + assert result.collect_schema()["a"] == nw.Int64 + expected = ( + [1, 2, -2, None, None] + if any(x in str(constructor) for x in DUCKDB_TRY_CAST_CONSTRUCTORS) + else [1, 2, None, None, None] + ) + assert_equal_data(result, {"a": expected}) + + +@pytest.mark.parametrize(("dtype", "lower", "upper"), INTEGER_CAST_CASES) +def test_cast_strict_false_string_to_integer_bounds( + constructor: Constructor, dtype: type[NonNestedDType], lower: int, upper: int +) -> None: + """Exercise the VARCHAR-to-integer portion of the cast matrix at each bound.""" + data = { + "a": [ + str(lower), + str(upper), + str(lower - 1), + str(upper + 1), + "not-a-number", + None, + ] + } + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(dtype, strict=False)).lazy().collect() + return + + if "pandas" in str(constructor) and dtype in {nw.Int64, nw.UInt64}: + # `to_numeric` may promote mixed in/out-of-range 64-bit integer text to + # Float64, at which point the valid boundary values are no longer exact. + with pytest.raises(NotImplementedError, match="exact Float64 range"): + df.select(nw.col("a").cast(dtype, strict=False)) + return + + result = df.select(nw.col("a").cast(dtype, strict=False)) + assert result.collect_schema()["a"] == dtype + assert_equal_data(result, {"a": [lower, upper, None, None, None, None]}) + + +def test_cast_strict_false_string_to_float(constructor: Constructor) -> None: + data = {"a": ["-1.5", "0", "2.25", "not-a-number", None]} + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(nw.Float64, strict=False)).lazy().collect() + return + + result = df.select(nw.col("a").cast(nw.Float64, strict=False)) + assert result.collect_schema()["a"] == nw.Float64 + assert_equal_data(result, {"a": [-1.5, 0.0, 2.25, None, None]}) + + +def test_cast_strict_false_string_to_datetime(constructor: Constructor) -> None: + data = {"a": ["2020-01-02 03:04:05", "not-a-datetime", None]} + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(nw.Datetime, strict=False)).lazy().collect() + return + + result = df.select(nw.col("a").cast(nw.Datetime, strict=False)) + assert result.collect_schema()["a"] == nw.Datetime + # Direct Polars casting is not datetime parsing. DuckDB TRY_CAST and Pandas' + # to_datetime do parse valid formatted timestamp strings. + expected = ( + [datetime(2020, 1, 2, 3, 4, 5), None, None] + if any(x in str(constructor) for x in DATETIME_PARSING_CONSTRUCTORS) + else [None, None, None] + ) + assert_equal_data(result, {"a": expected}) + + +def test_cast_strict_false_string_to_date(constructor: Constructor) -> None: + data = {"a": ["2020-01-02", "not-a-date", None]} + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(nw.Date, strict=False)).lazy().collect() + return + + result = df.select(nw.col("a").cast(nw.Date, strict=False)) + assert result.collect_schema()["a"] == nw.Date + assert_equal_data(result, {"a": [date(2020, 1, 2), None, None]}) + + +def test_cast_strict_false_integer_to_unsigned(constructor: Constructor) -> None: + data = {"a": [-1, 0, 255, 256, None]} + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(nw.UInt8, strict=False)).lazy().collect() + return + + result = df.select(nw.col("a").cast(nw.UInt8, strict=False)) + assert result.collect_schema()["a"] == nw.UInt8 + assert_equal_data(result, {"a": [None, 0, 255, None, None]}) + + +def test_cast_strict_false_float_to_int(constructor: Constructor) -> None: + data = {"a": [1.9, -1.9, 127.0, 1e30, None]} + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(nw.Int8, strict=False)).lazy().collect() + return + + result = df.select(nw.col("a").cast(nw.Int8, strict=False)) + assert result.collect_schema()["a"] == nw.Int8 + expected = ( + [2, -2, 127, None, None] + if any(x in str(constructor) for x in DUCKDB_TRY_CAST_CONSTRUCTORS) + else [1, -1, 127, None, None] + ) + assert_equal_data(result, {"a": expected}) + + +def test_cast_strict_false_numeric_to_numeric(constructor: Constructor) -> None: + # Plain, in-range numeric-to-numeric casts: `strict=False` should behave + # identically to `strict=True` when nothing is actually invalid. + data = {"a": [1, 2, 3]} + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(nw.Float64, strict=False)).lazy().collect() + return + + result = df.select(nw.col("a").cast(nw.Float64, strict=False)) + assert result.collect_schema()["a"] == nw.Float64 + assert_equal_data(result, data) + + +def test_cast_strict_false_numeric_to_boolean(constructor: Constructor) -> None: + data = {"a": [0, 1, 2, -1, None]} + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(nw.Boolean, strict=False)).lazy().collect() + return + + result = df.select(nw.col("a").cast(nw.Boolean, strict=False)) + assert result.collect_schema()["a"] == nw.Boolean + expected = ( + [False, True, True, True, True] + if "pandas_constructor" in str(constructor) + else [False, True, True, True, None] + ) + assert_equal_data(result, {"a": expected}) + + +def test_cast_strict_false_string_to_boolean(constructor: Constructor) -> None: + data = {"a": ["true", "false", "1", "not-a-boolean", None]} + df = nw.from_native(constructor(data)) + + if any(x in str(constructor) for x in DUCKDB_TRY_CAST_CONSTRUCTORS): + result = df.select(nw.col("a").cast(nw.Boolean, strict=False)) + assert_equal_data(result, {"a": [True, False, True, None, None]}) + elif "pandas_constructor" in str(constructor): + result = df.select(nw.col("a").cast(nw.Boolean, strict=False)) + assert_equal_data(result, {"a": [True, True, True, True, True]}) + else: + with pytest.raises(Exception): # noqa: B017, PT011 + df.select(nw.col("a").cast(nw.Boolean, strict=False)).lazy().collect() + + +def test_cast_strict_false_date_to_datetime(constructor: Constructor) -> None: + data = {"a": [date(2020, 1, 2), date(2024, 12, 31), None]} + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(nw.Datetime, strict=False)).lazy().collect() + return + + result = df.select(nw.col("a").cast(nw.Datetime, strict=False)) + assert result.collect_schema()["a"] == nw.Datetime + assert_equal_data(result, {"a": [datetime(2020, 1, 2), datetime(2024, 12, 31), None]}) + + +def test_cast_strict_false_datetime_to_date(constructor: Constructor) -> None: + data = { + "a": [datetime(2020, 1, 2, 3, 4, 5), datetime(2024, 12, 31, 23, 59, 59), None] + } + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(nw.Date, strict=False)).lazy().collect() + return + + result = df.select(nw.col("a").cast(nw.Date, strict=False)) + assert result.collect_schema()["a"] == nw.Date + assert_equal_data(result, {"a": [date(2020, 1, 2), date(2024, 12, 31), None]}) + + +def test_cast_strict_false_numeric_to_date(constructor: Constructor) -> None: + data = {"a": [0.0, 1.0, -1.0, 1.9, None]} + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(nw.Date, strict=False)).lazy().collect() + return + + result = df.select(nw.col("a").cast(nw.Date, strict=False)) + assert result.collect_schema()["a"] == nw.Date + if any(x in str(constructor) for x in DUCKDB_TRY_CAST_CONSTRUCTORS): + expected = [None, None, None, None, None] + elif "pandas" in str(constructor): + # Pandas interprets numeric datetime input as nanoseconds since the epoch; + # converting to Date consequently places all these small values near day 0. + expected = [ + date(1970, 1, 1), + date(1970, 1, 1), + date(1969, 12, 31), + date(1970, 1, 1), + None, + ] + else: + expected = [ + date(1970, 1, 1), + date(1970, 1, 2), + date(1969, 12, 31), + date(1970, 1, 2), + None, + ] + assert_equal_data(result, {"a": expected}) + + +def test_cast_strict_false_numeric_to_datetime(constructor: Constructor) -> None: + data = {"a": [0.0, 1.0, -1.0, 1.9, None]} + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(nw.Datetime, strict=False)).lazy().collect() + return + + result = df.select(nw.col("a").cast(nw.Datetime, strict=False)) + assert result.collect_schema()["a"] == nw.Datetime + if any(x in str(constructor) for x in DUCKDB_TRY_CAST_CONSTRUCTORS): + expected = [None, None, None, None, None] + elif "pandas" in str(constructor): + # Pandas treats numeric input as nanoseconds, then converts it to the + # requested Narwhals datetime unit. + expected = [ + datetime(1970, 1, 1), + datetime(1970, 1, 1), + datetime(1969, 12, 31, 23, 59, 59, 999999), + datetime(1970, 1, 1), + None, + ] + else: + expected = [ + datetime(1970, 1, 1), + datetime(1970, 1, 1, 0, 0, 0, 1), + datetime(1969, 12, 31, 23, 59, 59, 999999), + datetime(1970, 1, 1, 0, 0, 0, 1), + None, + ] + assert_equal_data(result, {"a": expected}) + + +@pytest.mark.parametrize( + "data", + [ + pytest.param([time(3, 4, 5), None], id="time"), + pytest.param([timedelta(days=1), None], id="duration"), + ], +) +def test_cast_strict_false_temporal_to_date_or_datetime_duckdb( + constructor: Constructor, data: list[time] | list[timedelta] +) -> None: + if not any(x in str(constructor) for x in DUCKDB_TRY_CAST_CONSTRUCTORS): + pytest.skip("DuckDB-specific TRY_CAST behavior") + + df = nw.from_native(constructor({"a": data})) + for dtype in (nw.Date, nw.Datetime): + result = df.select(nw.col("a").cast(dtype, strict=False)) + assert_equal_data(result, {"a": [None, None]}) + + +def test_cast_strict_false_string_to_binary(constructor: Constructor) -> None: + data = {"a": ["plain ASCII", "café", None]} + df = nw.from_native(constructor(data)) + + if any(backend in str(constructor) for backend in CAST_STRICT_FALSE_UNSUPPORTED): + with pytest.raises(NotImplementedError): + df.select(nw.col("a").cast(nw.Binary, strict=False)).lazy().collect() + return + + result = df.select(nw.col("a").cast(nw.Binary, strict=False)) + assert result.collect_schema()["a"] == nw.Binary + expected = ( + [b"plain ASCII", None, None] + if any(x in str(constructor) for x in DUCKDB_TRY_CAST_CONSTRUCTORS) + else [b"plain ASCII", b"caf\xc3\xa9", None] + ) + assert_equal_data(result, {"a": expected}) + + +def test_cast_strict_false_invalid_utf8_to_string_unsupported( + constructor: Constructor, +) -> None: + data = {"a": [b"valid UTF-8", b"\xff", None]} + df = nw.from_native(constructor(data)) + + if any(x in str(constructor) for x in DUCKDB_TRY_CAST_CONSTRUCTORS): + result = df.select(nw.col("a").cast(nw.String, strict=False)) + assert_equal_data(result, {"a": ["valid UTF-8", "\\xFF", None]}) + else: + with pytest.raises(Exception): # noqa: B017, PT011 + df.select(nw.col("a").cast(nw.String, strict=False)).lazy().collect() + + +def test_cast_strict_false_to_string_matches_duckdb_strict() -> None: + duckdb = pytest.importorskip("duckdb") + native = duckdb.sql( + """ + SELECT + [1, 2] AS list, + [1, 2]::INTEGER[2] AS array, + INTERVAL '1 day' AS duration + """ + ) + df = nw.from_native(native) + + strict = df.select(nw.all().cast(nw.String)) + non_strict = df.select(nw.all().cast(nw.String, strict=False)) + + assert nw.to_native(non_strict).fetchall() == nw.to_native(strict).fetchall() + + def test_cast_string() -> None: pytest.importorskip("pandas") import pandas as pd