Skip to content
Draft
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/api-reference/dataframe.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
- columns
- drop
- drop_nulls
- equals
- estimated_size
- explode
- filter
Expand Down
1 change: 1 addition & 0 deletions docs/api-reference/series.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
- diff
- drop_nulls
- dtype
- equals
- ewm_mean
- exp
- fill_nan
Expand Down
15 changes: 15 additions & 0 deletions src/narwhals/_arrow/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,21 @@ def schema(self) -> dict[str, DType]:
def collect_schema(self) -> dict[str, DType]:
return self.schema

def equals(self, other: Self, *, null_equal: bool) -> bool:
if self.shape != other.shape:
return False
if self.columns != other.columns:
return False
return all(
self.get_column(name).equals(
other.get_column(name),
check_dtypes=False,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we check also the dtype match? I think it's ok to turn off the check as long as above

- if self.columns != other.columns:
+ if self.schema != other.schema:

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

check_names=False,
null_equal=null_equal,
)
for name in self.columns
)

def estimated_size(self, unit: SizeUnit) -> int | float:
sz = self.native.nbytes
return scale_bytes(sz, unit)
Expand Down
20 changes: 20 additions & 0 deletions src/narwhals/_arrow/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -553,6 +553,26 @@ def cast(self, dtype: IntoDType) -> Self:
def null_count(self, *, _return_py_scalar: bool = True) -> int:
return maybe_extract_py_scalar(self.native.null_count, _return_py_scalar)

def equals(
self, other: Self, *, check_dtypes: bool, check_names: bool, null_equal: bool
) -> bool:

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should also compare NaNs, which will always be equivalent to one another.

You could do something like

lhs = ...
rhs = ...

try:
    both_nan = pc.fill_null(pc.and_(pc.is_nan(lhs), pc.is_nan(rhs)), False)
except ArrowNotImplementedError:
    both_nan = pa.scalar(False) # may need to broadcast this back out to a full array?

values_equal = pc.equal(lhs, rhs)
values_or_nan_equal = pc.fill_null(pc.or_(values_equal, both_nan), False) # no nulls in this array

if null_equal:
    # make the extra comparison with null values if null_equal is True
    ....
    return ...
return bool(pc.all(values_or_nan_equal).as_py())

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

16c7980

@camriddell: Thanks for the suggestion πŸ‘ŒπŸΌ Note that I slightly rewrote so that the question of broadcasting doesn't even arise in the first place

if check_names and self._name != other._name:
return False
if check_dtypes and self.dtype != other.dtype:
return False
if len(self) != len(other):
return False
lhs = self.native
rhs = other.native
if null_equal:
both_null = pc.and_(pc.is_null(lhs), pc.is_null(rhs))
values_equal = pc.equal(lhs, rhs)
result = pc.if_else(both_null, True, values_equal)
return bool(pc.all(result, skip_nulls=False).as_py())
if lhs.null_count or rhs.null_count:
return False
return bool(pc.all(pc.equal(lhs, rhs)).as_py())

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does pc.equal handle upcasting internally?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yep

>>> import pyarrow as pa
>>> import pyarrow.compute as pc
>>> pc.equal(pa.array([1,2,3], type=pa.int64()), pa.array([1.0, 2.0, 3.0], type=pa.float64()))
<pyarrow.lib.BooleanArray object at 0x7f46f53ceda0>
[
  true,
  true,
  true
]

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so no TODO here, right?


def head(self, n: int) -> Self:
if n >= 0:
return self._with_native(self.native.slice(0, n))
Expand Down
1 change: 1 addition & 0 deletions src/narwhals/_compliant/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ def __getitem__(
@property
def shape(self) -> tuple[int, int]: ...
def clone(self) -> Self: ...
def equals(self, other: Self, *, null_equal: bool) -> bool: ...
def estimated_size(self, unit: SizeUnit) -> int | float: ...
def gather_every(self, n: int, offset: int) -> Self: ...
def get_column(self, name: str) -> CompliantSeriesT: ...
Expand Down
3 changes: 3 additions & 0 deletions src/narwhals/_compliant/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,9 @@ def arg_max(self) -> int: ...
def arg_min(self) -> int: ...
def arg_true(self) -> Self: ...
def count(self) -> int: ...
def equals(
self, other: Self, *, check_dtypes: bool, check_names: bool, null_equal: bool
) -> bool: ...
def filter(self, predicate: Any) -> Self: ...
def first(self) -> PythonLiteral: ...
def last(self) -> PythonLiteral: ...
Expand Down
9 changes: 9 additions & 0 deletions src/narwhals/_pandas_like/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,15 @@ def drop_nulls(self, subset: Sequence[str] | None) -> Self:
mask = ~plx.any_horizontal(plx.col(*subset).is_null(), ignore_nulls=True)
return self.filter(mask)

def equals(self, other: Self, *, null_equal: bool) -> bool:
if self.shape != other.shape:
return False # not needed but can short-circuit the isna() scan below
if not null_equal and (
self.native.isna().any(axis=None) or other.native.isna().any(axis=None)
):
return False
return self.native.equals(other.native)

def estimated_size(self, unit: SizeUnit) -> int | float:
sz = self.native.memory_usage(deep=True).sum()
return scale_bytes(sz, unit=unit)
Expand Down
19 changes: 19 additions & 0 deletions src/narwhals/_pandas_like/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -1219,6 +1219,25 @@ def struct(self) -> PandasLikeSeriesStructNamespace:
raise TypeError(msg)
return PandasLikeSeriesStructNamespace(self)

def equals(
self, other: Self, *, check_dtypes: bool, check_names: bool, null_equal: bool
) -> bool:
if check_names and self._name != other._name:
return False
if check_dtypes and self.dtype != other.dtype:
return False
if not null_equal:
if len(self) != len(other):
return False # not needed but can short-circuit the isna() checks below
if self.native.isna().any() or other.native.isna().any():
return False
other_native = (
other.native.astype(self.native.dtype)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure how I feel about this, can't downcasting potentially raise an error? This is where a supertype(left_dtype, right_dtype) would be quite useful

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed we should avoid manually casting here.

>>> import pandas as pd
>>> self = pd.Series([1, 2, 3])
>>> other = pd.Series([*'123'])
>>> self.equals(other)
False
>>> self.equals(other.astype(self.dtype))
True
dtype: int64
>>> other.astype(self.dtype)
0    1
1    2
2    3

Instead of directly using pandas.Series.equals we should probably use the == operator along with a .isna() check on both sides since the == operator will take care of the casting for us:

>>> pd.Series([1,2,3], dtype="int64") == pd.Series([1.0, 2.0, 3.0], dtype="float64")
0    True
1    True
2    True
dtype: bool

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point! But I guess the supertype utility does not yet exist, right? Is this something for a separate PR? Also, for which backends would we need it? All of them, or just pandas, specifically for this PR? For the numpy backend, I think that np.result_type may do the job, but unclear how to best handle the pyarrow backend...

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry for the "not enough" context - that's correct get_supertype is still not here. Hopefully it will eventually get merged #3396

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, so I guess I'll be waiting then until this happens

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll be waiting then until this happens

Please don't! There is not enough consensus on it for now - I brought it up as a nice use case for it, not for blocking you

if not check_dtypes and self.native.dtype != other.native.dtype
else other.native
)
return self.native.equals(other_native)


class _PandasHist(EagerSeriesHist["pd.Series[Any]", "list[float]"]):
_series: PandasLikeSeries
Expand Down
2 changes: 2 additions & 0 deletions src/narwhals/_polars/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
[
"clone",
"drop_nulls",
"equals",
"estimated_size",
"explode",
"filter",
Expand Down Expand Up @@ -108,6 +109,7 @@

class PolarsBaseFrame(Generic[NativePolarsFrame]):
drop_nulls: Method[Self]
equals: Method[bool]
explode: Method[Self]
filter: Method[Self]
gather_every: Method[Self]
Expand Down
2 changes: 2 additions & 0 deletions src/narwhals/_polars/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@
"cum_sum",
"diff",
"drop_nulls",
"equals",
"exp",
"fill_null",
"fill_nan",
Expand Down Expand Up @@ -720,6 +721,7 @@ def struct(self) -> PolarsSeriesStructNamespace:
cum_sum: Method[Self]
diff: Method[Self]
drop_nulls: Method[Self]
equals: Method[bool]

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You might need to actually implement the method for backward compat. From the Series.equals docs:

Changed in version 0.20.31: The strict parameter was renamed check_dtypes.


You can find a lot of examples in the codebase for which we align past behavior to more recent one

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

exp: Method[Self]
fill_null: Method[Self]
fill_nan: Method[Self]
Expand Down
3 changes: 3 additions & 0 deletions src/narwhals/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -1295,6 +1295,9 @@ def drop_nulls(self, subset: str | list[str] | None = None) -> Self:
"""
return super().drop_nulls(subset=subset)

def equals(self, other: Self, *, null_equal: bool = True) -> bool:
return self._compliant_frame.equals(other._compliant_frame, null_equal=null_equal)

def with_row_index(
self, name: str = "index", *, order_by: str | Sequence[str] | None = None
) -> Self:
Expand Down
15 changes: 15 additions & 0 deletions src/narwhals/series.py
Original file line number Diff line number Diff line change
Expand Up @@ -2942,3 +2942,18 @@ def list(self) -> SeriesListNamespace[Self]:
@property
def struct(self) -> SeriesStructNamespace[Self]:
return SeriesStructNamespace(self)

def equals(
self,
other: Self,
*,
check_dtypes: bool = False,
check_names: bool = False,
null_equal: bool = True,
) -> bool:
return self._compliant_series.equals(
other._compliant_series,
check_dtypes=check_dtypes,
check_names=check_names,
null_equal=null_equal,
)
154 changes: 154 additions & 0 deletions tests/series_only/equals_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
from __future__ import annotations

from typing import TYPE_CHECKING

import pytest

import narwhals as nw

if TYPE_CHECKING:
from tests.utils import ConstructorEager

TBD = None
_NAN_HANDLING = pytest.mark.xfail(
strict=True, reason="NaN/null disambiguation not yet decided"
)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NaNs should always be equivalent to one another. The only exception to this rule will be when calling equals(…, null_equal=False) on pandas objects, since it cannot disambiguate Null from NaN.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I understand the problem, but not sure if I'm 100% what is the desired behavior for this case now. For the arrow backend, the problem could in principle be resolved, but not for the numpy backend. So what do we do?

  1. Resolve for pyarrow or
  2. Keep behavior consistent for both backends?

And then, regarding the failing case(s) that remain:

  1. Do we throw an error if the unsupported comparison is requested by the user?
  2. Or do we silently just return True even though we should return False for null-comparison when null_equals=False?
  3. Or do we document this somewhere?

_CROSS_TYPE = pytest.mark.xfail(
strict=True, reason="Inconsistent behavior across backends for incompatible types"
)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Backends like pyarrow error out if it does not have an available kernel to perform the operation. For these cases, start with a try/except and catch the ArrowNotImplementedError. I don't believe pyarrow exposes a way to interact with the type hierarchy to check what is comparable beforehand.

Polars returns False for these cases
>>> import polars as pl
>>> pl.Series([1,2,3]).equals(pl.Series([*'123']))
False
pyarrow raises errors when working with un-related dtypes
>>> import pyarrow.compute as pc
>>> import pyarrow as pa
>>> import pyarrow.compute as pc
>>> pc.equal(pa.array([1,2,3]), pa.array([*'123']))
Traceback (most recent call last):
  File "<python-input-5>", line 1, in <module>
    pc.equal(pa.array([1,2,3]), pa.array([*'123']))
    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/cameron/.pyenv/versions/3.14.2/lib/python3.14/site-packages/pyarrow/compute.py", line 252, in wrapper
    return func.call(args, None, memory_pool)
           ~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^
  File "pyarrow/_compute.pyx", line 399, in pyarrow._compute.Function.call
  File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
pyarrow.lib.ArrowNotImplementedError: Function 'equal' has no kernel matching input types (int64, string)

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

_DTYPE_INFERENCE = pytest.mark.xfail(
strict=True,
reason="Constructor dtype inference is backend-dependent for whole-number floats",
)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@FBruzzesi are there any test helpers that can help with this problem? We want the constructors to NOT eagerly downcast values. For example, these should each end up as floats dtype.

>>> pd.Series([1.0])
0    1.0
dtype: float64
>>> pd.Series([1.0]).convert_dtypes(dtype_backend="numpy_nullable")
0    1
dtype: Int64
>>> pd.Series([1.0]).convert_dtypes(dtype_backend="pyarrow")
0    1
dtype: int64[pyarrow]

There is convert_dtypes(convert_integer=False, dtype_backend="pyarrow"), but that causes ints to fall out of the requested dtype:

>>> pd.Series([1]).convert_dtypes(convert_integer=False, dtype_backend="pyarrow").dtype
dtype('int64')
>>> pd.Series([1.0]).convert_dtypes(convert_integer=False, dtype_backend="pyarrow").dtype
double[pyarrow]

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume you realized there was none and went ahead with #3749 πŸ™πŸΌ

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

_CHECK_NAMES = pytest.mark.xfail(
strict=True,
reason=(
"Bare ChunkedArray carries no name metadata, resulting in empty names - "
"TBD whether check_names=True should raise for pyarrow in case of empty names"
),
)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The names should exist on the narwhals._arrow.series.ArrowSeries object. The code seems to already have checks for names in them, unless this note was about something else?

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, the _name field exists on ArrowSeries, but my question is a different one: pa.ChunkedArray is an anonymous array. So when narwhals wraps one, _name is always set to "" regardless of what column it came from:

import pyarrow as pa
import narwhals as nw

t = pa.table({"left": [1, 2], "right": [1, 2]})
left  = nw.from_native(t["left"],  series_only=True)
right = nw.from_native(t["right"], series_only=True)

print(left.name)   # ""
print(right.name)  # ""
print(left.equals(right, check_names=True))  # True β€” names are both "", so check passes silently

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see the where the issue arises now. Can you rewrite the test in this fashion so the metadata is preserved appropriately?

left_native = constructor_eager({left_name: data})
right_native = constructor_eager({right_name: data})
left_nw = nw.from_native(left_native)[left_name]
right_nw = nw.from_native(right_native)[right_name]

result = left_nw.equals(right_nw, check_names=check_names)
assert result == expected



@pytest.mark.parametrize(
("left", "right", "expected"),
[
# --- Same dtype: int ---
([1, 2], [1, 2], True),
# --- Same dtype: float ---
([1.1, 2.2], [1.1, 2.2], True),
# --- Value differs ---
([1, 2], [1, 0], False),
# --- Cross-supertype: float vs int ---
([1.0, 2.0], [1, 2], True),
([1.1, 2.2], [1, 2], False),
# --- Cross-supertype: int vs string ---
# pandas: False, polars: False, pyarrow: raises
pytest.param([1, 2], ["1", "2"], TBD, marks=_CROSS_TYPE),
],
)
def test_series_equals(
constructor_eager: ConstructorEager,
left: list,
right: list,
expected: bool | None, # noqa: FBT001
) -> None:
left_native = constructor_eager({"left": left})["left"]
right_native = constructor_eager({"right": right})["right"]
left_nw = nw.from_native(left_native, series_only=True)
right_nw = nw.from_native(right_native, series_only=True)

result = left_nw.equals(right_nw)
assert result == expected


@pytest.mark.parametrize(
("left", "right", "null_equal", "expected"),
[
# --- Null vs value ---
([1, None], [1, 2], True, False),
([1, None], [1, 2], False, False),
# --- Null vs Null ---
([1, None], [1, None], True, True),
([1, None], [1, None], False, False),
# --- NaN vs NaN ---
# pandas: True, polars: True, pyarrow: False
pytest.param(
[1.0, float("nan")], [1.0, float("nan")], True, TBD, marks=_NAN_HANDLING
),
# pandas: True, polars: True, pyarrow: False
pytest.param(
[1.0, float("nan")], [1.0, float("nan")], False, TBD, marks=_NAN_HANDLING
),
# --- NaN vs value ---
# pandas: False, polars: False, pyarrow: False
([1.0, float("nan")], [1.0, 2.0], True, False),
],
)
def test_series_equals_null_equal(
constructor_eager: ConstructorEager,
left: list,
right: list,
null_equal: bool, # noqa: FBT001
expected: bool | None, # noqa: FBT001
) -> None:
left_native = constructor_eager({"left": left})["left"]
right_native = constructor_eager({"right": right})["right"]
left_nw = nw.from_native(left_native, series_only=True)
right_nw = nw.from_native(right_native, series_only=True)

result = left_nw.equals(right_nw, null_equal=null_equal)
assert result == expected


@pytest.mark.parametrize(
("left", "right", "check_dtypes", "expected"),
[
# --- Same values, different dtype ---
([1.1, 2.2], [1, 2], False, False),
# --- Whole-number floats, check_dtypes=False ---
([1.0, 2.0], [1, 2], False, True),
# --- Whole-number floats, check_dtypes=True ---
# pandas: False, polars: False, pyarrow: False, pandas[pyarrow]: True (infers Int64)
pytest.param([1.0, 2.0], [1, 2], True, TBD, marks=_DTYPE_INFERENCE),
],
)
def test_series_equals_check_dtypes(
constructor_eager: ConstructorEager,
left: list,
right: list,
check_dtypes: bool, # noqa: FBT001
expected: bool | None, # noqa: FBT001
) -> None:
left_native = constructor_eager({"a": left})["a"]
right_native = constructor_eager({"a": right})["a"]
left_nw = nw.from_native(left_native, series_only=True)
right_nw = nw.from_native(right_native, series_only=True)

result = left_nw.equals(right_nw, check_dtypes=check_dtypes)
assert result == expected


@pytest.mark.parametrize(
("left_name", "right_name", "check_names", "expected"),
[
# --- Different names ---
("left", "right", False, True),
# pandas: False, polars: False, pyarrow: True (ChunkedArray has no name metadata)
pytest.param("left", "right", True, TBD, marks=_CHECK_NAMES),
],
)
def test_series_equals_check_names(
constructor_eager: ConstructorEager,
left_name: str,
right_name: str,
check_names: bool, # noqa: FBT001
expected: bool | None, # noqa: FBT001
) -> None:
data = [1, 2]
left_native = constructor_eager({left_name: data})[left_name]
right_native = constructor_eager({right_name: data})[right_name]
left_nw = nw.from_native(left_native, series_only=True)
right_nw = nw.from_native(right_native, series_only=True)

result = left_nw.equals(right_nw, check_names=check_names)
assert result == expected
1 change: 1 addition & 0 deletions utils/check_api_reference.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ def read_documented_members(source: str | Path) -> list[str]:
"arg_max",
"arg_min",
"arg_true",
"equals",
"dtype",
"from_iterable",
"from_numpy",
Expand Down