From 9664de2e9d0d2c2a5140fef884855cdb7488991b Mon Sep 17 00:00:00 2001 From: "Md. Fazlul Karim" Date: Thu, 15 Oct 2026 16:34:30 +0600 Subject: [PATCH] fix(qdrant): apply all filter operators when multiple operators target the same field --- mem0/vector_stores/qdrant.py | 111 +++++++------- tests/vector_stores/test_qdrant.py | 228 ++++++++++++++++++++--------- 2 files changed, 217 insertions(+), 122 deletions(-) diff --git a/mem0/vector_stores/qdrant.py b/mem0/vector_stores/qdrant.py index b48e03c914..6d81e76a3e 100644 --- a/mem0/vector_stores/qdrant.py +++ b/mem0/vector_stores/qdrant.py @@ -1,6 +1,6 @@ import logging import re -from typing import Optional +from typing import List, Optional, Union from qdrant_client import QdrantClient, models from qdrant_client.models import ( @@ -95,6 +95,7 @@ def _get_bm25_encoder(self): if self._bm25_encoder is None: try: from fastembed import SparseTextEmbedding + self._bm25_encoder = SparseTextEmbedding(model_name="Qdrant/bm25") logger.info("BM25 encoder loaded (fastembed Qdrant/bm25)") except ImportError: @@ -175,9 +176,7 @@ def _create_filter_indexes(self): for field in common_fields: try: self.client.create_payload_index( - collection_name=self.collection_name, - field_name=field, - field_schema="keyword" + collection_name=self.collection_name, field_name=field, field_schema="keyword" ) logger.info(f"Created index for {field} in collection {self.collection_name}") except Exception as e: @@ -257,25 +256,22 @@ def insert(self, vectors: list, payloads: list = None, ids: list = None): @staticmethod def _is_datetime_range(range_kwargs: dict) -> bool: """Check if all values in range kwargs are ISO datetime strings.""" - return all( - isinstance(v, str) and Qdrant._ISO_DATETIME_RE.match(v) - for v in range_kwargs.values() - ) + return all(isinstance(v, str) and Qdrant._ISO_DATETIME_RE.match(v) for v in range_kwargs.values()) - def _build_field_condition(self, key: str, value) -> Optional[FieldCondition]: + def _build_field_condition(self, key: str, value) -> Optional[Union[FieldCondition, List[FieldCondition]]]: """ - Build a single FieldCondition from a key-value filter pair. + Build Qdrant FieldCondition(s) from a key-value filter pair. Supports the enhanced filter syntax documented at https://docs.mem0.ai/open-source/features/metadata-filtering Args: key (str): The payload field name. - value: A scalar for simple equality, or a dict with one operator key. + value: A scalar for simple equality, or a dict with operator keys. Returns: - Optional[FieldCondition]: The Qdrant field condition, or None if the - value is the wildcard '*' (match any / field exists — skip filter). + Optional[Union[FieldCondition, List[FieldCondition]]]: The Qdrant field condition(s), + or None if the value is the wildcard '*' (match any / field exists — skip filter). """ if not isinstance(value, dict): if value == "*": @@ -289,6 +285,12 @@ def _build_field_condition(self, key: str, value) -> Optional[FieldCondition]: return FieldCondition(key=key, match=MatchValue(value=value)) ops = set(value.keys()) + supported = {"eq", "ne", "gt", "gte", "lt", "lte", "in", "nin", "contains", "icontains"} + if (ops - supported) or not ops: + raise ValueError( + f"Unsupported filter operator(s) for field '{key}': {ops}. Supported operators: {supported}" + ) + range_ops = {"gt", "gte", "lt", "lte"} non_range_ops = ops - range_ops @@ -304,37 +306,30 @@ def _build_field_condition(self, key: str, value) -> Optional[FieldCondition]: try: return FieldCondition(key=key, range=DatetimeRange(**range_kwargs)) except (ValueError, TypeError) as e: - raise ValueError( - f"Invalid datetime value in range filter for field '{key}': {e}" - ) from e + raise ValueError(f"Invalid datetime value in range filter for field '{key}': {e}") from e return FieldCondition(key=key, range=Range(**range_kwargs)) - elif "eq" in value: - return FieldCondition(key=key, match=MatchValue(value=value["eq"])) - elif "ne" in value: - return FieldCondition(key=key, match=MatchExcept(**{"except": [value["ne"]]})) - elif "in" in value: - return FieldCondition(key=key, match=MatchAny(any=value["in"])) - elif "nin" in value: - return FieldCondition(key=key, match=MatchExcept(**{"except": value["nin"]})) - elif "contains" in value or "icontains" in value: - # MatchText: with a full-text index, tokenized matching (all words must appear). - # Without a full-text index, exact substring match. - op = "icontains" if "icontains" in value else "contains" - text = value[op] - if op == "icontains": - logger.debug( - "icontains on field '%s': Qdrant MatchText case sensitivity depends on " - "full-text index configuration. Without a full-text index this behaves " - "as a case-sensitive substring match (same as 'contains').", - key, - ) - return FieldCondition(key=key, match=MatchText(text=text)) - else: - supported = {"eq", "ne", "gt", "gte", "lt", "lte", "in", "nin", "contains", "icontains"} - raise ValueError( - f"Unsupported filter operator(s) for field '{key}': {ops}. " - f"Supported operators: {supported}" + + conditions = [] + if "eq" in value: + conditions.append(FieldCondition(key=key, match=MatchValue(value=value["eq"]))) + if "ne" in value: + conditions.append(FieldCondition(key=key, match=MatchExcept(**{"except": [value["ne"]]}))) + if "in" in value: + conditions.append(FieldCondition(key=key, match=MatchAny(any=value["in"]))) + if "nin" in value: + conditions.append(FieldCondition(key=key, match=MatchExcept(**{"except": value["nin"]}))) + if "contains" in value: + conditions.append(FieldCondition(key=key, match=MatchText(text=value["contains"]))) + if "icontains" in value: + logger.debug( + "icontains on field '%s': Qdrant MatchText case sensitivity depends on " + "full-text index configuration. Without a full-text index this behaves " + "as a case-sensitive substring match (same as 'contains').", + key, ) + conditions.append(FieldCondition(key=key, match=MatchText(text=value["icontains"]))) + + return conditions[0] if len(conditions) == 1 else conditions def _create_filter(self, filters: dict) -> Optional[Filter]: """ @@ -372,15 +367,11 @@ def _create_filter(self, filters: dict) -> Optional[Filter]: for key, value in normalized.items(): if key in ("AND", "OR", "NOT"): if not isinstance(value, list): - raise ValueError( - f"{key} filter value must be a list of filter dicts, " - f"got {type(value).__name__}" - ) + raise ValueError(f"{key} filter value must be a list of filter dicts, got {type(value).__name__}") for i, item in enumerate(value): if not isinstance(item, dict): raise ValueError( - f"{key} filter list item at index {i} must be a dict, " - f"got {type(item).__name__}: {item!r}" + f"{key} filter list item at index {i} must be a dict, got {type(item).__name__}: {item!r}" ) if key == "AND": @@ -399,9 +390,24 @@ def _create_filter(self, filters: dict) -> Optional[Filter]: if built: must_not.append(built) else: - condition = self._build_field_condition(key, value) - if condition is not None: - must.append(condition) + range_ops = {"gt", "gte", "lt", "lte"} + if isinstance(value, dict) and (set(value.keys()) & range_ops) and (set(value.keys()) - range_ops): + range_kwargs = {op: value[op] for op in range_ops if op in value} + cond = self._build_field_condition(key, range_kwargs) + if cond is not None: + must.append(cond) + non_range_dict = {op: value[op] for op in value if op not in range_ops} + cond = self._build_field_condition(key, non_range_dict) + if isinstance(cond, list): + must.extend(cond) + elif cond is not None: + must.append(cond) + else: + condition = self._build_field_condition(key, value) + if isinstance(condition, list): + must.extend(condition) + elif condition is not None: + must.append(condition) if not any([must, should, must_not]): return None @@ -438,8 +444,7 @@ def search_batch(self, queries: list, vectors_list: list, top_k: int = 1, filter """Batch search using Qdrant's query_batch_points for efficiency.""" query_filter = self._create_filter(filters) if filters else None requests = [ - models.QueryRequest(query=vec, filter=query_filter, limit=top_k, with_payload=True) - for vec in vectors_list + models.QueryRequest(query=vec, filter=query_filter, limit=top_k, with_payload=True) for vec in vectors_list ] try: results = self.client.query_batch_points( diff --git a/tests/vector_stores/test_qdrant.py b/tests/vector_stores/test_qdrant.py index ac45026f50..758b43ccf9 100644 --- a/tests/vector_stores/test_qdrant.py +++ b/tests/vector_stores/test_qdrant.py @@ -225,9 +225,7 @@ def test_search_with_filters(self): """Test search with agent_id and run_id filters.""" vectors = [[0.1, 0.2]] mock_point = MagicMock( - id=str(uuid.uuid4()), - score=0.95, - payload={"user_id": "alice", "agent_id": "agent1", "run_id": "run1"} + id=str(uuid.uuid4()), score=0.95, payload={"user_id": "alice", "agent_id": "agent1", "run_id": "run1"} ) self.client_mock.query_points.return_value = MagicMock(points=[mock_point]) @@ -240,7 +238,7 @@ def test_search_with_filters(self): self.assertEqual(call_args["collection_name"], "test_collection") self.assertEqual(call_args["query"], vectors) self.assertEqual(call_args["limit"], 1) - + # Verify that a Filter object was created query_filter = call_args["query_filter"] self.assertIsInstance(query_filter, Filter) @@ -254,11 +252,7 @@ def test_search_with_filters(self): def test_search_with_single_filter(self): """Test search with single filter.""" vectors = [[0.1, 0.2]] - mock_point = MagicMock( - id=str(uuid.uuid4()), - score=0.95, - payload={"user_id": "alice"} - ) + mock_point = MagicMock(id=str(uuid.uuid4()), score=0.95, payload={"user_id": "alice"}) self.client_mock.query_points.return_value = MagicMock(points=[mock_point]) filters = {"user_id": "alice"} @@ -290,10 +284,10 @@ def test_create_filter_multiple_filters(self): """Test _create_filter with multiple filters.""" filters = {"user_id": "alice", "agent_id": "agent1", "run_id": "run1"} result = self.qdrant._create_filter(filters) - + self.assertIsInstance(result, Filter) self.assertEqual(len(result.must), 3) - + # Check that all conditions are present conditions = [cond.key for cond in result.must] self.assertIn("user_id", conditions) @@ -304,7 +298,7 @@ def test_create_filter_single_filter(self): """Test _create_filter with single filter.""" filters = {"user_id": "alice"} result = self.qdrant._create_filter(filters) - + self.assertIsInstance(result, Filter) self.assertEqual(len(result.must), 1) self.assertEqual(result.must[0].key, "user_id") @@ -314,7 +308,7 @@ def test_create_filter_no_filters(self): """Test _create_filter with no filters.""" result = self.qdrant._create_filter(None) self.assertIsNone(result) - + result = self.qdrant._create_filter({}) self.assertIsNone(result) @@ -322,17 +316,17 @@ def test_create_filter_with_range_values(self): """Test _create_filter with range values.""" filters = {"user_id": "alice", "count": {"gte": 5, "lte": 10}} result = self.qdrant._create_filter(filters) - + self.assertIsInstance(result, Filter) self.assertEqual(len(result.must), 2) - + # Check that range condition is created - range_conditions = [cond for cond in result.must if hasattr(cond, 'range') and cond.range is not None] + range_conditions = [cond for cond in result.must if hasattr(cond, "range") and cond.range is not None] self.assertEqual(len(range_conditions), 1) self.assertEqual(range_conditions[0].key, "count") - + # Check that string condition is created - string_conditions = [cond for cond in result.must if hasattr(cond, 'match') and cond.match is not None] + string_conditions = [cond for cond in result.must if hasattr(cond, "match") and cond.match is not None] self.assertEqual(len(string_conditions), 1) self.assertEqual(string_conditions[0].key, "user_id") @@ -417,9 +411,7 @@ def test_list_cols(self): def test_list_with_filters(self): """Test list with agent_id and run_id filters.""" mock_point = MagicMock( - id=str(uuid.uuid4()), - score=0.95, - payload={"user_id": "alice", "agent_id": "agent1", "run_id": "run1"} + id=str(uuid.uuid4()), score=0.95, payload={"user_id": "alice", "agent_id": "agent1", "run_id": "run1"} ) self.client_mock.scroll.return_value = [mock_point] @@ -431,7 +423,7 @@ def test_list_with_filters(self): call_args = self.client_mock.scroll.call_args[1] self.assertEqual(call_args["collection_name"], "test_collection") self.assertEqual(call_args["limit"], 10) - + # Verify that a Filter object was created scroll_filter = call_args["scroll_filter"] self.assertIsInstance(scroll_filter, Filter) @@ -445,11 +437,7 @@ def test_list_with_filters(self): def test_list_with_single_filter(self): """Test list with single filter.""" - mock_point = MagicMock( - id=str(uuid.uuid4()), - score=0.95, - payload={"user_id": "alice"} - ) + mock_point = MagicMock(id=str(uuid.uuid4()), score=0.95, payload={"user_id": "alice"}) self.client_mock.scroll.return_value = [mock_point] filters = {"user_id": "alice"} @@ -786,18 +774,9 @@ def test_deeply_nested_logical(self): """3-level nesting: AND > OR > NOT.""" filters = { "AND": [ - { - "OR": [ - {"category": "work"}, - {"category": "personal"} - ] - }, + {"OR": [{"category": "work"}, {"category": "personal"}]}, {"priority": {"gte": 5}}, - { - "NOT": [ - {"status": "archived"} - ] - } + {"NOT": [{"status": "archived"}]}, ] } result = self.qdrant._create_filter(filters) @@ -830,9 +809,7 @@ def test_in_with_integers(self): def test_wildcard_inside_and(self): """Wildcard inside AND should be skipped, other conditions preserved.""" - result = self.qdrant._create_filter({ - "AND": [{"category": "*"}, {"user_id": "alice"}] - }) + result = self.qdrant._create_filter({"AND": [{"category": "*"}, {"user_id": "alice"}]}) self.assertIsInstance(result, Filter) # AND produces nested Filters; the wildcard sub-filter returns None and is skipped # Only the user_id sub-filter remains @@ -966,32 +943,24 @@ def test_iso_datetime_gte_lte_uses_datetime_range(self): def test_iso_date_only_uses_datetime_range(self): """Date-only strings (YYYY-MM-DD) should also use DatetimeRange.""" - cond = self.qdrant._build_field_condition( - "created_at", {"gte": "2025-01-01", "lt": "2025-02-01"} - ) + cond = self.qdrant._build_field_condition("created_at", {"gte": "2025-01-01", "lt": "2025-02-01"}) self.assertIsInstance(cond.range, DatetimeRange) def test_iso_datetime_with_offset_uses_datetime_range(self): """Datetime with timezone offset should use DatetimeRange.""" - cond = self.qdrant._build_field_condition( - "updated_at", {"gt": "2025-06-15T10:30:00+05:30"} - ) + cond = self.qdrant._build_field_condition("updated_at", {"gt": "2025-06-15T10:30:00+05:30"}) self.assertIsInstance(cond.range, DatetimeRange) def test_numeric_range_still_uses_range(self): """Numeric values should still use Range (not DatetimeRange).""" - cond = self.qdrant._build_field_condition( - "priority", {"gte": 5, "lte": 10} - ) + cond = self.qdrant._build_field_condition("priority", {"gte": 5, "lte": 10}) self.assertIsInstance(cond.range, Range) self.assertEqual(cond.range.gte, 5) self.assertEqual(cond.range.lte, 10) def test_float_range_still_uses_range(self): """Float values should still use Range.""" - cond = self.qdrant._build_field_condition( - "score", {"gt": 0.5, "lt": 0.9} - ) + cond = self.qdrant._build_field_condition("score", {"gt": 0.5, "lt": 0.9}) self.assertIsInstance(cond.range, Range) def test_datetime_range_via_create_filter(self): @@ -1006,9 +975,7 @@ def test_datetime_range_via_create_filter(self): def test_malformed_datetime_raises_with_field_context(self): """Malformed date-like string should raise ValueError with field name.""" with self.assertRaises(ValueError) as ctx: - self.qdrant._build_field_condition( - "created_at", {"gte": "2025-13-45"} - ) + self.qdrant._build_field_condition("created_at", {"gte": "2025-13-45"}) self.assertIn("created_at", str(ctx.exception)) def test_mixed_datetime_and_numeric_raises_error(self): @@ -1020,30 +987,26 @@ def test_mixed_datetime_and_numeric_raises_error(self): from pydantic import ValidationError with self.assertRaises(ValidationError): - self.qdrant._build_field_condition( - "field", {"gte": "2025-01-01", "lte": 100} - ) + self.qdrant._build_field_condition("field", {"gte": "2025-01-01", "lte": 100}) def test_iso_datetime_with_fractional_seconds(self): """Fractional seconds should use DatetimeRange.""" - cond = self.qdrant._build_field_condition( - "created_at", {"gte": "2025-01-01T00:00:00.123456Z"} - ) + cond = self.qdrant._build_field_condition("created_at", {"gte": "2025-01-01T00:00:00.123456Z"}) self.assertIsInstance(cond.range, DatetimeRange) def test_iso_datetime_space_separated(self): """Space-separated datetime should use DatetimeRange.""" - cond = self.qdrant._build_field_condition( - "created_at", {"gte": "2025-01-01 10:30:00"} - ) + cond = self.qdrant._build_field_condition("created_at", {"gte": "2025-01-01 10:30:00"}) self.assertIsInstance(cond.range, DatetimeRange) def test_datetime_with_numeric_mixed_filters(self): """Datetime and numeric range filters can coexist in same query.""" - result = self.qdrant._create_filter({ - "created_at": {"gte": "2025-01-01"}, - "priority": {"gte": 5}, - }) + result = self.qdrant._create_filter( + { + "created_at": {"gte": "2025-01-01"}, + "priority": {"gte": 5}, + } + ) self.assertIsInstance(result, Filter) self.assertEqual(len(result.must), 2) types = {type(c.range) for c in result.must} @@ -1065,3 +1028,130 @@ def test_missing_fastembed_warns_with_extras_install_hint(self): self.assertIn("mem0ai[extras]", joined) # points at the right install self.assertNotIn("pip install fastembed", joined) # not the bare package self.assertNotIn("\u2014", joined) # no em-dash + + +class TestQdrantEveryOperatorOnAField(unittest.TestCase): + """Regression tests for Issue #7468: multi-operator filters on a single field.""" + + def setUp(self): + self.client_mock = MagicMock(spec=QdrantClient) + self.qdrant = Qdrant( + collection_name="test_collection", + embedding_model_dims=4, + client=self.client_mock, + ) + + def test_ne_and_nin_on_one_field_both_apply(self): + """Filters with multiple match operators like ne and nin on a single field must emit conditions for both.""" + result = self.qdrant._create_filter({"status": {"ne": "archived", "nin": ["deleted"]}}) + self.assertIsInstance(result, Filter) + self.assertIsNotNone(result.must) + self.assertEqual(len(result.must), 2) + for cond in result.must: + self.assertEqual(cond.key, "status") + except_lists = [cond.match.except_ for cond in result.must if isinstance(cond.match, MatchExcept)] + self.assertEqual(len(except_lists), 2) + self.assertIn(["archived"], except_lists) + self.assertIn(["deleted"], except_lists) + + # In-memory Qdrant integration test + client = QdrantClient(":memory:") + store = Qdrant(collection_name="test_mem_ne_nin", embedding_model_dims=4, client=client) + store.insert( + vectors=[[0.1, 0.2, 0.3, 0.4]] * 3, + payloads=[{"status": "active"}, {"status": "archived"}, {"status": "deleted"}], + ids=[str(uuid.uuid4()) for _ in range(3)], + ) + res = store.list(filters={"status": {"ne": "archived", "nin": ["deleted"]}}, top_k=10) + self.assertEqual([p.payload["status"] for p in res[0]], ["active"]) + + search_res = store.search( + query="test", + vectors=[0.1, 0.2, 0.3, 0.4], + filters={"status": {"ne": "archived", "nin": ["deleted"]}}, + top_k=10, + ) + self.assertEqual([p.payload["status"] for p in search_res], ["active"]) + + def test_range_and_ne_on_one_field_both_apply(self): + """Filters combining range and match operators on the same field must emit both conditions.""" + result = self.qdrant._create_filter({"priority": {"gte": 1, "ne": 5}}) + self.assertIsInstance(result, Filter) + self.assertEqual(len(result.must), 2) + range_conds = [c for c in result.must if c.range is not None] + match_conds = [c for c in result.must if c.match is not None] + self.assertEqual(len(range_conds), 1) + self.assertEqual(len(match_conds), 1) + self.assertEqual(range_conds[0].key, "priority") + self.assertEqual(range_conds[0].range.gte, 1) + self.assertEqual(match_conds[0].key, "priority") + self.assertEqual(match_conds[0].match.except_, [5]) + + # In-memory Qdrant integration test + client = QdrantClient(":memory:") + store = Qdrant(collection_name="test_mem_range_ne", embedding_model_dims=4, client=client) + store.insert( + vectors=[[0.1, 0.2, 0.3, 0.4]] * 4, + payloads=[{"priority": 0}, {"priority": 1}, {"priority": 5}, {"priority": 10}], + ids=[str(uuid.uuid4()) for _ in range(4)], + ) + res = store.list(filters={"priority": {"gte": 1, "ne": 5}}, top_k=10) + returned_priorities = sorted(p.payload["priority"] for p in res[0]) + self.assertEqual(returned_priorities, [1, 10]) + + search_res = store.search( + query="test", + vectors=[0.1, 0.2, 0.3, 0.4], + filters={"priority": {"gte": 1, "ne": 5}}, + top_k=10, + ) + search_priorities = sorted(p.payload["priority"] for p in search_res) + self.assertEqual(search_priorities, [1, 10]) + + def test_and_merged_single_field_operators(self): + """AND logical filters on the same field should properly filter out excluded records.""" + client = QdrantClient(":memory:") + store = Qdrant(collection_name="test_mem_and_merged", embedding_model_dims=4, client=client) + store.insert( + vectors=[[0.1, 0.2, 0.3, 0.4]] * 3, + payloads=[{"status": "active"}, {"status": "archived"}, {"status": "deleted"}], + ids=[str(uuid.uuid4()) for _ in range(3)], + ) + res = store.list( + filters={"AND": [{"status": {"ne": "archived"}}, {"status": {"nin": ["deleted"]}}]}, + top_k=10, + ) + self.assertEqual([p.payload["status"] for p in res[0]], ["active"]) + + def test_build_field_condition_multiple_match_operators(self): + """_build_field_condition directly returning a list of conditions when given multiple match ops.""" + conds = self.qdrant._build_field_condition("status", {"ne": "archived", "nin": ["deleted"]}) + self.assertIsInstance(conds, list) + self.assertEqual(len(conds), 2) + self.assertEqual(conds[0].key, "status") + self.assertEqual(conds[1].key, "status") + self.assertEqual(conds[0].match.except_, ["archived"]) + self.assertEqual(conds[1].match.except_, ["deleted"]) + + def test_build_field_condition_contains_and_icontains(self): + """_build_field_condition with contains and icontains produces MatchText for both.""" + conds = self.qdrant._build_field_condition("bio", {"contains": "python", "icontains": "django"}) + self.assertIsInstance(conds, list) + self.assertEqual(len(conds), 2) + self.assertEqual(conds[0].match.text, "python") + self.assertEqual(conds[1].match.text, "django") + + def test_multi_operator_with_range_and_multiple_match(self): + """Field filter with range and multiple match operators applies all.""" + result = self.qdrant._create_filter({"priority": {"gte": 1, "lte": 10, "ne": 5, "nin": [7]}}) + self.assertIsInstance(result, Filter) + self.assertEqual(len(result.must), 3) + range_conds = [c for c in result.must if c.range is not None] + match_conds = [c for c in result.must if c.match is not None] + self.assertEqual(len(range_conds), 1) + self.assertEqual(len(match_conds), 2) + self.assertEqual(range_conds[0].range.gte, 1) + self.assertEqual(range_conds[0].range.lte, 10) + except_lists = [c.match.except_ for c in match_conds] + self.assertIn([5], except_lists) + self.assertIn([7], except_lists)