[MOD-17706] finalize SVSIndex::relabelVector

cmake/svs.cmake

@@ -123,6 +123,29 @@ if(USE_SVS)
         message("SVS LVQ implementation not found")
         add_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_LVQ=0")
     endif()
+
+    # `replace_external_id` is newer than the pre-built SVS releases that SVS_SHARED_LIB downloads,
+    # so whether it is available depends on which SVS this build ended up with rather than on the
+    # platform. Detect it in the header instead of assuming: it is a method, not a file, so this
+    # greps the header the LVQ check would have tested for existence.
+    set(SVS_DYNAMIC_INDEX_HEADER "svs/index/vamana/dynamic_index.h")
+    set(SVS_HAS_REPLACE_EXTERNAL_ID 0)
+    if(EXISTS "${svs_SOURCE_DIR}/include/${SVS_DYNAMIC_INDEX_HEADER}")
+        file(READ "${svs_SOURCE_DIR}/include/${SVS_DYNAMIC_INDEX_HEADER}" SVS_DYNAMIC_INDEX_SRC)
+        string(FIND "${SVS_DYNAMIC_INDEX_SRC}" "replace_external_id" SVS_REPLACE_EXTERNAL_ID_POS)
+        if(NOT SVS_REPLACE_EXTERNAL_ID_POS EQUAL -1)
+            set(SVS_HAS_REPLACE_EXTERNAL_ID 1)
+        endif()
+        unset(SVS_DYNAMIC_INDEX_SRC)
+    endif()
+
+    if(SVS_HAS_REPLACE_EXTERNAL_ID)
+        message("SVS replace_external_id found - SVS relabeling enabled")
+        add_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_REPLACE_EXTERNAL_ID=1")
+    else()
+        message("SVS replace_external_id not found - SVS relabeling reports unsupported")
+        add_compile_definitions(VectorSimilarity PUBLIC "HAVE_SVS_REPLACE_EXTERNAL_ID=0")
+    endif()
 else()
     message(STATUS "SVS support disabled")
     add_compile_definitions("HAVE_SVS=0")

src/VecSim/algorithms/svs/svs.h

@@ -549,6 +549,29 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
         return deleteVectorsImpl(labels, n);
     }
 
+#if HAVE_SVS_REPLACE_EXTERNAL_ID
+    // Only declared when the SVS this was built against offers `replace_external_id`. The
+    // pre-built SVS releases predate it, so where it is missing this override is left out and the
+    // interface default reports `VecSimRelabel_Unsupported` - which tells a caller to fall back to
+    // delete + insert rather than read it as a no-op.
+    VecSimRelabelCode relabelVector(labelType old_label, labelType new_label) override {
+        if (old_label == new_label) {
+            return VecSimRelabel_SameLabel;
+        }
+        // `isLabelExists` also covers the index that never held a vector, where `impl_` has not
+        // been created yet and so trivially holds nothing.
+        if (!isLabelExists(old_label)) {
+            return VecSimRelabel_OldLabelMissing;
+        }
+        if (isLabelExists(new_label)) {
+            return VecSimRelabel_NewLabelTaken;
+        }
+
+        impl_->replace_external_id(old_label, new_label);
+        return VecSimRelabel_OK;
+    }
+#endif // HAVE_SVS_REPLACE_EXTERNAL_ID
+
     bool isLabelExists(labelType label) const override {
         return impl_ ? impl_->has_id(label) : false;
     }

src/VecSim/algorithms/svs/svs_tiered.h

@@ -964,6 +964,71 @@ class TieredSVSIndex : public VecSimTieredIndex<DataType, float> {
         }
         return ret;
     }
+#if HAVE_SVS_REPLACE_EXTERNAL_ID
+    // Only declared when the SVS this was built against offers `replace_external_id`, mirroring
+    // `SVSIndex::relabelVector`. Left out otherwise, so the interface default reports
+    // `VecSimRelabel_Unsupported` for the whole tier rather than this moving a buffered label
+    // and refusing an ingested one -- a caller cannot act on a capability that depends on which
+    // tier happens to hold the label. It also keeps the runtime probe honest: an override that
+    // answered `SameLabel` before consulting the backend would look capable on a build that
+    // is not.
+    /**
+     * Move `old_label` onto `new_label`, leaving the vector where it is in whichever tier holds
+     * it. `new_label` must be unused in both tiers, not just the one holding `old_label`: a
+     * multi-value label routinely has copies in each, and a target taken in either would collide
+     * once the buffer drains.
+     *
+     * Reports `Unsupported` when the backend holds the label and cannot move it, which is the
+     * case when built against an SVS without `replace_external_id`. All-or-nothing: on any code
+     * other than `VecSimRelabel_OK` both tiers are untouched.
+     *
+     * `updateJobMutex` is taken first, in the order `updateSVSIndex` takes its own locks. Holding
+     * it is what makes this correct rather than merely serialised: an update job snapshots the
+     * buffer's labels *by value* and afterwards reconciles only id swaps and deletions, so a
+     * rename landing inside its window would be invisible to it and the vector would reach the
+     * backend under the old label.
+     */
+    VecSimRelabelCode relabelVector(labelType old_label, labelType new_label) override {
+        if (old_label == new_label) {
+            return VecSimRelabel_SameLabel;
+        }
+        auto *svs_index = GetSVSIndex();
+
+        std::lock_guard<std::mutex> update_lock(this->updateJobMutex);
+        std::unique_lock<std::shared_mutex> flat_lock(this->flatIndexGuard);
+        std::unique_lock<std::shared_mutex> main_lock(this->mainIndexGuard);
+
+        const bool in_flat = this->frontendIndex->isLabelExists(old_label);
+        const bool in_backend = svs_index->isLabelExists(old_label);
+        if (!in_flat && !in_backend) {
+            return VecSimRelabel_OldLabelMissing;
+        }
+        if (this->frontendIndex->isLabelExists(new_label) || svs_index->isLabelExists(new_label)) {
+            return VecSimRelabel_NewLabelTaken;
+        }
+
+        // The backend goes first because it is the tier that can refuse; refusing after the
+        // buffer had already moved would leave the label half applied.
+        if (in_backend) {
+            const VecSimRelabelCode backend_ret =
+                this->backendIndex->relabelVector(old_label, new_label);
+            if (backend_ret != VecSimRelabel_OK) {
+                return backend_ret;
+            }
+        }
+        if (in_flat) {
+            const VecSimRelabelCode flat_ret =
+                this->frontendIndex->relabelVector(old_label, new_label);
+#ifdef BUILD_TESTS
+            assert(flat_ret == VecSimRelabel_OK && "the buffer just reported holding this label");
+#endif
+            UNUSED(flat_ret);
+        }
+        return VecSimRelabel_OK;
+    }
+
+#endif // HAVE_SVS_REPLACE_EXTERNAL_ID
+
     size_t getNumMarkedDeleted() const override {
         return this->GetSVSIndex()->getNumMarkedDeleted();
     }

src/python_bindings/bindings.cpp

@@ -106,12 +106,11 @@ class PyVecSimIndex {
     template <typename DataType, typename DistType, typename NPArrayType = DataType>
     inline py::object rawVectorsAsNumpy(labelType label, size_t dim) {
         std::vector<std::vector<DataType>> vectors;
-        if (index->basicInfo().algo == VecSimAlgo_BF) {
-            dynamic_cast<BruteForceIndex<DataType, DistType> *>(this->index.get())
-                ->getDataByLabel(label, vectors);
+        if (auto *tiered =
+                dynamic_cast<VecSimTieredIndex<DataType, DistType> *>(this->index.get())) {
+            tiered->getDataByLabel(label, vectors);
         } else {
-            // index is HNSW
-            dynamic_cast<HNSWIndex<DataType, DistType> *>(this->index.get())
+            dynamic_cast<VecSimIndexAbstract<DataType, DistType> *>(this->index.get())
                 ->getDataByLabel(label, vectors);
         }
         size_t n_vectors = vectors.size();
@@ -216,6 +215,11 @@ class PyVecSimIndex {
 
     void runGC() { VecSimTieredIndex_GC(index.get()); }
 
+    VecSimRelabelCode relabelVector(labelType old_label, labelType new_label) {
+        py::gil_scoped_release py_gil;
+        return VecSimIndex_RelabelVector(index.get(), old_label, new_label);
+    }
+
     py::object getVector(labelType label) {
         VecSimIndexBasicInfo info = index->basicInfo();
         size_t dim = info.dim;
@@ -713,6 +717,14 @@ PYBIND11_MODULE(VecSim, m) {
         .def_readwrite("initialCapacity", &BFParams::initialCapacity)
         .def_readwrite("blockSize", &BFParams::blockSize);
 
+    py::enum_<VecSimRelabelCode>(m, "VecSimRelabelCode")
+        .value("VecSimRelabel_OK", VecSimRelabel_OK)
+        .value("VecSimRelabel_OldLabelMissing", VecSimRelabel_OldLabelMissing)
+        .value("VecSimRelabel_NewLabelTaken", VecSimRelabel_NewLabelTaken)
+        .value("VecSimRelabel_SameLabel", VecSimRelabel_SameLabel)
+        .value("VecSimRelabel_Unsupported", VecSimRelabel_Unsupported)
+        .export_values();
+
     py::enum_<VecSimSvsQuantBits>(m, "VecSimSvsQuantBits")
         .value("VecSimSvsQuant_NONE", VecSimSvsQuant_NONE)
         .value("VecSimSvsQuant_Scalar", VecSimSvsQuant_Scalar)
@@ -799,6 +811,8 @@ PYBIND11_MODULE(VecSim, m) {
         .def("create_batch_iterator", &PyVecSimIndex::createBatchIterator, py::arg("query_blob"),
              py::arg("query_param") = nullptr)
         .def("get_vector", &PyVecSimIndex::getVector)
+        .def("relabel_vector", &PyVecSimIndex::relabelVector, py::arg("old_label"),
+             py::arg("new_label"))
         .def("run_gc", &PyVecSimIndex::runGC);
 
     py::class_<PyHNSWLibIndex, PyVecSimIndex>(m, "HNSWIndex")

tests/flow/common.py

@@ -174,3 +174,10 @@ def fp32_expand_and_calc_cosine_dist(a, b):
     a_float32 = a.astype(np.float32)
     b_float32 = b.astype(np.float32)
     return spatial.distance.cosine(a_float32, b_float32)
+
+# The pre-built SVS releases predate `replace_external_id`, so a build that picked one up reports
+# relabeling as unsupported rather than implementing it. There is no python-visible build flag for
+# that, so ask the index itself on a throwaway label - a real implementation cannot answer
+# Unsupported, so this cannot mask a broken one.
+def svs_relabel_supported(index, label):
+    return index.relabel_vector(label, label) != VecSimRelabel_Unsupported

tests/flow/test_bruteforce.py

@@ -747,3 +747,61 @@ def test_range_query(self):
 
     def test_multi_value(self):
         self.multi_value(create_uint8_vectors)
+
+
+def test_relabel_vector(test_logger):
+    dim = 16
+    num_elements = 100
+    index = create_flat_index(dim, VecSimMetric_L2, VecSimType_FLOAT32)
+
+    data = np.float32(np.random.random((num_elements, dim)))
+    for label, vector in enumerate(data):
+        index.add_vector(vector, label)
+
+    old_label, new_label = 7, num_elements + 500
+    assert index.relabel_vector(old_label, new_label) == VecSimRelabel_OK
+
+    # The label moved without the data moving, and the index neither grew nor shrank.
+    assert index.index_size() == num_elements
+    assert_allclose(index.get_vector(new_label)[0], data[old_label], rtol=1e-6)
+    assert index.get_vector(old_label).shape == (0, dim)
+
+    # Still searchable, and answering under the new label.
+    labels, distances = index.knn_query(data[old_label], 1)
+    assert labels[0][0] == new_label
+    assert distances[0][0] < 1e-6
+
+    # Each rejection is reported distinctly, and none of them modifies the index.
+    assert index.relabel_vector(num_elements + 1, 0) == VecSimRelabel_OldLabelMissing
+    assert index.relabel_vector(0, 1) == VecSimRelabel_NewLabelTaken
+    assert index.relabel_vector(0, 0) == VecSimRelabel_SameLabel
+    assert index.index_size() == num_elements
+    test_logger.info("flat index relabel_vector moved the label, keeping the vector data")
+
+
+def test_relabel_vector_multi(test_logger):
+    dim = 16
+    num_labels = 20
+    per_label = 3
+    index = create_flat_index(dim, VecSimMetric_L2, VecSimType_FLOAT32, is_multi=True)
+
+    data = np.float32(np.random.random((num_labels, per_label, dim)))
+    for label in range(num_labels):
+        for vector in data[label]:
+            index.add_vector(vector, label)
+
+    old_label, new_label = 7, num_labels + 500
+    assert index.relabel_vector(old_label, new_label) == VecSimRelabel_OK
+
+    # Every vector under the label moves as a unit, keeping its data and insertion order.
+    assert index.index_size() == num_labels * per_label
+    assert_allclose(index.get_vector(new_label), data[old_label], rtol=1e-6)
+    assert index.get_vector(old_label).shape == (0, dim)
+
+    # Moving onto an occupied label must be rejected: accepting it would silently merge two
+    # labels' vectors, which is the failure mode unique to a multi index.
+    assert index.relabel_vector(new_label, 0) == VecSimRelabel_NewLabelTaken
+    assert index.get_vector(new_label).shape == (per_label, dim)
+    assert index.get_vector(0).shape == (per_label, dim)
+    assert index.index_size() == num_labels * per_label
+    test_logger.info("flat multi relabel_vector moved every vector under the label")

tests/flow/test_hnsw.py

@@ -1156,3 +1156,64 @@ def test_range_query(self, test_logger):
 
     def test_multi_value(self, test_logger):
         self.multi_value(create_uint8_vectors, test_logger)
+
+
+def test_relabel_vector(test_logger):
+    dim = 16
+    num_elements = 100
+    index = create_hnsw_index(dim, num_elements, VecSimMetric_L2, VecSimType_FLOAT32)
+
+    data = np.float32(np.random.random((num_elements, dim)))
+    for label, vector in enumerate(data):
+        index.add_vector(vector, label)
+
+    old_label, new_label = 7, num_elements + 500
+    assert index.relabel_vector(old_label, new_label) == VecSimRelabel_OK
+
+    # A relabel is bookkeeping only: the graph is untouched, so the vector keeps its data and its
+    # place in the index rather than being reinserted.
+    assert index.index_size() == num_elements
+    assert_allclose(index.get_vector(new_label)[0], data[old_label], rtol=1e-6)
+    assert index.get_vector(old_label).shape == (0, dim)
+    assert index.check_integrity()
+
+    labels, distances = index.knn_query(data[old_label], 1)
+    assert labels[0][0] == new_label
+    assert distances[0][0] < 1e-6
+
+    # Each rejection is reported distinctly, and none of them modifies the index.
+    assert index.relabel_vector(num_elements + 1, 0) == VecSimRelabel_OldLabelMissing
+    assert index.relabel_vector(0, 1) == VecSimRelabel_NewLabelTaken
+    assert index.relabel_vector(0, 0) == VecSimRelabel_SameLabel
+    assert index.index_size() == num_elements
+    assert index.check_integrity()
+    test_logger.info("HNSW relabel_vector moved the label, leaving the graph intact")
+
+
+def test_relabel_vector_multi(test_logger):
+    dim = 16
+    num_labels = 20
+    per_label = 3
+    index = create_hnsw_index(dim, num_labels * per_label, VecSimMetric_L2, VecSimType_FLOAT32,
+                              is_multi=True)
+
+    data = np.float32(np.random.random((num_labels, per_label, dim)))
+    for label in range(num_labels):
+        for vector in data[label]:
+            index.add_vector(vector, label)
+
+    old_label, new_label = 7, num_labels + 500
+    assert index.relabel_vector(old_label, new_label) == VecSimRelabel_OK
+
+    # All the label's internal ids are re-pointed together, and the graph is left alone.
+    assert index.index_size() == num_labels * per_label
+    assert_allclose(index.get_vector(new_label), data[old_label], rtol=1e-6)
+    assert index.get_vector(old_label).shape == (0, dim)
+    assert index.check_integrity()
+
+    # Accepting a move onto an occupied label would merge two labels' vectors.
+    assert index.relabel_vector(new_label, 0) == VecSimRelabel_NewLabelTaken
+    assert index.get_vector(new_label).shape == (per_label, dim)
+    assert index.get_vector(0).shape == (per_label, dim)
+    assert index.check_integrity()
+    test_logger.info("HNSW multi relabel_vector moved every vector under the label")