From 0d362bdf0cb0a395b7cd8fed425aa469877109ff Mon Sep 17 00:00:00 2001 From: TonyTonyCoder11 Date: Thu, 6 Aug 2026 09:45:21 +0200 Subject: [PATCH] Have kdrant migrate create the collection it migrates into The first release build caught it: the migration refused because the target did not exist, which is true of every collection nobody has made yet and therefore of the whole command. The workflow builds the binaries and proves them against a real Qdrant before anything is published, so nothing shipped. The target is created from the source's own vectors rather than from a size and a distance the person at the terminal did not choose. --shards and --replicas override, which is what turns a copy into the re-shard the help text was already promising. The decision is a function of its inputs rather than a lambda that writes into a builder, so it is asserted without a Qdrant to write to: sizes, distances, storage, named vectors, a multi-vector comparator and a sparse modifier all have to arrive intact. HNSW, quantization and optimizer settings deliberately do not, because they were tuned for a layout the target may not have. --- CHANGELOG.md | 5 + README.md | 8 +- .../kotlin/dev/kdrant/cli/Commands.kt | 79 +++++++++++- .../dev/kdrant/cli/MigrateTargetTest.kt | 116 ++++++++++++++++++ 4 files changed, 203 insertions(+), 5 deletions(-) create mode 100644 kdrant-cli/src/commonTest/kotlin/dev/kdrant/cli/MigrateTargetTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index f4ef868..2742f31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,11 @@ it means for a klib. A degraded cluster reports itself. And there is a binary. runners, run against a real Qdrant before they are allowed near a release, attested with SLSA provenance and attached to the GitHub Release with their checksums. It cannot embed, so it migrates what does not need re-embedding, and the help text says so. + `kdrant migrate` creates the target from the source's own vectors rather than asking for a size and a + distance the person at the terminal did not choose, and `--shards` and `--replicas` override, which is + what makes `kdrant migrate a b --shards 4` a re-shard. The first release build found that it did not: + the migration refused because the target did not exist, which is true of every collection nobody has + made yet and therefore of the whole command. - **The client contract against four Qdrant versions** (M50). `QdrantVersionMatrixIntegrationTest` runs the shared contract against the four most recent minors and writes the result into the README between generated markers. It does not fail the build: a cell that is red against an older server is the diff --git a/README.md b/README.md index 962f8d6..8a42387 100644 --- a/README.md +++ b/README.md @@ -491,9 +491,11 @@ export QDRANT_API_KEY=... # a key on a command line is a key in the she ./kdrant migrate articles articles-v2 --alias articles --checkpoint /var/tmp/articles.checkpoint ``` -That is the whole command to move a collection. It copies in id order, remembers where it got to, -checks the counts and the recall, and moves the alias only once the check passes. It cannot embed, so -it moves what does not need new vectors: a re-shard, a config change, a copy between clusters. +That is the whole command to move a collection. It creates the target from the source's own vectors, +copies in id order, remembers where it got to, checks the counts and the recall, and moves the alias +only once the check passes. `--shards` and `--replicas` override the source's layout, which is what +makes it a re-shard. It cannot embed, so it moves what does not need new vectors: a re-shard, a config +change, a copy between clusters. `kdrant collections`, `kdrant scroll` and `kdrant snapshot create|list|download|restore|delete` are the rest of it; `kdrant --help` prints the flags. It is not a query tool, because Qdrant's own dashboard is diff --git a/kdrant-cli/src/commonMain/kotlin/dev/kdrant/cli/Commands.kt b/kdrant-cli/src/commonMain/kotlin/dev/kdrant/cli/Commands.kt index 128b702..aecf5b1 100644 --- a/kdrant-cli/src/commonMain/kotlin/dev/kdrant/cli/Commands.kt +++ b/kdrant-cli/src/commonMain/kotlin/dev/kdrant/cli/Commands.kt @@ -1,8 +1,13 @@ package dev.kdrant.cli import dev.kdrant.QdrantClient +import dev.kdrant.dsl.CreateCollectionBuilder +import dev.kdrant.dsl.VectorParamsBuilder import dev.kdrant.migrate.MigrationVerification import dev.kdrant.migrate.migrateCollection +import dev.kdrant.model.CollectionParams +import dev.kdrant.model.VectorParams +import dev.kdrant.model.VectorsConfig import dev.kdrant.model.WithPayload import kotlinx.coroutines.flow.take import kotlinx.coroutines.flow.toList @@ -92,6 +97,11 @@ internal object Commands { /** * M42's procedure, with the checkpoint on disk and the recall threshold on the command line. * + * **The target is created from the source's own configuration.** A migration that does not + * re-embed keeps the same vectors, so asking the person at the terminal to restate a vector size + * and a distance they did not choose is asking them to get it wrong. `--shards` and `--replicas` + * override, which is what makes `kdrant migrate a b --shards 4` a re-shard rather than a copy. + * * **It cannot re-embed.** A CLI has no model, so it migrates what does not need new vectors: a * re-shard, a config change, a copy between clusters. Saying that plainly is better than a tool * that looks like it can move a collection onto a new embedding model and quietly copies the old @@ -105,7 +115,14 @@ internal object Commands { val batch = arguments.intOption("batch") ?: DEFAULT_MIGRATE_BATCH val checkpointPath = arguments.option("checkpoint") ?: "kdrant-migrate-$from-to-$to.checkpoint" + val source = client.getCollection(from).config?.params + ?: fail("$from reports no configuration, so there is nothing to create $to from") + val shards = arguments.intOption("shards") ?: source.shardNumber + val replicas = arguments.intOption("replicas") ?: source.replicationFactor + out("copying $from -> $to (batch $batch, recall >= $recall, checkpoint $checkpointPath)") + out("target: ${describe(source)}, ${shards ?: 1} shard(s), ${replicas ?: 1} replica(s)") + val report = client.migrateCollection( from = from, to = to, @@ -113,12 +130,67 @@ internal object Commands { batchSize = batch, checkpoints = FileCheckpointStore(files, checkpointPath), verification = MigrationVerification(minRecall = recall), + createTarget = { create(targetConfigurationOf(source, shards, replicas)) }, ) out("copied ${report.copied} point(s); source ${report.sourceCount}, target ${report.targetCount}") out("recall ${report.recall}") out(if (alias == null) "no alias was moved (pass --alias to move one)" else "alias '$alias' now points at $to") } + /** + * What the target should be created as: the source's vectors, with the sharding the caller asked + * for. A function of its inputs rather than a lambda that writes into a builder, so the decision + * can be asserted without a Qdrant to write it to. + * + * Only the vectors and the sharding, deliberately. HNSW tuning, quantization and optimizer + * settings are things an operator changed on the source for a reason, and carrying them onto a + * collection that is about to be re-sharded would carry a decision made for a different layout. + * They stay the server's defaults on the target and are set afterwards if they are wanted. + */ + internal fun targetConfigurationOf( + source: CollectionParams, + shards: Int?, + replicas: Int?, + ): CollectionParams = CollectionParams( + vectors = source.vectors, + sparseVectors = source.sparseVectors, + shardNumber = shards ?: source.shardNumber, + replicationFactor = replicas ?: source.replicationFactor, + onDiskPayload = source.onDiskPayload, + ) + + /** Puts [target] into the collection DSL. Mechanical, and the only part that needs a builder. */ + private fun CreateCollectionBuilder.create(target: CollectionParams) { + when (val vectors = target.vectors) { + is VectorsConfig.Single -> vector { copyFrom(vectors.params) } + is VectorsConfig.Named -> vectors.vectors.forEach { (name, params) -> + namedVector(name) { copyFrom(params) } + } + null -> Unit + } + target.sparseVectors?.forEach { (name, params) -> sparseVector(name) { modifier = params.modifier } } + target.onDiskPayload?.let { onDiskPayload = it } + target.shardNumber?.let { shardNumber = it } + target.replicationFactor?.let { replicationFactor = it } + } + + private fun VectorParamsBuilder.copyFrom(params: VectorParams) { + size = params.size + distance = params.distance + params.onDisk?.let { onDisk = it } + params.datatype?.let { datatype = it } + params.multivectorConfig?.let { multivector = it.comparator } + } + + /** What the target is about to be created as, printed before anything is copied. */ + private fun describe(source: CollectionParams): String = when (val vectors = source.vectors) { + is VectorsConfig.Single -> "one ${vectors.params.size}-dimension vector, ${vectors.params.distance}" + is VectorsConfig.Named -> vectors.vectors.entries.joinToString(", ") { (name, params) -> + "$name ${params.size}d ${params.distance}" + } + null -> "no dense vectors" + } + fun help(out: (String) -> Unit) { USAGE.trimIndent().lines().forEach(out) } @@ -139,7 +211,8 @@ internal object Commands { kdrant snapshot download [--out FILE] kdrant snapshot restore kdrant snapshot delete - kdrant migrate [--alias A] [--batch N] [--recall R] [--checkpoint FILE] + kdrant migrate [--alias A] [--shards N] [--replicas N] + [--batch N] [--recall R] [--checkpoint FILE] Connection: --host HOST default localhost, or ${'$'}QDRANT_HOST @@ -148,7 +221,9 @@ internal object Commands { --tls use HTTPS --ca-file FILE trust this PEM bundle instead of the system store - migrate copies points as they are. It cannot embed, so it moves what does not need new + migrate creates the target from the source's own vectors, so you do not restate a size and a + distance you did not choose; --shards and --replicas override, which is what makes it a + re-shard. It copies points as they are and cannot embed, so it moves what does not need new vectors: a re-shard, a config change, a copy between clusters. The alias moves only after the count and recall checks pass, and the checkpoint file makes an interrupted run resumable. """ diff --git a/kdrant-cli/src/commonTest/kotlin/dev/kdrant/cli/MigrateTargetTest.kt b/kdrant-cli/src/commonTest/kotlin/dev/kdrant/cli/MigrateTargetTest.kt new file mode 100644 index 0000000..751b352 --- /dev/null +++ b/kdrant-cli/src/commonTest/kotlin/dev/kdrant/cli/MigrateTargetTest.kt @@ -0,0 +1,116 @@ +package dev.kdrant.cli + +import dev.kdrant.model.CollectionParams +import dev.kdrant.model.Distance +import dev.kdrant.model.Modifier +import dev.kdrant.model.MultiVectorComparator +import dev.kdrant.model.MultiVectorConfig +import dev.kdrant.model.SparseVectorParams +import dev.kdrant.model.VectorDatatype +import dev.kdrant.model.VectorParams +import dev.kdrant.model.VectorsConfig +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +/** + * `kdrant migrate a b` has to create `b`, and the first release build found that it did not: the + * migration refused because the target did not exist, which is true of every collection that has not + * been made yet and therefore of the whole command. + * + * The configuration is taken from the source rather than asked for, so what is under test is that it + * arrives intact. A vector size carried wrong is a migration that fails on its first upsert; a + * distance carried wrong is one that succeeds and returns the wrong neighbours. + */ +class MigrateTargetTest { + + @Test + fun `a single anonymous vector keeps its size and distance and storage`() { + val target = Commands.targetConfigurationOf( + CollectionParams( + vectors = VectorsConfig.Single( + VectorParams( + size = 1536, + distance = Distance.COSINE, + onDisk = true, + datatype = VectorDatatype.FLOAT16, + ), + ), + onDiskPayload = true, + ), + shards = null, + replicas = null, + ) + + val vectors = target.vectors as VectorsConfig.Single + assertEquals(1536L, vectors.params.size) + assertEquals(Distance.COSINE, vectors.params.distance) + assertEquals(true, vectors.params.onDisk) + assertEquals(VectorDatatype.FLOAT16, vectors.params.datatype) + assertEquals(true, target.onDiskPayload) + } + + @Test + fun `named vectors keep their names and a multi-vector keeps its comparator`() { + val target = Commands.targetConfigurationOf( + CollectionParams( + vectors = VectorsConfig.Named( + mapOf( + "text" to VectorParams(size = 768, distance = Distance.COSINE), + "colbert" to VectorParams( + size = 128, + distance = Distance.DOT, + multivectorConfig = MultiVectorConfig(MultiVectorComparator.MAX_SIM), + ), + ), + ), + ), + shards = null, + replicas = null, + ) + + val vectors = target.vectors as VectorsConfig.Named + assertEquals(setOf("text", "colbert"), vectors.vectors.keys) + assertEquals(768L, vectors.vectors.getValue("text").size) + assertEquals( + MultiVectorComparator.MAX_SIM, + vectors.vectors.getValue("colbert").multivectorConfig?.comparator, + ) + } + + @Test + fun `a sparse vector keeps its modifier because IDF changes what the scores mean`() { + val target = Commands.targetConfigurationOf( + CollectionParams( + vectors = VectorsConfig.Single(VectorParams(size = 4, distance = Distance.DOT)), + sparseVectors = mapOf("keywords" to SparseVectorParams(modifier = Modifier.IDF)), + ), + shards = null, + replicas = null, + ) + + assertEquals(Modifier.IDF, target.sparseVectors?.getValue("keywords")?.modifier) + } + + @Test + fun `the sharding is the source's unless the caller overrides it`() { + val source = CollectionParams( + vectors = VectorsConfig.Single(VectorParams(size = 4, distance = Distance.DOT)), + shardNumber = 2, + replicationFactor = 1, + ) + + assertEquals(2, Commands.targetConfigurationOf(source, null, null).shardNumber) + assertEquals(1, Commands.targetConfigurationOf(source, null, null).replicationFactor) + assertEquals(8, Commands.targetConfigurationOf(source, shards = 8, replicas = null).shardNumber) + assertEquals(3, Commands.targetConfigurationOf(source, shards = null, replicas = 3).replicationFactor) + } + + @Test + fun `a source with no vectors at all produces no vectors rather than a guess`() { + val target = Commands.targetConfigurationOf(CollectionParams(), null, null) + + assertNull(target.vectors) + assertNull(target.sparseVectors) + } +}