diff --git a/CHANGELOG.md b/CHANGELOG.md index f4ef868..2742f31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,11 @@ it means for a klib. A degraded cluster reports itself. And there is a binary. runners, run against a real Qdrant before they are allowed near a release, attested with SLSA provenance and attached to the GitHub Release with their checksums. It cannot embed, so it migrates what does not need re-embedding, and the help text says so. + `kdrant migrate` creates the target from the source's own vectors rather than asking for a size and a + distance the person at the terminal did not choose, and `--shards` and `--replicas` override, which is + what makes `kdrant migrate a b --shards 4` a re-shard. The first release build found that it did not: + the migration refused because the target did not exist, which is true of every collection nobody has + made yet and therefore of the whole command. - **The client contract against four Qdrant versions** (M50). `QdrantVersionMatrixIntegrationTest` runs the shared contract against the four most recent minors and writes the result into the README between generated markers. It does not fail the build: a cell that is red against an older server is the diff --git a/README.md b/README.md index 962f8d6..8a42387 100644 --- a/README.md +++ b/README.md @@ -491,9 +491,11 @@ export QDRANT_API_KEY=... # a key on a command line is a key in the she ./kdrant migrate articles articles-v2 --alias articles --checkpoint /var/tmp/articles.checkpoint ``` -That is the whole command to move a collection. It copies in id order, remembers where it got to, -checks the counts and the recall, and moves the alias only once the check passes. It cannot embed, so -it moves what does not need new vectors: a re-shard, a config change, a copy between clusters. +That is the whole command to move a collection. It creates the target from the source's own vectors, +copies in id order, remembers where it got to, checks the counts and the recall, and moves the alias +only once the check passes. `--shards` and `--replicas` override the source's layout, which is what +makes it a re-shard. It cannot embed, so it moves what does not need new vectors: a re-shard, a config +change, a copy between clusters. `kdrant collections`, `kdrant scroll` and `kdrant snapshot create|list|download|restore|delete` are the rest of it; `kdrant --help` prints the flags. It is not a query tool, because Qdrant's own dashboard is diff --git a/kdrant-cli/src/commonMain/kotlin/dev/kdrant/cli/Commands.kt b/kdrant-cli/src/commonMain/kotlin/dev/kdrant/cli/Commands.kt index 128b702..aecf5b1 100644 --- a/kdrant-cli/src/commonMain/kotlin/dev/kdrant/cli/Commands.kt +++ b/kdrant-cli/src/commonMain/kotlin/dev/kdrant/cli/Commands.kt @@ -1,8 +1,13 @@ package dev.kdrant.cli import dev.kdrant.QdrantClient +import dev.kdrant.dsl.CreateCollectionBuilder +import dev.kdrant.dsl.VectorParamsBuilder import dev.kdrant.migrate.MigrationVerification import dev.kdrant.migrate.migrateCollection +import dev.kdrant.model.CollectionParams +import dev.kdrant.model.VectorParams +import dev.kdrant.model.VectorsConfig import dev.kdrant.model.WithPayload import kotlinx.coroutines.flow.take import kotlinx.coroutines.flow.toList @@ -92,6 +97,11 @@ internal object Commands { /** * M42's procedure, with the checkpoint on disk and the recall threshold on the command line. * + * **The target is created from the source's own configuration.** A migration that does not + * re-embed keeps the same vectors, so asking the person at the terminal to restate a vector size + * and a distance they did not choose is asking them to get it wrong. `--shards` and `--replicas` + * override, which is what makes `kdrant migrate a b --shards 4` a re-shard rather than a copy. + * * **It cannot re-embed.** A CLI has no model, so it migrates what does not need new vectors: a * re-shard, a config change, a copy between clusters. Saying that plainly is better than a tool * that looks like it can move a collection onto a new embedding model and quietly copies the old @@ -105,7 +115,14 @@ internal object Commands { val batch = arguments.intOption("batch") ?: DEFAULT_MIGRATE_BATCH val checkpointPath = arguments.option("checkpoint") ?: "kdrant-migrate-$from-to-$to.checkpoint" + val source = client.getCollection(from).config?.params + ?: fail("$from reports no configuration, so there is nothing to create $to from") + val shards = arguments.intOption("shards") ?: source.shardNumber + val replicas = arguments.intOption("replicas") ?: source.replicationFactor + out("copying $from -> $to (batch $batch, recall >= $recall, checkpoint $checkpointPath)") + out("target: ${describe(source)}, ${shards ?: 1} shard(s), ${replicas ?: 1} replica(s)") + val report = client.migrateCollection( from = from, to = to, @@ -113,12 +130,67 @@ internal object Commands { batchSize = batch, checkpoints = FileCheckpointStore(files, checkpointPath), verification = MigrationVerification(minRecall = recall), + createTarget = { create(targetConfigurationOf(source, shards, replicas)) }, ) out("copied ${report.copied} point(s); source ${report.sourceCount}, target ${report.targetCount}") out("recall ${report.recall}") out(if (alias == null) "no alias was moved (pass --alias to move one)" else "alias '$alias' now points at $to") } + /** + * What the target should be created as: the source's vectors, with the sharding the caller asked + * for. A function of its inputs rather than a lambda that writes into a builder, so the decision + * can be asserted without a Qdrant to write it to. + * + * Only the vectors and the sharding, deliberately. HNSW tuning, quantization and optimizer + * settings are things an operator changed on the source for a reason, and carrying them onto a + * collection that is about to be re-sharded would carry a decision made for a different layout. + * They stay the server's defaults on the target and are set afterwards if they are wanted. + */ + internal fun targetConfigurationOf( + source: CollectionParams, + shards: Int?, + replicas: Int?, + ): CollectionParams = CollectionParams( + vectors = source.vectors, + sparseVectors = source.sparseVectors, + shardNumber = shards ?: source.shardNumber, + replicationFactor = replicas ?: source.replicationFactor, + onDiskPayload = source.onDiskPayload, + ) + + /** Puts [target] into the collection DSL. Mechanical, and the only part that needs a builder. */ + private fun CreateCollectionBuilder.create(target: CollectionParams) { + when (val vectors = target.vectors) { + is VectorsConfig.Single -> vector { copyFrom(vectors.params) } + is VectorsConfig.Named -> vectors.vectors.forEach { (name, params) -> + namedVector(name) { copyFrom(params) } + } + null -> Unit + } + target.sparseVectors?.forEach { (name, params) -> sparseVector(name) { modifier = params.modifier } } + target.onDiskPayload?.let { onDiskPayload = it } + target.shardNumber?.let { shardNumber = it } + target.replicationFactor?.let { replicationFactor = it } + } + + private fun VectorParamsBuilder.copyFrom(params: VectorParams) { + size = params.size + distance = params.distance + params.onDisk?.let { onDisk = it } + params.datatype?.let { datatype = it } + params.multivectorConfig?.let { multivector = it.comparator } + } + + /** What the target is about to be created as, printed before anything is copied. */ + private fun describe(source: CollectionParams): String = when (val vectors = source.vectors) { + is VectorsConfig.Single -> "one ${vectors.params.size}-dimension vector, ${vectors.params.distance}" + is VectorsConfig.Named -> vectors.vectors.entries.joinToString(", ") { (name, params) -> + "$name ${params.size}d ${params.distance}" + } + null -> "no dense vectors" + } + fun help(out: (String) -> Unit) { USAGE.trimIndent().lines().forEach(out) } @@ -139,7 +211,8 @@ internal object Commands { kdrant snapshot download [--out FILE] kdrant snapshot restore kdrant snapshot delete - kdrant migrate [--alias A] [--batch N] [--recall R] [--checkpoint FILE] + kdrant migrate [--alias A] [--shards N] [--replicas N] + [--batch N] [--recall R] [--checkpoint FILE] Connection: --host HOST default localhost, or ${'$'}QDRANT_HOST @@ -148,7 +221,9 @@ internal object Commands { --tls use HTTPS --ca-file FILE trust this PEM bundle instead of the system store - migrate copies points as they are. It cannot embed, so it moves what does not need new + migrate creates the target from the source's own vectors, so you do not restate a size and a + distance you did not choose; --shards and --replicas override, which is what makes it a + re-shard. It copies points as they are and cannot embed, so it moves what does not need new vectors: a re-shard, a config change, a copy between clusters. The alias moves only after the count and recall checks pass, and the checkpoint file makes an interrupted run resumable. """ diff --git a/kdrant-cli/src/commonTest/kotlin/dev/kdrant/cli/MigrateTargetTest.kt b/kdrant-cli/src/commonTest/kotlin/dev/kdrant/cli/MigrateTargetTest.kt new file mode 100644 index 0000000..751b352 --- /dev/null +++ b/kdrant-cli/src/commonTest/kotlin/dev/kdrant/cli/MigrateTargetTest.kt @@ -0,0 +1,116 @@ +package dev.kdrant.cli + +import dev.kdrant.model.CollectionParams +import dev.kdrant.model.Distance +import dev.kdrant.model.Modifier +import dev.kdrant.model.MultiVectorComparator +import dev.kdrant.model.MultiVectorConfig +import dev.kdrant.model.SparseVectorParams +import dev.kdrant.model.VectorDatatype +import dev.kdrant.model.VectorParams +import dev.kdrant.model.VectorsConfig +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +/** + * `kdrant migrate a b` has to create `b`, and the first release build found that it did not: the + * migration refused because the target did not exist, which is true of every collection that has not + * been made yet and therefore of the whole command. + * + * The configuration is taken from the source rather than asked for, so what is under test is that it + * arrives intact. A vector size carried wrong is a migration that fails on its first upsert; a + * distance carried wrong is one that succeeds and returns the wrong neighbours. + */ +class MigrateTargetTest { + + @Test + fun `a single anonymous vector keeps its size and distance and storage`() { + val target = Commands.targetConfigurationOf( + CollectionParams( + vectors = VectorsConfig.Single( + VectorParams( + size = 1536, + distance = Distance.COSINE, + onDisk = true, + datatype = VectorDatatype.FLOAT16, + ), + ), + onDiskPayload = true, + ), + shards = null, + replicas = null, + ) + + val vectors = target.vectors as VectorsConfig.Single + assertEquals(1536L, vectors.params.size) + assertEquals(Distance.COSINE, vectors.params.distance) + assertEquals(true, vectors.params.onDisk) + assertEquals(VectorDatatype.FLOAT16, vectors.params.datatype) + assertEquals(true, target.onDiskPayload) + } + + @Test + fun `named vectors keep their names and a multi-vector keeps its comparator`() { + val target = Commands.targetConfigurationOf( + CollectionParams( + vectors = VectorsConfig.Named( + mapOf( + "text" to VectorParams(size = 768, distance = Distance.COSINE), + "colbert" to VectorParams( + size = 128, + distance = Distance.DOT, + multivectorConfig = MultiVectorConfig(MultiVectorComparator.MAX_SIM), + ), + ), + ), + ), + shards = null, + replicas = null, + ) + + val vectors = target.vectors as VectorsConfig.Named + assertEquals(setOf("text", "colbert"), vectors.vectors.keys) + assertEquals(768L, vectors.vectors.getValue("text").size) + assertEquals( + MultiVectorComparator.MAX_SIM, + vectors.vectors.getValue("colbert").multivectorConfig?.comparator, + ) + } + + @Test + fun `a sparse vector keeps its modifier because IDF changes what the scores mean`() { + val target = Commands.targetConfigurationOf( + CollectionParams( + vectors = VectorsConfig.Single(VectorParams(size = 4, distance = Distance.DOT)), + sparseVectors = mapOf("keywords" to SparseVectorParams(modifier = Modifier.IDF)), + ), + shards = null, + replicas = null, + ) + + assertEquals(Modifier.IDF, target.sparseVectors?.getValue("keywords")?.modifier) + } + + @Test + fun `the sharding is the source's unless the caller overrides it`() { + val source = CollectionParams( + vectors = VectorsConfig.Single(VectorParams(size = 4, distance = Distance.DOT)), + shardNumber = 2, + replicationFactor = 1, + ) + + assertEquals(2, Commands.targetConfigurationOf(source, null, null).shardNumber) + assertEquals(1, Commands.targetConfigurationOf(source, null, null).replicationFactor) + assertEquals(8, Commands.targetConfigurationOf(source, shards = 8, replicas = null).shardNumber) + assertEquals(3, Commands.targetConfigurationOf(source, shards = null, replicas = 3).replicationFactor) + } + + @Test + fun `a source with no vectors at all produces no vectors rather than a guess`() { + val target = Commands.targetConfigurationOf(CollectionParams(), null, null) + + assertNull(target.vectors) + assertNull(target.sparseVectors) + } +}