Have kdrant migrate create the collection it migrates into

CHANGELOG.md

@@ -85,6 +85,11 @@ it means for a klib. A degraded cluster reports itself. And there is a binary.
   runners, run against a real Qdrant before they are allowed near a release, attested with SLSA
   provenance and attached to the GitHub Release with their checksums. It cannot embed, so it migrates
   what does not need re-embedding, and the help text says so.
+  `kdrant migrate` creates the target from the source's own vectors rather than asking for a size and a
+  distance the person at the terminal did not choose, and `--shards` and `--replicas` override, which is
+  what makes `kdrant migrate a b --shards 4` a re-shard. The first release build found that it did not:
+  the migration refused because the target did not exist, which is true of every collection nobody has
+  made yet and therefore of the whole command.
 - **The client contract against four Qdrant versions** (M50). `QdrantVersionMatrixIntegrationTest` runs
   the shared contract against the four most recent minors and writes the result into the README between
   generated markers. It does not fail the build: a cell that is red against an older server is the

README.md

@@ -491,9 +491,11 @@ export QDRANT_API_KEY=...          # a key on a command line is a key in the she
 ./kdrant migrate articles articles-v2 --alias articles --checkpoint /var/tmp/articles.checkpoint
 ```
 
-That is the whole command to move a collection. It copies in id order, remembers where it got to,
-checks the counts and the recall, and moves the alias only once the check passes. It cannot embed, so
-it moves what does not need new vectors: a re-shard, a config change, a copy between clusters.
+That is the whole command to move a collection. It creates the target from the source's own vectors,
+copies in id order, remembers where it got to, checks the counts and the recall, and moves the alias
+only once the check passes. `--shards` and `--replicas` override the source's layout, which is what
+makes it a re-shard. It cannot embed, so it moves what does not need new vectors: a re-shard, a config
+change, a copy between clusters.
 
 `kdrant collections`, `kdrant scroll` and `kdrant snapshot create|list|download|restore|delete` are the
 rest of it; `kdrant --help` prints the flags. It is not a query tool, because Qdrant's own dashboard is

kdrant-cli/src/commonMain/kotlin/dev/kdrant/cli/Commands.kt

@@ -1,8 +1,13 @@
 package dev.kdrant.cli
 
 import dev.kdrant.QdrantClient
+import dev.kdrant.dsl.CreateCollectionBuilder
+import dev.kdrant.dsl.VectorParamsBuilder
 import dev.kdrant.migrate.MigrationVerification
 import dev.kdrant.migrate.migrateCollection
+import dev.kdrant.model.CollectionParams
+import dev.kdrant.model.VectorParams
+import dev.kdrant.model.VectorsConfig
 import dev.kdrant.model.WithPayload
 import kotlinx.coroutines.flow.take
 import kotlinx.coroutines.flow.toList
@@ -92,6 +97,11 @@ internal object Commands {
     /**
      * M42's procedure, with the checkpoint on disk and the recall threshold on the command line.
      *
+     * **The target is created from the source's own configuration.** A migration that does not
+     * re-embed keeps the same vectors, so asking the person at the terminal to restate a vector size
+     * and a distance they did not choose is asking them to get it wrong. `--shards` and `--replicas`
+     * override, which is what makes `kdrant migrate a b --shards 4` a re-shard rather than a copy.
+     *
      * **It cannot re-embed.** A CLI has no model, so it migrates what does not need new vectors: a
      * re-shard, a config change, a copy between clusters. Saying that plainly is better than a tool
      * that looks like it can move a collection onto a new embedding model and quietly copies the old
@@ -105,20 +115,82 @@ internal object Commands {
         val batch = arguments.intOption("batch") ?: DEFAULT_MIGRATE_BATCH
         val checkpointPath = arguments.option("checkpoint") ?: "kdrant-migrate-$from-to-$to.checkpoint"
 
+        val source = client.getCollection(from).config?.params
+            ?: fail("$from reports no configuration, so there is nothing to create $to from")
+        val shards = arguments.intOption("shards") ?: source.shardNumber
+        val replicas = arguments.intOption("replicas") ?: source.replicationFactor
+
         out("copying $from -> $to (batch $batch, recall >= $recall, checkpoint $checkpointPath)")
+        out("target: ${describe(source)}, ${shards ?: 1} shard(s), ${replicas ?: 1} replica(s)")
+
         val report = client.migrateCollection(
             from = from,
             to = to,
             alias = alias,
             batchSize = batch,
             checkpoints = FileCheckpointStore(files, checkpointPath),
             verification = MigrationVerification(minRecall = recall),
+            createTarget = { create(targetConfigurationOf(source, shards, replicas)) },
         )
         out("copied ${report.copied} point(s); source ${report.sourceCount}, target ${report.targetCount}")
         out("recall ${report.recall}")
         out(if (alias == null) "no alias was moved (pass --alias to move one)" else "alias '$alias' now points at $to")
     }
 
+    /**
+     * What the target should be created as: the source's vectors, with the sharding the caller asked
+     * for. A function of its inputs rather than a lambda that writes into a builder, so the decision
+     * can be asserted without a Qdrant to write it to.
+     *
+     * Only the vectors and the sharding, deliberately. HNSW tuning, quantization and optimizer
+     * settings are things an operator changed on the source for a reason, and carrying them onto a
+     * collection that is about to be re-sharded would carry a decision made for a different layout.
+     * They stay the server's defaults on the target and are set afterwards if they are wanted.
+     */
+    internal fun targetConfigurationOf(
+        source: CollectionParams,
+        shards: Int?,
+        replicas: Int?,
+    ): CollectionParams = CollectionParams(
+        vectors = source.vectors,
+        sparseVectors = source.sparseVectors,
+        shardNumber = shards ?: source.shardNumber,
+        replicationFactor = replicas ?: source.replicationFactor,
+        onDiskPayload = source.onDiskPayload,
+    )
+
+    /** Puts [target] into the collection DSL. Mechanical, and the only part that needs a builder. */
+    private fun CreateCollectionBuilder.create(target: CollectionParams) {
+        when (val vectors = target.vectors) {
+            is VectorsConfig.Single -> vector { copyFrom(vectors.params) }
+            is VectorsConfig.Named -> vectors.vectors.forEach { (name, params) ->
+                namedVector(name) { copyFrom(params) }
+            }
+            null -> Unit
+        }
+        target.sparseVectors?.forEach { (name, params) -> sparseVector(name) { modifier = params.modifier } }
+        target.onDiskPayload?.let { onDiskPayload = it }
+        target.shardNumber?.let { shardNumber = it }
+        target.replicationFactor?.let { replicationFactor = it }
+    }
+
+    private fun VectorParamsBuilder.copyFrom(params: VectorParams) {
+        size = params.size
+        distance = params.distance
+        params.onDisk?.let { onDisk = it }
+        params.datatype?.let { datatype = it }
+        params.multivectorConfig?.let { multivector = it.comparator }
+    }
+
+    /** What the target is about to be created as, printed before anything is copied. */
+    private fun describe(source: CollectionParams): String = when (val vectors = source.vectors) {
+        is VectorsConfig.Single -> "one ${vectors.params.size}-dimension vector, ${vectors.params.distance}"
+        is VectorsConfig.Named -> vectors.vectors.entries.joinToString(", ") { (name, params) ->
+            "$name ${params.size}d ${params.distance}"
+        }
+        null -> "no dense vectors"
+    }
+
     fun help(out: (String) -> Unit) {
         USAGE.trimIndent().lines().forEach(out)
     }
@@ -139,7 +211,8 @@ internal object Commands {
           kdrant snapshot download <collection> <snapshot> [--out FILE]
           kdrant snapshot restore <collection> <location>
           kdrant snapshot delete <collection> <snapshot>
-          kdrant migrate <from> <to> [--alias A] [--batch N] [--recall R] [--checkpoint FILE]
+          kdrant migrate <from> <to> [--alias A] [--shards N] [--replicas N]
+                                     [--batch N] [--recall R] [--checkpoint FILE]
 
         Connection:
           --host HOST        default localhost, or ${'$'}QDRANT_HOST
@@ -148,7 +221,9 @@ internal object Commands {
           --tls              use HTTPS
           --ca-file FILE     trust this PEM bundle instead of the system store
 
-        migrate copies points as they are. It cannot embed, so it moves what does not need new
+        migrate creates the target from the source's own vectors, so you do not restate a size and a
+        distance you did not choose; --shards and --replicas override, which is what makes it a
+        re-shard. It copies points as they are and cannot embed, so it moves what does not need new
         vectors: a re-shard, a config change, a copy between clusters. The alias moves only after the
         count and recall checks pass, and the checkpoint file makes an interrupted run resumable.
     """

kdrant-cli/src/commonTest/kotlin/dev/kdrant/cli/MigrateTargetTest.kt

@@ -0,0 +1,116 @@
+package dev.kdrant.cli
+
+import dev.kdrant.model.CollectionParams
+import dev.kdrant.model.Distance
+import dev.kdrant.model.Modifier
+import dev.kdrant.model.MultiVectorComparator
+import dev.kdrant.model.MultiVectorConfig
+import dev.kdrant.model.SparseVectorParams
+import dev.kdrant.model.VectorDatatype
+import dev.kdrant.model.VectorParams
+import dev.kdrant.model.VectorsConfig
+import kotlin.test.Test
+import kotlin.test.assertEquals
+import kotlin.test.assertNull
+
+/**
+ * `kdrant migrate a b` has to create `b`, and the first release build found that it did not: the
+ * migration refused because the target did not exist, which is true of every collection that has not
+ * been made yet and therefore of the whole command.
+ *
+ * The configuration is taken from the source rather than asked for, so what is under test is that it
+ * arrives intact. A vector size carried wrong is a migration that fails on its first upsert; a
+ * distance carried wrong is one that succeeds and returns the wrong neighbours.
+ */
+class MigrateTargetTest {
+
+    @Test
+    fun `a single anonymous vector keeps its size and distance and storage`() {
+        val target = Commands.targetConfigurationOf(
+            CollectionParams(
+                vectors = VectorsConfig.Single(
+                    VectorParams(
+                        size = 1536,
+                        distance = Distance.COSINE,
+                        onDisk = true,
+                        datatype = VectorDatatype.FLOAT16,
+                    ),
+                ),
+                onDiskPayload = true,
+            ),
+            shards = null,
+            replicas = null,
+        )
+
+        val vectors = target.vectors as VectorsConfig.Single
+        assertEquals(1536L, vectors.params.size)
+        assertEquals(Distance.COSINE, vectors.params.distance)
+        assertEquals(true, vectors.params.onDisk)
+        assertEquals(VectorDatatype.FLOAT16, vectors.params.datatype)
+        assertEquals(true, target.onDiskPayload)
+    }
+
+    @Test
+    fun `named vectors keep their names and a multi-vector keeps its comparator`() {
+        val target = Commands.targetConfigurationOf(
+            CollectionParams(
+                vectors = VectorsConfig.Named(
+                    mapOf(
+                        "text" to VectorParams(size = 768, distance = Distance.COSINE),
+                        "colbert" to VectorParams(
+                            size = 128,
+                            distance = Distance.DOT,
+                            multivectorConfig = MultiVectorConfig(MultiVectorComparator.MAX_SIM),
+                        ),
+                    ),
+                ),
+            ),
+            shards = null,
+            replicas = null,
+        )
+
+        val vectors = target.vectors as VectorsConfig.Named
+        assertEquals(setOf("text", "colbert"), vectors.vectors.keys)
+        assertEquals(768L, vectors.vectors.getValue("text").size)
+        assertEquals(
+            MultiVectorComparator.MAX_SIM,
+            vectors.vectors.getValue("colbert").multivectorConfig?.comparator,
+        )
+    }
+
+    @Test
+    fun `a sparse vector keeps its modifier because IDF changes what the scores mean`() {
+        val target = Commands.targetConfigurationOf(
+            CollectionParams(
+                vectors = VectorsConfig.Single(VectorParams(size = 4, distance = Distance.DOT)),
+                sparseVectors = mapOf("keywords" to SparseVectorParams(modifier = Modifier.IDF)),
+            ),
+            shards = null,
+            replicas = null,
+        )
+
+        assertEquals(Modifier.IDF, target.sparseVectors?.getValue("keywords")?.modifier)
+    }
+
+    @Test
+    fun `the sharding is the source's unless the caller overrides it`() {
+        val source = CollectionParams(
+            vectors = VectorsConfig.Single(VectorParams(size = 4, distance = Distance.DOT)),
+            shardNumber = 2,
+            replicationFactor = 1,
+        )
+
+        assertEquals(2, Commands.targetConfigurationOf(source, null, null).shardNumber)
+        assertEquals(1, Commands.targetConfigurationOf(source, null, null).replicationFactor)
+        assertEquals(8, Commands.targetConfigurationOf(source, shards = 8, replicas = null).shardNumber)
+        assertEquals(3, Commands.targetConfigurationOf(source, shards = null, replicas = 3).replicationFactor)
+    }
+
+    @Test
+    fun `a source with no vectors at all produces no vectors rather than a guess`() {
+        val target = Commands.targetConfigurationOf(CollectionParams(), null, null)
+
+        assertNull(target.vectors)
+        assertNull(target.sparseVectors)
+    }
+}