diff --git a/apps/rest-api/src/server.ts b/apps/rest-api/src/server.ts index 6df0642..45d2d27 100644 --- a/apps/rest-api/src/server.ts +++ b/apps/rest-api/src/server.ts @@ -79,6 +79,159 @@ async function generateEmbeddingSync( }; } +type FactWriteContext = { + note?: string; + relation_hint?: string; + related_fact_ids?: string[]; + tags?: string[]; + source_context?: string; +}; + +function deriveIngestSignals(content: string, context: FactWriteContext | null): { + ingest_priority: number; + confidence: number; +} { + const lengthScore = Math.min(content.length / 600, 1); + const contextBoost = context ? 0.15 : 0; + const linkedBoost = context?.related_fact_ids?.length + ? Math.min(context.related_fact_ids.length / 8, 0.2) + : 0; + const priority = Math.min(1, 0.35 + lengthScore * 0.35 + contextBoost + linkedBoost); + const confidence = Math.min(1, 0.45 + contextBoost + linkedBoost * 0.8); + return { + ingest_priority: Number(priority.toFixed(3)), + confidence: Number(confidence.toFixed(3)), + }; +} + +function normalizeFactWriteContext(raw: unknown): FactWriteContext | null { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + return null; + } + const input = raw as Record; + const relatedFactIds = Array.isArray(input.related_fact_ids) + ? input.related_fact_ids.map((v) => String(v)).filter(Boolean).slice(0, 20) + : []; + const tags = Array.isArray(input.tags) + ? input.tags.map((v) => String(v)).filter(Boolean).slice(0, 20) + : []; + return { + note: typeof input.note === "string" ? input.note : undefined, + relation_hint: + typeof input.relation_hint === "string" ? input.relation_hint : undefined, + related_fact_ids: relatedFactIds.length > 0 ? relatedFactIds : undefined, + tags: tags.length > 0 ? tags : undefined, + source_context: + typeof input.source_context === "string" ? input.source_context : undefined, + }; +} + +async function createFirstWaveRelations(args: { + factId: string; + factContent: string; + workspaceId: string; + createdBy: string; + namespace?: string; + context?: FactWriteContext | null; +}): Promise<{ created: number; considered: number }> { + const provider = process.env.OPENAI_API_KEY + ? createAIModelClient("openai", process.env.OPENAI_API_KEY).getProvider() + : undefined; + + const relationMinScore = parseFloat( + process.env.FIRST_WAVE_RELATION_MIN_SCORE || "0.72", + ); + const maxNeighbors = Math.max( + 1, + parseInt(process.env.FIRST_WAVE_MAX_NEIGHBORS || "4", 10), + ); + + const candidates = await Fact.search({ + query: args.factContent, + workspace_id: args.workspaceId, + namespace: args.namespace, + k: Math.max(8, maxNeighbors * 3), + offset: 0, + include_trashed: false, + use_vector_search: undefined, + embeddingProvider: provider, + }); + + let created = 0; + let considered = 0; + + // Explicit graph hints from write context have priority. + const hinted = new Set(args.context?.related_fact_ids || []); + for (const hintedFactId of hinted) { + if (hintedFactId === args.factId) { + continue; + } + const existing = await FactRelation.query({ + workspace_id: args.workspaceId, + from_fact: args.factId, + to_fact: hintedFactId, + limit: 1, + offset: 0, + }); + if (existing.length > 0) { + continue; + } + await FactRelation.create({ + from_fact: args.factId, + to_fact: hintedFactId, + type: "related_to", + workspace_id: args.workspaceId, + created_by: args.createdBy, + metadata: { + source: "write_context_hint", + relation_hint: args.context?.relation_hint || null, + confidence: 0.92, + }, + }); + created += 1; + } + + for (const c of candidates) { + if (c.id === args.factId) { + continue; + } + if (c.score < relationMinScore) { + continue; + } + if (created >= maxNeighbors) { + break; + } + considered += 1; + + const existing = await FactRelation.query({ + workspace_id: args.workspaceId, + from_fact: args.factId, + to_fact: c.id, + limit: 1, + offset: 0, + }); + if (existing.length > 0) { + continue; + } + + await FactRelation.create({ + from_fact: args.factId, + to_fact: c.id, + type: "related_to", + workspace_id: args.workspaceId, + created_by: args.createdBy, + metadata: { + source: "first_wave_similarity", + similarity_score: c.score, + confidence: Number(Math.min(0.9, 0.45 + c.score * 0.4).toFixed(3)), + }, + }); + created += 1; + } + + return { created, considered }; +} + function stripEmbeddings( record: T, ): Omit { @@ -304,9 +457,21 @@ export async function createServer(options?: { skipDbInit?: boolean }) { reply.code(401); return { error: "User ID is required for writes" }; } + const writeContext = normalizeFactWriteContext(body.context); + const writeNamespace = + body?.metadata && typeof body.metadata === "object" && !Array.isArray(body.metadata) + ? (body.metadata.namespace as string | undefined) + : undefined; + const ingestSignals = deriveIngestSignals(body.content, writeContext); + const metadataWithContext = { + ...(body.metadata || {}), + ...(writeContext ? { write_context: writeContext } : {}), + ingest_signals: ingestSignals, + }; + const fact = await Fact.write({ content: body.content, - metadata: body.metadata, + metadata: metadataWithContext, workspace_id: workspaceId, created_by: createdBy, last_updated_by: lastUpdatedBy, @@ -352,6 +517,22 @@ export async function createServer(options?: { skipDbInit?: boolean }) { fact: stripEmbeddings(fact), }; + // First-wave graph stitching on write: create immediate relations for + // semantic neighbors and optional user-provided context links. + try { + const firstWave = await createFirstWaveRelations({ + factId: fact.id, + factContent: body.content, + workspaceId, + createdBy, + namespace: writeNamespace, + context: writeContext, + }); + response.first_wave_relations = firstWave; + } catch (relationError: any) { + response.first_wave_relations_error = relationError.message || String(relationError); + } + // Include embedding status when sync_embedding was requested if (syncEmbedding) { response.embedding_generated = embeddingGenerated; @@ -460,6 +641,7 @@ export async function createServer(options?: { skipDbInit?: boolean }) { const results = await searchFacts({ query: body.query || "*", workspace_id: workspaceId, + namespace: body.namespace, k: body.k || 10, offset: body.offset || 0, include_trashed: body.include_trashed || false, diff --git a/docs/SPEC.md b/docs/SPEC.md index 2454c72..ef079df 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -23,11 +23,11 @@ Facts and categories – organize memory using facts and hierarchical categories Full-text search – keyword search using ArangoDB full-text indexes. -Vector embeddings – automatic generation of embeddings for facts, fact relations, and knowledge cards using OpenAI embeddings. Enables semantic search capabilities with manual cosine similarity calculation. Supports hybrid search combining full-text and vector search for optimal results. +Vector embeddings – automatic generation of embeddings for facts, fact relations, and knowledge cards using OpenAI embeddings. Enables semantic search capabilities with manual cosine similarity calculation. Hybrid fact retrieval now uses a query-adaptive rank-fusion pipeline (BM25 + vector + lexical coverage) to improve no-reranker relevance while keeping the same simple `facts_search` API. Graph database – ArangoDB provides native graph capabilities for modeling relationships between facts. -Relations – facts can be linked together with typed relationships (references, depends_on, related_to, part_of, etc.). +Relations – facts can be linked together with typed relationships (references, depends_on, related_to, part_of, etc.). New writes support optional `context` metadata and perform first-wave relation stitching at write time so the graph starts connected immediately. First-wave relation metadata now carries confidence/provenance signals for downstream replay/pruning. AQL queries – support for ArangoDB Query Language (AQL) for advanced graph queries and traversals. @@ -109,6 +109,18 @@ and should be treated as underscore equivalents. **facts_write Parameters:** - `content` (required): The content of the fact - `metadata` (optional): Key-value pairs of metadata +- `context` (optional): Additional write-time context used for first-wave graph linkage (for example `related_fact_ids`, `relation_hint`, tags, source context) +**Ingest Signals (internal):** +- Facts include `metadata.ingest_signals` with `ingest_priority` and `confidence` derived from write-time content/context. +- Retrieval logs an internal trace (`worker_logs`, `type: retrieval_trace`) with query/graph usage/result confidence for observability and future online calibration. + +**Retrieval/Graph Runtime Controls (env):** +- `GRAPH_EXPANSION_ENABLED` (default: enabled): global on/off switch for adaptive graph expansion +- `GRAPH_QUERY_PLANNER_AI` (default: disabled): enables optional LLM query planning for graph traversal hints +- `GRAPH_EXPANSION_BUDGET_MS` (default: `120`): max graph-expansion time budget per query +- `GRAPH_EXPANSION_MAX_CANDIDATES` (default: `40`): cap of graph candidates considered per query +- `FIRST_WAVE_RELATION_MIN_SCORE` (default: `0.72`): minimum similarity score for automatic first-wave relation creation +- `FIRST_WAVE_MAX_NEIGHBORS` (default: `4`): max first-wave auto-linked neighbors per newly written fact - `created_by` (optional): User ID of the creator. If not provided, inferred from authenticated session (OAuth token or API key) - `last_updated_by` (optional): User ID of the last updater. If not provided, inferred from authenticated session (OAuth token or API key) diff --git a/packages/api-core/src/index.ts b/packages/api-core/src/index.ts index 5817774..606d430 100644 --- a/packages/api-core/src/index.ts +++ b/packages/api-core/src/index.ts @@ -1,5 +1,6 @@ import { Fact, + FactRelation, KnowledgeCard, WorkspaceMember, collections, @@ -15,6 +16,12 @@ type KnowledgeCardSearchResult = { score: number; }; +type FactHit = Awaited>[number]; +type GraphPlan = { + expandedQuery: string; + relationTypes: string[]; +}; + function getProvider() { const client = createAIModelClient( (process.env.AI_PROVIDER as any) || "openai", @@ -26,6 +33,7 @@ function getProvider() { export async function searchFacts(args: { query: string; workspace_id?: string; + namespace?: string; k?: number; offset?: number; include_trashed?: boolean; @@ -33,18 +41,62 @@ export async function searchFacts(args: { const provider = getProvider(); const limit = Math.min(args.k || 5, 100); // Allow up to 100 for benchmarks const maxContentLength = 500; - - const hits = await Fact.search({ + const offset = args.offset || 0; + const seedWindow = Math.min(Math.max(limit * 3, 12), 120); + const dreamQueries = buildDreamQueries(args.query); + const perQueryWindow = Math.max(8, Math.floor(seedWindow / Math.max(1, dreamQueries.length))); + const variantHitLists = await Promise.all( + dreamQueries.map((q) => + Fact.search({ + query: q, + workspace_id: args.workspace_id, + namespace: args.namespace, + k: perQueryWindow, + offset: 0, + include_trashed: args.include_trashed, + use_vector_search: undefined, + embeddingProvider: provider, + }), + ), + ); + const seedHits = fuseDreamHitLists(args.query, variantHitLists, seedWindow); + + const graphEnabled = shouldUseGraphExpansion(args.query, seedHits); + const graphPlan = graphEnabled + ? await buildGraphQueryPlan(args.query, seedHits, provider) + : { expandedQuery: args.query, relationTypes: ["related_to"] }; + const graphExpandedHits = graphEnabled + ? await getGraphExpandedHits({ + seedHits, + relationTypes: graphPlan.relationTypes, + workspaceId: args.workspace_id, + namespace: args.namespace, + includeTrashed: args.include_trashed, + queryForFilter: graphPlan.expandedQuery || args.query, + }) + : []; + + const allCandidates = graphEnabled + ? fuseFactCandidates( + args.query, + seedHits, + graphExpandedHits, + limit + offset, + ).slice(offset, offset + limit) + : seedHits.slice(offset, offset + limit); + + await logRetrievalTrace({ query: args.query, workspace_id: args.workspace_id, - k: limit, - offset: args.offset, - include_trashed: args.include_trashed, - use_vector_search: undefined, - embeddingProvider: provider, + namespace: args.namespace, + graph_enabled: graphEnabled, + seed_count: seedHits.length, + graph_count: graphExpandedHits.length, + result_count: allCandidates.length, + top_score: allCandidates[0]?.score ?? 0, }); - const optimizedHits = hits.map((hit) => { + const optimizedHits = allCandidates.map((hit) => { const { embedding, embedding_model, _key, _id, ...rest } = hit; const content = rest.content.length > maxContentLength @@ -61,12 +113,318 @@ export async function searchFacts(args: { hits: optimizedHits, total_returned: optimizedHits.length, limit_used: limit, + graph_enriched: graphEnabled, + graph_query: graphPlan.expandedQuery, note: optimizedHits.some((h) => h.content_truncated) ? "Some facts have truncated content. Fetch the fact by ID for full content." : undefined, }; } +async function buildGraphQueryPlan( + query: string, + seedHits: FactHit[], + provider: ReturnType, +): Promise { + const fallbackTerms = tokenizeQueryTerms(query).slice(0, 8); + const seedTerms = seedHits + .slice(0, 5) + .flatMap((h) => tokenizeQueryTerms(h.content)) + .slice(0, 12); + const fallbackExpanded = Array.from(new Set([...fallbackTerms, ...seedTerms])).join(" "); + const fallbackPlan: GraphPlan = { + expandedQuery: fallbackExpanded || query, + relationTypes: ["related_to"], + }; + + const aiPlannerEnabled = process.env.GRAPH_QUERY_PLANNER_AI === "true"; + if (!aiPlannerEnabled || !process.env.OPENAI_API_KEY || seedHits.length === 0) { + return fallbackPlan; + } + + try { + const seedContext = seedHits + .slice(0, 4) + .map((h, i) => `hit_${i + 1}: ${h.content.slice(0, 220)}`) + .join("\n"); + + const response = await provider.chatCompletion( + [ + { + role: "system", + content: + "You are a retrieval query planner. Return compact JSON only.", + }, + { + role: "user", + content: [ + `query: ${query}`, + "candidate relation types: related_to, depends_on, references, part_of", + "Use the seed hits to propose graph traversal hints.", + "Return JSON with keys expanded_query (string) and relation_types (array of strings, max 2).", + "Seed hits:", + seedContext, + ].join("\n"), + }, + ], + { + model: getChatModel(), + temperature: 0.1, + responseFormat: "json_object", + }, + ); + + if (!response.content) { + return fallbackPlan; + } + + const parsed = JSON.parse(response.content); + const expandedQuery = String(parsed.expanded_query || "").trim(); + const relationTypes = Array.isArray(parsed.relation_types) + ? parsed.relation_types + .map((t: unknown) => String(t).trim()) + .filter(Boolean) + .slice(0, 2) + : []; + + return { + expandedQuery: expandedQuery || fallbackPlan.expandedQuery, + relationTypes: relationTypes.length > 0 ? relationTypes : fallbackPlan.relationTypes, + }; + } catch (error: any) { + console.warn("Graph query planning failed, using fallback:", error.message); + return fallbackPlan; + } +} + +async function getGraphExpandedHits(args: { + seedHits: FactHit[]; + relationTypes: string[]; + workspaceId?: string; + namespace?: string; + includeTrashed?: boolean; + queryForFilter: string; +}): Promise { + const startMs = Date.now(); + const budgetMs = Math.max( + 20, + parseInt(process.env.GRAPH_EXPANSION_BUDGET_MS || "120", 10), + ); + const maxGraphCandidates = Math.max( + 8, + parseInt(process.env.GRAPH_EXPANSION_MAX_CANDIDATES || "40", 10), + ); + const relationTypes = args.relationTypes.length > 0 ? args.relationTypes : ["related_to"]; + const collected = new Map(); + const seedSubset = args.seedHits.slice(0, 3); + const relationSubset = relationTypes.slice(0, 1); + + const tasks: Array> = []; + for (const hit of seedSubset) { + for (const relType of relationSubset) { + tasks.push(FactRelation.getRelatedFacts(hit.id, relType)); + } + } + const allNeighborSets = await Promise.all(tasks); + + for (const neighbors of allNeighborSets) { + for (const n of neighbors) { + if (Date.now() - startMs > budgetMs) { + return Array.from(collected.values()).slice(0, maxGraphCandidates); + } + const fact = n.fact; + if (!fact || !fact.id) { + continue; + } + if (args.workspaceId && fact.workspace_id !== args.workspaceId) { + continue; + } + if ( + args.namespace && + (!fact.metadata || fact.metadata.namespace !== args.namespace) + ) { + continue; + } + if (!args.includeTrashed && (fact.trashed || fact.deleted_at)) { + continue; + } + const overlap = lexicalOverlap(args.queryForFilter, fact.content); + if (overlap < 0.15) { + continue; + } + if (!collected.has(fact.id)) { + collected.set(fact.id, { + ...fact, + score: 0.12 + overlap, + } as FactHit); + if (collected.size >= maxGraphCandidates) { + return Array.from(collected.values()); + } + } + } + } + + return Array.from(collected.values()).slice(0, maxGraphCandidates); +} + +function tokenizeQueryTerms(text: string): string[] { + return (text || "") + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((t) => t.length >= 3); +} + +function lexicalOverlap(query: string, content: string): number { + const q = new Set(tokenizeQueryTerms(query)); + if (q.size === 0) { + return 0; + } + const c = new Set(tokenizeQueryTerms(content)); + let overlap = 0; + for (const t of q) { + if (c.has(t)) { + overlap += 1; + } + } + return overlap / q.size; +} + +function fuseFactCandidates( + query: string, + seedHits: FactHit[], + graphHits: FactHit[], + window: number, +): FactHit[] { + const map = new Map(); + for (let i = 0; i < seedHits.length; i++) { + map.set(seedHits[i].id, { fact: seedHits[i], seedRank: i + 1 }); + } + for (let i = 0; i < graphHits.length; i++) { + const existing = map.get(graphHits[i].id); + if (existing) { + existing.graphRank = i + 1; + } else { + map.set(graphHits[i].id, { fact: graphHits[i], graphRank: i + 1 }); + } + } + + const rrfK = 50; + const scored = Array.from(map.values()).map((entry) => { + const seed = entry.seedRank ? 1 / (rrfK + entry.seedRank) : 0; + const graph = entry.graphRank ? 1 / (rrfK + entry.graphRank) : 0; + const lexical = lexicalOverlap(query, entry.fact.content) * 0.08; + const score = seed * 0.88 + graph * 0.12 + lexical; + return { ...entry.fact, score }; + }); + + scored.sort((a, b) => b.score - a.score); + return scored.slice(0, window); +} + +function shouldUseGraphExpansion(query: string, seedHits: FactHit[]): boolean { + if (seedHits.length === 0) { + return false; + } + if (process.env.GRAPH_EXPANSION_ENABLED === "false") { + return false; + } + const queryTokens = tokenizeQueryTerms(query); + const isLikelyCompositional = + /\b(and|both|before|after|between|connected|relation|related|caused|because|impact)\b/i.test( + query, + ); + const sparseTopRecall = seedHits + .slice(0, 3) + .some((h) => lexicalOverlap(query, h.content) < 0.22); + + // Confidence-based trigger: if top scores are flat, graph expansion may surface + // missing bridge facts and reduce retrieval uncertainty. + const top = seedHits.slice(0, 4).map((h) => h.score || 0); + const scoreSpread = top.length >= 2 ? top[0] - top[top.length - 1] : 0; + const uncertainRanking = scoreSpread < 0.12; + + return isLikelyCompositional || (queryTokens.length >= 6 && (sparseTopRecall || uncertainRanking)); +} + +async function logRetrievalTrace(trace: { + query: string; + workspace_id?: string; + namespace?: string; + graph_enabled: boolean; + seed_count: number; + graph_count: number; + result_count: number; + top_score: number; +}) { + try { + await collections.worker_logs.save({ + worker_name: "retrieval-engine", + status: "completed", + metadata: { + type: "retrieval_trace", + ...trace, + }, + created_at: new Date().toISOString(), + updated_at: new Date().toISOString(), + } as any); + } catch { + // Observability should never break retrieval. + } +} + +function buildDreamQueries(query: string): string[] { + const base = query.trim(); + const variants = new Set([base]); + const tokens = tokenizeQueryTerms(base); + if (tokens.length > 3) { + variants.add(tokens.join(" ")); + variants.add(tokens.slice(0, Math.min(6, tokens.length)).join(" ")); + } + if (/\bwho\b/i.test(base)) { + variants.add(`${base} person identity biography`); + } + if (/\bwhen\b/i.test(base)) { + variants.add(`${base} date year timeline`); + } + if (/\bwhere\b/i.test(base)) { + variants.add(`${base} location place country city`); + } + + // Generic decomposition query to improve recall on multi-facet queries. + if (tokens.length >= 5) { + variants.add(tokens.slice(0, Math.ceil(tokens.length / 2)).join(" ")); + variants.add(tokens.slice(Math.floor(tokens.length / 2)).join(" ")); + } + return Array.from(variants).slice(0, 4); +} + +function fuseDreamHitLists( + originalQuery: string, + hitLists: FactHit[][], + window: number, +): FactHit[] { + const rrfK = 40; + const map = new Map(); + for (const list of hitLists) { + for (let i = 0; i < list.length; i++) { + const hit = list[i]; + const rankScore = 1 / (rrfK + i + 1); + const lexical = lexicalOverlap(originalQuery, hit.content) * 0.06; + const existing = map.get(hit.id); + const nextScore = rankScore + lexical; + if (existing) { + existing.score += nextScore; + } else { + map.set(hit.id, { fact: hit, score: nextScore }); + } + } + } + const scored = Array.from(map.values()) + .map((entry) => ({ ...entry.fact, score: entry.score })) + .sort((a, b) => b.score - a.score); + return scored.slice(0, window); +} + export async function searchKnowledgeCards(args: { query: string; workspace_id?: string; diff --git a/packages/db/src/models/Fact.ts b/packages/db/src/models/Fact.ts index b1e4c6c..e586f70 100644 --- a/packages/db/src/models/Fact.ts +++ b/packages/db/src/models/Fact.ts @@ -36,6 +36,7 @@ export interface FactSearchResult extends FactRecord { export interface FactSearchParams { query: string; workspace_id?: string; // Workspace ID for filtering + namespace?: string; // Optional metadata.namespace filter k?: number; offset?: number; include_trashed?: boolean; @@ -51,6 +52,28 @@ export interface FactUpdateInput { } export class Fact { + private static tokenizeForIntent(text: string): string[] { + return (text || "") + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((token) => token.length >= 3); + } + + private static lexicalCoverageScore(query: string, content: string): number { + const queryTokens = new Set(this.tokenizeForIntent(query)); + if (queryTokens.size === 0) { + return 0; + } + const contentTokens = new Set(this.tokenizeForIntent(content)); + let overlap = 0; + for (const token of queryTokens) { + if (contentTokens.has(token)) { + overlap += 1; + } + } + return overlap / queryTokens.size; + } + static async write(input: FactInput): Promise { const now = new Date().toISOString(); @@ -292,6 +315,10 @@ export class Fact { filters.push(`fact.workspace_id == @workspaceId`); bindVars.workspaceId = params.workspace_id; } + if (params.namespace) { + filters.push(`fact.metadata.namespace == @namespace`); + bindVars.namespace = params.namespace; + } filters.push(`(fact.trashed == false || @includeTrashed == true)`); const filterClause = filters.length > 0 ? `FILTER ${filters.join(" && ")}` : ""; @@ -355,6 +382,10 @@ export class Fact { postFilters.push(`fact.workspace_id == @workspaceId`); bindVars.workspaceId = params.workspace_id; } + if (params.namespace) { + postFilters.push(`fact.metadata.namespace == @namespace`); + bindVars.namespace = params.namespace; + } // Use ArangoSearch view with BM25 scoring // TOKENS(@query, "text_en") tokenizes the query using the text_en analyzer @@ -412,6 +443,10 @@ export class Fact { filters.push(`fact.workspace_id == @workspaceId`); bindVars.workspaceId = params.workspace_id; } + if (params.namespace) { + filters.push(`fact.metadata.namespace == @namespace`); + bindVars.namespace = params.namespace; + } filters.push(`(fact.trashed == false || @includeTrashed == true)`); const filterClause = `FILTER ${filters.join(" && ")}`; @@ -455,6 +490,10 @@ export class Fact { filters.push(`fact.workspace_id == @workspaceId`); bindVars.workspaceId = params.workspace_id; } + if (params.namespace) { + filters.push(`fact.metadata.namespace == @namespace`); + bindVars.namespace = params.namespace; + } filters.push(`(fact.trashed == false || @includeTrashed == true)`); filters.push(`LOWER(fact.content) LIKE LOWER(CONCAT("%", @query, "%"))`); @@ -569,6 +608,12 @@ export class Fact { if (params.workspace_id && fact.workspace_id !== params.workspace_id) { return false; } + if ( + params.namespace && + (!fact.metadata || fact.metadata.namespace !== params.namespace) + ) { + return false; + } // Filter trashed if (fact.trashed && !includeTrashed) { return false; @@ -646,6 +691,10 @@ export class Fact { filters.push(`fact.workspace_id == @workspaceId`); bindVars.workspaceId = params.workspace_id; } + if (params.namespace) { + filters.push(`fact.metadata.namespace == @namespace`); + bindVars.namespace = params.namespace; + } const aql = ` FOR fact IN facts @@ -700,6 +749,7 @@ export class Fact { params: FactSearchParams, ): Promise { const limit = params.k || 5; + const offset = params.offset || 0; const provider = params.embeddingProvider; // If no provider, use full-text only @@ -708,71 +758,112 @@ export class Fact { } try { - // Get results from both full-text and vector search + // Get a wider candidate pool from both channels before fusion. + const candidateWindow = Math.min((limit + offset) * 4, 300); const [fullTextResults, vectorResults] = await Promise.all([ - this._fullTextSearch({ ...params, k: limit * 2 }), // Get more results to merge - this._vectorSearch({ ...params, k: limit * 2 }), + this._fullTextSearch({ ...params, k: candidateWindow, offset: 0 }), + this._vectorSearch({ ...params, k: candidateWindow, offset: 0 }), ]); - // Create a map to deduplicate and combine scores + // Reciprocal Rank Fusion (RRF) is more stable than averaging raw scores + // when BM25 and vector scales drift on heterogeneous corpora. + const rrfK = 60; const resultMap = new Map< string, - { fact: FactRecord; bm25Score: number | null; vectorScore: number | null } + { + fact: FactRecord; + bm25Rank: number | null; + vectorRank: number | null; + bm25Score: number | null; + vectorScore: number | null; + } >(); - // Add full-text results - // BM25 scores are unbounded (0 to ~20+), normalize to 0-1 using: score / (score + 1) - for (const result of fullTextResults) { + for (let i = 0; i < fullTextResults.length; i++) { + const result = fullTextResults[i]; const normalizedBM25 = result.score / (result.score + 1); resultMap.set(result.id, { fact: result, + bm25Rank: i + 1, + vectorRank: null, bm25Score: normalizedBM25, vectorScore: null, }); } - // Add vector results (already normalized 0-1) - for (const result of vectorResults) { + for (let i = 0; i < vectorResults.length; i++) { + const result = vectorResults[i]; const existing = resultMap.get(result.id); if (existing) { + existing.vectorRank = i + 1; existing.vectorScore = result.score; } else { resultMap.set(result.id, { fact: result, + bm25Rank: null, + vectorRank: i + 1, bm25Score: null, vectorScore: result.score, }); } } - // Combine scores: weighted average of BM25 and vector scores - // If only one score exists, use it; otherwise average them + // Query-adaptive weighting: + // - shorter/sparser queries keep a lexical anchor + // - descriptive queries shift harder to semantic retrieval + const queryTokenCount = this.tokenizeForIntent(params.query).length; + const lexicalWeight = queryTokenCount <= 4 ? 0.4 : 0.32; + const semanticWeight = 1 - lexicalWeight; + const combinedResults: FactSearchResult[] = Array.from( resultMap.values(), ).map((item) => { - let combinedScore: number; - if (item.bm25Score !== null && item.vectorScore !== null) { - // Both scores exist - average them (equal weight) - combinedScore = (item.bm25Score + item.vectorScore) / 2; - } else if (item.bm25Score !== null) { - // Only BM25 score - combinedScore = item.bm25Score * 0.8; // Slight penalty for single-source - } else if (item.vectorScore !== null) { - // Only vector score - combinedScore = item.vectorScore * 0.8; // Slight penalty for single-source - } else { - combinedScore = 0; + const bm25Rrf = + item.bm25Rank !== null ? 1 / (rrfK + item.bm25Rank) : 0; + const vectorRrf = + item.vectorRank !== null ? 1 / (rrfK + item.vectorRank) : 0; + + let fused = lexicalWeight * bm25Rrf + semanticWeight * vectorRrf; + + const lexicalCoverage = this.lexicalCoverageScore( + params.query, + item.fact.content, + ); + fused += lexicalCoverage * 0.04; + + const appearsInBoth = item.bm25Rank !== null && item.vectorRank !== null; + if (appearsInBoth) { + fused += 0.015; + } + + // Add a small direct semantic confidence term for better recall. + if (item.vectorScore !== null) { + fused += item.vectorScore * 0.08; } + return { ...item.fact, - score: combinedScore, + score: fused, }; }); - // Sort by combined score and limit - combinedResults.sort((a, b) => b.score - a.score); + // Semantic confidence gate: for low-confidence vector queries, preserve + // a lexical anchor to avoid semantic drift while still favoring recall. + if (vectorResults.length > 0 && fullTextResults.length > 0) { + const topVectorScore = vectorResults[0]?.score || 0; + if (topVectorScore < 0.25) { + const lexicalAnchor = fullTextResults[0]; + const exists = combinedResults.find((r) => r.id === lexicalAnchor.id); + if (!exists) { + combinedResults.push({ + ...lexicalAnchor, + score: lexicalAnchor.score / (lexicalAnchor.score + 1), + }); + } + } + } - const offset = params.offset || 0; + combinedResults.sort((a, b) => b.score - a.score); return combinedResults.slice(offset, offset + limit); } catch (error: any) { console.error("Hybrid search error:", error.message); diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index 2bbde1a..fbb0fab 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -103,6 +103,13 @@ runs/ hotpotqa_summary.json ``` +Generate a cross-run high-level summary (with regression flags) as Markdown: +```bash +npx tsx tests/benchmarks/scripts/summarize-benchmarks.ts \ + --runs-dir tests/benchmarks/runs \ + --output-md tests/benchmarks/output/benchmark_highlevel_summary.md +``` + ## Prerequisites 1. **Docker** - All benchmarks run in containers diff --git a/tests/benchmarks/docs/spec.md b/tests/benchmarks/docs/spec.md index bb78c1e..7fb46eb 100644 --- a/tests/benchmarks/docs/spec.md +++ b/tests/benchmarks/docs/spec.md @@ -202,6 +202,7 @@ Create `tests/benchmarks/vector_baseline.py`: - **Step 4:** KP adapters (HTTP + Mock adapters, helpers) - **Step 5:** Vector baseline (563 lines + tests + demo + docs) - **Step 6:** Master runner script (run_all.py with combined reporting) +- Added `scripts/summarize-benchmarks.ts` (Node.js/TypeScript) to compile high-level cross-run Markdown summaries and flag regressions for debugging/optimization. ### In Progress 🔄 - None diff --git a/tests/benchmarks/scripts/summarize-benchmarks.ts b/tests/benchmarks/scripts/summarize-benchmarks.ts new file mode 100644 index 0000000..5259f4b --- /dev/null +++ b/tests/benchmarks/scripts/summarize-benchmarks.ts @@ -0,0 +1,308 @@ +#!/usr/bin/env node + +import { promises as fs } from "node:fs"; +import path from "node:path"; + +type BetterDirection = "higher" | "lower"; + +type MetricSpec = { + key: string; + better: BetterDirection; + label: string; +}; + +type RunRecord = { + run: string; + benchmark: "msmarco" | "hotpotqa" | "freshness"; + gitCommit: string | null; + gitBranch: string | null; + metrics: Record; +}; + +const METRICS: Record = { + msmarco: [ + { key: "avg_mrr", better: "higher", label: "MRR" }, + { key: "avg_recall_at_k", better: "higher", label: "Recall@K" }, + { key: "avg_ndcg_at_k", better: "higher", label: "NDCG@K" }, + { key: "avg_latency_ms", better: "lower", label: "Latency (ms)" }, + ], + hotpotqa: [ + { key: "avg_sf_f1", better: "higher", label: "Supporting Facts F1" }, + { key: "avg_sf_recall", better: "higher", label: "Supporting Facts Recall" }, + { key: "avg_sf_precision", better: "higher", label: "Supporting Facts Precision" }, + { key: "avg_doc_recall", better: "higher", label: "Document Recall" }, + { key: "avg_mrr", better: "higher", label: "MRR" }, + { key: "avg_f1", better: "higher", label: "Answer F1" }, + { key: "avg_latency_ms", better: "lower", label: "Latency (ms)" }, + ], + freshness: [ + { key: "mean_seconds", better: "lower", label: "Mean Time-to-Truth (s)" }, + { key: "p95_seconds", better: "lower", label: "P95 Time-to-Truth (s)" }, + { key: "max_seconds", better: "lower", label: "Max Time-to-Truth (s)" }, + { key: "n_successful", better: "higher", label: "Successful Lookups" }, + ], +}; + +function parseArgs(argv: string[]) { + const defaults = { + runsDir: "tests/benchmarks/runs", + outputMd: "tests/benchmarks/output/benchmark_highlevel_summary.md", + limit: 10, + }; + + let runsDir = defaults.runsDir; + let outputMd = defaults.outputMd; + let limit = defaults.limit; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === "--runs-dir" && argv[i + 1]) { + runsDir = argv[i + 1]; + i += 1; + } else if (arg === "--output-md" && argv[i + 1]) { + outputMd = argv[i + 1]; + i += 1; + } else if (arg === "--limit" && argv[i + 1]) { + const parsed = Number.parseInt(argv[i + 1], 10); + if (!Number.isNaN(parsed) && parsed > 0) { + limit = parsed; + } + i += 1; + } + } + + return { runsDir, outputMd, limit }; +} + +async function readJson(filePath: string): Promise | null> { + try { + const raw = await fs.readFile(filePath, "utf8"); + return JSON.parse(raw) as Record; + } catch { + return null; + } +} + +function detectBenchmark(runName: string, metadata: Record | null): RunRecord["benchmark"] | null { + const benchmarkMeta = typeof metadata?.benchmark === "string" ? metadata.benchmark.toLowerCase() : runName.toLowerCase(); + if (benchmarkMeta.includes("msmarco")) { + return "msmarco"; + } + if (benchmarkMeta.includes("hotpot")) { + return "hotpotqa"; + } + if (benchmarkMeta.includes("freshness")) { + return "freshness"; + } + return null; +} + +function formatMetric(value: unknown): string { + if (typeof value === "number") { + return Number.isInteger(value) ? `${value}` : value.toFixed(4); + } + return "n/a"; +} + +function statusVsPrevious( + better: BetterDirection, + current: number, + previous: number, +): { status: "improved" | "regressed" | "unchanged"; delta: number } { + const delta = current - previous; + if (delta === 0) { + return { status: "unchanged", delta }; + } + if (better === "higher") { + return { status: delta > 0 ? "improved" : "regressed", delta }; + } + return { status: delta < 0 ? "improved" : "regressed", delta }; +} + +function benchmarkDisplayName(benchmark: RunRecord["benchmark"]): string { + if (benchmark === "msmarco") { + return "MS MARCO"; + } + if (benchmark === "hotpotqa") { + return "HotpotQA"; + } + return "Freshness"; +} + +async function collectRuns(runsDir: string): Promise> { + const grouped: Record = { + msmarco: [], + hotpotqa: [], + freshness: [], + }; + + const entries = await fs.readdir(runsDir, { withFileTypes: true }); + const runDirs = entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort(); + + for (const runName of runDirs) { + const runPath = path.join(runsDir, runName); + const metadata = await readJson(path.join(runPath, "metadata.json")); + const benchmark = detectBenchmark(runName, metadata); + if (!benchmark) { + continue; + } + + const summaryFile = + benchmark === "msmarco" + ? "msmarco_summary.json" + : benchmark === "hotpotqa" + ? "hotpotqa_summary.json" + : "freshness_batch.json"; + + const summaryJson = await readJson(path.join(runPath, summaryFile)); + if (!summaryJson) { + continue; + } + + const kpMetrics = + summaryJson.kp && typeof summaryJson.kp === "object" + ? (summaryJson.kp as Record) + : null; + if (!kpMetrics) { + continue; + } + + grouped[benchmark].push({ + run: runName, + benchmark, + gitCommit: typeof metadata?.git_commit === "string" ? metadata.git_commit : null, + gitBranch: typeof metadata?.git_branch === "string" ? metadata.git_branch : null, + metrics: kpMetrics, + }); + } + + return grouped; +} + +function buildMarkdown(grouped: Record, limit: number): string { + const lines: string[] = []; + const now = new Date().toISOString(); + + let improvedTotal = 0; + let regressedTotal = 0; + let unchangedTotal = 0; + let benchmarkCount = 0; + + const benchmarkSections: string[] = []; + + for (const benchmark of ["msmarco", "hotpotqa", "freshness"] as const) { + const runs = grouped[benchmark]; + if (runs.length === 0) { + continue; + } + + benchmarkCount += 1; + const section: string[] = []; + + const recent = runs.slice(-limit); + const latest = recent[recent.length - 1]; + const previous = recent.length > 1 ? recent[recent.length - 2] : null; + const regressions: string[] = []; + const improvements: string[] = []; + const unchanged: string[] = []; + + section.push(`## ${benchmarkDisplayName(benchmark)} (${benchmark})`); + section.push(""); + section.push(`Latest run: \`${latest.run}\` on branch \`${latest.gitBranch ?? "unknown"}\` at commit \`${latest.gitCommit ?? "unknown"}\`.`); + section.push(""); + + if (!previous) { + section.push("No previous run found for direct comparison yet."); + section.push(""); + benchmarkSections.push(section.join("\n")); + continue; + } + + section.push(`Compared against previous run: \`${previous.run}\`.`); + section.push(""); + section.push("### What changed"); + section.push(""); + + for (const spec of METRICS[benchmark]) { + const current = latest.metrics[spec.key]; + const prev = previous.metrics[spec.key]; + if (typeof current !== "number" || typeof prev !== "number") { + continue; + } + + const { status, delta } = statusVsPrevious(spec.better, current, prev); + const signedDelta = `${delta >= 0 ? "+" : ""}${delta.toFixed(4)}`; + const bullet = `- ${spec.label}: ${formatMetric(current)} (prev ${formatMetric(prev)}, delta ${signedDelta}) -> **${status}**`; + section.push(bullet); + + if (status === "regressed") { + regressions.push(spec.label); + regressedTotal += 1; + } else if (status === "improved") { + improvements.push(spec.label); + improvedTotal += 1; + } else { + unchanged.push(spec.label); + unchangedTotal += 1; + } + } + + section.push(""); + section.push("### Interpretation"); + section.push(""); + if (regressions.length > 0) { + section.push(`- Regressions to investigate: ${regressions.join(", ")}.`); + } else { + section.push("- No regressions in this benchmark."); + } + section.push( + improvements.length > 0 + ? `- Improvements observed: ${improvements.join(", ")}.` + : "- No metric improvements in this benchmark.", + ); + if (unchanged.length > 0) { + section.push(`- Stable metrics: ${unchanged.join(", ")}.`); + } + section.push(`- Runs included in trend window (${recent.length}): ${recent.map((run) => `\`${run.run}\``).join(", ")}.`); + section.push(""); + + benchmarkSections.push(section.join("\n")); + } + + lines.push("# Benchmark High-Level Summary"); + lines.push(""); + lines.push(`Generated at: \`${now}\``); + lines.push(""); + lines.push(`Built from archived runs for reasoning, debugging, and optimization.`); + lines.push(""); + lines.push("## Executive Summary"); + lines.push(""); + lines.push(`- Benchmarks analyzed: ${benchmarkCount}`); + lines.push(`- Metrics improved: ${improvedTotal}`); + lines.push(`- Metrics regressed: ${regressedTotal}`); + lines.push(`- Metrics unchanged: ${unchangedTotal}`); + lines.push( + regressedTotal > 0 + ? "- Overall signal: some regressions are present and should be treated as expected debugging targets." + : "- Overall signal: no regressions detected in latest comparisons.", + ); + lines.push(""); + lines.push(...benchmarkSections); + + return `${lines.join("\n")}\n`; +} + +async function main() { + const { runsDir, outputMd, limit } = parseArgs(process.argv.slice(2)); + const grouped = await collectRuns(runsDir); + const markdown = buildMarkdown(grouped, limit); + + await fs.mkdir(path.dirname(outputMd), { recursive: true }); + await fs.writeFile(outputMd, markdown, "utf8"); + console.log(`Wrote markdown summary: ${outputMd}`); +} + +main().catch((error) => { + console.error("Failed to generate summary:", error); + process.exit(1); +}); diff --git a/tests/benchmarks/src/lib/adapter.py b/tests/benchmarks/src/lib/adapter.py index d764553..1b842ad 100644 --- a/tests/benchmarks/src/lib/adapter.py +++ b/tests/benchmarks/src/lib/adapter.py @@ -390,6 +390,8 @@ def query( 'k': search_k, 'include_trashed': False, } + if namespace: + payload['namespace'] = namespace response = self.session.post( url,