Fix #34 conservative pi-dream consolidation

README.md

@@ -41,8 +41,8 @@ pi install npm:pi-memory
 pi install ./pi-memory
 ```
 
-That's it — the six core tools (`memory_write`, `memory_forget`, `memory_restore`,
-`memory_read`, `scratchpad`, `memory_status`) work immediately with no other setup.
+That's it — the core tools (`memory_write`, `memory_forget`, `memory_restore`,
+`memory_dream`, `memory_read`, `scratchpad`, `memory_status`) work immediately with no other setup.
 Search is opt-in below.
 
 ### Optional: enable search with qmd
@@ -80,6 +80,7 @@ Without qmd, the core tools still work fully — only `memory_search` and select
 | `memory_write` | Write to MEMORY.md (long-term) or daily log |
 | `memory_forget` | Delete matching entries and create a durable recovery record |
 | `memory_restore` | Restore a deletion using the recovery ID returned by `memory_forget` |
+| `memory_dream` | Consolidate MEMORY.md: report/apply near-duplicate and superseded entries (pi-dream) |
 | `memory_read` | Read any memory file or list daily logs |
 | `scratchpad` | Add/done/undo/clear/list checklist items |
 | `memory_search` | Search across all memory files (requires qmd) |
@@ -175,6 +176,7 @@ This ensures in-progress context survives compaction and is visible in the next
 
 - **Persistence**: Memory files are plain markdown on disk — readable, editable, and git-friendly.
 - **Recoverable deletion**: `memory_forget` stores complete deleted entries under `recovery/` before changing memory and returns a recovery ID that `memory_restore` can use. Recovery JSON is outside qmd's `**/*.md` index.
+- **pi-dream consolidation** (`memory_dream` tool, `/pi-dream` command): detects near-duplicate entries (Jaccard similarity over word tokens) and older entries superseded by newer ones about the same topic. `mode='report'` analyzes without touching files; `mode='apply'` removes redundant older entries through the same recovery-record pipeline as `memory_forget`, so consolidation is undoable. Thresholds are tunable via `duplicateSimilarity` / `supersedeSimilarity` parameters.
 - **Tool response previews**: Write/scratchpad tools return size-capped previews instead of full file contents.
 - **qmd auto-setup**: On first session start with qmd available, the extension creates the collection and path contexts automatically.
 - **qmd re-indexing**: After every write, a debounced `qmd update` runs in the background (fire-and-forget, non-blocking) unless disabled via `PI_MEMORY_QMD_UPDATE`.

index.ts

@@ -724,6 +724,202 @@ export function forgetBlocks(content: string, match: string): { content: string;
 	};
 }
 
+// ---------------------------------------------------------------------------
+// pi-dream: memory consolidation analysis
+// ---------------------------------------------------------------------------
+
+export interface DreamBlock {
+	/** Timestamp meta comment line ("" for unstamped paragraph blocks). */
+	meta: string;
+	body: string;
+	timestamp: Date | null;
+}
+
+export interface DreamAnalysisOptions {
+	/** Jaccard similarity at/below which two entries are considered duplicates. Default 0.75. */
+	duplicateSimilarity?: number;
+	/** Jaccard similarity at which an older entry counts as superseded by a newer one. Default 0.6. */
+	supersedeSimilarity?: number;
+	/** Minimum age gap in days between a superseded entry and its newer replacement. Default 7. */
+	supersedeMinAgeDays?: number;
+}
+
+export interface DreamAnalysis {
+	blocks: DreamBlock[];
+	/** Groups of block indices (length > 1) whose text is near-identical. */
+	duplicateGroups: number[][];
+	/** Pairs where the older entry is considered superseded by the newer one. */
+	superseded: Array<{ olderIndex: number; newerIndex: number }>;
+}
+
+const DREAM_TIMESTAMP_REGEX =
+	/^<!-- (?:(?:last updated: )?(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})|HANDOFF (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})) /;
+
+/** Parse MEMORY.md content into stamped/unstamped blocks (same unit as forgetBlocks removes). */
+export function parseMemoryBlocks(content: string): DreamBlock[] {
+	const normalized = content.replace(/\r\n?/g, "\n").replace(/^\uFEFF/, "");
+	const rawBlocks: string[] = [];
+	let currentLines: string[] = [];
+	let currentIsStamped = false;
+	const flushCurrent = () => {
+		const current = currentLines.join("\n").trim();
+		if (!current) return;
+		if (currentIsStamped) {
+			rawBlocks.push(current);
+		} else {
+			rawBlocks.push(
+				...current
+					.split(/\n{2,}/)
+					.map((block) => block.trim())
+					.filter(Boolean),
+			);
+		}
+	};
+	for (const line of normalized.split("\n")) {
+		if (MEMORY_ENTRY_META_COMMENT_REGEX.test(line)) {
+			flushCurrent();
+			currentLines = [line];
+			currentIsStamped = true;
+		} else {
+			currentLines.push(line);
+		}
+	}
+	flushCurrent();
+
+	return rawBlocks.map((raw) => {
+		const lines = raw.split("\n");
+		const first = lines[0] ?? "";
+		if (MEMORY_ENTRY_META_COMMENT_REGEX.test(first)) {
+			const match = DREAM_TIMESTAMP_REGEX.exec(first);
+			const ts = match ? (match[1] ?? match[2]) : undefined;
+			return {
+				meta: first,
+				body: lines.slice(1).join("\n").trim() || raw,
+				timestamp: ts ? new Date(ts.replace(" ", "T")) : null,
+			};
+		}
+		return { meta: "", body: raw, timestamp: null };
+	});
+}
+
+function dreamTokenSet(text: string): Set<string> {
+	const tokens = text
+		.toLowerCase()
+		.split(/[^a-z0-9]+/)
+		.filter((word) => word.length >= 3);
+	return new Set(tokens);
+}
+
+/** Jaccard similarity over lowercase word tokens (words < 3 chars ignored). */
+export function dreamSimilarity(a: string, b: string): number {
+	const setA = dreamTokenSet(a);
+	const setB = dreamTokenSet(b);
+	if (setA.size === 0 && setB.size === 0) return 1;
+	let intersection = 0;
+	for (const token of setA) {
+		if (setB.has(token)) intersection++;
+	}
+	const union = setA.size + setB.size - intersection;
+	return union === 0 ? 0 : intersection / union;
+}
+
+function dreamDaysBetween(a: Date, b: Date): number {
+	return Math.abs(a.getTime() - b.getTime()) / 86_400_000;
+}
+
+/** Analyze MEMORY.md content for duplicate groups and superseded older entries. */
+function dreamThreshold(value: number | undefined, fallback: number, label: string): number {
+	if (value === undefined) return fallback;
+	if (!Number.isFinite(value) || value < 0 || value > 1) throw new Error(`${label} must be between 0 and 1`);
+	return value;
+}
+
+export function dreamAnalyze(content: string, opts: DreamAnalysisOptions = {}): DreamAnalysis {
+	const duplicateThreshold = dreamThreshold(opts.duplicateSimilarity, 0.75, "duplicateSimilarity");
+	const supersedeThreshold = dreamThreshold(opts.supersedeSimilarity, 0.6, "supersedeSimilarity");
+	const supersedeMinAgeDays = opts.supersedeMinAgeDays ?? 7;
+	if (!Number.isFinite(supersedeMinAgeDays) || supersedeMinAgeDays < 0)
+		throw new Error("supersedeMinAgeDays must be a non-negative number");
+	const blocks = parseMemoryBlocks(content);
+	const n = blocks.length;
+
+	const duplicateGroups: number[][] = [];
+	const assigned = new Set<number>();
+	for (let i = 0; i < n; i++) {
+		if (assigned.has(i)) continue;
+		const group = [i];
+		for (let j = i + 1; j < n; j++) {
+			if (assigned.has(j)) continue;
+			if (group.every((member) => dreamSimilarity(blocks[member].body, blocks[j].body) >= duplicateThreshold)) {
+				group.push(j);
+			}
+		}
+		if (group.length > 1) {
+			duplicateGroups.push(group);
+			for (const member of group) assigned.add(member);
+		}
+	}
+
+	const duplicatePairs = new Set<string>();
+	for (const group of duplicateGroups) {
+		for (let i = 0; i < group.length; i++) {
+			for (let j = i + 1; j < group.length; j++) duplicatePairs.add(`${group[i]}:${group[j]}`);
+		}
+	}
+
+	const superseded: Array<{ olderIndex: number; newerIndex: number }> = [];
+	for (let i = 0; i < n; i++) {
+		for (let j = i + 1; j < n; j++) {
+			if (duplicatePairs.has(`${i}:${j}`)) continue;
+			const sim = dreamSimilarity(blocks[i].body, blocks[j].body);
+			const ti = blocks[i].timestamp;
+			const tj = blocks[j].timestamp;
+			if (sim >= supersedeThreshold && ti && tj && dreamDaysBetween(ti, tj) >= supersedeMinAgeDays) {
+				const olderIsFirst = ti <= tj;
+				superseded.push(olderIsFirst ? { olderIndex: i, newerIndex: j } : { olderIndex: j, newerIndex: i });
+			}
+		}
+	}
+	return { blocks, duplicateGroups, superseded };
+}
+
+/** Compute which block indices pi-dream would remove: older members of duplicate groups plus superseded entries. */
+export function dreamDropIndices(analysis: DreamAnalysis): number[] {
+	const drops = new Set<number>();
+	for (const group of analysis.duplicateGroups) {
+		const keep = [...group].sort((a, b) => {
+			const ta = analysis.blocks[a].timestamp?.getTime();
+			const tb = analysis.blocks[b].timestamp?.getTime();
+			if (ta !== undefined && tb !== undefined && ta !== tb) return tb - ta;
+			if (ta !== undefined && tb === undefined) return -1;
+			if (ta === undefined && tb !== undefined) return 1;
+			return b - a;
+		})[0]!;
+		for (const index of group) if (index !== keep) drops.add(index);
+	}
+	for (const pair of analysis.superseded) drops.add(pair.olderIndex);
+	return [...drops].sort((a, b) => a - b);
+}
+
+/** Apply consolidation to content: returns surviving content plus complete removed blocks. */
+export function dreamApply(
+	content: string,
+	opts: DreamAnalysisOptions = {},
+): { keptContent: string; removed: string[] } {
+	const newline = content.includes("\r\n") ? "\r\n" : "\n";
+	const analysis = dreamAnalyze(content, opts);
+	const drops = dreamDropIndices(analysis);
+	if (drops.length === 0) return { keptContent: content, removed: [] };
+	const removed: string[] = [];
+	const kept: string[] = [];
+	analysis.blocks.forEach((block, index) => {
+		const raw = block.meta ? `${block.meta}\n${block.body}` : block.body;
+		if (drops.includes(index)) removed.push(raw.replace(/\n/g, newline));
+		else kept.push(raw.replace(/\n/g, newline));
+	});
+	return { keptContent: kept.length ? `${kept.join("\n\n")}\n` : "", removed };
+}
+
 function recoveryPath(recoveryId: string): string | null {
 	if (!RECOVERY_ID_REGEX.test(recoveryId)) return null;
 	return path.join(RECOVERY_DIR, `${recoveryId}.json`);
@@ -2149,6 +2345,150 @@ export default function (pi: ExtensionAPI) {
 		},
 	});
 
+	// --- memory_dream (pi-dream) tool ---
+	pi.registerTool({
+		name: "memory_dream",
+		label: "Memory Dream",
+		description: [
+			"Consolidate long-term memory (MEMORY.md): detect near-duplicate entries and older entries",
+			"superseded by newer ones about the same topic. mode='report' (default) analyzes only and",
+			"returns findings; mode='apply' removes redundant older entries via the standard recovery-record",
+			"pipeline (undo with memory_restore). Run this when MEMORY.md has grown bloated or contradictory",
+			"after many sessions.",
+		].join("\n"),
+		parameters: Type.Object({
+			mode: Type.Optional(
+				StringEnum(["report", "apply"] as const, {
+					description: "'report' (default) analyzes without changing files; 'apply' removes redundant entries",
+				}),
+			),
+			duplicateSimilarity: Type.Optional(
+				Type.Number({ minimum: 0, maximum: 1, description: "Jaccard threshold for duplicates (0-1, default 0.75)" }),
+			),
+			supersedeSimilarity: Type.Optional(
+				Type.Number({ minimum: 0, maximum: 1, description: "Jaccard threshold for superseded entries (0-1, default 0.6)" }),
+			),
+		}),
+		async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
+			ensureDirs();
+			const existing = readFileSafe(MEMORY_FILE);
+			if (!existing?.trim()) {
+				return {
+					content: [{ type: "text", text: `Nothing stored in ${MEMORY_FILE} — nothing to dream about.` }],
+					details: { path: MEMORY_FILE, removed: 0 },
+				};
+			}
+			const opts: DreamAnalysisOptions = {};
+			if (params.duplicateSimilarity !== undefined) opts.duplicateSimilarity = params.duplicateSimilarity;
+			if (params.supersedeSimilarity !== undefined) opts.supersedeSimilarity = params.supersedeSimilarity;
+
+			const analysis = dreamAnalyze(existing, opts);
+			const dropCount = dreamDropIndices(analysis).length;
+			if (dropCount === 0) {
+				return {
+					content: [
+						{
+							type: "text",
+							text: `pi-dream: no duplicates or superseded entries found in ${MEMORY_FILE} (${analysis.blocks.length} entries analyzed). Memory looks healthy.`,
+						},
+					],
+					details: { path: MEMORY_FILE, analyzed: analysis.blocks.length, removed: 0 },
+				};
+			}
+
+			const previewBlock = (index: number): string => {
+				const block = analysis.blocks[index];
+				const raw = block.meta ? `${block.meta}\n${block.body}` : block.body;
+				const preview = buildPreview(raw, { maxLines: 4, maxChars: 300, mode: "start" });
+				return preview.preview;
+			};
+			const findings: string[] = [];
+			for (const group of analysis.duplicateGroups) {
+				findings.push(`Duplicate group (keeping newest #${Math.max(...group)}):`);
+				for (const index of group) findings.push(`  [#${index}] ${previewBlock(index)}`);
+			}
+			for (const pair of analysis.superseded) {
+				findings.push(`Superseded: [#${pair.olderIndex}] replaced by newer [#${pair.newerIndex}]:`);
+				findings.push(`  [#${pair.olderIndex}] ${previewBlock(pair.olderIndex)}`);
+			}
+			const findingsText = findings.join("\n");
+
+			if ((params.mode ?? "report") === "report") {
+				return {
+					content: [
+						{
+							type: "text",
+							text:
+								`pi-dream report for ${MEMORY_FILE}: ${dropCount} of ${analysis.blocks.length} entries are removable ` +
+								`(${analysis.duplicateGroups.length} duplicate group(s), ${analysis.superseded.length} superseded).\n\n` +
+								`${findingsText}\n\n` +
+								`Review the findings, then re-run with mode='apply' to remove them (recovery record included).`,
+						},
+					],
+					details: {
+						path: MEMORY_FILE,
+						analyzed: analysis.blocks.length,
+						removable: dropCount,
+						duplicateGroups: analysis.duplicateGroups.length,
+						superseded: analysis.superseded.length,
+					},
+				};
+			}
+
+			const applied = dreamApply(existing, opts);
+			const recovery = writeRecoveryRecord("long_term", undefined, applied.removed);
+			fs.writeFileSync(MEMORY_FILE, applied.keptContent, "utf-8");
+			snapshotDirty = true;
+			await ensureQmdAvailableForUpdate();
+			scheduleQmdUpdate();
+			return {
+				content: [
+					{
+						type: "text",
+						text:
+							`pi-dream: consolidated ${MEMORY_FILE}. Removed ${applied.removed.length} redundant entr${applied.removed.length === 1 ? "y" : "ies"} ` +
+							`(was ${analysis.blocks.length}, now ${analysis.blocks.length - applied.removed.length}). ` +
+							`Recovery ID: ${recovery.id}. To undo this consolidation, call memory_restore with that ID.\n\n` +
+							`Removed:\n${findingsText}`,
+					},
+				],
+				details: {
+					path: MEMORY_FILE,
+					analyzed: analysis.blocks.length,
+					removed: applied.removed.length,
+					recoveryId: recovery.id,
+					recoveryPath: recoveryPath(recovery.id),
+				},
+			};
+		},
+	});
+
+	// --- /pi-dream command: drives the agent to run the memory_dream tool ---
+	pi.registerCommand("pi-dream", {
+		description:
+			"Memory consolidation: /pi-dream [auto|report|apply] (default auto is report-only; apply must be explicit)",
+		handler: async (args, ctx) => {
+			const mode = (args ?? "").trim().toLowerCase();
+			const existing = readFileSafe(MEMORY_FILE);
+			if (!existing?.trim()) {
+				ctx.ui.notify("pi-dream: memory is empty — nothing to consolidate.", "info");
+				return;
+			}
+			if (mode !== "" && mode !== "report" && mode !== "apply" && mode !== "auto") {
+				ctx.ui.notify("pi-dream: unknown argument. Usage: /pi-dream [report|apply]", "warning");
+				return;
+			}
+			const effective = mode === "" ? "auto" : mode;
+			pi.sendUserMessage(
+				effective === "apply"
+					? "Run the memory_dream tool with mode='apply' to consolidate MEMORY.md. Show me what was removed and the recovery ID."
+					: effective === "auto"
+						? "Run the memory_dream tool in report mode on MEMORY.md and show me the findings. Do not modify anything. If consolidation looks useful, tell me to re-run /pi-dream apply explicitly."
+						: "Run the memory_dream tool in report mode and show me the full findings for MEMORY.md — duplicate groups and superseded entries with previews. Do not modify anything.",
+			);
+		},
+	});
+
 	// --- memory_restore tool ---
 	pi.registerTool({
 		name: "memory_restore",