diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b6f948d..508fe43a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,9 @@ jobs: if: github.actor != 'release-please[bot]' services: ollama: - image: ollama/ollama:latest + # Keep semantic-search snapshots reproducible. New Ollama runtimes can + # change embedding execution even when the all-minilm model is unchanged. + image: ollama/ollama@sha256:a6149234667efc71d37766d61c1a16f24c33e4cd7a0bf4125c44a7e47e2419c4 ports: - 11434:11434 steps: diff --git a/CLAUDE.md b/CLAUDE.md index 526dde3a..ec0aed4c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -152,7 +152,7 @@ Codex, Cursor, and OpenCode reuse the same repo-root `skills/`, `hooks/`, and │ ├── root.go # Cobra root command │ ├── stdio.go # MCP server │ ├── hook.go # Hook handlers -│ ├── purge.go # Index data cleanup +│ ├── clean.go # Age-based index data cleanup │ └── index.go # CLI indexing ├── internal/ │ ├── config/ # Config loading & paths @@ -168,7 +168,7 @@ Codex, Cursor, and OpenCode reuse the same repo-root `skills/`, `hooks/`, and Lumen has two execution contexts with distinct output strategies: -**Interactive** (`lumen index`, `lumen purge`, `lumen search`): +**Interactive** (`lumen index`, `lumen clean`, `lumen search`): - Progress and status → `tui.Progress` (pterm) on **stderr** - Completion summaries → `fmt.Printf` on **stdout** - Errors → `fmt.Fprintf(os.Stderr, ...)` diff --git a/README.md b/README.md index e5e5c8e4..8866db60 100644 --- a/README.md +++ b/README.md @@ -392,8 +392,18 @@ and binary version. Different models or Lumen versions automatically get separate indexes. No files are added to your repo, no `.gitignore` modifications needed. -You can safely delete the entire `lumen` directory to clear all indexes, or use -`lumen purge` to do it automatically. +You can safely delete the entire `lumen` directory to clear all indexes, or let +Lumen reclaim the space for you: + +```bash +lumen clean # remove indexes unused for 30 days or whose project is gone +lumen clean --days 7 # tighten the cutoff to a week +lumen clean --days 0 # remove every cached index on this host +``` + +An index counts as used every time Lumen opens it (search, indexing, status, or +session start), so indexes for projects you still work on are never removed. +Indexes with an indexer currently running are always kept. **Git worktrees** are detected automatically. When you create a new worktree (`git worktree add` or `claude --worktree`), Lumen finds a sibling worktree's @@ -432,7 +442,7 @@ In Cursor, Codex, or OpenCode, use the shared `doctor` skill or call Run `/lumen:reindex` inside Claude Code to force a full re-index, or: ```bash -lumen purge && lumen index . +lumen index --force . ``` In Codex, use the bundled `reindex` skill to refresh the index through the MCP diff --git a/cmd/clean.go b/cmd/clean.go new file mode 100644 index 00000000..5c4b801a --- /dev/null +++ b/cmd/clean.go @@ -0,0 +1,198 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/ory/lumen/internal/config" + "github.com/ory/lumen/internal/indexlock" + "github.com/ory/lumen/internal/store" + "github.com/spf13/cobra" +) + +// defaultCleanDays is how long an index may go unused before `lumen clean` +// removes it. +const defaultCleanDays = 30 + +func init() { + addCleanFlags(cleanCmd) + rootCmd.AddCommand(cleanCmd) +} + +// addCleanFlags registers the clean flags. Shared with the tests so the flag +// definition never drifts from what runClean reads. +func addCleanFlags(cmd *cobra.Command) { + cmd.Flags().Int("days", defaultCleanDays, + "remove indexes not used in the last N days (0 removes every index that is not currently being written)") +} + +var cleanCmd = &cobra.Command{ + Use: "clean", + Short: "Remove unused or orphaned lumen indexes", + Long: fmt.Sprintf(`Deletes unused lumen index databases under ~/.local/share/lumen/. + +An index is removed when it has not been opened for --days days (default %d), +or when the project it was built for no longer exists — indexes are keyed by +project path, embedding model, and index version, so renamed projects, deleted +checkouts, and abandoned models leave behind data that is never read again. + +Indexes written by older binaries that never recorded an access time fall back +to their last indexing time; those without any usable timestamp are removed. + +Use "lumen clean --days 0" to drop every cached index on this host, and +"lumen index --force " to rebuild a single project from scratch. + +Indexes with an indexer currently running are always kept.`, defaultCleanDays), + Args: cobra.NoArgs, + RunE: runClean, +} + +func runClean(cmd *cobra.Command, _ []string) error { + days, err := cmd.Flags().GetInt("days") + if err != nil { + return err + } + if days < 0 { + return fmt.Errorf("--days must not be negative, got %d", days) + } + dataDir := filepath.Join(config.XDGDataDir(), "lumen") + return cleanIndexes(cmd.ErrOrStderr(), cmd.OutOrStdout(), dataDir, days, time.Now()) +} + +// cleanIndexes removes every stale index directory directly under dataDir, +// reporting each decision on stderr and a summary on stdout. now is injected so +// the age cutoff is testable. Failures to remove a single directory are +// reported and the sweep continues; the first such failure is returned once +// every directory has been considered. +func cleanIndexes(stderr, stdout io.Writer, dataDir string, days int, now time.Time) error { + entries, err := os.ReadDir(dataDir) + if err != nil { + if os.IsNotExist(err) { + _, _ = fmt.Fprintln(stderr, "No index data found — nothing to clean.") + return nil + } + return fmt.Errorf("read data dir: %w", err) + } + + cutoff := now.Add(-time.Duration(days) * 24 * time.Hour) + removed, skipped := 0, 0 + var firstErr error + + for _, entry := range entries { + // Only hash-named index directories are candidates; the shared + // debug.log lives in the same data dir. + if !entry.IsDir() { + continue + } + hashDir := filepath.Join(dataDir, entry.Name()) + dbPath := filepath.Join(hashDir, "index.db") + + if indexlock.IsHeld(indexlock.LockPathForDB(dbPath)) { + _, _ = fmt.Fprintf(stderr, "Keeping %s: an indexer is currently running.\n", entry.Name()) + skipped++ + continue + } + + stale, reason := isIndexStale(dbPath, days, cutoff) + if !stale { + skipped++ + continue + } + if err := os.RemoveAll(hashDir); err != nil { + _, _ = fmt.Fprintf(stderr, "Failed to remove %s: %v\n", hashDir, err) + if firstErr == nil { + firstErr = fmt.Errorf("remove %s: %w", hashDir, err) + } + skipped++ + continue + } + _, _ = fmt.Fprintf(stderr, "Removed %s (%s).\n", entry.Name(), reason) + removed++ + } + + _, _ = fmt.Fprintf(stdout, "Removed %d index director%s, skipped %d.\n", + removed, pluralY(removed), skipped) + return firstErr +} + +// isIndexStale reports whether the index at dbPath is no longer worth keeping, +// along with a human-readable reason. The metadata read is read-only so it does +// not itself count as an access. +func isIndexStale(dbPath string, days int, cutoff time.Time) (bool, string) { + if days == 0 { + return true, "--days 0" + } + + meta, err := store.ReadMetaAt(dbPath, "project_path", store.MetaLastAccessedAt, "last_indexed_at") + if err != nil { + // Missing, truncated, or non-lumen database: nothing here can be read + // again, so it is pure waste. + return true, "no readable index metadata" + } + + projectPath := meta["project_path"] + if projectPath == "" { + return true, "no project path recorded" + } + info, statErr := os.Stat(projectPath) + switch { + case statErr == nil && !info.IsDir(): + return true, fmt.Sprintf("project path %s is not a directory", projectPath) + case os.IsNotExist(statErr): + return true, fmt.Sprintf("project %s no longer exists", projectPath) + } + // Any other stat error (e.g. an unreadable parent directory) is + // inconclusive — the project may well still be there, so fall through to + // the age check rather than deleting a live index. + + if ts, ok := parseIndexTime(meta[store.MetaLastAccessedAt]); ok { + if ts.After(cutoff) { + return false, "" + } + return true, fmt.Sprintf("not accessed since %s", ts.Format(time.RFC3339)) + } + if ts, ok := parseIndexTime(meta["last_indexed_at"]); ok { + if ts.After(cutoff) { + return false, "" + } + return true, fmt.Sprintf("not indexed since %s", ts.Format(time.RFC3339)) + } + return true, "no usable access timestamp" +} + +// parseIndexTime parses an RFC3339 metadata timestamp, reporting whether the +// value was present and well-formed. +func parseIndexTime(value string) (time.Time, bool) { + if value == "" { + return time.Time{}, false + } + ts, err := time.Parse(time.RFC3339, value) + if err != nil { + return time.Time{}, false + } + return ts, true +} + +func pluralY(n int) string { + if n == 1 { + return "y" + } + return "ies" +} diff --git a/cmd/clean_test.go b/cmd/clean_test.go new file mode 100644 index 00000000..2f7abe32 --- /dev/null +++ b/cmd/clean_test.go @@ -0,0 +1,413 @@ +// Copyright 2026 Aeneas Rekkas +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bytes" + "database/sql" + "os" + "path/filepath" + "testing" + "time" + + "github.com/ory/lumen/internal/config" + "github.com/ory/lumen/internal/embedder" + "github.com/ory/lumen/internal/indexlock" + "github.com/ory/lumen/internal/store" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// cleanNow is the fixed "current time" used by the clean tests so age +// comparisons never depend on the wall clock. +var cleanNow = time.Date(2026, 6, 15, 12, 0, 0, 0, time.UTC) + +// daysAgo formats a timestamp n days before cleanNow the way the indexer does. +func daysAgo(n int) string { + return cleanNow.AddDate(0, 0, -n).Format(time.RFC3339) +} + +// seedIndex creates a real SQLite index DB in the hash directory for +// (projectPath, model) and applies meta to project_meta, so tests exercise the +// same metadata-scan code path as production. Note that store.New stamps +// last_accessed_at on open; pass an explicit value in meta to override it, or +// use deleteMeta to simulate an index written by an older binary. +func seedIndex(t *testing.T, projectPath, model string, meta map[string]string) string { + t.Helper() + dbPath := config.DBPathForProject(projectPath, model) + require.NoError(t, os.MkdirAll(filepath.Dir(dbPath), 0o755)) + s, err := store.New(dbPath, 4) + require.NoError(t, err) + require.NoError(t, s.SetMeta("project_path", projectPath)) + for k, v := range meta { + require.NoError(t, s.SetMeta(k, v)) + } + require.NoError(t, s.Close()) + return filepath.Dir(dbPath) +} + +// deleteMeta removes keys from the index's project_meta table, simulating +// indexes written by binaries that never recorded them. +func deleteMeta(t *testing.T, hashDir string, keys ...string) { + t.Helper() + db, err := sql.Open("sqlite3", filepath.Join(hashDir, "index.db")) + require.NoError(t, err) + defer func() { _ = db.Close() }() + for _, k := range keys { + _, err := db.Exec("DELETE FROM project_meta WHERE key = ?", k) + require.NoError(t, err) + } +} + +// projectDir creates a stand-in project directory that exists on disk. +func projectDir(t *testing.T, name string) string { + t.Helper() + dir := filepath.Join(resolvedTempDir(t), name) + require.NoError(t, os.MkdirAll(dir, 0o755)) + return dir +} + +// runCleanIndexes invokes the cleanup sweep against the data dir under tmp. +func runCleanIndexes(t *testing.T, tmp string, days int) (stdout, stderr string, err error) { + t.Helper() + outBuf := new(bytes.Buffer) + errBuf := new(bytes.Buffer) + err = cleanIndexes(errBuf, outBuf, filepath.Join(tmp, "lumen"), days, cleanNow) + return outBuf.String(), errBuf.String(), err +} + +// runCleanCmd invokes runClean through a command carrying the real clean flags. +func runCleanCmd(t *testing.T, args ...string) (stdout, stderr string, err error) { + t.Helper() + outBuf := new(bytes.Buffer) + errBuf := new(bytes.Buffer) + cmd := &cobra.Command{Use: "clean"} + addCleanFlags(cmd) + cmd.SetOut(outBuf) + cmd.SetErr(errBuf) + if err := cmd.Flags().Parse(args); err != nil { + return "", "", err + } + err = runClean(cmd, cmd.Flags().Args()) + return outBuf.String(), errBuf.String(), err +} + +func TestClean_KeepsRecentlyAccessedIndex(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "fresh") + hashDir := seedIndex(t, project, embedder.DefaultModel, map[string]string{ + "last_accessed_at": daysAgo(3), + }) + + stdoutOut, _, err := runCleanIndexes(t, tmp, 30) + require.NoError(t, err) + assert.DirExists(t, hashDir, "recently accessed index must survive") + assert.Contains(t, stdoutOut, "Removed 0 index") + assert.Contains(t, stdoutOut, "skipped 1") +} + +func TestClean_RemovesStaleIndex(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "stale") + hashDir := seedIndex(t, project, embedder.DefaultModel, map[string]string{ + "last_accessed_at": daysAgo(45), + }) + + stdoutOut, stderrOut, err := runCleanIndexes(t, tmp, 30) + require.NoError(t, err) + assert.NoDirExists(t, hashDir, "index unused for 45 days must be removed") + assert.Contains(t, stderrOut, "not accessed since") + assert.Contains(t, stdoutOut, "Removed 1 index") +} + +// TestClean_ExactCutoffIsStale pins the boundary: an index last accessed +// exactly on the cutoff counts as stale. +func TestClean_ExactCutoffIsStale(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "boundary") + hashDir := seedIndex(t, project, embedder.DefaultModel, map[string]string{ + "last_accessed_at": cleanNow.Add(-30 * 24 * time.Hour).Format(time.RFC3339), + }) + + _, _, err := runCleanIndexes(t, tmp, 30) + require.NoError(t, err) + assert.NoDirExists(t, hashDir, "index exactly on the cutoff must be removed") +} + +// TestClean_FallsBackToLastIndexedAt covers indexes written before +// last_accessed_at existed: their indexing timestamp decides staleness. +func TestClean_FallsBackToLastIndexedAt(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + freshProject := projectDir(t, "legacy-fresh") + freshDir := seedIndex(t, freshProject, embedder.DefaultModel, map[string]string{ + "last_indexed_at": daysAgo(2), + }) + deleteMeta(t, freshDir, "last_accessed_at") + + staleProject := projectDir(t, "legacy-stale") + staleDir := seedIndex(t, staleProject, embedder.DefaultModel, map[string]string{ + "last_indexed_at": daysAgo(90), + }) + deleteMeta(t, staleDir, "last_accessed_at") + + _, stderrOut, err := runCleanIndexes(t, tmp, 30) + require.NoError(t, err) + assert.DirExists(t, freshDir, "recently indexed legacy index must survive") + assert.NoDirExists(t, staleDir, "long-unindexed legacy index must be removed") + assert.Contains(t, stderrOut, "not indexed since") +} + +func TestClean_RemovesIndexWithNoTimestamps(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "no-timestamps") + hashDir := seedIndex(t, project, embedder.DefaultModel, nil) + deleteMeta(t, hashDir, "last_accessed_at", "last_indexed_at") + + _, stderrOut, err := runCleanIndexes(t, tmp, 30) + require.NoError(t, err) + assert.NoDirExists(t, hashDir, "index without any timestamp must be removed") + assert.Contains(t, stderrOut, "no usable access timestamp") +} + +func TestClean_RemovesIndexWithInvalidTimestamps(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "invalid-timestamps") + hashDir := seedIndex(t, project, embedder.DefaultModel, map[string]string{ + "last_accessed_at": "not-a-timestamp", + "last_indexed_at": "also-not-a-timestamp", + }) + + _, _, err := runCleanIndexes(t, tmp, 30) + require.NoError(t, err) + assert.NoDirExists(t, hashDir, "index with unparseable timestamps must be removed") +} + +// TestClean_InvalidAccessTimestampFallsBackToIndexedAt verifies a corrupt +// last_accessed_at does not discard a perfectly good last_indexed_at. +func TestClean_InvalidAccessTimestampFallsBackToIndexedAt(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "invalid-access") + hashDir := seedIndex(t, project, embedder.DefaultModel, map[string]string{ + "last_accessed_at": "garbage", + "last_indexed_at": daysAgo(1), + }) + + _, _, err := runCleanIndexes(t, tmp, 30) + require.NoError(t, err) + assert.DirExists(t, hashDir, "recent last_indexed_at must keep the index alive") +} + +func TestClean_RemovesIndexWhenProjectIsGone(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "deleted-project") + hashDir := seedIndex(t, project, embedder.DefaultModel, map[string]string{ + "last_accessed_at": daysAgo(1), + }) + require.NoError(t, os.RemoveAll(project)) + + _, stderrOut, err := runCleanIndexes(t, tmp, 30) + require.NoError(t, err) + assert.NoDirExists(t, hashDir, "index of a deleted project must be removed regardless of age") + assert.Contains(t, stderrOut, "no longer exists") +} + +func TestClean_RemovesIndexWhenProjectPathIsNotADirectory(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := filepath.Join(resolvedTempDir(t), "a-file") + require.NoError(t, os.WriteFile(project, []byte("not a project"), 0o600)) + hashDir := seedIndex(t, project, embedder.DefaultModel, map[string]string{ + "last_accessed_at": daysAgo(1), + }) + + _, stderrOut, err := runCleanIndexes(t, tmp, 30) + require.NoError(t, err) + assert.NoDirExists(t, hashDir, "index whose project path is a file must be removed") + assert.Contains(t, stderrOut, "not a directory") +} + +func TestClean_RemovesIndexWithoutProjectPath(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "unrecorded") + hashDir := seedIndex(t, project, embedder.DefaultModel, map[string]string{ + "last_accessed_at": daysAgo(1), + }) + deleteMeta(t, hashDir, "project_path") + + _, stderrOut, err := runCleanIndexes(t, tmp, 30) + require.NoError(t, err) + assert.NoDirExists(t, hashDir, "index without a recorded project path must be removed") + assert.Contains(t, stderrOut, "no project path recorded") +} + +func TestClean_RemovesMalformedIndexDirectory(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + dataDir := filepath.Join(tmp, "lumen") + empty := filepath.Join(dataDir, "0000000000000000") + require.NoError(t, os.MkdirAll(empty, 0o755)) + garbage := filepath.Join(dataDir, "1111111111111111") + require.NoError(t, os.MkdirAll(garbage, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(garbage, "index.db"), []byte("not sqlite"), 0o600)) + + stdoutOut, _, err := runCleanIndexes(t, tmp, 30) + require.NoError(t, err) + assert.NoDirExists(t, empty, "index directory without a database must be removed") + assert.NoDirExists(t, garbage, "unreadable index database must be removed") + assert.Contains(t, stdoutOut, "Removed 2 index") +} + +// TestClean_LeavesNonIndexFilesAlone verifies the sweep only touches index +// directories — the shared debug log lives in the same data dir. +func TestClean_LeavesNonIndexFilesAlone(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + dataDir := filepath.Join(tmp, "lumen") + require.NoError(t, os.MkdirAll(dataDir, 0o755)) + logPath := filepath.Join(dataDir, "debug.log") + require.NoError(t, os.WriteFile(logPath, []byte("log line\n"), 0o600)) + + _, _, err := runCleanIndexes(t, tmp, 0) + require.NoError(t, err) + assert.FileExists(t, logPath, "debug.log must not be removed") +} + +// TestClean_HandlesMultipleModelsPerProject verifies each model's index is aged +// independently, since switching models creates a separate index directory. +func TestClean_HandlesMultipleModelsPerProject(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "multi-model") + current := seedIndex(t, project, embedder.DefaultModel, map[string]string{ + "last_accessed_at": daysAgo(1), + }) + abandoned := seedIndex(t, project, "some-other-model", map[string]string{ + "last_accessed_at": daysAgo(200), + }) + require.NotEqual(t, current, abandoned, "models must map to distinct index dirs") + + stdoutOut, _, err := runCleanIndexes(t, tmp, 30) + require.NoError(t, err) + assert.DirExists(t, current, "index for the model in use must survive") + assert.NoDirExists(t, abandoned, "index for the abandoned model must be removed") + assert.Contains(t, stdoutOut, "Removed 1 index") +} + +func TestClean_DaysZeroRemovesEveryIndex(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "in-use") + hashDir := seedIndex(t, project, embedder.DefaultModel, map[string]string{ + "last_accessed_at": cleanNow.Format(time.RFC3339), + }) + + stdoutOut, _, err := runCleanIndexes(t, tmp, 0) + require.NoError(t, err) + assert.NoDirExists(t, hashDir, "--days 0 must remove even a just-used index") + assert.Contains(t, stdoutOut, "Removed 1 index") +} + +func TestClean_SkipsLockedIndex(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "indexing-now") + hashDir := seedIndex(t, project, embedder.DefaultModel, map[string]string{ + "last_accessed_at": daysAgo(500), + }) + + lockPath := indexlock.LockPathForDB(filepath.Join(hashDir, "index.db")) + lock, err := indexlock.TryAcquire(lockPath) + require.NoError(t, err) + require.NotNil(t, lock) + defer lock.Release() + require.True(t, indexlock.IsHeld(lockPath), "precondition: lock must read as held") + + stdoutOut, stderrOut, err := runCleanIndexes(t, tmp, 0) + require.NoError(t, err) + assert.DirExists(t, hashDir, "an index being written must not be removed") + assert.Contains(t, stderrOut, "indexer is currently running") + assert.Contains(t, stdoutOut, "Removed 0 index") + assert.Contains(t, stdoutOut, "skipped 1") +} + +func TestClean_NoIndexDataIsNoOp(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + _, stderrOut, err := runCleanIndexes(t, tmp, 30) + require.NoError(t, err) + assert.Contains(t, stderrOut, "No index data found") +} + +func TestClean_NegativeDaysFails(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "kept") + hashDir := seedIndex(t, project, embedder.DefaultModel, nil) + + _, _, err := runCleanCmd(t, "--days", "-1") + require.Error(t, err) + assert.Contains(t, err.Error(), "--days") + assert.DirExists(t, hashDir, "a rejected invocation must not delete anything") +} + +func TestClean_RejectsPositionalArgs(t *testing.T) { + require.Error(t, cleanCmd.Args(cleanCmd, []string{"/some/project"}), + "clean takes no positional arguments") + require.NoError(t, cleanCmd.Args(cleanCmd, nil)) +} + +// TestClean_DefaultDays pins the documented 30-day default. +func TestClean_DefaultDays(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "default-days") + fresh := seedIndex(t, project, embedder.DefaultModel, nil) + stale := seedIndex(t, project, "stale-model", map[string]string{ + "last_accessed_at": time.Now().UTC().AddDate(0, 0, -31).Format(time.RFC3339), + }) + + _, _, err := runCleanCmd(t) + require.NoError(t, err) + assert.DirExists(t, fresh, "index accessed just now must survive the default cutoff") + assert.NoDirExists(t, stale, "index unused for 31 days must be removed by default") +} diff --git a/cmd/purge.go b/cmd/purge.go deleted file mode 100644 index 37ec3cc8..00000000 --- a/cmd/purge.go +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2026 Aeneas Rekkas -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - "io" - "os" - "path/filepath" - "strings" - - "github.com/ory/lumen/internal/config" - "github.com/ory/lumen/internal/git" - "github.com/ory/lumen/internal/store" - "github.com/spf13/cobra" -) - -func init() { - rootCmd.AddCommand(purgeCmd) -} - -var purgeCmd = &cobra.Command{ - Use: "purge [path...]", - Short: "Remove lumen index data", - Long: `Deletes lumen index databases under ~/.local/share/lumen/. - -With no arguments, removes every index (irreversible — all indexes will be -rebuilt on the next search). - -With one or more paths, removes only the index directories associated with -those projects. Each path is normalized to its git root first, then matched -against the project_path recorded inside each index database, so switching -embedding models or using custom models never leaves orphan indexes. - -Indexes created by older binaries that did not record project_path cannot be -matched by path; run "lumen purge" with no arguments to wipe those. - -Note: a concurrently running indexer for a purged project may log a write -error and exit; re-run "lumen index" afterwards to rebuild.`, - Args: cobra.ArbitraryArgs, - RunE: runPurge, -} - -func runPurge(cmd *cobra.Command, args []string) error { - if len(args) == 0 { - return purgeAll(cmd.ErrOrStderr()) - } - return purgeProjects(cmd.ErrOrStderr(), cmd.OutOrStdout(), args) -} - -func purgeAll(stderr io.Writer) error { - dataDir := filepath.Join(config.XDGDataDir(), "lumen") - - info, err := os.Stat(dataDir) - if err != nil { - if os.IsNotExist(err) { - _, _ = fmt.Fprintln(stderr, "No index data found — nothing to purge.") - return nil - } - return fmt.Errorf("stat data directory: %w", err) - } - if !info.IsDir() { - return fmt.Errorf("%s is not a directory", dataDir) - } - - if err := os.RemoveAll(dataDir); err != nil { - return fmt.Errorf("remove index data: %w", err) - } - _, _ = fmt.Fprintf(stderr, "Removed all index data (%s)\n", dataDir) - return nil -} - -func purgeProjects(stderr, stdout io.Writer, args []string) error { - dataDir := filepath.Join(config.XDGDataDir(), "lumen") - indexMap, err := scanIndexes(dataDir) - if err != nil { - return err - } - - seen := make(map[string]bool) - totalRemoved := 0 - for _, arg := range args { - removed, err := purgeOneTarget(stderr, indexMap, seen, arg) - if err != nil { - return err - } - totalRemoved += removed - } - _, _ = fmt.Fprintf(stdout, "Removed %d index director%s.\n", totalRemoved, pluralY(totalRemoved)) - return nil -} - -// scanIndexes walks dataDir (one level deep) and returns a map of stored -// project_path → list of hash directories for that project. Hash directories -// that can't be read or lack project_path metadata are silently skipped so a -// single broken index never blocks purging of others. -func scanIndexes(dataDir string) (map[string][]string, error) { - result := make(map[string][]string) - entries, err := os.ReadDir(dataDir) - if err != nil { - if os.IsNotExist(err) { - return result, nil - } - return nil, fmt.Errorf("read data dir: %w", err) - } - for _, entry := range entries { - if !entry.IsDir() { - continue - } - hashDir := filepath.Join(dataDir, entry.Name()) - dbPath := filepath.Join(hashDir, "index.db") - stored, err := store.ReadMetaAt(dbPath, "project_path") - if err != nil || stored == "" { - continue - } - result[stored] = append(result[stored], hashDir) - } - return result, nil -} - -// purgeOneTarget resolves arg to a project root and removes every hash -// directory whose stored project_path matches. seen tracks hash directories -// already deleted during this invocation so two args resolving to the same -// project are not double-counted. -func purgeOneTarget(stderr io.Writer, indexMap map[string][]string, seen map[string]bool, arg string) (int, error) { - abs, err := filepath.Abs(arg) - if err != nil { - return 0, fmt.Errorf("resolve %q: %w", arg, err) - } - if resolved, err := filepath.EvalSymlinks(abs); err == nil { - abs = resolved - } - - target := abs - inGitRepo := false - if root, err := git.RepoRoot(abs); err == nil { - target = root - inGitRepo = true - } - - match := "" - if _, ok := indexMap[target]; ok { - match = target - } else if !inGitRepo { - // Non-git fallback: match the deepest stored path that contains the - // target. Mirrors `findAncestorIndex` semantics used by index/search. - match = longestAncestor(indexMap, target) - } - - if match == "" { - _, _ = fmt.Fprintf(stderr, "No index found for %s.\n", abs) - return 0, nil - } - - removed := 0 - for _, hashDir := range indexMap[match] { - if seen[hashDir] { - continue - } - seen[hashDir] = true - if err := os.RemoveAll(hashDir); err != nil { - return removed, fmt.Errorf("remove %s: %w", hashDir, err) - } - removed++ - } - _, _ = fmt.Fprintf(stderr, "Removed %d index director%s for %s.\n", - removed, pluralY(removed), match) - return removed, nil -} - -// longestAncestor returns the longest key in indexMap that is either equal to -// target or an ancestor directory of target, or "" if no such key exists. -func longestAncestor(indexMap map[string][]string, target string) string { - best := "" - for stored := range indexMap { - if stored == target || strings.HasPrefix(target, stored+string(filepath.Separator)) { - if len(stored) > len(best) { - best = stored - } - } - } - return best -} - -func pluralY(n int) string { - if n == 1 { - return "y" - } - return "ies" -} diff --git a/cmd/purge_test.go b/cmd/purge_test.go deleted file mode 100644 index 1a1badf8..00000000 --- a/cmd/purge_test.go +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2026 Aeneas Rekkas -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "bytes" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/ory/lumen/internal/config" - "github.com/ory/lumen/internal/embedder" - "github.com/ory/lumen/internal/store" - "github.com/spf13/cobra" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// seedIndex creates a real SQLite index DB at the hash-named directory for -// (projectPath, model) with project_path recorded in project_meta, so that -// tests exercise the same metadata-scan code path as production. -func seedIndex(t *testing.T, projectPath, model string) string { - t.Helper() - dbPath := config.DBPathForProject(projectPath, model) - require.NoError(t, os.MkdirAll(filepath.Dir(dbPath), 0o755)) - s, err := store.New(dbPath, 4) - require.NoError(t, err) - require.NoError(t, s.SetMeta("project_path", projectPath)) - require.NoError(t, s.Close()) - return filepath.Dir(dbPath) -} - -// runPurgeCmd invokes runPurge with the provided args and returns captured -// stdout, stderr, and the error (if any). -func runPurgeCmd(t *testing.T, args []string) (stdout, stderr string, err error) { - t.Helper() - outBuf := new(bytes.Buffer) - errBuf := new(bytes.Buffer) - cmd := &cobra.Command{} - cmd.SetOut(outBuf) - cmd.SetErr(errBuf) - err = runPurge(cmd, args) - return outBuf.String(), errBuf.String(), err -} - -func TestPurge_NoArgs_RemovesEverything(t *testing.T) { - tmp := resolvedTempDir(t) - t.Setenv("XDG_DATA_HOME", tmp) - - seedIndex(t, "/project/a", embedder.DefaultModel) - seedIndex(t, "/project/b", embedder.DefaultModel) - - lumenRoot := filepath.Join(tmp, "lumen") - entries, err := os.ReadDir(lumenRoot) - require.NoError(t, err) - require.Len(t, entries, 2, "should have seeded two hash dirs") - - _, stderrOut, err := runPurgeCmd(t, nil) - require.NoError(t, err) - assert.Contains(t, stderrOut, "Removed all index data") - - _, err = os.Stat(lumenRoot) - assert.True(t, os.IsNotExist(err), "lumen data dir should be gone, got err=%v", err) -} - -func TestPurge_NoArgs_NothingToPurge(t *testing.T) { - tmp := resolvedTempDir(t) - t.Setenv("XDG_DATA_HOME", tmp) - - _, stderrOut, err := runPurgeCmd(t, nil) - require.NoError(t, err) - assert.Contains(t, stderrOut, "No index data found") -} - -func TestPurge_SinglePath_RemovesOnlyThatProject(t *testing.T) { - tmp := resolvedTempDir(t) - t.Setenv("XDG_DATA_HOME", tmp) - - // Two independent projects with seeded indexes. - projectA := filepath.Join(tmp, "projectA") - projectB := filepath.Join(tmp, "projectB") - require.NoError(t, os.MkdirAll(projectA, 0o755)) - require.NoError(t, os.MkdirAll(projectB, 0o755)) - runGit(t, projectA, "init") - runGit(t, projectB, "init") - - hashDirA := seedIndex(t, projectA, embedder.DefaultModel) - hashDirB := seedIndex(t, projectB, embedder.DefaultModel) - - _, stderrOut, err := runPurgeCmd(t, []string{projectA}) - require.NoError(t, err) - assert.Contains(t, stderrOut, projectA, "should log the purged project path") - - _, err = os.Stat(hashDirA) - assert.True(t, os.IsNotExist(err), "project A hash dir should be gone") - _, err = os.Stat(hashDirB) - assert.NoError(t, err, "project B hash dir should be untouched") -} - -func TestPurge_PathInsideGitRepo_ResolvesToGitRoot(t *testing.T) { - tmp := resolvedTempDir(t) - t.Setenv("XDG_DATA_HOME", tmp) - - repoDir := filepath.Join(tmp, "repo") - subDir := filepath.Join(repoDir, "sub") - require.NoError(t, os.MkdirAll(subDir, 0o755)) - runGit(t, repoDir, "init") - - hashDir := seedIndex(t, repoDir, embedder.DefaultModel) - - _, _, err := runPurgeCmd(t, []string{subDir}) - require.NoError(t, err) - - _, err = os.Stat(hashDir) - assert.True(t, os.IsNotExist(err), "git-root hash dir should be removed when passing a subdirectory") -} - -func TestPurge_PathWithAncestorIndex_ResolvesToAncestor(t *testing.T) { - tmp := resolvedTempDir(t) - t.Setenv("XDG_DATA_HOME", tmp) - - grandparent := filepath.Join(tmp, "workspace") - child := filepath.Join(grandparent, "a", "b") - require.NoError(t, os.MkdirAll(child, 0o755)) - - hashDir := seedIndex(t, grandparent, embedder.DefaultModel) - - _, _, err := runPurgeCmd(t, []string{child}) - require.NoError(t, err) - - _, err = os.Stat(hashDir) - assert.True(t, os.IsNotExist(err), "ancestor hash dir should be removed") -} - -func TestPurge_PathWithoutIndex_ReportsNoneAndExitsZero(t *testing.T) { - tmp := resolvedTempDir(t) - t.Setenv("XDG_DATA_HOME", tmp) - - // Path exists but has no index anywhere up the tree. - dir := filepath.Join(tmp, "empty") - require.NoError(t, os.MkdirAll(dir, 0o755)) - - _, stderrOut, err := runPurgeCmd(t, []string{dir}) - require.NoError(t, err) - assert.Contains(t, strings.ToLower(stderrOut), "no index found") -} - -func TestPurge_MultiplePaths_RemovesEach(t *testing.T) { - tmp := resolvedTempDir(t) - t.Setenv("XDG_DATA_HOME", tmp) - - projectA := filepath.Join(tmp, "projectA") - projectB := filepath.Join(tmp, "projectB") - require.NoError(t, os.MkdirAll(projectA, 0o755)) - require.NoError(t, os.MkdirAll(projectB, 0o755)) - runGit(t, projectA, "init") - runGit(t, projectB, "init") - - hashDirA := seedIndex(t, projectA, embedder.DefaultModel) - hashDirB := seedIndex(t, projectB, embedder.DefaultModel) - - _, _, err := runPurgeCmd(t, []string{projectA, projectB}) - require.NoError(t, err) - - _, err = os.Stat(hashDirA) - assert.True(t, os.IsNotExist(err), "project A hash dir should be gone") - _, err = os.Stat(hashDirB) - assert.True(t, os.IsNotExist(err), "project B hash dir should be gone") -} - -func TestPurge_UnknownModelName_StillPurgedByStoredMetadata(t *testing.T) { - // Indexes created with custom or aliased model names (not in KnownModels) - // must still be purged — the match is by stored project_path, not by - // enumerating known models and recomputing the hash. - tmp := resolvedTempDir(t) - t.Setenv("XDG_DATA_HOME", tmp) - - project := filepath.Join(tmp, "project") - require.NoError(t, os.MkdirAll(project, 0o755)) - runGit(t, project, "init") - - hashDir := seedIndex(t, project, "some-custom-alias-model-not-in-registry") - - _, _, err := runPurgeCmd(t, []string{project}) - require.NoError(t, err) - - _, err = os.Stat(hashDir) - assert.True(t, os.IsNotExist(err), "custom-model hash dir should be removed via stored project_path") -} - -func TestPurge_MultiplePaths_MixedHitsAndMisses(t *testing.T) { - tmp := resolvedTempDir(t) - t.Setenv("XDG_DATA_HOME", tmp) - - projectA := filepath.Join(tmp, "projectA") - empty := filepath.Join(tmp, "empty") - require.NoError(t, os.MkdirAll(projectA, 0o755)) - require.NoError(t, os.MkdirAll(empty, 0o755)) - runGit(t, projectA, "init") - - hashDirA := seedIndex(t, projectA, embedder.DefaultModel) - - _, stderrOut, err := runPurgeCmd(t, []string{projectA, empty}) - require.NoError(t, err) - - _, err = os.Stat(hashDirA) - assert.True(t, os.IsNotExist(err), "project A hash dir should be gone") - assert.Contains(t, strings.ToLower(stderrOut), "no index found", "miss should be reported") -} diff --git a/internal/store/store.go b/internal/store/store.go index 4b9c29f9..5e07114b 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "strings" + "time" sqlite_vec "github.com/asg017/sqlite-vec-go-bindings/cgo" _ "github.com/mattn/go-sqlite3" // register sqlite3 driver @@ -28,6 +29,15 @@ import ( "github.com/ory/lumen/internal/chunker" ) +// MetaLastAccessedAt is the project_meta key holding the RFC3339 UTC timestamp +// of the last time this index was opened. `lumen clean` reads it to decide +// whether an index is still in use. +const MetaLastAccessedAt = "last_accessed_at" + +// accessStampBusyTimeoutMS bounds how long opening a store waits for the write +// lock to record its access time before giving up on the stamp. +const accessStampBusyTimeoutMS = 250 + func init() { sqlite_vec.Auto() } @@ -90,7 +100,47 @@ func New(dsn string, dimensions int) (*Store, error) { deleteDBFiles(dsn) s, err = openStore(dsn, dimensions) } - return s, err + if err != nil { + return s, err + } + s.stampAccess(dsn) + return s, nil +} + +// stampAccess records the current time as this index's last access so +// `lumen clean` can tell indexes that are still in use apart from abandoned +// ones. +// +// The write runs on a throwaway connection with a short busy timeout instead of +// the store's own connection: a concurrently running indexer holds the SQLite +// write lock for the duration of every insert batch, and the sqlite3 driver does +// not honour context cancellation while waiting out busy_timeout — so reusing +// the store's 120s timeout would stall opening (and therefore search) for +// minutes. Failing to stamp is harmless: the indexer stamps its own open, and +// `clean` falls back to last_indexed_at. +func (s *Store) stampAccess(dsn string) { + value := time.Now().UTC().Format(time.RFC3339) + if dsn == ":memory:" { + // Nothing else can hold the lock on a private in-memory database, and a + // second connection would open a different one. + _ = s.SetMeta(MetaLastAccessedAt, value) + return + } + + db, err := sql.Open("sqlite3", dsn) + if err != nil { + return + } + defer func() { _ = db.Close() }() + db.SetMaxOpenConns(1) + if _, err := db.Exec(fmt.Sprintf("PRAGMA busy_timeout=%d", accessStampBusyTimeoutMS)); err != nil { + return + } + _, _ = db.Exec( + `INSERT INTO project_meta (key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value`, + MetaLastAccessedAt, value, + ) } func openStore(dsn string, dimensions int) (*Store, error) { @@ -296,34 +346,28 @@ func resetAndRecreateVecTable(db *sql.DB, dimensions int) error { } // ReadMetaAt opens the SQLite database at dbPath read-only and returns the -// project_meta value for key, or "" if the key is missing. It is safe to call -// on databases created by older binaries with a different schema version; the -// function performs no migrations and holds no write lock. Returns an error -// only when the file cannot be opened (missing file, permission denied) or -// when the project_meta table is missing (the file is not a lumen index). -func ReadMetaAt(dbPath, key string) (string, error) { +// project_meta values for keys in a single open. Missing keys are absent from +// the returned map. It is safe to call on databases created by older binaries +// with a different schema version; the function performs no migrations, holds +// no write lock, and — unlike New — does not count as an index access, so +// scanning metadata never keeps an abandoned index alive. Returns an error only +// when the file cannot be opened (missing file, permission denied) or when the +// project_meta table is missing (the file is not a lumen index). +func ReadMetaAt(dbPath string, keys ...string) (map[string]string, error) { if _, err := os.Stat(dbPath); err != nil { - return "", fmt.Errorf("stat %s: %w", dbPath, err) + return nil, fmt.Errorf("stat %s: %w", dbPath, err) } db, err := sql.Open("sqlite3", dbPath) if err != nil { - return "", fmt.Errorf("open %s: %w", dbPath, err) + return nil, fmt.Errorf("open %s: %w", dbPath, err) } defer func() { _ = db.Close() }() db.SetMaxOpenConns(1) if _, err := db.Exec("PRAGMA query_only=ON"); err != nil { - return "", fmt.Errorf("set query_only: %w", err) + return nil, fmt.Errorf("set query_only: %w", err) } - var val string - err = db.QueryRow("SELECT value FROM project_meta WHERE key = ?", key).Scan(&val) - if err == sql.ErrNoRows { - return "", nil - } - if err != nil { - return "", fmt.Errorf("query project_meta: %w", err) - } - return val, nil + return queryMeta(db, keys) } // SetMeta upserts a key-value pair in the project_meta table. @@ -351,9 +395,17 @@ func (s *Store) GetMeta(key string) (string, error) { // Missing keys are absent from the returned map. Uses the read-only connection // when available for concurrency with writes. func (s *Store) GetMetaBatch(keys []string) (map[string]string, error) { + return queryMeta(s.reader(), keys) +} + +// queryMeta fetches keys from project_meta in a single query. Missing keys are +// absent from the returned map. +func queryMeta(db *sql.DB, keys []string) (map[string]string, error) { + result := make(map[string]string, len(keys)) if len(keys) == 0 { - return map[string]string{}, nil + return result, nil } + placeholders := make([]string, len(keys)) args := make([]any, len(keys)) for i, k := range keys { @@ -364,21 +416,23 @@ func (s *Store) GetMetaBatch(keys []string) (map[string]string, error) { "SELECT key, value FROM project_meta WHERE key IN (%s)", strings.Join(placeholders, ","), ) - rows, err := s.reader().Query(query, args...) + rows, err := db.Query(query, args...) if err != nil { - return nil, fmt.Errorf("query meta batch: %w", err) + return nil, fmt.Errorf("query project_meta: %w", err) } defer func() { _ = rows.Close() }() - result := make(map[string]string, len(keys)) for rows.Next() { var k, v string if err := rows.Scan(&k, &v); err != nil { - return nil, fmt.Errorf("scan meta: %w", err) + return nil, fmt.Errorf("scan project_meta: %w", err) } result[k] = v } - return result, rows.Err() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("query project_meta: %w", err) + } + return result, nil } // UpsertFile inserts or updates a file path and its content hash. diff --git a/internal/store/store_test.go b/internal/store/store_test.go index f8d13037..c24180b1 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -16,8 +16,11 @@ package store import ( "context" + "database/sql" "path/filepath" + "strings" "testing" + "time" "github.com/ory/lumen/internal/chunker" ) @@ -540,8 +543,41 @@ func TestReadMetaAt_ReturnsStoredValue(t *testing.T) { if err != nil { t.Fatalf("ReadMetaAt: %v", err) } - if got != "/some/project" { - t.Fatalf("ReadMetaAt(project_path) = %q, want %q", got, "/some/project") + if got["project_path"] != "/some/project" { + t.Fatalf("ReadMetaAt(project_path) = %q, want %q", got["project_path"], "/some/project") + } +} + +// TestReadMetaAt_MultipleKeys verifies a single read-only open can fetch every +// key `lumen clean` needs to classify an index. +func TestReadMetaAt_MultipleKeys(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "index.db") + s, err := New(dbPath, 4) + if err != nil { + t.Fatal(err) + } + if err := s.SetMeta("project_path", "/some/project"); err != nil { + t.Fatal(err) + } + if err := s.SetMeta("last_indexed_at", "2026-01-02T03:04:05Z"); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + + got, err := ReadMetaAt(dbPath, "project_path", "last_accessed_at", "last_indexed_at") + if err != nil { + t.Fatalf("ReadMetaAt: %v", err) + } + if got["project_path"] != "/some/project" { + t.Errorf("project_path = %q, want %q", got["project_path"], "/some/project") + } + if got["last_indexed_at"] != "2026-01-02T03:04:05Z" { + t.Errorf("last_indexed_at = %q, want %q", got["last_indexed_at"], "2026-01-02T03:04:05Z") + } + if got["last_accessed_at"] == "" { + t.Error("last_accessed_at should be stamped by New") } } @@ -559,8 +595,8 @@ func TestReadMetaAt_MissingKeyReturnsEmpty(t *testing.T) { if err != nil { t.Fatalf("ReadMetaAt on empty meta: %v", err) } - if got != "" { - t.Fatalf("ReadMetaAt missing key = %q, want empty", got) + if _, ok := got["project_path"]; ok { + t.Fatalf("ReadMetaAt missing key = %q, want absent", got["project_path"]) } } @@ -570,3 +606,144 @@ func TestReadMetaAt_MissingFileReturnsError(t *testing.T) { t.Fatal("expected error for missing DB file") } } + +// TestNew_StampsLastAccessedAt verifies that opening a store records an +// RFC3339 UTC access timestamp, which `lumen clean` uses to decide whether an +// index is still in use. +func TestNew_StampsLastAccessedAt(t *testing.T) { + before := time.Now().UTC().Truncate(time.Second) + dbPath := filepath.Join(t.TempDir(), "index.db") + s, err := New(dbPath, 4) + if err != nil { + t.Fatal(err) + } + defer func() { _ = s.Close() }() + + got, err := s.GetMeta("last_accessed_at") + if err != nil { + t.Fatalf("GetMeta(last_accessed_at): %v", err) + } + if !strings.HasSuffix(got, "Z") { + t.Errorf("last_accessed_at = %q, want a UTC (Z-suffixed) timestamp", got) + } + ts, err := time.Parse(time.RFC3339, got) + if err != nil { + t.Fatalf("parse last_accessed_at %q: %v", got, err) + } + if ts.Before(before) { + t.Errorf("last_accessed_at = %v, want at or after %v", ts, before) + } +} + +// TestNew_RefreshesLastAccessedAt verifies that reopening an existing index +// bumps the recorded access time rather than leaving a stale value behind. +func TestNew_RefreshesLastAccessedAt(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "index.db") + s, err := New(dbPath, 4) + if err != nil { + t.Fatal(err) + } + stale := "2020-01-01T00:00:00Z" + if err := s.SetMeta("last_accessed_at", stale); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + + reopened, err := New(dbPath, 4) + if err != nil { + t.Fatal(err) + } + defer func() { _ = reopened.Close() }() + + got, err := reopened.GetMeta("last_accessed_at") + if err != nil { + t.Fatalf("GetMeta(last_accessed_at): %v", err) + } + if got == stale { + t.Error("reopening the store should refresh last_accessed_at") + } +} + +// TestNew_AccessStampDoesNotBlockOnBusyDatabase verifies that opening an index +// while another writer holds the SQLite write lock — the normal state during a +// background reindex — still returns promptly. The access stamp is bookkeeping; +// blocking on it for the 120s busy_timeout would stall search. +func TestNew_AccessStampDoesNotBlockOnBusyDatabase(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "index.db") + s, err := New(dbPath, 4) + if err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + + // Hold the write lock from an independent connection, as a concurrently + // running indexer process would. + blocker, err := sql.Open("sqlite3", dbPath) + if err != nil { + t.Fatal(err) + } + defer func() { _ = blocker.Close() }() + tx, err := blocker.Begin() + if err != nil { + t.Fatal(err) + } + if _, err := tx.Exec("INSERT INTO project_meta (key, value) VALUES ('blocker', '1')"); err != nil { + t.Fatal(err) + } + defer func() { _ = tx.Rollback() }() + + done := make(chan error, 1) + start := time.Now() + go func() { + reopened, err := New(dbPath, 4) + if reopened != nil { + _ = reopened.Close() + } + done <- err + }() + + select { + case err := <-done: + if err != nil { + t.Fatalf("New on a busy database: %v", err) + } + if elapsed := time.Since(start); elapsed > 30*time.Second { + t.Errorf("New took %v on a busy database, want a bounded wait", elapsed) + } + case <-time.After(30 * time.Second): + t.Fatal("New blocked on a busy database; the access stamp must not wait for the write lock") + } +} + +// TestReadMetaAt_DoesNotRefreshLastAccessedAt guards the invariant that the +// read-only metadata scan used by `lumen clean` never counts as an access. +func TestReadMetaAt_DoesNotRefreshLastAccessedAt(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "index.db") + s, err := New(dbPath, 4) + if err != nil { + t.Fatal(err) + } + stale := "2020-01-01T00:00:00Z" + if err := s.SetMeta("last_accessed_at", stale); err != nil { + t.Fatal(err) + } + if err := s.Close(); err != nil { + t.Fatal(err) + } + + if _, err := ReadMetaAt(dbPath, "last_accessed_at"); err != nil { + t.Fatalf("ReadMetaAt: %v", err) + } + + got, err := ReadMetaAt(dbPath, "last_accessed_at") + if err != nil { + t.Fatalf("ReadMetaAt: %v", err) + } + if got["last_accessed_at"] != stale { + t.Errorf("last_accessed_at = %q, want unchanged %q", got["last_accessed_at"], stale) + } +} diff --git a/skills/reindex/SKILL.md b/skills/reindex/SKILL.md index 4c190469..71f152e3 100644 --- a/skills/reindex/SKILL.md +++ b/skills/reindex/SKILL.md @@ -17,8 +17,11 @@ Refresh or rebuild the bundled Lumen index for the current project. missing indexes automatically. 3. If the user explicitly asks for a clean rebuild, explain the options and run one via the shell: - - `lumen purge . && lumen index .` — deletes only the current project's - cached index before rebuilding. Prefer this. - - `lumen purge && lumen index .` — deletes every cached index on the host - before rebuilding. Use only when the user asks for a full wipe. + - `lumen index --force .` — rebuilds only the current project's index from + scratch. Prefer this. + - `lumen clean --days 0 && lumen index .` — deletes every cached index on the + host before rebuilding. Use only when the user asks for a full wipe. + - `lumen clean` — removes indexes for projects that no longer exist or have + not been used in 30 days. Use to reclaim disk space, not to rebuild the + current project. 4. After the refresh or rebuild, report the new index status.