diff --git a/README.md b/README.md index 8866db60..3f2b8c8a 100644 --- a/README.md +++ b/README.md @@ -398,7 +398,7 @@ Lumen reclaim the space for you: ```bash lumen clean # remove indexes unused for 30 days or whose project is gone lumen clean --days 7 # tighten the cutoff to a week -lumen clean --days 0 # remove every cached index on this host +lumen clean --days 0 # remove every eligible index except actively locked indexes ``` An index counts as used every time Lumen opens it (search, indexing, status, or diff --git a/cmd/clean.go b/cmd/clean.go index 5c4b801a..01685a1d 100644 --- a/cmd/clean.go +++ b/cmd/clean.go @@ -24,12 +24,18 @@ import ( "github.com/ory/lumen/internal/config" "github.com/ory/lumen/internal/indexlock" "github.com/ory/lumen/internal/store" + "github.com/ory/lumen/internal/tui" "github.com/spf13/cobra" ) // defaultCleanDays is how long an index may go unused before `lumen clean` // removes it. -const defaultCleanDays = 30 +const ( + defaultCleanDays = 30 + maxCleanDays = 106751 +) + +var removeIndexDir = os.RemoveAll func init() { addCleanFlags(cleanCmd) @@ -40,7 +46,7 @@ func init() { // definition never drifts from what runClean reads. func addCleanFlags(cmd *cobra.Command) { cmd.Flags().Int("days", defaultCleanDays, - "remove indexes not used in the last N days (0 removes every index that is not currently being written)") + "remove indexes not used in the last N days (0 removes every eligible index except those protected by active locks)") } var cleanCmd = &cobra.Command{ @@ -56,7 +62,8 @@ checkouts, and abandoned models leave behind data that is never read again. Indexes written by older binaries that never recorded an access time fall back to their last indexing time; those without any usable timestamp are removed. -Use "lumen clean --days 0" to drop every cached index on this host, and +Use "lumen clean --days 0" to drop every eligible cached index except those +protected by active locks, and "lumen index --force " to rebuild a single project from scratch. Indexes with an indexer currently running are always kept.`, defaultCleanDays), @@ -72,6 +79,9 @@ func runClean(cmd *cobra.Command, _ []string) error { if days < 0 { return fmt.Errorf("--days must not be negative, got %d", days) } + if days > maxCleanDays { + return fmt.Errorf("--days must not exceed %d, got %d", maxCleanDays, days) + } dataDir := filepath.Join(config.XDGDataDir(), "lumen") return cleanIndexes(cmd.ErrOrStderr(), cmd.OutOrStdout(), dataDir, days, time.Now()) } @@ -82,10 +92,11 @@ func runClean(cmd *cobra.Command, _ []string) error { // reported and the sweep continues; the first such failure is returned once // every directory has been considered. func cleanIndexes(stderr, stdout io.Writer, dataDir string, days int, now time.Time) error { + progress := tui.NewProgress(stderr) entries, err := os.ReadDir(dataDir) if err != nil { if os.IsNotExist(err) { - _, _ = fmt.Fprintln(stderr, "No index data found — nothing to clean.") + progress.Info("No index data found — nothing to clean.") return nil } return fmt.Errorf("read data dir: %w", err) @@ -102,29 +113,17 @@ func cleanIndexes(stderr, stdout io.Writer, dataDir string, days int, now time.T continue } hashDir := filepath.Join(dataDir, entry.Name()) - dbPath := filepath.Join(hashDir, "index.db") - - if indexlock.IsHeld(indexlock.LockPathForDB(dbPath)) { - _, _ = fmt.Fprintf(stderr, "Keeping %s: an indexer is currently running.\n", entry.Name()) + wasRemoved, err := cleanIndex(progress, entry.Name(), hashDir, days, cutoff) + if wasRemoved { + removed++ + } else { skipped++ - continue - } - - stale, reason := isIndexStale(dbPath, days, cutoff) - if !stale { - skipped++ - continue } - if err := os.RemoveAll(hashDir); err != nil { - _, _ = fmt.Fprintf(stderr, "Failed to remove %s: %v\n", hashDir, err) + if err != nil { if firstErr == nil { - firstErr = fmt.Errorf("remove %s: %w", hashDir, err) + firstErr = err } - skipped++ - continue } - _, _ = fmt.Fprintf(stderr, "Removed %s (%s).\n", entry.Name(), reason) - removed++ } _, _ = fmt.Fprintf(stdout, "Removed %d index director%s, skipped %d.\n", @@ -132,6 +131,28 @@ func cleanIndexes(stderr, stdout io.Writer, dataDir string, days int, now time.T return firstErr } +// cleanIndex evaluates and removes one index while holding its writer lock. +func cleanIndex(progress *tui.Progress, name, hashDir string, days int, cutoff time.Time) (bool, error) { + dbPath := filepath.Join(hashDir, "index.db") + lock, err := indexlock.TryAcquire(indexlock.LockPathForDB(dbPath)) + if err != nil || lock == nil { + progress.Info(fmt.Sprintf("Keeping %s: an indexer is currently running.", name)) + return false, nil + } + defer lock.Release() + + stale, reason := isIndexStale(dbPath, days, cutoff) + if !stale { + return false, nil + } + if err := removeIndexDir(hashDir); err != nil { + progress.Info(fmt.Sprintf("Failed to remove %s: %v", hashDir, err)) + return false, fmt.Errorf("remove %s: %w", hashDir, err) + } + progress.Info(fmt.Sprintf("Removed %s (%s).", name, reason)) + return true, nil +} + // isIndexStale reports whether the index at dbPath is no longer worth keeping, // along with a human-readable reason. The metadata read is read-only so it does // not itself count as an access. diff --git a/cmd/clean_test.go b/cmd/clean_test.go index 2f7abe32..904ad9b6 100644 --- a/cmd/clean_test.go +++ b/cmd/clean_test.go @@ -17,6 +17,7 @@ package cmd import ( "bytes" "database/sql" + "errors" "os" "path/filepath" "testing" @@ -389,6 +390,61 @@ func TestClean_NegativeDaysFails(t *testing.T) { assert.DirExists(t, hashDir, "a rejected invocation must not delete anything") } +func TestClean_MaxDaysAccepted(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "max-days") + hashDir := seedIndex(t, project, embedder.DefaultModel, nil) + + _, _, err := runCleanCmd(t, "--days", "106751") + require.NoError(t, err) + assert.DirExists(t, hashDir, "the largest safe whole-day duration must be accepted") +} + +func TestClean_TooManyDaysFailsWithoutDeleting(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "overflow-days") + hashDir := seedIndex(t, project, embedder.DefaultModel, nil) + + _, _, err := runCleanCmd(t, "--days", "106752") + require.Error(t, err) + assert.Contains(t, err.Error(), "106751") + assert.DirExists(t, hashDir, "an overflowing duration must be rejected before cleanup") +} + +func TestClean_HoldsLockDuringRemovalAndReleasesItAfterFailure(t *testing.T) { + tmp := resolvedTempDir(t) + t.Setenv("XDG_DATA_HOME", tmp) + + project := projectDir(t, "remove-failure") + hashDir := seedIndex(t, project, embedder.DefaultModel, map[string]string{ + "last_accessed_at": daysAgo(45), + }) + lockPath := indexlock.LockPathForDB(filepath.Join(hashDir, "index.db")) + removeErr := errors.New("injected removal failure") + lockHeldDuringRemoval := false + originalRemoveIndexDir := removeIndexDir + removeIndexDir = func(path string) error { + assert.Equal(t, hashDir, path) + lockHeldDuringRemoval = indexlock.IsHeld(lockPath) + return removeErr + } + t.Cleanup(func() { removeIndexDir = originalRemoveIndexDir }) + + _, _, err := runCleanIndexes(t, tmp, 30) + require.ErrorIs(t, err, removeErr) + assert.True(t, lockHeldDuringRemoval, "cleanup must hold the index lock while removing the directory") + assert.DirExists(t, hashDir, "a failed removal must leave the index directory in place") + + lock, lockErr := indexlock.TryAcquire(lockPath) + require.NoError(t, lockErr) + require.NotNil(t, lock, "cleanup must release the index lock after a removal failure") + lock.Release() +} + func TestClean_RejectsPositionalArgs(t *testing.T) { require.Error(t, cleanCmd.Args(cleanCmd, []string{"/some/project"}), "clean takes no positional arguments") diff --git a/skills/reindex/SKILL.md b/skills/reindex/SKILL.md index 71f152e3..e9c534c6 100644 --- a/skills/reindex/SKILL.md +++ b/skills/reindex/SKILL.md @@ -19,8 +19,9 @@ Refresh or rebuild the bundled Lumen index for the current project. run one via the shell: - `lumen index --force .` — rebuilds only the current project's index from scratch. Prefer this. - - `lumen clean --days 0 && lumen index .` — deletes every cached index on the - host before rebuilding. Use only when the user asks for a full wipe. + - `lumen clean --days 0 && lumen index .` — deletes every eligible cached + index except those protected by active locks, then rebuilds. Use only when + the user asks for a full wipe. - `lumen clean` — removes indexes for projects that no longer exist or have not been used in 30 days. Use to reclaim disk space, not to rebuild the current project.