diff --git a/docs/commands/reference.mdx b/docs/commands/reference.mdx index 509f234a..5a5ef88d 100644 --- a/docs/commands/reference.mdx +++ b/docs/commands/reference.mdx @@ -19,7 +19,7 @@ When invoked without arguments, kache prints `--help` and exits — it does not | `kache list [] [--sort ] [--no-pager]` | List cache entries or show details for one crate | | `kache why-miss ` | Diagnose why a crate keeps missing the cache | | `kache report [--format ] [--since ] [--root ] [--output ] [--top ]` | Build report in text / json / trace / markdown / github format | -| `kache gc [--max-age ]` | Evict entries by LRU or age | +| `kache gc [--max-age \| --stale-schema]` | Evict entries by LRU, age, or obsolete key schema | | `kache purge [--crate-name ]` | Wipe entire cache or entries for one crate | | `kache clean [-n \| --dry-run] [-y \| --yes]` | Find and remove `target/` directories (interactive; `-n` previews, `-y` removes all non-interactively) | | `kache sync [flags]` | Sync local cache with a configured remote | @@ -210,17 +210,20 @@ Saving a manifest is cheap; it is safe to run at the end of a CI build alongside ## `kache gc` ```sh -kache gc [--max-age ] +kache gc [--max-age | --stale-schema] ``` -Runs garbage collection. Without `--max-age`, applies the automatic policy: opt-in age retention from `KACHE_GC_MAX_AGE_HOURS` (default `0`, disabled) first, then duplicate and size-pressure eviction against the recomputed physical store size. Size pressure fires above `KACHE_MAX_SIZE` and evicts down to 90% of that cap. Duplicate cleanup is bounded by the same target and removes an entry only when backfilled metadata proves positive physical reclaim; unknown legacy entries are kept. The command reports each policy separately. With `--max-age`, runs only the requested age policy regardless of store size. +Runs garbage collection. Without an option, applies the automatic policy: opt-in age retention from `KACHE_GC_MAX_AGE_HOURS` (default `0`, disabled) first, then duplicate and size-pressure eviction against the recomputed physical store size. Size pressure fires above `KACHE_MAX_SIZE` and evicts down to 90% of that cap. Duplicate cleanup is bounded by the same target and removes an entry only when backfilled metadata proves positive physical reclaim; unknown legacy entries are kept. The command reports each policy separately. With `--max-age`, runs only the requested age policy regardless of store size. -During a rolling upgrade, both modes verify GC policy v2 before mutation and use a `gc_v2` command that older daemons reject without evicting. If the daemon cannot provide v2 semantics and per-policy reporting, the command reruns the requested policy locally under the same cross-process GC lock. +`--stale-schema` is an explicit key-upgrade cleanup. It retains entries created by the running cache-key recipe and removes entries from older recipes, including legacy entries whose recipe was not recorded. Ordinary upgrades and automatic GC do not remove those legacy entries. This mode runs locally under the cross-process GC lock and cannot be combined with `--max-age`. + +During a rolling upgrade, the automatic and `--max-age` modes verify GC policy v2 before mutation and use a `gc_v2` command that older daemons reject without evicting. If the daemon cannot provide v2 semantics and per-policy reporting, the command reruns the requested policy locally under the same cross-process GC lock. The stale-schema mode always runs locally because older daemons do not know the running binary's key recipe. ```sh kache gc # automatic configured policy (age first, then pressure) kache gc --max-age 30d # remove anything unused for 30 days kache gc --max-age 7d # aggressive cleanup before a release build +kache gc --stale-schema # reclaim entries orphaned by cache-key recipe changes ``` If `clean_incremental` is enabled (the default), GC removes tracked incremental compilation directories that previous wrapper invocations registered. Normal cached invocations also remove the current Cargo incremental dir eagerly. Adaptive and forced preserve-incremental compiles use separate state that kache never registers for GC; `cargo clean` still removes it with the target profile. diff --git a/src/cli.rs b/src/cli.rs index 8ba34e96..b7f56e51 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -2491,7 +2491,28 @@ pub fn run_gc_local(config: &Config, mode: GcMode) -> Result<()> { } /// Run garbage collection via the daemon. -pub fn gc(config: &Config, max_age_hours: Option) -> Result<()> { +pub fn gc(config: &Config, max_age_hours: Option, stale_schema: bool) -> Result<()> { + if stale_schema { + let store = Store::open(config)?; + let _gc_lock = match store.try_gc_lock()? { + Some(lock) => lock, + None => { + println!("Another GC is already running; skipping."); + return Ok(()); + } + }; + let stats = store.evict_stale_key_schemas(crate::cache_key::CACHE_KEY_VERSION)?; + println!( + "Stale-schema GC:{}\nCurrent key schema: {}.", + describe_eviction(&stats, false), + crate::cache_key::CACHE_KEY_VERSION, + ); + let total_size = store.total_size()?; + let entry_count = store.entry_count()?; + println!("Store: {} ({} entries)", ByteSize(total_size), entry_count); + return Ok(()); + } + let mode = GcMode::from_env(); if mode == GcMode::Background { let sleep_secs = std::env::var("KACHE_AUTO_GC_RETRY_DELAY_SECS") @@ -8194,6 +8215,7 @@ mod tests { ) -> crate::store::EntryMeta { crate::store::EntryMeta { cache_key: "k".to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "c".to_string(), crate_types: crate_types.iter().map(|v| (*v).to_string()).collect(), files: Vec::new(), diff --git a/src/main.rs b/src/main.rs index 3fa86d40..ed6c0248 100644 --- a/src/main.rs +++ b/src/main.rs @@ -102,11 +102,15 @@ enum Commands { no_pager: bool, }, - /// Run garbage collection (LRU eviction) + /// Run garbage collection Gc { /// Evict entries older than this duration (e.g. 7d, 24h) - #[arg(long)] + #[arg(long, conflicts_with = "stale_schema")] max_age: Option, + + /// Remove entries from old or unrecorded cache-key schemas + #[arg(long)] + stale_schema: bool, }, /// Wipe entire cache or entries for a specific crate @@ -499,7 +503,10 @@ fn main() -> Result<()> { sort, no_pager, }) => cli::list(&config, crate_name.as_deref(), &sort, no_pager), - Some(Commands::Gc { max_age }) => { + Some(Commands::Gc { + max_age, + stale_schema, + }) => { let hours = max_age .as_deref() .map(|value| { @@ -510,7 +517,7 @@ fn main() -> Result<()> { }) }) .transpose()?; - cli::gc(&config, hours) + cli::gc(&config, hours, stale_schema) } Some(Commands::Purge { crate_name }) => cli::purge(&config, crate_name.as_deref()), Some(Commands::Clean { dry_run, yes }) => cli::clean(dry_run, yes), diff --git a/src/store.rs b/src/store.rs index 87074ee3..1a10bcce 100644 --- a/src/store.rs +++ b/src/store.rs @@ -502,6 +502,13 @@ fn run_tmutil_addexclusion_bounded(dir: &str) { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct EntryMeta { pub cache_key: String, + /// Cache-key recipe version that produced `cache_key`. + /// + /// Entries written before this field existed deserialize as `0` (unknown), + /// so an explicit stale-schema sweep can reclaim them without making old + /// stores unreadable during an ordinary upgrade. + #[serde(default)] + pub key_schema: u32, pub crate_name: String, pub crate_types: Vec, pub files: Vec, @@ -960,6 +967,11 @@ fn initialize_db(db: &Connection) -> rusqlite::Result<()> { // this column the cache has no way to weigh what it is about to destroy. let _ = db .execute_batch("ALTER TABLE entries ADD COLUMN compile_time_ms INTEGER NOT NULL DEFAULT 0"); + // Cache-key recipe version for targeted reclamation after a key bump + // (kunobi-ninja/kache#750). Legacy rows are `0` = unknown and remain usable + // until the user explicitly requests a stale-schema sweep. + let _ = + db.execute_batch("ALTER TABLE entries ADD COLUMN key_schema INTEGER NOT NULL DEFAULT 0"); db.execute_batch( "CREATE TABLE IF NOT EXISTS blobs ( @@ -1764,6 +1776,7 @@ impl Store { // Write metadata (only meta.json in the entry directory) let meta = EntryMeta { cache_key: cache_key.to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: crate_name.to_string(), crate_types: crate_types.to_vec(), files: cached_files, @@ -1808,8 +1821,8 @@ impl Store { } record_entry_blobs(&tx, cache_key, &meta.files)?; tx.execute( - "INSERT OR REPLACE INTO entries (cache_key, crate_name, crate_type, profile, num_features, size, content_hash, compile_time_ms, committed) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 1)", - params![cache_key, crate_name, crate_type_str, profile, num_features, total_size as i64, content_hash, compile_time_ms as i64], + "INSERT OR REPLACE INTO entries (cache_key, crate_name, crate_type, profile, num_features, size, content_hash, compile_time_ms, key_schema, committed) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 1)", + params![cache_key, crate_name, crate_type_str, profile, num_features, total_size as i64, content_hash, compile_time_ms as i64, crate::cache_key::CACHE_KEY_VERSION], )?; tx.commit()?; @@ -1979,8 +1992,8 @@ impl Store { } record_entry_blobs(&tx, cache_key, &meta.files)?; tx.execute( - "INSERT OR REPLACE INTO entries (cache_key, crate_name, crate_type, profile, num_features, size, content_hash, compile_time_ms, committed) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 1)", - params![cache_key, meta.crate_name, crate_type_str, meta.profile, num_features, total_size as i64, content_hash, meta.compile_time_ms as i64], + "INSERT OR REPLACE INTO entries (cache_key, crate_name, crate_type, profile, num_features, size, content_hash, compile_time_ms, key_schema, committed) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 1)", + params![cache_key, meta.crate_name, crate_type_str, meta.profile, num_features, total_size as i64, content_hash, meta.compile_time_ms as i64, meta.key_schema], )?; tx.commit()?; @@ -2124,7 +2137,7 @@ impl Store { // Claim the entry row first. If it is already there, a concurrent or // earlier rebuild owns this entry's refcounts and we must not add more. let inserted = tx.execute( - "INSERT OR IGNORE INTO entries (cache_key, crate_name, crate_type, profile, num_features, size, content_hash, compile_time_ms, committed) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 1)", + "INSERT OR IGNORE INTO entries (cache_key, crate_name, crate_type, profile, num_features, size, content_hash, compile_time_ms, key_schema, committed) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 1)", params![ cache_key, meta.crate_name, @@ -2133,7 +2146,8 @@ impl Store { num_features, total_size as i64, content_hash, - meta.compile_time_ms as i64 + meta.compile_time_ms as i64, + meta.key_schema ], )?; if inserted == 0 { @@ -2638,6 +2652,39 @@ impl Store { self.evict_with(&crate::eviction::OlderThanPolicy { hours }, None) } + /// Remove entries written by a different (or unknown legacy) cache-key + /// recipe while retaining every entry from the running recipe. + /// + /// This is deliberately explicit rather than part of ordinary GC: rows + /// created before key-schema recording use `0`, and an upgrade must not + /// discard a still-reachable cache merely because its metadata predates + /// this field. `kache gc --stale-schema` is the user's opt-in boundary. + pub fn evict_stale_key_schemas(&self, current_schema: u32) -> Result { + let keys = { + let mut stmt = self.db.prepare( + "SELECT cache_key FROM entries + WHERE committed = 1 AND key_schema != ?1 + ORDER BY cache_key", + )?; + stmt.query_map(params![current_schema], |row| row.get::<_, String>(0))? + .collect::>>()? + }; + let candidates = self.eviction_candidates()?; + let by_key = candidates + .iter() + .map(|entry| (entry.key.as_str(), entry)) + .collect::>(); + let durable_upload_keys = self.durable_upload_keys()?; + Ok(self.apply_eviction( + &keys, + &by_key, + "stale_schema", + None, + None, + &durable_upload_keys, + )) + } + /// Evict duplicate entries that share the same content_hash. /// Keeps the most recently accessed entry for each content_hash group /// (consistent with LRU eviction policy). @@ -5644,6 +5691,7 @@ mod tests { fs::create_dir_all(&entry_dir).unwrap(); let meta = EntryMeta { cache_key: cache_key.to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "empty".to_string(), crate_types: vec!["rlib".to_string()], files: vec![], @@ -6190,6 +6238,103 @@ mod tests { assert!(store.contains("k1")); } + #[test] + fn evict_stale_key_schemas_keeps_only_the_running_schema() { + let dir = tempfile::tempdir().unwrap(); + let config = test_config(dir.path()); + let store = Store::open(&config).unwrap(); + + for (key, content) in [ + ("current", b"current artifact".as_slice()), + ("old", b"old artifact".as_slice()), + ("legacy", b"legacy artifact".as_slice()), + ] { + let output = dir.path().join(format!("{key}.rlib")); + std::fs::write(&output, content).unwrap(); + store + .put( + key, + key, + &["lib".into()], + &[], + "", + "dev", + &[(output, format!("{key}.rlib"))], + "", + "", + ) + .unwrap(); + } + + let prior_schema = crate::cache_key::CACHE_KEY_VERSION.saturating_sub(1); + store + .db + .execute( + "UPDATE entries + SET key_schema = ?1, last_accessed = datetime('now', '-1 day') + WHERE cache_key = 'old'", + params![prior_schema], + ) + .unwrap(); + store + .db + .execute( + "UPDATE entries + SET key_schema = 0, last_accessed = datetime('now', '-1 day') + WHERE cache_key = 'legacy'", + [], + ) + .unwrap(); + + let stats = store + .evict_stale_key_schemas(crate::cache_key::CACHE_KEY_VERSION) + .unwrap(); + assert_eq!(stats.entries_evicted, 2); + assert!(stats.bytes_freed > 0); + assert_eq!(stats.blobs_removed, 2); + assert_eq!(stats.entries_pinned, 0); + assert!(store.contains("current")); + assert!(!store.contains("old")); + assert!(!store.contains("legacy")); + assert_eq!(store.entry_count().unwrap(), 1); + + let second = store + .evict_stale_key_schemas(crate::cache_key::CACHE_KEY_VERSION) + .unwrap(); + assert_eq!(second.entries_evicted, 0); + } + + #[test] + fn entry_meta_key_schema_defaults_to_unknown_for_legacy_json() { + let dir = tempfile::tempdir().unwrap(); + let config = test_config(dir.path()); + let store = Store::open(&config).unwrap(); + let output = dir.path().join("lib.rlib"); + std::fs::write(&output, b"artifact").unwrap(); + store + .put( + "key", + "crate", + &["lib".into()], + &[], + "", + "dev", + &[(output, "lib.rlib".into())], + "", + "", + ) + .unwrap(); + + let content = std::fs::read_to_string(store.entry_dir("key").join("meta.json")).unwrap(); + let current: EntryMeta = serde_json::from_str(&content).unwrap(); + assert_eq!(current.key_schema, crate::cache_key::CACHE_KEY_VERSION); + + let mut legacy: serde_json::Value = serde_json::from_str(&content).unwrap(); + legacy.as_object_mut().unwrap().remove("key_schema"); + let parsed: EntryMeta = serde_json::from_value(legacy).unwrap(); + assert_eq!(parsed.key_schema, 0); + } + #[test] fn test_store_import_downloaded_entry() { let dir = tempfile::tempdir().unwrap(); @@ -6205,8 +6350,10 @@ mod tests { // Real content hash — the import trust boundary re-hashes and rejects a // mismatch (kunobi-ninja/kache#211). let hash = crate::cache_key::hash_file(&entry_dir.join("lib.rlib")).unwrap(); + let prior_schema = crate::cache_key::CACHE_KEY_VERSION.saturating_sub(1); let meta = EntryMeta { cache_key: "downloaded_key".to_string(), + key_schema: prior_schema, crate_name: "downloaded_crate".to_string(), crate_types: vec!["lib".to_string()], files: vec![CachedFile { @@ -6229,6 +6376,15 @@ mod tests { store.import_downloaded_entry("downloaded_key").unwrap(); assert!(store.contains("downloaded_key")); assert_eq!(store.entry_count().unwrap(), 1); + let indexed_schema: u32 = store + .db + .query_row( + "SELECT key_schema FROM entries WHERE cache_key = 'downloaded_key'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(indexed_schema, prior_schema); } #[test] @@ -6243,6 +6399,7 @@ mod tests { let meta = EntryMeta { cache_key: "incomplete_key".to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "incomplete_crate".to_string(), crate_types: vec!["lib".to_string()], files: vec![CachedFile { @@ -6286,6 +6443,7 @@ mod tests { let hash = crate::cache_key::hash_file(&entry_dir.join("lib.rlib")).unwrap(); let meta = EntryMeta { cache_key: "dl_key".to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "dl_crate".to_string(), crate_types: vec!["lib".to_string()], files: vec![CachedFile { @@ -6635,6 +6793,7 @@ mod tests { let meta = EntryMeta { cache_key: "mismatch_key".to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "mismatch_crate".to_string(), crate_types: vec!["lib".to_string()], files: vec![CachedFile { @@ -6685,6 +6844,7 @@ mod tests { mutate(&mut file); let meta = EntryMeta { cache_key: key.to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "c".to_string(), crate_types: vec!["lib".to_string()], files: vec![file], @@ -8180,6 +8340,7 @@ mod tests { let hash = crate::cache_key::hash_file(&entry_dir.join("lib.rlib")).unwrap(); let meta = EntryMeta { cache_key: "old_key".to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "old_crate".to_string(), crate_types: vec!["lib".to_string()], files: vec![CachedFile { @@ -8238,6 +8399,7 @@ mod tests { let hash = crate::cache_key::hash_file(&artifact).unwrap(); let meta = EntryMeta { cache_key: "old_bad_key".to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "old_bad_crate".to_string(), crate_types: vec!["lib".to_string()], files: vec![CachedFile { @@ -8303,6 +8465,7 @@ mod tests { let meta = EntryMeta { cache_key: key.to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "shared_crate".to_string(), crate_types: vec!["lib".to_string()], files: vec![CachedFile { @@ -8849,6 +9012,7 @@ mod tests { let meta = EntryMeta { cache_key: "legacy_key".to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "legacy_crate".to_string(), crate_types: vec!["lib".to_string()], files: vec![ @@ -8940,6 +9104,7 @@ mod tests { let size = fs::metadata(&artifact).unwrap().len(); let meta = EntryMeta { cache_key: "legacy_race".to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "legacy_crate".to_string(), crate_types: vec!["lib".to_string()], files: vec![CachedFile { @@ -9194,6 +9359,7 @@ mod tests { fn covers_requested_emit_semantics() { let mk = |kinds: &[&str]| EntryMeta { cache_key: "k".into(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "c".into(), crate_types: vec![], files: vec![], @@ -9252,6 +9418,7 @@ mod tests { let meta = EntryMeta { cache_key: "k".into(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "rococo_runtime".into(), crate_types: vec!["cdylib".into()], files, @@ -9325,6 +9492,7 @@ mod tests { let meta = EntryMeta { cache_key: "dl_ch_test".to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "dlcrate".to_string(), crate_types: vec!["lib".to_string()], files: vec![CachedFile { diff --git a/src/wrapper.rs b/src/wrapper.rs index e4b007e2..0615b1fb 100644 --- a/src/wrapper.rs +++ b/src/wrapper.rs @@ -5143,6 +5143,7 @@ mod tests { fn meta_with_diagnostics(stdout: &str, stderr: &str) -> crate::store::EntryMeta { crate::store::EntryMeta { cache_key: "k".to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "c".to_string(), crate_types: vec![], files: vec![], @@ -5237,6 +5238,7 @@ mod tests { ) -> crate::store::EntryMeta { crate::store::EntryMeta { cache_key: cache_key.to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "foo".to_string(), crate_types: vec!["lib".to_string()], files, @@ -5628,6 +5630,7 @@ mod tests { fn meta(names: &[&str]) -> crate::store::EntryMeta { crate::store::EntryMeta { cache_key: "key".to_string(), + key_schema: crate::cache_key::CACHE_KEY_VERSION, crate_name: "foo.c".to_string(), crate_types: vec![], files: names diff --git a/tests/cli_commands_test.rs b/tests/cli_commands_test.rs index a81dfa8b..7919c769 100644 --- a/tests/cli_commands_test.rs +++ b/tests/cli_commands_test.rs @@ -470,6 +470,22 @@ fn gc_on_empty_cache_succeeds() { e.cmd().arg("gc").assert().success(); let e = env(); e.cmd().args(["gc", "--max-age", "7d"]).assert().success(); + let e = env(); + e.cmd() + .args(["gc", "--stale-schema"]) + .assert() + .success() + .stdout(predicates::str::contains("Stale-schema GC:")); +} + +#[test] +fn gc_rejects_overlapping_explicit_policies() { + let e = env(); + e.cmd() + .args(["gc", "--max-age", "7d", "--stale-schema"]) + .assert() + .failure() + .stderr(predicates::str::contains("cannot be used with")); } #[test]