Skip to content

Commit aa9cbb9

Browse files
committed
Observability commit 23: enforce disk cache initialization
1 parent 12f4e3f commit aa9cbb9

44 files changed

Lines changed: 1079 additions & 276 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎crates/fluxheim-server/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ mod native_http1;
1616
#[cfg(feature = "acme")]
1717
mod native_http1_acme;
1818
mod native_http1_cache;
19+
mod native_http1_cache_lifecycle;
1920
mod native_http1_client;
2021
mod native_http1_cors;
2122
mod native_http1_forwarded;

‎crates/fluxheim-server/src/native_http1_cache.rs‎

Lines changed: 31 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ pub(crate) use native_http1_cache_backend::prepare_native_storage_bin_root_for_l
3939
use native_http1_cache_backend::{
4040
NativeDiskCacheBackend, NativeDiskCacheLocation, NativeDiskCacheRecord, NativeDiskCacheState,
4141
};
42+
pub(crate) use native_http1_cache_disk_path::prepare_native_disk_cache_root;
4243
use native_http1_cache_disk_path::{
4344
native_cache_path_contains_symlink, native_disk_cache_read_limit, read_native_disk_cache_file,
4445
};
@@ -103,6 +104,13 @@ pub(crate) struct NativeDiskCache {
103104
index_flush: Option<NativeStorageBinIndexFlush>,
104105
}
105106

107+
impl Drop for NativeDiskCache {
108+
fn drop(&mut self) {
109+
// Close background writes and checkpoint while the backend still owns its lease.
110+
drop(self.index_flush.take());
111+
}
112+
}
113+
106114
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
107115
pub(crate) struct NativeDiskCacheStats {
108116
pub(crate) entries: u64,
@@ -125,34 +133,26 @@ impl NativeDiskCache {
125133
}
126134

127135
pub(crate) fn from_config(config: &CacheConfig) -> Option<Self> {
128-
if !config.disk.enabled || !native_disk_cache_supported(config) {
129-
return None;
136+
Self::try_from_config(config).ok().flatten()
137+
}
138+
139+
pub(crate) fn try_from_config(config: &CacheConfig) -> std::io::Result<Option<Self>> {
140+
if !config.disk.enabled {
141+
return Ok(None);
130142
}
131-
let (root, backend) = match NativeDiskCacheBackend::from_config(config) {
132-
Ok(backend) => backend,
133-
Err(error) => {
134-
log::error!(
135-
target: "fluxheim::native_http1",
136-
"native disk cache backend: {error}"
137-
);
138-
return None;
139-
}
140-
};
141-
let encryption =
142-
match NativeDiskCacheEncryption::from_config(&config.disk.encryption, &root) {
143-
Ok(encryption) => encryption,
144-
Err(error) => {
145-
log::error!(
146-
target: "fluxheim::native_http1",
147-
"native disk cache encryption {}: {error}",
148-
root.display()
149-
);
150-
fluxheim_observability::operational::failure(
151-
fluxheim_observability::event::FailureReason::CacheEncryption,
152-
);
153-
return None;
154-
}
155-
};
143+
if !native_disk_cache_supported(config) {
144+
return Err(std::io::Error::new(
145+
std::io::ErrorKind::Unsupported,
146+
"unsupported disk cache policy",
147+
));
148+
}
149+
let (root, backend) = NativeDiskCacheBackend::from_config(config)?;
150+
let encryption = NativeDiskCacheEncryption::from_config(&config.disk.encryption, &root)
151+
.inspect_err(|_| {
152+
fluxheim_observability::operational::failure(
153+
fluxheim_observability::event::FailureReason::CacheEncryption,
154+
)
155+
})?;
156156
let mut cache = Self {
157157
root,
158158
max_bytes: config.disk.max_size_bytes.as_u64(),
@@ -175,36 +175,18 @@ impl NativeDiskCache {
175175
.ok_or_else(|| std::io::Error::other("cache encryption disappeared"))?
176176
.complete_root_migration(&cache.root)
177177
});
178-
if let Err(error) = migration {
179-
log::error!(
180-
target: "fluxheim::security",
181-
"native disk cache encryption-root migration {}: {error}",
182-
cache.root.display()
183-
);
184-
return None;
185-
}
186-
}
187-
if let Err(error) = cache.rebuild_index() {
188-
log::warn!(
189-
target: "fluxheim::native_http1",
190-
"native disk cache index rebuild {}: {error}",
191-
cache.root.display()
192-
);
178+
migration?;
193179
}
180+
cache.rebuild_index()?;
194181
cache.index_flush = NativeStorageBinIndexFlush::start(
195182
&cache.backend,
196183
&cache.state,
197184
cache
198185
.encryption
199186
.as_ref()
200187
.map(NativeDiskCacheEncryption::index_key),
201-
)
202-
.inspect_err(|error| {
203-
log::error!(target: "fluxheim::native_http1", "cache index service: {error}");
204-
})
205-
.ok()?;
206-
cache.persist_storage_bin_index();
207-
Some(cache)
188+
)?;
189+
Ok(Some(cache))
208190
}
209191

210192
pub(crate) fn stats(&self) -> NativeDiskCacheStats {

‎crates/fluxheim-server/src/native_http1_cache_disk_path.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use std::path::{Path, PathBuf};
55

66
const NATIVE_DISK_CACHE_READ_OVERHEAD_BYTES: u64 = 1024 * 1024;
77

8-
pub(super) fn prepare_native_disk_cache_root(root: &Path) -> std::io::Result<PathBuf> {
8+
pub(crate) fn prepare_native_disk_cache_root(root: &Path) -> std::io::Result<PathBuf> {
99
if native_configured_cache_path_contains_symlink(root)? {
1010
return Err(std::io::Error::new(
1111
std::io::ErrorKind::InvalidInput,

‎crates/fluxheim-server/src/native_http1_cache_disk_path_windows.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_REPARSE_POINT;
66

77
const NATIVE_DISK_CACHE_READ_OVERHEAD_BYTES: u64 = 1024 * 1024;
88

9-
pub(super) fn prepare_native_disk_cache_root(root: &Path) -> std::io::Result<PathBuf> {
9+
pub(crate) fn prepare_native_disk_cache_root(root: &Path) -> std::io::Result<PathBuf> {
1010
if native_configured_cache_path_contains_symlink(root)? {
1111
return Err(unsafe_path_error(
1212
"native disk cache root must not cross reparse points",
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
use std::sync::{Arc, Mutex, Weak};
2+
3+
use fluxheim_config::CacheConfig;
4+
5+
use crate::NativeHttp1ProxyConfigError;
6+
use crate::native_http1_cache::NativeDiskCache;
7+
use crate::native_http1_cache::prepare_native_disk_cache_root;
8+
9+
struct SharedDiskCache {
10+
policy: CacheConfig,
11+
vhost: String,
12+
route: Option<String>,
13+
cache: Weak<NativeDiskCache>,
14+
}
15+
16+
static DISK_CACHES: Mutex<Vec<SharedDiskCache>> = Mutex::new(Vec::new());
17+
const MAX_LIVE_DISK_CACHES: usize = 4096;
18+
19+
pub(crate) fn shared_disk_cache(
20+
config: &CacheConfig,
21+
vhost: &str,
22+
route: Option<&str>,
23+
) -> Result<Option<Arc<NativeDiskCache>>, NativeHttp1ProxyConfigError> {
24+
if !config.disk.enabled {
25+
return Ok(None);
26+
}
27+
let initialization = || NativeHttp1ProxyConfigError::CacheInitialization;
28+
let path = config.disk.path.as_deref().ok_or_else(initialization)?;
29+
let root = prepare_native_disk_cache_root(path).map_err(|_| initialization())?;
30+
let mut policy = config.clone();
31+
policy.disk.path = Some(root);
32+
// Presentation-only changes do not alter persisted objects or allocator state.
33+
policy.status_header = None;
34+
policy.status_reason_header = None;
35+
36+
// Serialize initialization, not requests. Weak entries never prolong a lease.
37+
let mut caches = DISK_CACHES.lock().map_err(|_| initialization())?;
38+
caches.retain(|entry| entry.cache.strong_count() > 0);
39+
for entry in caches.iter() {
40+
if entry.policy.disk.path != policy.disk.path {
41+
continue;
42+
}
43+
if let Some(cache) = entry.cache.upgrade() {
44+
if entry.policy != policy
45+
|| entry.vhost != vhost
46+
|| entry.route.as_deref() != route
47+
|| config.disk.encryption.enabled
48+
{
49+
return Err(NativeHttp1ProxyConfigError::CacheRestartRequired);
50+
}
51+
return Ok(Some(cache));
52+
}
53+
}
54+
if caches.len() >= MAX_LIVE_DISK_CACHES {
55+
return Err(initialization());
56+
}
57+
let cache = Arc::new(
58+
NativeDiskCache::try_from_config(&policy)
59+
.map_err(|_| initialization())?
60+
.ok_or_else(initialization)?,
61+
);
62+
caches.push(SharedDiskCache {
63+
policy,
64+
vhost: vhost.to_owned(),
65+
route: route.map(str::to_owned),
66+
cache: Arc::downgrade(&cache),
67+
});
68+
Ok(Some(cache))
69+
}

‎crates/fluxheim-server/src/native_http1_cache_purge.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,12 @@ pub(crate) fn register_native_disk_cache_purge_handle(
228228
);
229229
}
230230
};
231+
registry.retain(|entry| entry.cache.strong_count() > 0);
232+
if registry.iter().any(|entry| {
233+
entry.vhost == vhost && entry.route == route && entry.cache.ptr_eq(&Arc::downgrade(cache))
234+
}) {
235+
return;
236+
}
231237
registry.push(NativeDiskCachePurgeHandle {
232238
vhost,
233239
route,

‎crates/fluxheim-server/src/native_http1_cache_storage_bin.rs‎

Lines changed: 68 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@ struct NativeStorageBinIndexTask {
4141
layout: StorageBinLayoutPlan,
4242
state: Arc<Mutex<NativeDiskCacheState>>,
4343
dirty: AtomicBool,
44-
flush_lock: Mutex<()>,
44+
// The final synchronous flush closes the task while holding this lock.
45+
flush_lock: Mutex<bool>,
4546
index_key: Option<Arc<SecretBytes<32>>>,
4647
}
4748

@@ -59,9 +60,15 @@ impl NativeStorageBinIndexFlush {
5960
layout: storage_bin.layout.clone(),
6061
state: Arc::clone(state),
6162
dirty: AtomicBool::new(false),
62-
flush_lock: Mutex::new(()),
63+
flush_lock: Mutex::new(false),
6364
index_key,
6465
});
66+
let entries = task
67+
.state
68+
.lock()
69+
.map_err(|_| std::io::Error::other("cache index state is unavailable"))
70+
.map(|state| native_storage_bin_index_entries(&state, task.index_key.as_deref()))?;
71+
write_storage_bin_index(&task.layout, &entries)?;
6572
service.register(&task);
6673
Ok(Some(Self { task }))
6774
}
@@ -113,13 +120,19 @@ pub(super) fn native_storage_bin_index_worker_count() -> usize {
113120

114121
impl NativeStorageBinIndexTask {
115122
fn flush(&self, force: bool) {
116-
let _flush = self.flush_lock.lock().unwrap_or_else(|error| {
123+
let mut closed = self.flush_lock.lock().unwrap_or_else(|error| {
117124
log::error!(
118125
target: "fluxheim::security",
119126
"native storage-bin index flush lock poisoned: {error}; aborting"
120127
);
121128
fluxheim_common::fatal::abort(fluxheim_common::fatal::Reason::StorageBinFlushPoisoned);
122129
});
130+
if *closed {
131+
return;
132+
}
133+
if force {
134+
*closed = true;
135+
}
123136
let was_dirty = self.dirty.swap(false, Ordering::AcqRel);
124137
if !force && !was_dirty {
125138
return;
@@ -377,6 +390,58 @@ impl NativeDiskCache {
377390
mod confidential_index_tests {
378391
use super::*;
379392

393+
#[test]
394+
fn final_index_flush_finishes_before_lease_release_and_closes_retained_tasks() {
395+
let root = tempfile::tempdir().unwrap();
396+
let mut config = fluxheim_config::CacheConfig::default();
397+
config.disk.enabled = true;
398+
config.disk.backend = fluxheim_config::CacheDiskBackend::StorageBin;
399+
config.disk.path = Some(root.path().to_path_buf());
400+
let cache = NativeDiskCache::try_from_config(&config).unwrap().unwrap();
401+
let task = Arc::clone(&cache.index_flush.as_ref().unwrap().task);
402+
let guard = task.flush_lock.lock().unwrap();
403+
let (started, ready) = std::sync::mpsc::channel();
404+
let dropper = std::thread::spawn(move || {
405+
started.send(()).unwrap();
406+
drop(cache);
407+
});
408+
ready.recv_timeout(Duration::from_secs(5)).unwrap();
409+
assert!(
410+
super::super::native_http1_cache_backend::acquire_native_storage_bin_lease(root.path())
411+
.is_err()
412+
);
413+
drop(guard);
414+
dropper.join().unwrap();
415+
// Parallel subprocess tests can briefly inherit a Unix lease across fork,
416+
// until exec closes CLOEXEC descriptors. A retained worker must not own it.
417+
let deadline = std::time::Instant::now() + Duration::from_secs(5);
418+
loop {
419+
match super::super::native_http1_cache_backend::acquire_native_storage_bin_lease(
420+
root.path(),
421+
) {
422+
Ok(lease) => {
423+
drop(lease);
424+
break;
425+
}
426+
Err(error)
427+
if error.kind() == std::io::ErrorKind::AlreadyExists
428+
&& std::time::Instant::now() < deadline =>
429+
{
430+
std::thread::sleep(Duration::from_millis(10))
431+
}
432+
Err(error) => {
433+
panic!("lease was not released after closing the index task: {error}")
434+
}
435+
}
436+
}
437+
let index = root.path().join(".fluxheim-storage-bin-index-v1");
438+
std::fs::write(&index, b"new generation").unwrap();
439+
task.dirty.store(true, Ordering::Release);
440+
task.flush(false);
441+
task.flush(true);
442+
assert_eq!(std::fs::read(index).unwrap(), b"new generation");
443+
}
444+
380445
#[test]
381446
fn encrypted_index_entries_do_not_persist_combined_keys() {
382447
let mut state = NativeDiskCacheState::default();

‎crates/fluxheim-server/src/native_http1_proxy_builder.rs‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -253,13 +253,15 @@ impl NativeHttp1Proxy {
253253
.with_metrics_scope("root", None)
254254
.with_header_policy(&config.headers)
255255
});
256-
let native = native.map(|proxy| {
257-
if Self::proxy_cache_supported_for_proxy(&config.cache, &config.proxy) {
258-
proxy.with_proxy_cache_config_for(&config.cache, "root", None)
259-
} else {
260-
proxy
261-
}
262-
});
256+
let native = native
257+
.map(|proxy| {
258+
if Self::proxy_cache_supported_for_proxy(&config.cache, &config.proxy) {
259+
proxy.with_proxy_cache_config_for(&config.cache, "root", None)
260+
} else {
261+
Ok(proxy)
262+
}
263+
})
264+
.transpose()?;
263265
#[cfg(any(
264266
feature = "compression-brotli",
265267
feature = "compression-gzip",

‎crates/fluxheim-server/src/native_http1_proxy_cache_support.rs‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,23 +18,21 @@ impl NativeHttp1Proxy {
1818
Self::proxy_cache_supported(cache) && native_slice_cache_supported_for_proxy(cache, proxy)
1919
}
2020

21-
pub fn with_proxy_cache_config(mut self, cache: &CacheConfig) -> Self {
22-
if let Some(cache) = NativeProxyMemoryCache::from_config(cache) {
23-
self.cache = Some(cache);
24-
}
25-
self
21+
pub fn with_proxy_cache_config(
22+
self,
23+
cache: &CacheConfig,
24+
) -> Result<Self, crate::NativeHttp1ProxyConfigError> {
25+
self.with_proxy_cache_config_for(cache, "native", None)
2626
}
2727

2828
pub fn with_proxy_cache_config_for(
2929
mut self,
3030
cache: &CacheConfig,
3131
vhost: &str,
3232
route: Option<&str>,
33-
) -> Self {
34-
if let Some(cache) = NativeProxyMemoryCache::from_config_with_metrics(cache, vhost, route) {
35-
self.cache = Some(cache);
36-
}
37-
self
33+
) -> Result<Self, crate::NativeHttp1ProxyConfigError> {
34+
self.cache = NativeProxyMemoryCache::try_from_config_with_metrics(cache, vhost, route)?;
35+
Ok(self)
3836
}
3937
}
4038

0 commit comments

Comments
 (0)