diff --git a/.gitignore b/.gitignore index 5c90e550648..87f77acf0e1 100644 --- a/.gitignore +++ b/.gitignore @@ -31,5 +31,7 @@ Cargo.lock # Generated for testing tutorial code build /src/plan/mygc +/*.log +/_* # Python script cache __pycache__ diff --git a/Cargo.toml b/Cargo.toml index 7f74d72cb3b..9f4f810f1c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,13 +31,16 @@ doctest = false [dependencies] # MMTk macros - we have to specify a version here in order to publish the crate, even though we use the dependency from a local path. -mmtk-macros = { version="0.33.0", path = "macros/" } +mmtk-macros = { version = "0.33.0", path = "macros/" } # Third party dependencies atomic = "0.6.0" atomic_refcell = "0.1.7" atomic-traits = "0.4.0" -bytemuck = { version = "1.14.0", features = ["derive", "zeroable_maybe_uninit"] } +bytemuck = { version = "1.14.0", features = [ + "derive", + "zeroable_maybe_uninit", +] } cfg-if = "1.0" crossbeam = "0.8.1" delegate = "0.13.2" @@ -46,7 +49,9 @@ enum-map = "2.7.3" env_logger = { version = "0.11.3", optional = true } is-terminal = "0.4.7" itertools = "0.14.0" -jemalloc-sys = { version = "0.5.3", features = ["disable_initial_exec_tls"], optional = true } +jemalloc-sys = { version = "0.5.3", features = [ + "disable_initial_exec_tls", +], optional = true } lazy_static = "1.1" libc = "0.2" log = { version = "0.4", features = ["max_level_trace"] } @@ -106,6 +111,14 @@ builtin_env_logger = ["dep:env_logger"] # Enable this feature if you want to use nightly features and compiler nightly = [] +# LXR Features +lxr_no_cm = [] +lxr_no_lazy = [] +lxr_stw = ["lxr_no_cm", "lxr_no_lazy"] +lxr_no_nursery_evac = [] +lxr_no_mature_evac = [] +lxr_no_evac = ["lxr_no_nursery_evac", "lxr_no_mature_evac"] + # This feature is only supported on x86-64 for now # It's manually added to CI scripts perf_counter = ["dep:pfm"] @@ -141,7 +154,7 @@ set_unlog_bits_vm_space = [] # TODO: This is not properly implemented yet. We currently use an immortal space instead, and do not guarantee read-only semantics. ro_space = [] # A code space with execution permission. -code_space = [] +code_space = [] # By default, we only allow execution permission for code spaces. With this feature, all the spaces have execution permission. # Use with care. diff --git a/README.md b/README.md index 3b557084604..9ad9bf8d799 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ MMTk uses a pinned Rust version in the repository (recorded in the `rust-toolcha our tests and benchmarks using the pinned Rust version. We recommend using the pinned Rust version for development. We update the pinned Rust version between releases of mmtk-core to keep it close to the latest Rust stable release. The release cycle of mmtk-core is six weeks, roughly the same as -Rust itself. +Rust itself. Our minimum support Rust version (MSRV) policy is "N-1" (note that N is *NOT* the current stable Rust release). That means we also ensure mmtk-core works properly with the Rust toolchain that is one minor version before the version specified in diff --git a/docs/dummyvm/include/mmtk.h b/docs/dummyvm/include/mmtk.h index 33f77e9408d..5e12576bad7 100644 --- a/docs/dummyvm/include/mmtk.h +++ b/docs/dummyvm/include/mmtk.h @@ -71,8 +71,8 @@ extern bool mmtk_is_in_mmtk_spaces(void* object); // Return if the address pointed to by `addr` is in memory that is mapped by MMTk extern bool mmtk_is_mapped_address(void* addr); -// Request MMTk to trigger a GC. Note that this may not actually trigger a GC -extern void mmtk_handle_user_collection_request(void* tls); +// Request MMTk to trigger a GC. Note that this may not actually trigger a GC unless `force` is true +extern void mmtk_handle_user_collection_request(void* tls, bool force); // Add a reference to the list of weak references extern void mmtk_add_weak_candidate(void* ref); diff --git a/docs/dummyvm/src/api.rs b/docs/dummyvm/src/api.rs index 3fae076fa6b..bc0c24a0280 100644 --- a/docs/dummyvm/src/api.rs +++ b/docs/dummyvm/src/api.rs @@ -167,8 +167,8 @@ pub extern "C" fn mmtk_is_mapped_address(address: Address) -> bool { } #[no_mangle] -pub extern "C" fn mmtk_handle_user_collection_request(tls: VMMutatorThread) { - memory_manager::handle_user_collection_request::(mmtk(), tls); +pub extern "C" fn mmtk_handle_user_collection_request(tls: VMMutatorThread, force: bool) { + memory_manager::handle_user_collection_request::(mmtk(), tls, force); } #[no_mangle] diff --git a/docs/dummyvm/src/object_model.rs b/docs/dummyvm/src/object_model.rs index e63a4e85ff7..84ffb2a22e3 100644 --- a/docs/dummyvm/src/object_model.rs +++ b/docs/dummyvm/src/object_model.rs @@ -19,6 +19,8 @@ impl ObjectModel for VMObjectModel { // Global metadata const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); + const GLOBAL_FIELD_UNLOG_BIT_SPEC: VMGlobalFieldUnlogBitSpec = + VMGlobalFieldUnlogBitSpec::side_after(Self::GLOBAL_LOG_BIT_SPEC.as_spec()); // Local metadata diff --git a/docs/dummyvm/src/scanning.rs b/docs/dummyvm/src/scanning.rs index 0465acbabcc..ff393175867 100644 --- a/docs/dummyvm/src/scanning.rs +++ b/docs/dummyvm/src/scanning.rs @@ -21,10 +21,10 @@ impl Scanning for VMScanning { fn scan_vm_specific_roots(_tls: VMWorkerThread, _factory: impl RootsWorkFactory) { unimplemented!() } - fn scan_object>( + fn scan_object( _tls: VMWorkerThread, _object: ObjectReference, - _slot_visitor: &mut SV, + _slot_visitor: &mut impl SlotVisitor, ) { unimplemented!() } diff --git a/docs/userguide/src/tutorial/mygc/ss/alloc.md b/docs/userguide/src/tutorial/mygc/ss/alloc.md index 7f7182f881f..54875842c7a 100644 --- a/docs/userguide/src/tutorial/mygc/ss/alloc.md +++ b/docs/userguide/src/tutorial/mygc/ss/alloc.md @@ -126,7 +126,7 @@ collection for our GC plan. The trait `Plan` has a `common()` method (and a `common_mut()` counterpart) that should return a (mutable) reference to the common plan. Implement these methods now. Several default -method implementations in the `Plan` trait, such as `end_of_pause()`, rely on `common_mut()` to do +method implementations in the `Plan` trait, such as `on_pause_end()`, rely on `common_mut()` to do the right thing, so once these are implemented, `MyGC` does not need to override those methods. ```rust diff --git a/src/lib.rs b/src/lib.rs index a2ff7c4c6cb..6170cdc7112 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +#![allow(static_mut_refs)] // Use the `{likely, unlikely}` provided by compiler when using nightly #![cfg_attr(feature = "nightly", feature(core_intrinsics))] @@ -39,6 +40,7 @@ pub use mmtk::MMTK; mod global_state; pub use crate::global_state::{GcStatus, LiveBytesStats}; +#[macro_use] mod policy; pub mod build_info; diff --git a/src/memory_manager.rs b/src/memory_manager.rs index 1e8e8d15d72..058849bd927 100644 --- a/src/memory_manager.rs +++ b/src/memory_manager.rs @@ -28,6 +28,17 @@ use crate::vm::slot::MemorySlice; use crate::vm::ReferenceGlue; use crate::vm::VMBinding; +/// Notify MMTk that a GC has started, so that MMTk can update its internal statistics (e.g. GC counts +/// and timers) to reflect this. A binding does not normally need to call this directly, as MMTk calls it +/// itself when it triggers a GC; it is only needed if the binding drives GC start/stop outside of MMTk's +/// own scheduling. +/// +/// Arguments: +/// * `mmtk`: A reference to an MMTk instance. +pub fn report_gc_start(mmtk: &MMTK) { + mmtk.stats.start_gc(); +} + use std::collections::HashMap; /// Initialize an MMTk instance. A VM should call this method after creating an [`crate::MMTK`] @@ -677,8 +688,9 @@ pub fn total_bytes(mmtk: &MMTK) -> usize { pub fn handle_user_collection_request( mmtk: &MMTK, tls: VMMutatorThread, + force: bool, ) -> bool { - mmtk.handle_user_collection_request(tls, false, false) + mmtk.handle_user_collection_request(tls, force, false) } /// Is the object alive? diff --git a/src/mmtk.rs b/src/mmtk.rs index 86398ecb70a..a076a0aeef3 100644 --- a/src/mmtk.rs +++ b/src/mmtk.rs @@ -114,13 +114,15 @@ impl Default for MMTKBuilder { /// An MMTk instance. MMTk allows multiple instances to run independently, and each instance gives users a separate heap. /// *Note that multi-instances is not fully supported yet* pub struct MMTK { - pub(crate) options: Arc, + /// The command line options for this MMTk instance, shared with the VM binding. + pub options: Arc, pub(crate) state: Arc, pub(crate) plan: UnsafeCell>>, pub(crate) reference_processors: ReferenceProcessors, pub(crate) finalizable_processor: Mutex>::FinalizableType>>, - pub(crate) scheduler: Arc>, + /// The GC work scheduler that schedules and executes GC work packets on GC worker threads. + pub scheduler: Arc>, #[cfg(feature = "sanity")] pub(crate) sanity_checker: Mutex>, #[cfg(feature = "extreme_assertions")] @@ -345,6 +347,10 @@ impl MMTK { pub fn harness_begin(&self, tls: VMMutatorThread) { probe!(mmtk, harness_begin); self.handle_user_collection_request(tls, true, true); + if tls.0 .0.is_null() { + use crate::vm::Collection; + VM::VMCollection::block_for_gc(tls); + } self.state.inside_harness.store(true, Ordering::SeqCst); self.stats.start_all(); self.scheduler.enable_stat(); @@ -452,7 +458,9 @@ impl MMTK { .handle_user_collection_request(force, exhaustive) { use crate::vm::Collection; - VM::VMCollection::block_for_gc(tls); + if !tls.0 .0.is_null() { + VM::VMCollection::block_for_gc(tls); + } true } else { false diff --git a/src/plan/barriers.rs b/src/plan/barriers.rs index 6035fa0d670..89eab46ba27 100644 --- a/src/plan/barriers.rs +++ b/src/plan/barriers.rs @@ -21,6 +21,8 @@ pub enum BarrierSelector { NoBarrier, /// Object remembering post-write barrier is used. ObjectBarrier, + /// Field remembering post-write barrier is used, using a per-field (rather than per-object) unlogged bit. + FieldBarrier, /// Object remembering pre-write barrier with weak reference loading barrier. // TODO: We might be able to generalize this to object remembering pre-write barrier. SATBBarrier, @@ -149,8 +151,11 @@ impl Barrier for NoBarrier {} /// A barrier is a combination of fast-path behaviour + slow-path semantics. /// The fast-path code will decide whether to call the slow-path calls. pub trait BarrierSemantics: 'static + Send { + /// The VM binding type that this barrier semantics is specialized for. type VM: VMBinding; + /// The metadata spec used to store the unlogged bit that this barrier's fast-path checks and + /// slow-path clears/sets. const UNLOG_BIT_SPEC: MetadataSpec = *::VMObjectModel::GLOBAL_LOG_BIT_SPEC.as_spec(); @@ -271,6 +276,77 @@ impl Barrier for ObjectBarrier { } } +/// Generic object barrier with a type argument defining it's slow-path behaviour. +pub struct FieldBarrier { + semantics: S, +} + +impl FieldBarrier { + /// Create a new FieldBarrier with the given semantics. + pub fn new(semantics: S) -> Self { + Self { semantics } + } +} + +impl Barrier for FieldBarrier { + fn flush(&mut self) { + self.semantics.flush(); + } + + fn load_weak_reference(&mut self, o: ObjectReference) { + self.semantics.load_weak_reference(o) + } + + fn object_probable_write(&mut self, obj: ObjectReference) { + self.semantics.object_probable_write_slow(obj); + } + + fn object_reference_write_pre( + &mut self, + src: ObjectReference, + slot: ::VMSlot, + target: Option, + ) { + self.semantics + .object_reference_write_slow(src, slot, target); + } + + fn object_reference_write_post( + &mut self, + _src: ObjectReference, + _slot: ::VMSlot, + _target: Option, + ) { + unimplemented!() + } + + fn object_reference_write_slow( + &mut self, + src: ObjectReference, + slot: ::VMSlot, + target: Option, + ) { + self.semantics + .object_reference_write_slow(src, slot, target); + } + + fn memory_region_copy_pre( + &mut self, + src: ::VMMemorySlice, + dst: ::VMMemorySlice, + ) { + self.semantics.memory_region_copy_slow(src, dst); + } + + fn memory_region_copy_post( + &mut self, + _src: ::VMMemorySlice, + _dst: ::VMMemorySlice, + ) { + unimplemented!() + } +} + /// A SATB (Snapshot-At-The-Beginning) barrier implementation. /// This barrier is basically a pre-write object barrier with a weak reference loading barrier. pub struct SATBBarrier { diff --git a/src/plan/concurrent/barrier.rs b/src/plan/concurrent/barrier.rs index 0bd89955642..127fb1c79fa 100644 --- a/src/plan/concurrent/barrier.rs +++ b/src/plan/concurrent/barrier.rs @@ -15,6 +15,10 @@ use crate::{ MMTK, }; +/// A snapshot-at-the-beginning (SATB) barrier, used by concurrent plans (e.g. concurrent Immix) +/// to preserve the objects that were reachable at the start of concurrent marking. Old values +/// overwritten by mutators, and referents loaded from weak references, are buffered and later +/// enqueued as roots for concurrent marking, so they are not incorrectly reclaimed as garbage. pub struct SATBBarrierSemantics< VM: VMBinding, P: ConcurrentPlan + PlanTraceObject, @@ -30,6 +34,7 @@ pub struct SATBBarrierSemantics< impl + PlanTraceObject, const KIND: TraceKind> SATBBarrierSemantics { + /// Create a new SATB barrier for the given mutator, with empty SATB/weak-reference buffers. pub fn new(mmtk: &'static MMTK, tls: VMMutatorThread) -> Self { Self { mmtk, @@ -156,7 +161,7 @@ impl + PlanTraceObject, const KIND } fn object_probable_write_slow(&mut self, obj: ObjectReference) { - crate::plan::tracing::SlotIterator::::iterate_fields(obj, self.tls.0, |s| { + obj.iterate_fields::(self.tls.0, |s| { self.enqueue_node(Some(obj), s, None); }); } diff --git a/src/plan/concurrent/concurrent_marking_work.rs b/src/plan/concurrent/concurrent_marking_work.rs index b8b24f46f77..143b5295ef8 100644 --- a/src/plan/concurrent/concurrent_marking_work.rs +++ b/src/plan/concurrent/concurrent_marking_work.rs @@ -3,7 +3,7 @@ use crate::plan::concurrent::Pause; use crate::plan::tracing::{PlanTrace, Trace}; use crate::plan::PlanTraceObject; use crate::policy::gc_work::TraceKind; -use crate::scheduler::{GCWork, GCWorker, WorkBucketStage}; +use crate::scheduler::{gc_work::RootKind, GCWork, GCWorker, WorkBucketStage}; use crate::util::{scanning_helper, ObjectReference}; use crate::vm::slot::Slot; use crate::vm::{RootsKind, RootsWorkFactory, VMBinding}; @@ -202,7 +202,11 @@ impl + PlanTraceObject, const KIND impl + PlanTraceObject, const KIND: TraceKind> RootsWorkFactory for ConcurrentMarkingRootsWorkFactory { - fn create_process_roots_work(&mut self, slots: Vec) { + fn create_process_roots_work_with_root_kind( + &mut self, + slots: Vec, + _kind: RootKind, + ) { probe!(mmtk, roots, RootsKind::NORMAL, slots.len()); self.debug_assert_initial_mark(); diff --git a/src/plan/concurrent/immix/global.rs b/src/plan/concurrent/immix/global.rs index fb5c7f5d90f..53c084d5535 100644 --- a/src/plan/concurrent/immix/global.rs +++ b/src/plan/concurrent/immix/global.rs @@ -41,10 +41,12 @@ use mmtk_macros::{HasSpaces, PlanTraceObject}; /// The concurrent GC consists of two STW pauses (initial mark and final mark) with concurrent marking in between. #[derive(HasSpaces, PlanTraceObject)] pub struct ConcurrentImmix { + /// The Immix space in which all objects for this plan are allocated. #[post_scan] #[space] #[copy_semantics(CopySemantics::DefaultCopy)] pub immix_space: ImmixSpace, + /// The common plan state (e.g. the immortal, large object and VM spaces) shared by plans. #[parent] pub common: CommonPlan, last_gc_was_defrag: AtomicBool, @@ -172,6 +174,7 @@ impl Plan for ConcurrentImmix { } Pause::InitialMark => self.schedule_concurrent_marking_initial_pause(scheduler), Pause::FinalMark => self.schedule_concurrent_marking_final_pause(scheduler), + Pause::RefCount => unreachable!(), } } @@ -205,6 +208,7 @@ impl Plan for ConcurrentImmix { .schedule_unlog_bits_op(UnlogBitsOperation::BulkSet); } Pause::FinalMark => (), + Pause::RefCount => unreachable!(), } } @@ -233,10 +237,11 @@ impl Plan for ConcurrentImmix { // we will need to clear the unlog bits at an appropriate place. } } + Pause::RefCount => unreachable!(), } } - fn end_of_pause(&mut self, mmtk: &'static MMTK, _tls: VMWorkerThread) { + fn on_pause_end(&mut self, mmtk: &'static MMTK, _tls: VMWorkerThread) { self.last_gc_was_defrag .store(self.immix_space.end_of_gc(), Ordering::Relaxed); @@ -288,7 +293,7 @@ impl Plan for ConcurrentImmix { &self.common } - fn notify_mutators_paused(&self, mmtk: &'static MMTK) { + fn on_pause_start(&self, mmtk: &'static MMTK) { use crate::vm::ActivePlan; let pause = self.current_pause().unwrap(); match pause { @@ -310,6 +315,7 @@ impl Plan for ConcurrentImmix { } self.set_concurrent_marking_state(false); } + Pause::RefCount => unreachable!(), } // Every pause starts a new GC cycle, except `FinalMark`, which continues the cycle @@ -327,6 +333,8 @@ impl Plan for ConcurrentImmix { } impl ConcurrentImmix { + /// Create a new `ConcurrentImmix` plan, setting up the Immix space and disabling the + /// scheduler work buckets that are not used by this plan (e.g. forwarding and compaction). pub fn new(args: CreateGeneralPlanArgs) -> Self { if *args.options.concurrent_immix_disable_concurrent_marking { warn!("Option 'concurrent_immix_disable_concurrent_marking' is set to true. Concurrent marking is disabled for ConcurrentImmix. This will make ConcurrentImmix behave exactly like full heap Immix."); @@ -446,6 +454,8 @@ impl ConcurrentImmix { .set_sentinel(Box::new(VMProcessWeakRefs::>::new())); } + /// Return whether concurrent marking is currently active (i.e. an `InitialMark` pause has + /// happened and the corresponding `FinalMark` has not yet completed). pub fn concurrent_marking_in_progress(&self) -> bool { self.concurrent_marking_active.load(Ordering::Acquire) } diff --git a/src/plan/concurrent/mod.rs b/src/plan/concurrent/mod.rs index 9cb46d7104b..082b30fef5e 100644 --- a/src/plan/concurrent/mod.rs +++ b/src/plan/concurrent/mod.rs @@ -1,6 +1,10 @@ +/// The SATB (snapshot-at-the-beginning) barrier semantics shared by concurrent plans. pub mod barrier; pub(super) mod concurrent_marking_work; -pub(super) mod global; +/// The `ConcurrentPlan` trait, shared by plans that support a concurrent marking phase. +pub mod global; + +pub use self::global::ConcurrentPlan; pub mod immix; @@ -22,6 +26,10 @@ pub enum Pause { InitialMark, /// The pause after concurrent marking. FinalMark, + /// A reference-counting-only pause. Used by plans that combine reference counting with + /// optional concurrent marking (e.g. LXR). Concurrent marking is NOT active during this + /// pause — any in-flight concurrent work is postponed. + RefCount, } unsafe impl bytemuck::ZeroableInOption for Pause {} diff --git a/src/plan/generational/copying/global.rs b/src/plan/generational/copying/global.rs index 8f76e7875ef..108b0e794d9 100644 --- a/src/plan/generational/copying/global.rs +++ b/src/plan/generational/copying/global.rs @@ -118,9 +118,9 @@ impl Plan for GenCopy { } } - fn end_of_pause(&mut self, mmtk: &'static MMTK, tls: VMWorkerThread) { + fn on_pause_end(&mut self, mmtk: &'static MMTK, tls: VMWorkerThread) { let next_gc_full_heap = CommonGenPlan::should_next_gc_be_full_heap(self); - self.gen.end_of_pause(tls, next_gc_full_heap); + self.gen.on_pause_end(tls, next_gc_full_heap); mmtk.gc_trigger.policy.on_gc_end(mmtk); } diff --git a/src/plan/generational/global.rs b/src/plan/generational/global.rs index 98d8a73c87f..5c207ccfbe8 100644 --- a/src/plan/generational/global.rs +++ b/src/plan/generational/global.rs @@ -78,9 +78,9 @@ impl CommonGenPlan { self.nursery.release(); } - pub fn end_of_pause(&mut self, tls: VMWorkerThread, next_gc_full_heap: bool) { + pub fn on_pause_end(&mut self, tls: VMWorkerThread, next_gc_full_heap: bool) { self.set_next_gc_full_heap(next_gc_full_heap); - self.common.end_of_pause(tls); + self.common.on_pause_end(tls); } /// Independent of how many pages remain in the page budget (a function of heap size), we must diff --git a/src/plan/generational/immix/global.rs b/src/plan/generational/immix/global.rs index 8e822d31b29..203c1bf2688 100644 --- a/src/plan/generational/immix/global.rs +++ b/src/plan/generational/immix/global.rs @@ -157,9 +157,9 @@ impl Plan for GenImmix { .store(full_heap, Ordering::Relaxed); } - fn end_of_pause(&mut self, mmtk: &'static MMTK, tls: VMWorkerThread) { + fn on_pause_end(&mut self, mmtk: &'static MMTK, tls: VMWorkerThread) { let next_gc_full_heap = CommonGenPlan::should_next_gc_be_full_heap(self); - self.gen.end_of_pause(tls, next_gc_full_heap); + self.gen.on_pause_end(tls, next_gc_full_heap); let did_defrag = self.immix_space.end_of_gc(); self.last_gc_was_defrag.store(did_defrag, Ordering::Relaxed); diff --git a/src/plan/global.rs b/src/plan/global.rs index faf18e19616..d278820d26c 100644 --- a/src/plan/global.rs +++ b/src/plan/global.rs @@ -60,6 +60,7 @@ pub fn create_mutator( PlanSelector::StickyImmix => { crate::plan::sticky::immix::mutator::create_stickyimmix_mutator(tls, mmtk) } + PlanSelector::LXR => crate::plan::lxr::mutator::create_lxr_mutator(tls, mmtk), PlanSelector::ConcurrentImmix => { crate::plan::concurrent::immix::mutator::create_concurrent_immix_mutator(tls, mmtk) } @@ -104,6 +105,7 @@ pub fn create_plan( PlanSelector::StickyImmix => { Box::new(crate::plan::sticky::immix::StickyImmix::new(args)) as Box> } + PlanSelector::LXR => crate::plan::lxr::LXR::new(args) as Box>, PlanSelector::ConcurrentImmix => { Box::new(crate::plan::concurrent::immix::ConcurrentImmix::new(args)) as Box> @@ -204,13 +206,18 @@ pub trait Plan: 'static + HasSpaces + Sync + Downcast { /// This defines what space this plan will allocate objects into for different semantics. fn get_allocator_mapping(&self) -> &'static EnumMap; - /// Called when all mutators are paused. This is called before prepare. + /// Called once all mutators have been stopped. This is called before `Prepare`, which is right + /// before root scanning starts, at the beginning of a GC pause. /// - /// A plan that overrides this function need to manage the invocation of `GCTriggerPolicy::on_gc_start` at the proper timing for the plan. - fn notify_mutators_paused(&self, mmtk: &'static MMTK) { + /// Plans that need to do per-pause setup (e.g. resetting mark tables, flushing mutator state) + /// can override this. + /// + /// A plan that overrides this function need to manage the invocation of + /// `GCTriggerPolicy::on_gc_start` at the proper timing for the plan. + fn on_pause_start(&self, mmtk: &'static MMTK) { assert!( self.concurrent().is_none(), - "ConcurrentPlan must override notify_mutators_paused" + "ConcurrentPlan must override on_pause_start" ); mmtk.gc_trigger.policy.on_gc_start(mmtk); } @@ -228,17 +235,21 @@ pub trait Plan: 'static + HasSpaces + Sync + Downcast { /// This is invoked once per GC by one worker thread. `tls` is the worker thread that executes this method. fn release(&mut self, tls: VMWorkerThread); - /// Inform the plan about the end of a pause. It is guaranteed that there is no further work - /// for this pause. This is invoked once per pause by one worker thread. `tls` is the worker - /// thread that executes this method. + /// Called at the end of a GC pause. It is guaranteed that there is no further work for this + /// pause. This is invoked once per pause by one worker thread. `tls` is the worker thread + /// that executes this method. + /// + /// Plans that need to do per-pause teardown (e.g. recording pause-end statistics) can override + /// this. /// - /// A plan that overrides this function need to do whatever the default implementation does at the proper timing - /// for the plan, such as calling `CommonPlan::end_of_pause` and `GCTriggerPolicy::on_gc_end`. - fn end_of_pause(&mut self, mmtk: &'static MMTK, tls: VMWorkerThread) { - self.common_mut().end_of_pause(tls); + /// A plan that overrides this function need to do whatever the default implementation does at + /// the proper timing for the plan, such as calling `CommonPlan::on_pause_end`, and selectively + /// call `GCTriggerPolicy::on_gc_end` if the pause is the end of a GC. + fn on_pause_end(&mut self, mmtk: &'static MMTK, tls: VMWorkerThread) { + self.common_mut().on_pause_end(tls); assert!( self.concurrent().is_none(), - "ConcurrentPlan must override end_of_pause" + "ConcurrentPlan must override on_pause_end" ); mmtk.gc_trigger.policy.on_gc_end(mmtk); } @@ -349,6 +360,14 @@ pub trait Plan: 'static + HasSpaces + Sync + Downcast { true } + /// Return the work bucket stage in which mutator (and VM) roots should be scanned for this + /// plan. By default, roots are scanned in the `Prepare` stage, but concurrent/incremental + /// plans may schedule root scanning into a different stage (e.g. alongside reference + /// counting increments). + fn root_scanning_stage(&self) -> WorkBucketStage { + WorkBucketStage::Prepare + } + /// Return whether the current GC may move any object. The VM binding can make use of this /// information and choose to or not to update some data structures that record the addresses /// of objects. @@ -684,8 +703,8 @@ impl BasePlan { self.vm_space.set_side_log_bits(); } - pub fn end_of_pause(&mut self, _tls: VMWorkerThread) { - // Do nothing here. None of the spaces needs end_of_pause. + pub fn on_pause_end(&mut self, _tls: VMWorkerThread) { + // Do nothing here. None of the spaces needs on_pause_end. } pub(crate) fn collection_required(&self, plan: &P, space_full: bool) -> bool { @@ -780,14 +799,14 @@ impl CommonPlan { pub fn prepare(&mut self, tls: VMWorkerThread, full_heap: bool) { self.immortal.prepare(); self.los.prepare(full_heap); - self.prepare_nonmoving_space(full_heap); + // self.prepare_nonmoving_space(full_heap); self.base.prepare(tls, full_heap) } pub fn release(&mut self, tls: VMWorkerThread, full_heap: bool) { self.immortal.release(); self.los.release(full_heap); - self.release_nonmoving_space(full_heap); + // self.release_nonmoving_space(full_heap); self.base.release(tls, full_heap) } @@ -822,9 +841,9 @@ impl CommonPlan { self.base.set_side_log_bits(); } - pub fn end_of_pause(&mut self, tls: VMWorkerThread) { + pub fn on_pause_end(&mut self, tls: VMWorkerThread) { self.end_of_gc_nonmoving_space(); - self.base.end_of_pause(tls); + self.base.on_pause_end(tls); } pub fn get_immortal(&self) -> &ImmortalSpace { @@ -857,6 +876,7 @@ impl CommonPlan { } } + #[allow(dead_code)] fn prepare_nonmoving_space(&mut self, _full_heap: bool) { cfg_if::cfg_if! { if #[cfg(feature = "immortal_as_nonmoving")] { @@ -869,6 +889,7 @@ impl CommonPlan { } } + #[allow(dead_code)] fn release_nonmoving_space(&mut self, _full_heap: bool) { cfg_if::cfg_if! { if #[cfg(feature = "immortal_as_nonmoving")] { diff --git a/src/plan/immix/global.rs b/src/plan/immix/global.rs index b1fb75f6c0d..2f873ec1e38 100644 --- a/src/plan/immix/global.rs +++ b/src/plan/immix/global.rs @@ -27,12 +27,16 @@ use enum_map::EnumMap; use mmtk_macros::{HasSpaces, PlanTraceObject}; +/// The vanilla (non-concurrent, non-generational) Immix plan. It always does a full-heap, +/// stop-the-world collection, optionally with defragmentation. #[derive(HasSpaces, PlanTraceObject)] pub struct Immix { + /// The Immix space in which all objects for this plan are allocated. #[post_scan] #[space] #[copy_semantics(CopySemantics::DefaultCopy)] pub immix_space: ImmixSpace, + /// The common plan state (e.g. the immortal, large object and VM spaces) shared by plans. #[parent] pub common: CommonPlan, last_gc_was_defrag: AtomicBool, @@ -93,10 +97,10 @@ impl Plan for Immix { self.release_inner(tls, UnlogBitsOperation::NoOp); } - fn end_of_pause(&mut self, mmtk: &'static MMTK, tls: VMWorkerThread) { + fn on_pause_end(&mut self, mmtk: &'static MMTK, tls: VMWorkerThread) { self.last_gc_was_defrag .store(self.immix_space.end_of_gc(), Ordering::Relaxed); - self.common.end_of_pause(tls); + self.common.on_pause_end(tls); mmtk.gc_trigger.policy.on_gc_end(mmtk); } @@ -126,6 +130,8 @@ impl Plan for Immix { } impl Immix { + /// Create a new `Immix` plan with the default (non-mixed-age, movable) Immix space + /// configuration. pub fn new(args: CreateGeneralPlanArgs) -> Self { let plan_args = CreateSpecificPlanArgs { global_args: args, @@ -141,6 +147,9 @@ impl Immix { ) } + /// Create a new `Immix` plan, using the given plan args and Immix space configuration. This + /// allows Immix-derived plans (e.g. generational/sticky Immix) to customize the Immix space + /// (e.g. to enable mixed-age allocation) while reusing the rest of the plan setup. pub fn new_with_args( mut plan_args: CreateSpecificPlanArgs, space_args: ImmixSpaceArgs, diff --git a/src/plan/lxr/barrier.rs b/src/plan/lxr/barrier.rs new file mode 100644 index 00000000000..456525f51ed --- /dev/null +++ b/src/plan/lxr/barrier.rs @@ -0,0 +1,239 @@ +//! Read/Write barrier implementations. + +use std::sync::Arc; + +use atomic::Ordering; + +use super::LazySweepingJobsCounter; +use super::LXR; +use crate::plan::barriers::BarrierSemantics; +use crate::plan::concurrent::global::ConcurrentPlan; +use crate::plan::concurrent::Pause; +use crate::plan::lxr::gc_work::rc::ProcessDecs; +use crate::plan::lxr::gc_work::rc::ProcessIncs; +use crate::plan::lxr::gc_work::rc::EDGE_KIND_MATURE; +use crate::plan::lxr::gc_work::tracing::ProcessModBufSATB; +use crate::plan::VectorQueue; +use crate::scheduler::WorkBucketStage; +use crate::util::metadata::log_bit::{LOGGED_VALUE, UNLOGGED_VALUE}; +use crate::util::metadata::side_metadata::address_to_meta_address; +use crate::util::metadata::side_metadata::SideMetadataSpec; +use crate::util::*; +use crate::vm::slot::MemorySlice; +use crate::vm::slot::Slot; +use crate::vm::*; +use crate::MMTK; + +pub struct LXRFieldBarrierSemantics { + mmtk: &'static MMTK, + tls: VMMutatorThread, + incs: VectorQueue, + decs: VectorQueue, + refs: VectorQueue, + lxr: &'static LXR, +} + +impl LXRFieldBarrierSemantics { + const UNLOG_BITS: SideMetadataSpec = *VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC + .as_spec() + .extract_side_spec(); + + #[allow(unused)] + pub fn new(mmtk: &'static MMTK, tls: VMMutatorThread) -> Self { + Self { + mmtk, + tls, + incs: VectorQueue::default(), + decs: VectorQueue::default(), + refs: VectorQueue::default(), + lxr: mmtk.get_plan().downcast_ref::>().unwrap(), + } + } + + fn get_slot_logging_state(&self, slot: VM::VMSlot) -> u8 { + unsafe { Self::UNLOG_BITS.load(slot.to_address()) } + } + + fn attempt_to_log_field(&self, slot: VM::VMSlot) -> bool { + loop { + // Bailout if logged + if self.get_slot_logging_state(slot) == LOGGED_VALUE { + return false; + } + // Attempt to log the slots + match Self::UNLOG_BITS.compare_exchange_atomic( + slot.to_address(), + UNLOGGED_VALUE, + LOGGED_VALUE, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => return true, + Err(current) => { + if current == LOGGED_VALUE { + return false; + } + } + } + // Failed to log the slot. Spin. + std::hint::spin_loop(); + } + } + + fn log_slot_and_get_old_target(&self, slot: VM::VMSlot) -> Result, ()> { + if self.get_slot_logging_state(slot) == LOGGED_VALUE { + return Err(()); + } + let old = slot.load(); + if self.attempt_to_log_field(slot) { + Ok(old) + } else { + Err(()) + } + } + + fn slow( + &mut self, + _src: Option, + slot: VM::VMSlot, + old: Option, + ) { + // Reference counting + if let Some(old) = old { + self.decs.push(old); + if self.decs.is_full() { + self.flush_decs_and_satb(); + } + } + self.incs.push(slot); + if self.incs.is_full() { + self.flush_incs(); + } + } + + fn enqueue_node( + &mut self, + src: Option, + slot: VM::VMSlot, + _new: Option, + ) -> bool { + if let Ok(old) = self.log_slot_and_get_old_target(slot) { + self.slow(src, slot, old); + true + } else { + false + } + } + + fn should_create_satb_packets(&self) -> bool { + self.lxr.cm_enabled() + && (self.lxr.concurrent_work_in_progress() + || self.lxr.current_pause() == Some(Pause::FinalMark)) + } + + #[cold] + fn flush_incs(&mut self) { + if !self.incs.is_empty() { + let incs = self.incs.take(); + self.lxr.rc.increase_inc_buffer_size(incs.len()); + self.mmtk.scheduler.work_buckets[WorkBucketStage::RCProcessIncs].add(ProcessIncs::< + _, + EDGE_KIND_MATURE, + >::new( + incs, self.lxr + )); + } + } + + #[cold] + fn flush_decs_and_satb(&mut self) { + if !self.decs.is_empty() { + let w = if self.should_create_satb_packets() { + let decs = Arc::new(self.decs.take()); + self.mmtk.scheduler.work_buckets[WorkBucketStage::FinishConcurrentWork] + .add(ProcessModBufSATB::new_arc(decs.clone())); + ProcessDecs::new_arc(decs, LazySweepingJobsCounter::new_decs()) + } else { + let decs = self.decs.take(); + ProcessDecs::new(decs, LazySweepingJobsCounter::new_decs()) + }; + if super::LAZY_DECREMENTS { + self.mmtk.scheduler.work_buckets[WorkBucketStage::Concurrent] + .add_deferred(Box::new(w)); + } else { + self.mmtk.scheduler.work_buckets[WorkBucketStage::STWRCDecsAndSweep].add(w); + } + } + } + + #[cold] + fn flush_weak_refs(&mut self) { + if !self.refs.is_empty() { + debug_assert!(self.should_create_satb_packets()); + let nodes = self.refs.take(); + self.mmtk.scheduler.work_buckets[WorkBucketStage::FinishConcurrentWork] + .add(ProcessModBufSATB::new(nodes)); + } + } +} + +impl BarrierSemantics for LXRFieldBarrierSemantics { + type VM = VM; + + #[cold] + fn flush(&mut self) { + self.flush_weak_refs(); + self.flush_incs(); + self.flush_decs_and_satb(); + } + + fn object_reference_write_slow( + &mut self, + src: ObjectReference, + slot: VM::VMSlot, + target: Option, + ) { + self.enqueue_node(Some(src), slot, target); + } + + fn memory_region_copy_slow(&mut self, _src: VM::VMMemorySlice, dst: VM::VMMemorySlice) { + // Quickly check if all fields are logged. If yes, skip the barrier. + let unlog_bits_start = address_to_meta_address(&Self::UNLOG_BITS, dst.start()); + let unlog_bits_start_aligned = unlog_bits_start.align_down(16); + let unlog_bits_end = + address_to_meta_address(&Self::UNLOG_BITS, dst.start() + dst.bytes() - 1); + let unlog_bits_end_aligned = unlog_bits_end.align_down(16); + let mut cursor = unlog_bits_start_aligned; + let mut all_logged = true; + while cursor <= unlog_bits_end_aligned { + if unsafe { cursor.load::() } != 0 { + all_logged = false; + break; + } + cursor += 16usize; + } + if all_logged { + return; + } + + for s in dst.iter_slots() { + let _succ = self.enqueue_node(None, s, None); + } + } + + fn load_weak_reference(&mut self, o: ObjectReference) { + if !self.lxr.concurrent_work_in_progress() || self.lxr.is_marked(o) { + return; + } + self.refs.push(o); + if self.refs.is_full() { + self.flush_weak_refs(); + } + } + + fn object_probable_write_slow(&mut self, obj: ObjectReference) { + obj.iterate_fields::(self.tls.0, |s| { + let _succ = self.enqueue_node(Some(obj), s, None); + }); + } +} diff --git a/src/plan/lxr/block_allocation.rs b/src/plan/lxr/block_allocation.rs new file mode 100644 index 00000000000..ab443752d2e --- /dev/null +++ b/src/plan/lxr/block_allocation.rs @@ -0,0 +1,205 @@ +use super::gc_work::nursery_sweeping::{RCLazySweepNurseryBlocks, RCSTWSweepNurseryBlocks}; +use super::LXR; +use crate::plan::concurrent::global::ConcurrentPlan; +use crate::plan::concurrent::Pause; +use crate::policy::immix::block::{Block, BlockState}; +use crate::policy::immix::{ImmixHooks, ImmixSpace}; +use crate::scheduler::{GCWork, GCWorkScheduler, WorkBucketStage}; +use crate::util::constants::LOG_BYTES_IN_PAGE; +use crate::util::linear_scan::Region; +use crate::vm::VMBinding; +use atomic::{Atomic, Ordering}; +use std::cell::UnsafeCell; +use std::sync::atomic::AtomicUsize; +use std::sync::RwLock; + +struct BlockCache { + cursor: AtomicUsize, + buffer: RwLock>>, +} + +impl BlockCache { + fn new() -> Self { + Self { + cursor: AtomicUsize::new(0), + buffer: RwLock::new((0..32768).map(|_| Atomic::new(Block::ZERO)).collect()), + } + } + + fn len(&self) -> usize { + self.cursor.load(Ordering::SeqCst) + } + + fn push(&self, block: Block) { + let i = self.cursor.fetch_add(1, Ordering::SeqCst); + let buffer = self.buffer.read().unwrap(); + if i < buffer.len() { + buffer[i].store(block, Ordering::SeqCst); + } else { + std::mem::drop(buffer); + let mut buffer = self.buffer.write().unwrap(); + if i >= buffer.len() { + buffer.resize_with(i << 1, || Atomic::new(Block::ZERO)); + } + buffer[i].store(block, Ordering::Relaxed); + } + } + + fn visit_slice(&self, f: impl Fn(&[Atomic])) { + let count = self.cursor.load(Ordering::SeqCst); + let blocks = self.buffer.read().unwrap(); + f(&blocks[0..count]) + } + + fn reset(&self) { + self.cursor.store(0, Ordering::SeqCst); + } +} + +pub struct BlockAllocation { + space: UnsafeCell<*const ImmixSpace>, + lxr: UnsafeCell<*const LXR>, + nursery_blocks: BlockCache, + reused_blocks: BlockCache, +} + +unsafe impl Sync for BlockAllocation {} +unsafe impl Send for BlockAllocation {} + +impl BlockAllocation { + pub fn new() -> Self { + Self { + space: UnsafeCell::new(std::ptr::null()), + lxr: UnsafeCell::new(std::ptr::null()), + nursery_blocks: BlockCache::new(), + reused_blocks: BlockCache::new(), + } + } + + pub fn init(&self, space: &ImmixSpace, lxr: &'static LXR) { + unsafe { + *self.space.get() = space as *const ImmixSpace; + *self.lxr.get() = lxr as *const LXR; + } + } + + fn space(&self) -> &'static ImmixSpace { + unsafe { &**self.space.get() } + } + + fn lxr(&self) -> &'static LXR { + unsafe { &**self.lxr.get() } + } + + pub fn clean_nursery_mb(&self) -> usize { + self.nursery_blocks.len() << Block::LOG_BYTES >> 20 + } + + pub fn total_young_allocation_in_bytes(&self) -> usize { + (self.nursery_blocks.len() << Block::LOG_BYTES) + + (self.space().get_mutator_recycled_lines_in_pages() << LOG_BYTES_IN_PAGE) + } + + pub fn reset_block_mark_for_mutator_reused_blocks(&self, _pause: Pause) { + // SATB sweep has problem scanning mutator recycled blocks. + // Remaing the block state as "reusing" and reset them here. + self.reused_blocks.visit_slice(|blocks| { + for b in blocks { + let b = b.load(Ordering::Relaxed); + b.set_state(BlockState::Marked); + } + }); + } + + pub fn sweep_mutator_reused_blocks(&self, pause: Pause) { + if pause == Pause::Full || pause == Pause::FinalMark { + self.reused_blocks.reset(); + return; + } + self.reused_blocks.visit_slice(|blocks| { + for b in blocks { + let block = b.load(Ordering::Relaxed); + self.lxr().add_to_possibly_dead_mature_blocks(block, false); + } + }); + self.reused_blocks.reset(); + } + + /// Reset allocated_block_buffer and free nursery blocks. + pub fn sweep_nursery_blocks(&self, scheduler: &GCWorkScheduler, pause: Pause) { + const PARALLEL_STW_SWEEPING: bool = false; + let max_stw_sweep_blocks: usize = usize::MAX; + let space = self.space(); + self.nursery_blocks.visit_slice(|blocks| { + if PARALLEL_STW_SWEEPING { + return self.parallel_sweep_all_nursery_blocks(scheduler, blocks); + } + let total_nursery_blocks = blocks.len(); + let stw_limit = if pause == Pause::Full { + total_nursery_blocks + } else { + usize::min(total_nursery_blocks, max_stw_sweep_blocks) + }; + for b in &blocks[0..stw_limit] { + let block = b.load(Ordering::Relaxed); + debug_assert_ne!(block.get_state(), BlockState::Unallocated); + block.rc_sweep_nursery(space); + } + if total_nursery_blocks > stw_limit { + let packets = blocks[stw_limit..total_nursery_blocks] + .chunks(1024) + .map(|c| { + let blocks: Vec = + c.iter().map(|x| x.load(Ordering::Relaxed)).collect(); + Box::new(RCLazySweepNurseryBlocks::new(blocks)) as Box> + }) + .collect(); + scheduler.work_buckets[WorkBucketStage::Concurrent].bulk_add_deferred(packets); + } + }); + self.nursery_blocks.reset(); + } + + fn parallel_sweep_all_nursery_blocks( + &self, + scheduler: &GCWorkScheduler, + blocks: &[Atomic], + ) { + let total_nursery_blocks = blocks.len(); + let packets = blocks[..total_nursery_blocks] + .chunks(1024) + .map(|c| { + let blocks: Vec = c.iter().map(|x| x.load(Ordering::Relaxed)).collect(); + Box::new(RCSTWSweepNurseryBlocks::new(blocks)) as Box> + }) + .collect(); + scheduler.work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets); + } +} + +impl ImmixHooks for BlockAllocation { + fn on_clean_block_acquired(&self, block: Block, copy: bool) { + if !copy { + self.nursery_blocks.push(block); + } + if copy { + block.initialize_field_unlog_table_as_unlogged::(); + } + if self.cm_in_progress_or_final_mark() { + block.initialize_mark_table_as_marked::(); + } else { + block.clear_mark_table::(); + } + } + + fn on_reusable_block_acquired(&self, block: Block, copy: bool) { + if !copy { + self.reused_blocks.push(block); + } + } + + fn cm_in_progress_or_final_mark(&self) -> bool { + let lxr = self.lxr(); + lxr.concurrent_work_in_progress() || lxr.current_pause() == Some(Pause::FinalMark) + } +} diff --git a/src/plan/lxr/gc_work/mature_evac.rs b/src/plan/lxr/gc_work/mature_evac.rs new file mode 100644 index 00000000000..b6014fbde25 --- /dev/null +++ b/src/plan/lxr/gc_work/mature_evac.rs @@ -0,0 +1,178 @@ +use std::marker::PhantomData; +use std::ops::Range; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use super::tracing::LXRStopTheWorldProcessEdges; +use crate::plan::lxr::mature_evac::MatureEvacuationSet; +use crate::policy::immix::line::Line; +use crate::util::heap::chunk_map::Chunk; +use crate::util::linear_scan::Region; +use crate::util::metadata::side_metadata::spec_defs::{IX_LINE_REUSE_COUNT, LOS_PAGE_REUSE_COUNT}; +use crate::vm::slot::Slot; +use crate::{ + plan::concurrent::Pause, + policy::{ + immix::block::{Block, BlockState}, + space::Space, + }, + scheduler::{GCWork, GCWorker, WorkBucketStage}, + vm::VMBinding, + MMTK, +}; + +use super::super::mature_evac::RemSetEntry; +use super::super::LXR; + +pub static SELECT_DEFRAG_BLOCK_JOB_COUNTER: AtomicUsize = AtomicUsize::new(0); + +pub struct SelectDefragBlocks { + pub chunks: Range, + #[allow(unused)] + pub defrag_threshold: usize, +} + +impl GCWork for SelectDefragBlocks { + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let mut fragmented_blocks = vec![]; + let lxr = mmtk.get_plan().downcast_ref::>().unwrap(); + + // Iterate over all blocks in this chunk + let num_chunks = (self.chunks.end.start() - self.chunks.start.start()) >> Chunk::LOG_BYTES; + let ix_space = &mmtk + .get_plan() + .downcast_ref::>() + .unwrap() + .immix_space; + for i in 0..num_chunks { + let chunk = self.chunks.start.next_nth(i); + if !ix_space.chunk_map.is_allocated(chunk) { + continue; + } + for block in chunk.iter_region::() { + // Skip unallocated blocks. + if MatureEvacuationSet::skip_block(block) { + continue; + } + // This is a fragmented block? + let score = block.calc_dead_lines() << Line::LOG_BYTES; + if lxr.current_pause().unwrap() == Pause::Full || score >= (Block::BYTES >> 1) { + fragmented_blocks.push((block, score)); + } + } + } + // Flush to global fragmented_blocks + if !fragmented_blocks.is_empty() { + lxr.evac_set + .fragmented_blocks_size + .fetch_add(fragmented_blocks.len(), Ordering::SeqCst); + lxr.evac_set.fragmented_blocks.push(fragmented_blocks); + } + + if SELECT_DEFRAG_BLOCK_JOB_COUNTER.fetch_sub(1, Ordering::SeqCst) == 1 { + lxr.evac_set.select_mature_evacuation_candidates(lxr) + } + } +} + +pub struct EvacuateMatureObjects { + remset: Vec>, + _p: PhantomData, +} + +impl EvacuateMatureObjects { + pub const CAPACITY: usize = 1024; + + #[allow(clippy::assertions_on_constants)] + pub fn new(remset: Vec>) -> Self { + debug_assert!(super::super::MATURE_EVACUATION); + Self { + remset, + _p: PhantomData, + } + } + + fn address_is_valid_oop_slot(&self, s: VM::VMSlot, original_reuse: u8, lxr: &LXR) -> bool { + // Keep slots not in the mmtk heap + // These should be slots in the c++ `ClassLoaderData` objects. We remember these slots + // in the remembered-set to avoid expensive CLD scanning. + let addr = s.to_address(); + // Check reuse count + if lxr.immix_space.address_in_space(addr) { + let reuse = IX_LINE_REUSE_COUNT.load_atomic::(addr, atomic::Ordering::SeqCst); + if reuse != original_reuse { + return false; + } + } else if lxr.los().address_in_space(addr) { + let reuse = LOS_PAGE_REUSE_COUNT.load_atomic::(addr, atomic::Ordering::SeqCst); + if reuse != original_reuse { + return false; + } + } else { + return false; + } + // Skip slots in collection set + if lxr.address_in_defrag(addr) { + return false; + } + // Check if it is a real oop field + if lxr.immix_space.address_in_space(s.to_address()) { + let block = Block::from_unaligned_address(s.to_address()); + if block.get_state() == BlockState::Unallocated { + return false; + } + } + true + } + + fn process_slot(&mut self, s: VM::VMSlot, reuse: u8, lxr: &LXR) -> bool { + // Skip slots that does not contain a real oop + if !self.address_is_valid_oop_slot(s, reuse, lxr) { + return false; + } + // Skip objects that are dead or out of the collection set. + let v = unsafe { s.to_address().load::() }; + if v & 0b111 != 0 { + panic!("Invalid slot: {s:?} -> {v:#x}"); + } + let Some(o) = s.load() else { + return false; + }; + if !o.is_in_any_space() || !lxr.immix_space.in_space(o) { + return false; + } + if !lxr.rc.is_dead(o) && Block::in_defrag_block(o) { + return true; + } + false + } + + fn process_slots(&mut self, mmtk: &'static MMTK) -> Option>> { + let lxr = mmtk.get_plan().downcast_ref::>().unwrap(); + assert_eq!(lxr.current_pause(), Some(Pause::FinalMark)); + let remset = std::mem::take(&mut self.remset); + let mut slots = vec![]; + for entry in remset { + let (s, reuse) = entry.decode(); + if self.process_slot(s, reuse, lxr) { + slots.push(s); + } + } + if !slots.is_empty() { + Some(Box::new( + LXRStopTheWorldProcessEdges::<_, false>::new_remset(slots, mmtk), + )) + } else { + None + } + } +} + +impl GCWork for EvacuateMatureObjects { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { + let Some(work) = self.process_slots(mmtk) else { + return; + }; + // transitive closure + worker.add_boxed_work(WorkBucketStage::Closure, work) + } +} diff --git a/src/plan/lxr/gc_work/mature_sweeping.rs b/src/plan/lxr/gc_work/mature_sweeping.rs new file mode 100644 index 00000000000..7b497435a5b --- /dev/null +++ b/src/plan/lxr/gc_work/mature_sweeping.rs @@ -0,0 +1,129 @@ +use std::ops::Range; +use std::sync::atomic::Ordering; + +use crate::plan::lxr::{LazySweepingJobsCounter, LXR}; +use crate::policy::immix::block::{Block, BlockState}; +use crate::policy::immix::line::Line; +use crate::policy::immix::ImmixSpace; +use crate::scheduler::{GCWork, GCWorker}; +use crate::util::heap::chunk_map::Chunk; +use crate::util::linear_scan::Region; +use crate::util::rc::{self, RefCountHelper}; +use crate::util::ObjectReference; +use crate::vm::VMBinding; +use crate::MMTK; + +/// Chunk sweeping work packet. +pub struct SweepDeadCycles { + chunks: Range, + _counter: LazySweepingJobsCounter, + rc: RefCountHelper, +} + +#[allow(unused)] +impl SweepDeadCycles { + const CAPACITY: usize = 1024; + + pub fn new(chunks: Range, counter: LazySweepingJobsCounter) -> Self { + Self { + chunks, + _counter: counter, + rc: RefCountHelper::NEW, + } + } + + fn process_dead_object(&mut self, o: ObjectReference) { + if RefCountHelper::::SANITY { + unsafe { + o.to_raw_address().store(0xdeadusize); + } + } + + // Clear the VO bit. + // Note that if the object is in the LOS, + // the VO bit will be cleared in `LargeObjectSpace::release_object`. + #[cfg(feature = "vo_bit")] + crate::util::metadata::vo_bit::unset_vo_bit(o); + + self.rc.unmark_straddle_object(o); + self.rc.set(o, 0); + } + + fn process_block(&mut self, block: Block, lxr: &LXR, immix_space: &ImmixSpace) { + let mut has_dead_object = false; + let mut has_live = false; + let mut cursor = block.start(); + let limit = block.end(); + while cursor < limit { + let o = unsafe { cursor.to_object_reference::() }; + cursor += rc::MIN_OBJECT_SIZE; + let c = self.rc.count(o); + if c != 0 && !immix_space.is_marked(o) { + if Line::is_aligned(o.to_raw_address()) { + if c == 1 && self.rc.is_straddle_line(Line::containing_obj_ref(o)) { + continue; + } else { + std::sync::atomic::fence(Ordering::SeqCst); + if self.rc.count(o) == 0 { + continue; + } + } + } + self.process_dead_object(o); + has_dead_object = true; + } else if c != 0 { + has_live = true; + } + } + if has_dead_object || !has_live { + lxr.add_to_possibly_dead_mature_blocks(block, false); + } + } +} + +impl GCWork for SweepDeadCycles { + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let lxr = mmtk.get_plan().downcast_ref::>().unwrap(); + let immix_space = &lxr.immix_space; + let num_chunks = (self.chunks.end.start() - self.chunks.start.start()) >> Chunk::LOG_BYTES; + let ix_space = &mmtk + .get_plan() + .downcast_ref::>() + .unwrap() + .immix_space; + for i in 0..num_chunks { + let chunk = self.chunks.start.next_nth(i); + if !ix_space.chunk_map.is_allocated(chunk) { + continue; + } + + for block in chunk + .iter_region::() + .filter(|block| block.get_state() != BlockState::Unallocated) + { + if block.is_defrag_source() || block.get_state() == BlockState::Nursery { + continue; + } else { + self.process_block(block, lxr, immix_space) + } + } + } + } +} + +pub struct RCSweepMatureAfterSATBLOS { + _counter: LazySweepingJobsCounter, +} + +impl RCSweepMatureAfterSATBLOS { + pub fn new(counter: LazySweepingJobsCounter) -> Self { + Self { _counter: counter } + } +} + +impl GCWork for RCSweepMatureAfterSATBLOS { + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let los = mmtk.get_plan().common().get_los(); + los.sweep_rc_mature_objects_after_satb(&|o| los.is_marked(o) || los.rc.count(o) == 0); + } +} diff --git a/src/plan/lxr/gc_work/mod.rs b/src/plan/lxr/gc_work/mod.rs new file mode 100644 index 00000000000..9286c768b8c --- /dev/null +++ b/src/plan/lxr/gc_work/mod.rs @@ -0,0 +1,134 @@ +use super::global::LXR; +use crate::plan::tracing::UnsupportedTrace; +use crate::plan::VectorObjectQueue; +use crate::scheduler::gc_work::RootKind; +use crate::scheduler::{GCWorker, WorkBucketStage}; +use crate::util::ObjectReference; +use crate::vm::{RootsWorkFactory, VMBinding}; +use crate::{Plan, MMTK}; +use std::marker::PhantomData; + +pub mod mature_evac; +pub mod mature_sweeping; +pub mod nursery_sweeping; +pub mod prepare; +pub mod rc; +pub mod tracing; + +use rc::CollectRoots; + +/// Common base fields shared by LXR's custom root/closure work packets. +/// +/// This used to be `crate::scheduler::gc_work::ProcessEdgesBase`. After upstream replaced +/// `ProcessEdgesWork` with the stateless `Trace` API, LXR keeps its own work-packet based +/// closures, so this helper lives locally in the LXR plan. +pub struct ProcessEdgesBase { + pub slots: Vec, + pub nodes: VectorObjectQueue, + mmtk: &'static MMTK, + // Use raw pointer for fast pointer dereferencing, instead of using `Option<&'static mut GCWorker>`. + // Because a copying gc will dereference this pointer at least once for every object copy. + worker: *mut GCWorker, + pub roots: bool, + pub root_kind: Option, + pub bucket: WorkBucketStage, +} + +unsafe impl Send for ProcessEdgesBase {} + +impl ProcessEdgesBase { + pub fn new( + slots: Vec, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { + #[cfg(feature = "extreme_assertions")] + if crate::util::slot_logger::should_check_duplicate_slots(mmtk.get_plan()) { + for slot in &slots { + // log slot, panic if already logged + mmtk.slot_logger.log_slot(*slot); + } + } + Self { + slots, + nodes: VectorObjectQueue::new(), + mmtk, + worker: std::ptr::null_mut(), + roots, + root_kind: if roots { Some(RootKind::Strong) } else { None }, + bucket, + } + } + + pub fn set_worker(&mut self, worker: &mut GCWorker) { + self.worker = worker; + } + + pub fn worker(&self) -> &'static mut GCWorker { + unsafe { &mut *self.worker } + } + + pub fn mmtk(&self) -> &'static MMTK { + self.mmtk + } + + pub fn plan(&self) -> &'static dyn Plan { + self.mmtk.get_plan() + } +} + +/// The [`crate::scheduler::GCWorkContext`] for LXR. +/// +/// LXR does not use the generic `Trace`-based closures. Instead it schedules its own custom work +/// packets. The `DefaultTrace`/`PinningTrace` members are therefore set to [`UnsupportedTrace`], +/// and root scanning is routed through [`LXRRootsWorkFactory`]. +pub struct LXRGCWorkContext(PhantomData); + +impl crate::scheduler::GCWorkContext for LXRGCWorkContext { + type VM = VM; + type PlanType = LXR; + type DefaultTrace = UnsupportedTrace; + type PinningTrace = UnsupportedTrace; + + fn make_roots_work_factory( + mmtk: &'static MMTK, + ) -> impl RootsWorkFactory<::VMSlot> { + LXRRootsWorkFactory::new(mmtk) + } +} + +/// The [`RootsWorkFactory`] used by LXR. Roots are processed by reference-counting them through +/// [`CollectRoots`] (which spawns `ProcessIncs`). +pub struct LXRRootsWorkFactory { + mmtk: &'static MMTK, +} + +impl Clone for LXRRootsWorkFactory { + fn clone(&self) -> Self { + Self { mmtk: self.mmtk } + } +} + +impl LXRRootsWorkFactory { + fn new(mmtk: &'static MMTK) -> Self { + Self { mmtk } + } +} + +impl RootsWorkFactory for LXRRootsWorkFactory { + fn create_process_roots_work_with_root_kind(&mut self, slots: Vec, kind: RootKind) { + let stage = self.mmtk.get_plan().root_scanning_stage(); + let mut w = CollectRoots::new(slots, true, self.mmtk, stage); + w.root_kind = Some(kind); + crate::memory_manager::add_work_packet(self.mmtk, stage, w); + } + + fn create_process_pinning_roots_work(&mut self, _nodes: Vec) { + unreachable!("LXR does not support pinning roots"); + } + + fn create_process_tpinning_roots_work(&mut self, _nodes: Vec) { + unreachable!("LXR does not support transitive pinning roots"); + } +} diff --git a/src/plan/lxr/gc_work/nursery_sweeping.rs b/src/plan/lxr/gc_work/nursery_sweeping.rs new file mode 100644 index 00000000000..383b2e2f18f --- /dev/null +++ b/src/plan/lxr/gc_work/nursery_sweeping.rs @@ -0,0 +1,117 @@ +use atomic::Ordering; + +use crate::plan::lxr::{LazySweepingJobsCounter, LXR}; +use crate::policy::immix::block::Block; +use crate::scheduler::WorkBucketStage; +use crate::scheduler::{GCWork, GCWorker}; +use crate::vm::VMBinding; +use crate::MMTK; + +pub struct RCLazySweepNurseryBlocks { + blocks: Vec, + _counter: LazySweepingJobsCounter, +} + +impl RCLazySweepNurseryBlocks { + pub fn new(blocks: Vec) -> Self { + Self { + blocks, + _counter: LazySweepingJobsCounter::new_decs(), + } + } +} + +impl GCWork for RCLazySweepNurseryBlocks { + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let lxr = &mmtk.get_plan().downcast_ref::>().unwrap(); + let mut released_blocks = 0; + for block in &self.blocks { + if block.rc_sweep_nursery(&lxr.immix_space) { + released_blocks += 1; + } + } + lxr.num_clean_blocks_released_lazy + .fetch_add(released_blocks, Ordering::SeqCst); + } +} + +pub struct RCSTWSweepNurseryBlocks { + blocks: Vec, + _counter: LazySweepingJobsCounter, +} + +impl RCSTWSweepNurseryBlocks { + pub fn new(blocks: Vec) -> Self { + Self { + blocks, + _counter: LazySweepingJobsCounter::new_decs(), + } + } +} + +impl GCWork for RCSTWSweepNurseryBlocks { + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let space = &mmtk + .get_plan() + .downcast_ref::>() + .unwrap() + .immix_space; + for block in &self.blocks { + block.rc_sweep_nursery(space); + } + } +} + +pub struct SweepBlocksAfterDecs { + blocks: Vec<(Block, bool)>, + _counter: LazySweepingJobsCounter, +} + +impl SweepBlocksAfterDecs { + pub fn new(blocks: Vec<(Block, bool)>, counter: LazySweepingJobsCounter) -> Self { + Self { + blocks, + _counter: counter, + } + } +} + +impl GCWork for SweepBlocksAfterDecs { + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let lxr = mmtk.get_plan().downcast_ref::>().unwrap(); + if self.blocks.is_empty() { + return; + } + let mut count = 0; + for (block, defrag) in &self.blocks { + block.unlog(); + if block.rc_sweep_mature::(&lxr.immix_space, *defrag) { + count += 1; + } else { + assert!( + !*defrag, + "defrag block is freed? {:?} {:?} {}", + block, + block.get_state(), + block.is_defrag_source() + ); + } + } + if count != 0 + && (lxr.current_pause().is_none() + || mmtk.scheduler.work_buckets[WorkBucketStage::STWRCDecsAndSweep].is_open()) + { + lxr.num_clean_blocks_released_lazy + .fetch_add(count, Ordering::Relaxed); + } + } +} + +pub struct ReleaseLOSNursery; + +impl GCWork for ReleaseLOSNursery { + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let lxr = mmtk.get_plan().downcast_ref::>().unwrap(); + lxr.los().release_rc_nursery_objects(); + } +} diff --git a/src/plan/lxr/gc_work/prepare.rs b/src/plan/lxr/gc_work/prepare.rs new file mode 100644 index 00000000000..3550e3c6e8b --- /dev/null +++ b/src/plan/lxr/gc_work/prepare.rs @@ -0,0 +1,100 @@ +use super::super::LXR; +use crate::policy::immix::block::{Block, BlockState}; +use crate::scheduler::{GCWork, GCWorker}; +use crate::util::heap::chunk_map::Chunk; +use crate::util::linear_scan::Region; +use crate::{vm::*, Plan, MMTK}; +use std::ops::Range; + +pub struct FastRCPrepare; + +impl GCWork for FastRCPrepare { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { + let lxr = mmtk.get_plan().downcast_ref::>().unwrap(); + #[allow(invalid_reference_casting)] + let lxr = unsafe { &mut *(lxr as *const LXR as *mut LXR) }; + lxr.prepare(worker.tls) + } +} + +pub struct ConcurrentChunkMetadataZeroing { + pub chunks: Range, +} + +impl ConcurrentChunkMetadataZeroing { + /// Clear object mark table + #[allow(unused)] + fn reset_object_mark(chunk: Chunk) { + VM::VMObjectModel::LOCAL_MARK_BIT_SPEC + .extract_side_spec() + .bzero_metadata(chunk.start(), Chunk::BYTES); + } +} + +impl GCWork for ConcurrentChunkMetadataZeroing { + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let num_chunks = (self.chunks.end.start() - self.chunks.start.start()) >> Chunk::LOG_BYTES; + let ix_space = &mmtk + .get_plan() + .downcast_ref::>() + .unwrap() + .immix_space; + for i in 0..num_chunks { + let chunk = self.chunks.start.next_nth(i); + if !ix_space.chunk_map.is_allocated(chunk) { + continue; + } + Self::reset_object_mark::(chunk); + } + } +} + +/// A work packet to prepare each block for GC. +/// Performs the action on a range of chunks. +pub struct PrepareChunksForFullGC { + pub chunks: Range, +} + +impl PrepareChunksForFullGC { + /// Clear object mark table + #[allow(unused)] + fn reset_object_mark(chunk: Chunk) { + VM::VMObjectModel::LOCAL_MARK_BIT_SPEC + .extract_side_spec() + .bzero_metadata(chunk.start(), Chunk::BYTES); + } +} + +impl GCWork for PrepareChunksForFullGC { + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let num_chunks = (self.chunks.end.start() - self.chunks.start.start()) >> Chunk::LOG_BYTES; + let ix_space = &mmtk + .get_plan() + .downcast_ref::>() + .unwrap() + .immix_space; + for i in 0..num_chunks { + let chunk = self.chunks.start.next_nth(i); + if !ix_space.chunk_map.is_allocated(chunk) { + continue; + } + // Iterate over all blocks in this chunk + for block in chunk.iter_region::() { + let state = block.get_state(); + // Skip unallocated blocks. + if state == BlockState::Unallocated { + continue; + } + // Clear defrag state + assert!(!block.is_defrag_source()); + // Clear block mark data. + if block.get_state() != BlockState::Nursery { + block.set_state(BlockState::Unmarked); + } + debug_assert!(!block.get_state().is_reusable()); + // debug_assert_ne!(block.get_state(), BlockState::Marked); + // debug_assert_ne!(block.get_state(), BlockState::Nursery); + } + } + } +} diff --git a/src/plan/lxr/gc_work/rc.rs b/src/plan/lxr/gc_work/rc.rs new file mode 100644 index 00000000000..b769c0f644e --- /dev/null +++ b/src/plan/lxr/gc_work/rc.rs @@ -0,0 +1,761 @@ +use super::super::LazySweepingJobsCounter; +use super::super::SurvivalRatioPredictorLocal; +use super::super::LXR; +use super::super::{LAZY_DECREMENTS, MATURE_EVACUATION, NO_EVAC, NURSERY_EVACUATION}; +use super::tracing::LXRConcurrentTraceObjects; +use super::tracing::LXRStopTheWorldProcessEdges; +use super::ProcessEdgesBase; +use crate::plan::VectorQueue; +use crate::policy::immix::block::BlockState; +use crate::scheduler::gc_work::RootKind; +use crate::util::copy::CopySemantics; +use crate::util::copy::GCWorkerCopyContext; +use crate::util::linear_scan::UnstraddlableRegion; +use crate::util::metadata::side_metadata::SideMetadataSpec; +use crate::util::rc::*; +use crate::vm::slot::Slot; +use crate::{ + plan::concurrent::global::ConcurrentPlan, + plan::concurrent::Pause, + policy::{immix::block::Block, space::Space}, + scheduler::{GCWork, GCWorker, WorkBucketStage}, + util::{metadata::side_metadata, object_forwarding, ObjectReference}, + vm::*, + MMTK, +}; +use atomic::Ordering; +use std::ops::{Deref, DerefMut}; +use std::sync::Arc; + +pub struct ProcessIncs { + /// Increments to process + incs: Vec, + /// Recursively generated new increments + new_incs: VectorQueue, + new_incs_count: u32, + pause: Pause, + in_cm: bool, + no_evac: bool, + pub root_kind: Option, + depth: u32, + lxr: &'static LXR, + rc: RefCountHelper, + survival_ratio_predictor_local: SurvivalRatioPredictorLocal, + copy_context: *mut GCWorkerCopyContext, +} + +unsafe impl Send for ProcessIncs {} + +impl ProcessIncs { + const CAPACITY: usize = 1024; + const UNLOG_BITS: SideMetadataSpec = *VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC + .as_spec() + .extract_side_spec(); + + fn worker(&self) -> &'static mut GCWorker { + GCWorker::::current() + } + + #[allow(clippy::mut_from_ref)] + fn copy_context(&self) -> &mut GCWorkerCopyContext { + unsafe { &mut *self.copy_context } + } + + fn __default(lxr: &'static LXR) -> Self { + Self { + incs: vec![], + new_incs: VectorQueue::default(), + new_incs_count: 0, + lxr, + pause: Pause::RefCount, + in_cm: false, + no_evac: false, + depth: 1, + rc: RefCountHelper::NEW, + root_kind: None, + survival_ratio_predictor_local: SurvivalRatioPredictorLocal::default(), + copy_context: std::ptr::null_mut(), + } + } + + fn add_new_slot(&mut self, s: VM::VMSlot) { + self.new_incs.push(s); + self.new_incs_count += 1; + if self.new_incs_count as usize >= Self::CAPACITY { + self.flush(); + } + } + + pub fn new(incs: Vec, lxr: &'static LXR) -> Self { + Self { + incs, + ..Self::__default(lxr) + } + } + + fn promote(&mut self, o: ObjectReference, copied: bool, los: bool, depth: u32) { + let size = o.get_size::(); + + if !los { + let block = Block::containing(o); + let in_nursery_block = block.get_state() == BlockState::Nursery; + if !copied && in_nursery_block { + block.set_as_in_place_promoted(); + } + self.rc.promote_with_size(o, size); + if copied { + self.survival_ratio_predictor_local + .record_copied_promotion(size); + } + } else { + // println!("promote los {:?} {}", o, self.immix().is_marked(o)); + } + // Don't mark copied objects in initial mark pause. The concurrent marker will do it (and can also resursively mark the old objects). + if self.in_cm || self.pause == Pause::FinalMark { + debug_assert!(self.lxr.is_marked(o), "{:?} is not marked", o); + } + self.scan_nursery_object(o, los, !copied, depth, size); + } + + fn record_mature_evac_remset2( + &mut self, + slot_in_defrag: bool, + s: VM::VMSlot, + o: ObjectReference, + ) { + if !(MATURE_EVACUATION && (self.in_cm || self.pause == Pause::FinalMark)) { + return; + } + if !slot_in_defrag && self.lxr.in_defrag(o) { + self.lxr.mature_evac_remset.record(s, o, self.lxr); + } + } + + fn record_mature_evac_remset(&mut self, s: VM::VMSlot, o: ObjectReference) { + if !(MATURE_EVACUATION && (self.in_cm || self.pause == Pause::FinalMark)) { + return; + } + self.record_mature_evac_remset2(self.lxr.address_in_defrag(s.to_address()), s, o); + } + + fn scan_nursery_object( + &mut self, + o: ObjectReference, + los: bool, + in_place_promotion: bool, + _depth: u32, + size: usize, + ) { + let heap_bytes_per_unlog_byte = if VM::VMObjectModel::COMPRESSED_PTR_ENABLED { + 32usize + } else { + 64 + }; + if los { + let start = + side_metadata::address_to_meta_address(&Self::UNLOG_BITS, o.to_raw_address()) + .to_mut_ptr::(); + let limit = side_metadata::address_to_meta_address( + &Self::UNLOG_BITS, + (o.to_raw_address() + size).align_up(heap_bytes_per_unlog_byte), + ) + .to_mut_ptr::(); + unsafe { + let bytes = limit.offset_from(start) as usize; + std::ptr::write_bytes(start, 0xffu8, bytes); + } + o.to_raw_address().unlog_field_relaxed::(); + } else if in_place_promotion { + let header_size = if VM::VMObjectModel::COMPRESSED_PTR_ENABLED { + 12usize + } else { + 16 + }; + let step = heap_bytes_per_unlog_byte << 2; + let end = o.to_raw_address() + size; + let aligned_end = end.align_up(step); + let cursor = o.to_raw_address() + header_size; + let mut cursor = cursor.align_down(step); + let mut meta = side_metadata::address_to_meta_address(&Self::UNLOG_BITS, cursor); + while cursor < aligned_end { + unsafe { meta.store(0xffffffffu32) } + meta += 4usize; + cursor += step; + } + }; + let obj_in_defrag = !los && Block::in_defrag_block(o); + o.iterate_fields::(self.worker().tls.0, |slot| { + let Some(target) = slot.load() else { + return; + }; + debug_assert!( + target.to_raw_address().is_mapped(), + "Unmapped obj {:?}.{:?} -> {:?}", + o, + slot, + target + ); + debug_assert!( + target.is_in_any_space(), + "Unmapped obj {:?}.{:?} -> {:?}", + o, + slot, + target + ); + let rc = self.rc.count(target); + if rc == 0 { + self.add_new_slot(slot); + } else { + if rc != crate::util::rc::MAX_REF_COUNT { + let _ = self.rc.inc(target); + } + self.record_mature_evac_remset2(obj_in_defrag, slot, target); + } + }); + } + + #[cold] + fn flush(&mut self) { + if !self.new_incs.is_empty() { + let new_incs = self.new_incs.take(); + let mut w = ProcessIncs::::new(new_incs, self.lxr); + w.depth += 1; + self.worker().add_work(WorkBucketStage::Unconstrained, w); + } + self.new_incs_count = 0; + } + + fn inc(&self, o: ObjectReference) -> bool { + self.rc.inc(o) == Ok(0) + } + + fn dont_evacuate(&self, o: ObjectReference, los: bool) -> bool { + if los { + return true; + } + // Skip mature object + if self.rc.count(o) != 0 { + return true; + } + // Skip recycled lines + if Block::containing(o).get_state() != BlockState::Nursery { + return true; + } + if cfg!(debug_assertions) { + let cls = unsafe { (o.to_raw_address() + 8usize).load::() }; + assert!(cls != 0, "ERROR {:?} rc={}", o, self.rc.count(o)); + } + false + } + + fn process_inc_and_evacuate(&mut self, o: ObjectReference, depth: u32) -> ObjectReference { + let los = self.lxr.los().in_space(o); + if NURSERY_EVACUATION && !los && object_forwarding::is_forwarded_or_being_forwarded::(o) + { + while object_forwarding::is_being_forwarded::(o) { + std::hint::spin_loop(); + } + let new = if object_forwarding::is_forwarded::(o) { + object_forwarding::read_forwarding_pointer::(o) + } else { + o + }; + let promoted = self.inc(new); + if promoted && new == o { + self.promote(o, false, los, depth); + } + return new; + } + if !NURSERY_EVACUATION || self.dont_evacuate(o, los) { + if self.inc(o) { + self.promote(o, false, los, depth); + } + return o; + } + let forwarding_status = object_forwarding::attempt_to_forward::(o); + if object_forwarding::state_is_forwarded_or_being_forwarded(forwarding_status) { + // Object is moved to a new location. + let new = object_forwarding::spin_and_get_forwarded_object::(o, forwarding_status); + self.inc(new); + new + } else { + let is_nursery = self.rc.count(o) == 0; + if is_nursery && !self.no_evac { + // Evacuate the object + let new = object_forwarding::try_forward_object::( + o, + CopySemantics::DefaultCopy, + self.copy_context(), + |_new| { + #[cfg(feature = "vo_bit")] + { + // Set the VO bit of the new object. + crate::util::metadata::vo_bit::set_vo_bit(_new); + // Clear the VO bit of the old object. + // Note that sweeping can also clear the VO bit when the line is freed, + // but no RC inc/dec should be performed on the old object from now on. + // We clear it eagerly to detect inc/dec errors. + crate::util::metadata::vo_bit::unset_vo_bit(o); + } + }, + ); + if let Some(new) = new { + self.inc(new); + self.promote(new, true, false, depth); + new + } else { + warn!("to-space overflow"); + // Object is not moved. + let promoted = self.inc(o); + object_forwarding::clear_forwarding_bits::(o); + if promoted { + self.promote(o, false, los, depth); + } + NO_EVAC.store(true, Ordering::Relaxed); + self.no_evac = true; + o + } + } else { + // Object is not moved. + let promoted = self.inc(o); + object_forwarding::clear_forwarding_bits::(o); + if promoted { + self.promote(o, false, los, depth); + } + o + } + } + } + + /// Return `None` if the increment of the slot should be delayed + fn unlog_and_load_rc_object( + &mut self, + s: VM::VMSlot, + ) -> Option { + let o = s.load(); + // unlog slot + if K == EDGE_KIND_MATURE { + s.to_address().unlog_field_relaxed::(); + } + o + } + + fn process_slot( + &mut self, + s: VM::VMSlot, + depth: u32, + add_root_to_remset: bool, + ) -> Option { + let o = match self.unlog_and_load_rc_object::(s) { + Some(o) => o, + _ => { + return None; + } + }; + // println!(" - inc {:?}: {:?} rc={}", s, o, self.rc.count(o)); + let new = self.process_inc_and_evacuate(o, depth); + // Put this into remset if this is a mature slot, or a weak root + if K != EDGE_KIND_ROOT || add_root_to_remset { + self.record_mature_evac_remset(s, new); + } + if new != o { + s.store(new) + } + Some(new) + } + + fn process_incs( + &mut self, + mut incs: AddressBuffer<'_, VM::VMSlot>, + depth: u32, + add_root_to_remset: bool, + ) -> Option> { + if K == EDGE_KIND_ROOT { + let roots = incs.as_mut_ptr() as *mut ObjectReference; + let mut num_roots = 0usize; + for s in incs.iter() { + if let Some(new) = self.process_slot::(*s, depth, add_root_to_remset) { + unsafe { + roots.add(num_roots).write(new); + } + num_roots += 1; + } + } + if num_roots != 0 { + let cap = incs.capacity(); + std::mem::forget(incs); + let roots = + unsafe { Vec::::from_raw_parts(roots, num_roots, cap) }; + Some(roots) + } else { + None + } + } else { + for s in incs.iter() { + self.process_slot::(*s, depth, false); + } + None + } + } +} + +pub type EdgeKind = u8; +pub const EDGE_KIND_ROOT: u8 = 0; +pub const EDGE_KIND_NURSERY: u8 = 1; +pub const EDGE_KIND_MATURE: u8 = 2; + +enum AddressBuffer<'a, S: Slot> { + Owned(Vec), + Ref(&'a mut Vec), +} + +impl Deref for AddressBuffer<'_, S> { + type Target = Vec; + fn deref(&self) -> &Self::Target { + match self { + Self::Owned(x) => x, + Self::Ref(x) => x, + } + } +} + +impl DerefMut for AddressBuffer<'_, S> { + fn deref_mut(&mut self) -> &mut Self::Target { + match self { + Self::Owned(x) => x, + Self::Ref(x) => x, + } + } +} + +impl GCWork for ProcessIncs { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { + self.lxr = mmtk.get_plan().downcast_ref::>().unwrap(); + self.pause = self.lxr.current_pause().unwrap(); + self.in_cm = self.lxr.concurrent_work_in_progress(); + self.copy_context = self.worker().get_copy_context_mut() as *mut GCWorkerCopyContext; + if NO_EVAC.load(Ordering::Relaxed) { + self.no_evac = true; + } else { + let over_space = mmtk.get_plan().get_used_pages() + - mmtk.get_plan().get_collection_reserved_pages() + > mmtk.get_plan().get_total_pages(); + if over_space { + self.no_evac = true; + NO_EVAC.store(true, Ordering::Relaxed); + } + } + // Process main buffer + let root_slots = if KIND == EDGE_KIND_ROOT + && (self.pause == Pause::FinalMark || self.pause == Pause::Full) + { + self.incs.clone() + } else { + vec![] + }; + let roots = { + let incs = std::mem::take(&mut self.incs); + self.process_incs::(AddressBuffer::Owned(incs), self.depth, false) + }; + if let Some(roots) = roots { + if self.lxr.cm_enabled() + && self.pause == Pause::InitialMark + && !self.root_kind.unwrap().should_skip_mark_and_decs() + { + if cfg!(any(feature = "sanity", debug_assertions)) { + for r in &roots { + assert!( + r.to_raw_address().is_mapped(), + "Invalid object {:?}: address is not mapped", + r + ); + } + } + worker.scheduler().work_buckets[WorkBucketStage::ConcurrentResumable] + .add(LXRConcurrentTraceObjects::new(roots.clone(), mmtk)); + } + if self.pause == Pause::FinalMark || self.pause == Pause::Full { + if !root_slots.is_empty() && self.root_kind != Some(RootKind::Weak) { + if self.pause == Pause::FinalMark { + let mut w = LXRStopTheWorldProcessEdges::<_, false>::new( + root_slots, + true, + mmtk, + WorkBucketStage::Closure, + ); + w.root_kind = self.root_kind; + worker.add_work(WorkBucketStage::Closure, w) + } else { + let mut w = LXRStopTheWorldProcessEdges::<_, true>::new( + root_slots, + true, + mmtk, + WorkBucketStage::Closure, + ); + w.root_kind = self.root_kind; + worker.add_work(WorkBucketStage::Closure, w) + }; + } + } else if !self.root_kind.unwrap().should_skip_decs() { + self.lxr.curr_roots.read().unwrap().push(roots); + } + } + // Process recursively generated buffer + let mut depth = self.depth; + let mut incs = vec![]; + const ACTIVE_PACKET_SPLIT: bool = false; + while !self.new_incs.is_empty() { + self.new_incs_count = 0; + depth += 1; + incs.clear(); + self.new_incs.swap(&mut incs); + if ACTIVE_PACKET_SPLIT && depth >= 16 && incs.len() > 1 { + let (a, b) = incs.split_at(incs.len() / 2); + let mut w = ProcessIncs::::new(b.to_vec(), self.lxr); + w.depth = depth; + self.worker().add_work(WorkBucketStage::Unconstrained, w); + incs = a.to_vec(); + } + if !incs.is_empty() { + self.process_incs::(AddressBuffer::Ref(&mut incs), depth, false); + } + } + self.survival_ratio_predictor_local.sync(); + } +} + +pub struct ProcessDecs { + /// Decrements to process + decs: Option>, + decs_arc: Option>>, + /// Recursively generated new decrements + new_decs: VectorQueue, + counter: LazySweepingJobsCounter, + mark_objects: VectorQueue, + mark_dead_objects: bool, + mature_sweeping_in_progress: bool, + rc: RefCountHelper, +} + +impl ProcessDecs { + fn worker(&self) -> &mut GCWorker { + GCWorker::::current() + } + + pub fn new(decs: Vec, counter: LazySweepingJobsCounter) -> Self { + Self { + decs: Some(decs), + decs_arc: None, + new_decs: VectorQueue::default(), + counter, + mark_objects: VectorQueue::default(), + mark_dead_objects: false, + mature_sweeping_in_progress: false, + rc: RefCountHelper::NEW, + } + } + + pub fn new_arc(decs: Arc>, counter: LazySweepingJobsCounter) -> Self { + Self { + decs: None, + decs_arc: Some(decs), + new_decs: VectorQueue::default(), + counter, + mark_objects: VectorQueue::default(), + mark_dead_objects: false, + mature_sweeping_in_progress: false, + rc: RefCountHelper::NEW, + } + } + + fn recursive_dec(&mut self, o: ObjectReference) { + self.new_decs.push(o); + if self.new_decs.is_full() { + self.flush() + } + } + + fn new_work(&self, w: ProcessDecs) { + self.worker().add_work(WorkBucketStage::Unconstrained, w); + } + + fn flush(&mut self) { + let mmtk = GCWorker::::current().mmtk; + if !self.new_decs.is_empty() { + let new_decs = self.new_decs.take(); + self.new_work(ProcessDecs::new(new_decs, self.counter.clone_with_decs())); + } + if !self.mark_objects.is_empty() { + let objects = self.mark_objects.take(); + let w = LXRConcurrentTraceObjects::new(objects, mmtk); + if LAZY_DECREMENTS { + self.worker().add_work(WorkBucketStage::Unconstrained, w); + } else { + self.worker().scheduler().work_buckets[WorkBucketStage::ConcurrentResumable].add(w); + } + } + } + + #[cold] + fn process_dead_object(&mut self, o: ObjectReference, lxr: &LXR) -> bool { + if self.mark_dead_objects { + lxr.mark(o); + } + // Recursively decrease field ref counts + o.iterate_fields::(self.worker().tls.0, |slot| { + if let Some(x) = slot.load() { + // println!(" -- rec dec {:?}.{:?} -> {:?}", o, slot, x); + let rc = self.rc.count(x); + if rc != MAX_REF_COUNT && rc != 0 { + self.recursive_dec(x); + } + if self.mark_dead_objects && !lxr.is_marked(x) { + if cfg!(any(feature = "sanity", debug_assertions)) { + assert!( + x.to_raw_address().is_mapped(), + "Invalid object {:?}.{:?} -> {:?}: address is not mapped", + o, + slot, + x + ); + } + self.mark_objects.push(x); + if self.mark_objects.is_full() { + self.flush(); + } + } + } + }); + let in_ix_space = lxr.immix_space.in_space(o); + if in_ix_space { + // Clear the VO bit if `o` is in the immix space. + // Note that if the object is in the LOS, + // the VO bit will be cleared in `LargeObjectSpace::release_object`. + #[cfg(feature = "vo_bit")] + crate::util::metadata::vo_bit::unset_vo_bit(o); + + self.rc.unmark_straddle_object(o); + } + if RefCountHelper::::SANITY { + unsafe { o.to_raw_address().store(0xdeadusize) }; + } + if in_ix_space { + let block = Block::containing(o); + lxr.add_to_possibly_dead_mature_blocks(block, false); + false + } else { + true + } + } + + fn process_decs(&mut self, decs: &[ObjectReference], lxr: &LXR) { + for o in decs.iter() { + if self.rc.is_dead_or_stuck(*o) + || (self.mature_sweeping_in_progress && !lxr.is_marked(*o)) + { + continue; + } + let o = if MATURE_EVACUATION && object_forwarding::is_forwarded::(*o) { + object_forwarding::read_forwarding_pointer::(*o) + } else { + *o + }; + let mut dead = false; + let mut is_los = false; + let mut already_run = false; + let result = self.rc.clone().fetch_update(o, |c| { + if already_run { + log::warn!("fetch_update is re-run! o: {o}"); + } else { + already_run = true; + } + if c == 1 && !dead { + dead = true; + is_los = self.process_dead_object(o, lxr); + } + debug_assert!(c <= MAX_REF_COUNT); + if c == 0 || c == MAX_REF_COUNT { + None /* sticky */ + } else { + Some(c - 1) + } + }); + if result == Ok(1) && is_los { + lxr.los().rc_free(o); + } + } + } +} + +impl GCWork for ProcessDecs { + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let lxr = mmtk.get_plan().downcast_ref::>().unwrap(); + self.mark_dead_objects = if LAZY_DECREMENTS { + lxr.concurrent_work_in_progress() && lxr.previous_pause() != Some(Pause::InitialMark) + } else { + lxr.concurrent_work_in_progress() && lxr.current_pause() != Some(Pause::InitialMark) + }; + self.mature_sweeping_in_progress = if LAZY_DECREMENTS { + lxr.previous_pause() == Some(Pause::FinalMark) + || lxr.current_pause() == Some(Pause::Full) + } else { + lxr.current_pause() == Some(Pause::FinalMark) + || lxr.current_pause() == Some(Pause::Full) + }; + if let Some(decs) = std::mem::take(&mut self.decs) { + self.process_decs(&decs, lxr); + } else if let Some(decs) = std::mem::take(&mut self.decs_arc) { + self.process_decs(&decs, lxr); + } + let mut decs = vec![]; + while !self.new_decs.is_empty() { + decs.clear(); + self.new_decs.swap(&mut decs); + self.process_decs(&decs, lxr); + } + self.flush(); + } +} + +pub struct CollectRoots { + base: ProcessEdgesBase, +} + +impl CollectRoots { + pub fn new( + slots: Vec, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { + debug_assert!(roots); + let base = ProcessEdgesBase::new(slots, roots, mmtk, bucket); + Self { base } + } +} + +impl GCWork for CollectRoots { + fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static MMTK) { + self.set_worker(worker); + if !self.slots.is_empty() { + let lxr = self.mmtk().get_plan().downcast_ref::>().unwrap(); + let roots = std::mem::take(&mut self.slots); + let mut w = ProcessIncs::<_, EDGE_KIND_ROOT>::new(roots, lxr); + w.root_kind = self.root_kind; + GCWork::do_work(&mut w, self.worker(), self.mmtk()); + } + } +} + +impl Deref for CollectRoots { + type Target = ProcessEdgesBase; + fn deref(&self) -> &Self::Target { + &self.base + } +} + +impl DerefMut for CollectRoots { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.base + } +} diff --git a/src/plan/lxr/gc_work/tracing.rs b/src/plan/lxr/gc_work/tracing.rs new file mode 100644 index 00000000000..dcb0ff5b783 --- /dev/null +++ b/src/plan/lxr/gc_work/tracing.rs @@ -0,0 +1,469 @@ +use super::super::LXR; +use super::ProcessEdgesBase; +use crate::plan::concurrent::Pause; +use crate::plan::VectorQueue; +use crate::policy::immix::block::Block; +use crate::policy::space::Space; +use crate::scheduler::RootKind; +use crate::util::copy::CopySemantics; +use crate::util::linear_scan::UnstraddlableRegion; +use crate::util::rc::RefCountHelper; +use crate::util::{ObjectReference, VMThread}; +use crate::vm::slot::Slot; +use crate::{ + plan::ObjectQueue, + scheduler::{GCWork, GCWorker, WorkBucketStage}, + vm::*, + MMTK, +}; +use atomic::Ordering; +use std::ops::{Deref, DerefMut}; +use std::sync::Arc; + +pub struct LXRConcurrentTraceObjects { + plan: &'static LXR, + // objects to mark and scan + objects: Option>, + objects_arc: Option>>, + // recursively generated objects + next_objects: VectorQueue, + rc: RefCountHelper, + worker: *mut GCWorker, +} + +impl LXRConcurrentTraceObjects { + const SATB_BUFFER_SIZE: usize = 8192; + + pub fn new(objects: Vec, mmtk: &'static MMTK) -> Self { + let plan = mmtk.get_plan().downcast_ref::>().unwrap(); + super::super::NUM_CONCURRENT_TRACING_PACKETS.fetch_add(1, Ordering::SeqCst); + Self { + plan, + objects: Some(objects), + objects_arc: None, + next_objects: VectorQueue::default(), + rc: RefCountHelper::NEW, + worker: std::ptr::null_mut(), + } + } + + pub fn new_arc(objects: Arc>, mmtk: &'static MMTK) -> Self { + let plan = mmtk.get_plan().downcast_ref::>().unwrap(); + super::super::NUM_CONCURRENT_TRACING_PACKETS.fetch_add(1, Ordering::SeqCst); + Self { + plan, + objects: None, + objects_arc: Some(objects), + next_objects: VectorQueue::default(), + rc: RefCountHelper::NEW, + worker: std::ptr::null_mut(), + } + } + + #[cold] + fn flush(&mut self) { + if !self.next_objects.is_empty() { + let objects = self.next_objects.take(); + let worker = GCWorker::::current(); + debug_assert!(self.plan.cm_enabled()); + let w = Self::new(objects, worker.mmtk); + worker.add_work(WorkBucketStage::ConcurrentResumable, w); + } + } + + fn trace_object(&mut self, object: ObjectReference) -> ObjectReference { + if self.rc.count(object) == 0 { + return object; + } + if self.plan.immix_space.in_space(object) { + self.plan + .immix_space + .trace_object_without_moving_rc(self, object); + } else { + self.plan.los().trace_object(self, object); + } + object + } + + fn trace_objects(&mut self, objects: &[ObjectReference]) { + for o in objects { + self.trace_object(*o); + } + } + + fn scan_and_enqueue(&mut self, object: ObjectReference) { + object.iterate_fields::(unsafe { (*self.worker).tls }.0, |s| { + let Some(t) = s.load() else { + return; + }; + if super::super::MATURE_EVACUATION && CHECK_REMSET && self.plan.in_defrag(t) { + self.plan.mature_evac_remset.record(s, t, self.plan); + } + self.next_objects.push(t); + if self.next_objects.len() > Self::SATB_BUFFER_SIZE { + self.flush(); + } + }); + } +} + +impl ObjectQueue for LXRConcurrentTraceObjects { + fn enqueue(&mut self, object: ObjectReference) { + if cfg!(feature = "sanity") { + assert!( + object.to_raw_address().is_mapped(), + "Invalid obj {:?}: address is not mapped", + object + ); + } + let should_check_remset = !self.plan.in_defrag(object); + if should_check_remset { + self.scan_and_enqueue::(object) + } else { + self.scan_and_enqueue::(object) + } + } +} + +unsafe impl Send for LXRConcurrentTraceObjects {} + +impl GCWork for LXRConcurrentTraceObjects { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { + self.worker = worker; + debug_assert!(!mmtk.scheduler.work_buckets[WorkBucketStage::RCProcessIncs].is_open()); + // mark objects + if let Some(objects) = self.objects.take() { + self.trace_objects(&objects) + } else if let Some(objects) = self.objects_arc.take() { + self.trace_objects(&objects) + } + let pause_opt = self.plan.current_pause(); + if pause_opt == Some(Pause::FinalMark) || pause_opt.is_none() { + let mut next_objects = vec![]; + while !self.next_objects.is_empty() { + let pause_opt = self.plan.current_pause(); + if !(pause_opt == Some(Pause::FinalMark) || pause_opt.is_none()) { + break; + } + next_objects.clear(); + self.next_objects.swap(&mut next_objects); + self.trace_objects(&next_objects); + } + } + self.flush(); + // CM: Decrease counter + super::super::NUM_CONCURRENT_TRACING_PACKETS.fetch_sub(1, Ordering::SeqCst); + debug_assert!(!mmtk.scheduler.work_buckets[WorkBucketStage::RCProcessIncs].is_open()); + } +} + +pub struct ProcessModBufSATB { + nodes: Option>, + nodes_arc: Option>>, +} + +impl ProcessModBufSATB { + pub fn new(nodes: Vec) -> Self { + // super::NUM_CONCURRENT_TRACING_PACKETS.fetch_add(1, Ordering::SeqCst); + Self { + nodes: Some(nodes), + nodes_arc: None, + } + } + pub fn new_arc(nodes: Arc>) -> Self { + // super::NUM_CONCURRENT_TRACING_PACKETS.fetch_add(1, Ordering::SeqCst); + Self { + nodes: None, + nodes_arc: Some(nodes), + } + } +} + +impl GCWork for ProcessModBufSATB { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { + let mut w = if let Some(nodes) = self.nodes.take() { + if nodes.is_empty() { + return; + } + if cfg!(any(feature = "sanity", debug_assertions)) { + for o in &nodes { + assert!( + o.to_raw_address().is_mapped(), + "Invalid object {:?}: address is not mapped", + o + ); + } + } + LXRConcurrentTraceObjects::new(nodes, mmtk) + } else if let Some(nodes) = self.nodes_arc.take() { + if nodes.is_empty() { + return; + } + if cfg!(any(feature = "sanity", debug_assertions)) { + for o in &*nodes { + assert!( + o.to_raw_address().is_mapped(), + "Invalid object {:?}: address is not mapped", + o + ); + } + } + LXRConcurrentTraceObjects::new_arc(nodes, mmtk) + } else { + return; + }; + + let current_pause = mmtk + .get_plan() + .downcast_ref::>() + .unwrap() + .current_pause(); + if current_pause != Some(Pause::FinalMark) { + worker.scheduler().work_buckets[WorkBucketStage::ConcurrentResumable].add(w); + } else { + GCWork::do_work(&mut w, worker, mmtk); + } + } +} + +pub struct LXRStopTheWorldProcessEdges { + lxr: &'static LXR, + pause: Pause, + base: ProcessEdgesBase, + forwarded_roots: Vec, + next_slots: VectorQueue, + next_slot_count: u32, + remset_recorded_slots: bool, + should_record_forwarded_roots: bool, +} + +impl LXRStopTheWorldProcessEdges { + const OVERWRITE_REFERENCE: bool = super::super::MATURE_EVACUATION; + + pub fn new_remset(slots: Vec, mmtk: &'static MMTK) -> Self { + let mut me = Self::new(slots, false, mmtk, WorkBucketStage::Closure); + me.remset_recorded_slots = true; + me + } + + pub fn new( + slots: Vec, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { + let base = ProcessEdgesBase::new(slots, roots, mmtk, bucket); + let lxr = base.plan().downcast_ref::>().unwrap(); + Self { + lxr, + base, + pause: Pause::RefCount, + forwarded_roots: vec![], + next_slots: VectorQueue::new(), + next_slot_count: 0, + remset_recorded_slots: false, + should_record_forwarded_roots: false, + } + } + + #[cold] + fn flush(&mut self) { + if !self.next_slots.is_empty() { + let slots = self.next_slots.take(); + let w = Self::new(slots, false, self.mmtk(), self.bucket); + self.worker() + .add_boxed_work(WorkBucketStage::Unconstrained, Box::new(w)); + } + assert!(self.nodes.is_empty()); + self.next_slot_count = 0; + } + + fn process_slots(&mut self) { + self.should_record_forwarded_roots = self.roots + && !self + .root_kind + .map(|r| r.should_skip_decs()) + .unwrap_or_default(); + self.pause = self.lxr.current_pause().unwrap(); + if self.should_record_forwarded_roots { + self.forwarded_roots.reserve(self.slots.len()); + } + let slots = std::mem::take(&mut self.slots); + if self.roots && self.root_kind == Some(RootKind::Weak) { + self.process_slots_impl::(&slots); + } else if self.remset_recorded_slots { + self.process_slots_impl::(&slots); + } else { + self.process_slots_impl::(&slots); + } + self.roots = false; + self.remset_recorded_slots = false; + let should_record_forwarded_roots = self.should_record_forwarded_roots; + self.should_record_forwarded_roots = false; + let mut slots = vec![]; + while !self.next_slots.is_empty() { + self.next_slot_count = 0; + slots.clear(); + self.next_slots.swap(&mut slots); + self.process_slots_impl::(&slots); + } + self.flush(); + if should_record_forwarded_roots { + let roots = std::mem::take(&mut self.forwarded_roots); + self.lxr.curr_roots.read().unwrap().push(roots); + } + } +} + +impl GCWork for LXRStopTheWorldProcessEdges { + fn do_work(&mut self, worker: &mut GCWorker, _mmtk: &'static MMTK) { + self.set_worker(worker); + self.process_slots(); + if !self.nodes.is_empty() { + self.flush(); + } + } +} + +impl LXRStopTheWorldProcessEdges { + #[inline] + fn full_gc_trace_object( + &mut self, + object: ObjectReference, + ) -> ObjectReference { + debug_assert!(FULL_GC); + debug_assert!(object.is_in_any_space()); + debug_assert!(object.to_raw_address().is_aligned_to(8)); + // debug_assert!(object.class_is_valid::()); + if WEAK_ROOT && !Block::containing(object).is_defrag_source() { + return object; + } + let x = if self.lxr.immix_space.in_space(object) { + let pause = self.pause; + let worker = self.worker(); + self.lxr.immix_space.rc_trace_object( + self, + object, + CopySemantics::DefaultCopy, + pause, + true, + worker, + ) + } else { + self.lxr.los().trace_object(self, object) + }; + if self.should_record_forwarded_roots { + self.forwarded_roots.push(x) + } + x + } + + #[inline] + fn mature_evac_trace_object( + &mut self, + object: ObjectReference, + ) -> ObjectReference { + debug_assert!(!FULL_GC); + // The memory (lines) of these slots can be reused at any time during mature evacuation. + // Filter out invalid target objects. + if REMSET && (!object.is_in_any_space() || !object.to_raw_address().is_aligned_to(8)) { + return object; + } + if self.lxr.rc.count(object) == 0 { + return object; + } + if WEAK_ROOT && !Block::containing(object).is_defrag_source() { + return object; + } + debug_assert!(object.is_in_any_space(), "Invalid {:?}", object); + debug_assert!( + object.to_raw_address().is_aligned_to(8), + "Invalid {:?} remset={}", + object, + self.remset_recorded_slots + ); + let object = object.get_forwarded_object().unwrap_or(object); + let new_object = if self.lxr.immix_space.in_space(object) { + if self + .lxr + .rc + .address_is_in_straddle_line(object.to_raw_address()) + { + return object; + } + let pause = self.pause; + let worker = self.worker(); + self.lxr.immix_space.rc_trace_object( + self, + object, + CopySemantics::DefaultCopy, + pause, + true, + worker, + ) + } else { + self.lxr.los().trace_object(self, object) + }; + if self.should_record_forwarded_roots { + self.forwarded_roots.push(new_object) + } + new_object + } + + #[inline] + fn __process_slot(&mut self, slot: VM::VMSlot) { + let Some(object) = slot.load() else { + return; + }; + let new_object = if !FULL_GC { + self.mature_evac_trace_object::(object) + } else { + self.full_gc_trace_object::(object) + }; + if Self::OVERWRITE_REFERENCE && new_object != object { + slot.store(new_object); + } + } + + fn process_slots_impl( + &mut self, + slots: &[VM::VMSlot], + ) { + for s in slots { + self.__process_slot::(*s); + } + } +} + +impl ObjectQueue for LXRStopTheWorldProcessEdges { + fn enqueue(&mut self, object: ObjectReference) { + let limit: usize = if FULL_GC { 8192 } else { 1024 }; + // TODO: Use actual TLS. + object.iterate_fields::(VMThread::UNINITIALIZED, |s| { + let Some(o) = s.load() else { + return; + }; + if self.lxr.is_marked(o) && !self.lxr.in_defrag(o) { + return; + } + self.next_slots.push(s); + self.next_slot_count += 1; + if self.next_slot_count as usize >= limit { + self.flush(); + } + }); + } +} + +impl Deref for LXRStopTheWorldProcessEdges { + type Target = ProcessEdgesBase; + fn deref(&self) -> &Self::Target { + &self.base + } +} + +impl DerefMut for LXRStopTheWorldProcessEdges { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.base + } +} diff --git a/src/plan/lxr/global.rs b/src/plan/lxr/global.rs new file mode 100644 index 00000000000..b6f5ba9a5f5 --- /dev/null +++ b/src/plan/lxr/global.rs @@ -0,0 +1,865 @@ +use super::block_allocation::BlockAllocation; +use super::gc_work::nursery_sweeping::ReleaseLOSNursery; +use super::gc_work::prepare::FastRCPrepare; +use super::gc_work::rc::ProcessDecs; +use super::gc_work::LXRGCWorkContext; +use super::mature_evac::MatureEvacuationSet; +use super::mutator::ALLOCATOR_MAPPING; +use super::{LazySweepingJobsCounter, LAZY_SWEEPING_JOBS}; +use crate::plan::concurrent::global::ConcurrentPlan; +use crate::plan::concurrent::Pause; +use crate::plan::global::CommonPlan; +use crate::plan::global::{BasePlan, CreateGeneralPlanArgs, CreateSpecificPlanArgs}; +use crate::plan::lxr::gc_work::mature_sweeping::{RCSweepMatureAfterSATBLOS, SweepDeadCycles}; +use crate::plan::lxr::gc_work::nursery_sweeping::SweepBlocksAfterDecs; +use crate::plan::lxr::gc_work::prepare::{ConcurrentChunkMetadataZeroing, PrepareChunksForFullGC}; +use crate::plan::lxr::mature_evac::MatureEvecRemSet; +use crate::plan::AllocationSemantics; +use crate::plan::MutatorContext; +use crate::plan::Plan; +use crate::plan::PlanConstraints; +use crate::policy::immix::block::Block; +use crate::policy::immix::ImmixSpaceArgs; +use crate::policy::largeobjectspace::LargeObjectSpace; +use crate::policy::space::Space; +use crate::scheduler::gc_work::*; +use crate::util::alloc::allocators::AllocatorSelector; +#[cfg(feature = "analysis")] +use crate::util::analysis::GcHookWork; +use crate::util::constants::*; +use crate::util::copy::*; +use crate::util::heap::{SpaceStats, VMRequest}; +use crate::util::metadata::side_metadata::SideMetadataContext; +use crate::util::metadata::MetadataSpec; +use crate::util::options::Options; +use crate::util::rc::{RefCountHelper, RC_TABLE}; +#[cfg(feature = "sanity")] +use crate::util::sanity::sanity_checker::*; +use crate::util::{metadata, Address, ObjectReference}; +use crate::vm::ActivePlan; +use crate::vm::{Collection, ObjectModel, VMBinding}; +use crate::BarrierSelector; +use crate::{policy::immix::ImmixSpace, util::opaque_pointer::VMWorkerThread}; +use crate::{scheduler::*, MMTK}; +use atomic::{Atomic, Ordering}; +use crossbeam::queue::SegQueue; +use enum_map::EnumMap; +use spin::Lazy; +use std::sync::atomic::{AtomicBool, AtomicUsize}; +use std::sync::{Condvar, Mutex, RwLock}; + +const LOG_CONSERVATIVE_SURVIVAL_RATIO_MULTIPLER: usize = 1; + +static HEAP_AFTER_GC: AtomicUsize = AtomicUsize::new(0); + +use mmtk_macros::{HasSpaces, PlanTraceObject}; + +/// The LXR plan: a low-latency garbage collector that uses reference counting +/// to reclaim most garbage immediately, and periodically runs a concurrent +/// tracing cycle (backed by an Immix mature space) to collect reference cycles. +#[derive(HasSpaces, PlanTraceObject)] +pub struct LXR { + #[post_scan] + #[space] + #[copy_semantics(CopySemantics::DefaultCopy)] + /// The Immix-based space holding both nursery and mature (RC-managed) objects, + /// and used as the evacuation destination for both nursery and mature copying. + pub immix_space: ImmixSpace, + #[parent] + /// The common plan state (base plan, large object space, etc.) shared with + /// other plans. + pub common: CommonPlan, + /// Always true for non-rc immix. + /// For RC immix, this is used for enable backup tracing. + perform_cycle_collection: AtomicBool, + current_pause: Atomic>, + previous_pause: Atomic>, + hint_cycle_gc: AtomicBool, + hint_emergency_gc: AtomicBool, + avail_pages_at_end_of_last_gc: AtomicUsize, + zeroing_packets_scheduled: AtomicBool, + decide_cycle_collection: (Mutex, Condvar), + in_concurrent_marking: AtomicBool, + /// Roots collected during the previous GC pause, still awaiting reference-count + /// decrement processing at the start of the next pause. + pub prev_roots: RwLock>>, + /// Roots collected during the current GC pause. Swapped into `prev_roots` at + /// the end of the pause (in `release`) so they can be processed next time. + pub curr_roots: RwLock>>, + /// Helper for accessing the per-object reference-count metadata. + pub rc: RefCountHelper, + block_allocation: BlockAllocation, + pub(super) evac_set: MatureEvacuationSet, + pub(super) mature_evac_remset: MatureEvecRemSet, + pub(super) num_clean_blocks_released_lazy: AtomicUsize, + pub(super) possibly_dead_mature_blocks: SegQueue<(Block, bool)>, +} + +/// The static plan constraints for LXR: it moves objects, uses a field-level +/// write barrier with log bits, and enables reference counting. +pub static LXR_CONSTRAINTS: Lazy = Lazy::new(|| PlanConstraints { + moves_objects: true, + // Max immix object size is half of a block. + max_non_los_default_alloc_bytes: crate::policy::immix::MAX_IMMIX_OBJECT_SIZE, + barrier: BarrierSelector::FieldBarrier, + needs_log_bit: true, + needs_field_log_bit: true, + rc_enabled: true, + needs_prepare_mutator: false, + ..PlanConstraints::default() +}); + +impl Plan for LXR { + fn current_gc_may_move_object(&self) -> bool { + true + } + + fn collection_required(&self, space_full: bool, _space: Option>) -> bool { + // Spaces or heap full + if self.base().collection_required(self, space_full) { + return true; + } + // SATB is finished + if self.concurrent_work_in_progress() && super::concurrent_marking_packets_drained() { + return true; + } + // Survival limits + let total_young_alloc_pages = + self.block_allocation.total_young_allocation_in_bytes() >> LOG_BYTES_IN_MBYTE; + let predicted_survival_mb: usize = + ((total_young_alloc_pages as f64 * super::SURVIVAL_RATIO_PREDICTOR.ratio()) as usize) + << LOG_CONSERVATIVE_SURVIVAL_RATIO_MULTIPLER; + if predicted_survival_mb >= super::MAX_SURVIVAL_MB { + return true; + } + if !self.immix_space.common().contiguous { + let available_to_space = self.get_total_pages() - self.get_used_pages(); + if predicted_survival_mb >= available_to_space { + return true; + } + } + false + } + + fn last_collection_was_exhaustive(&self) -> bool { + self.previous_pause.load(Ordering::SeqCst) == Some(Pause::Full) + } + + fn constraints(&self) -> &'static PlanConstraints { + &LXR_CONSTRAINTS + } + + fn create_copy_config(&'static self) -> CopyConfig { + use enum_map::enum_map; + CopyConfig { + copy_mapping: enum_map! { + CopySemantics::DefaultCopy => CopySelector::Immix(0), + _ => CopySelector::Unused, + }, + space_mapping: vec![(CopySelector::Immix(0), &self.immix_space)], + constraints: &LXR_CONSTRAINTS, + } + } + + fn schedule_collection(&'static self, scheduler: &GCWorkScheduler) { + if !super::LazySweepingJobs::all_finished() { + warn!("LXR Lazy Sweeping Not Finished"); + } + let pause = self.select_collection_kind(); + // Wait for concurrent packets + // Mark table zeroing + if pause == Pause::InitialMark || pause == Pause::Full { + self.schedule_mark_table_zeroing_tasks(Some(pause)) + } + self.zeroing_packets_scheduled + .store(false, Ordering::SeqCst); + // Set current pause kind + self.current_pause.store(Some(pause), Ordering::SeqCst); + self.perform_cycle_collection + .store(pause != Pause::RefCount, Ordering::SeqCst); + // Schedule work + match pause { + Pause::Full => self.schedule_emergency_full_heap_collection(scheduler), + Pause::RefCount => self.schedule_rc_collection(scheduler), + Pause::InitialMark => self.schedule_concurrent_marking_initial_pause(scheduler), + Pause::FinalMark => self.schedule_concurrent_marking_final_pause(scheduler), + } + // Analysis routine that is ran. It is generally recommended to take advantage + // of the scheduling system we have in place for more performance + #[cfg(feature = "analysis")] + scheduler.work_buckets[WorkBucketStage::Unconstrained].add(GcHookWork); + // Resume mutators + if pause == Pause::Full || pause == Pause::FinalMark { + #[cfg(feature = "sanity")] + scheduler.work_buckets[WorkBucketStage::Final].add(ScheduleSanityGC::::new(self)); + } + } + + fn get_allocator_mapping(&self) -> &'static EnumMap { + &ALLOCATOR_MAPPING + } + + fn prepare(&mut self, tls: VMWorkerThread) { + let pause = self.current_pause().unwrap(); + if pause == Pause::FinalMark || pause == Pause::Full { + self.common.los.is_end_of_satb_or_full_gc = true; + // release nursery memory before mature evacuation, to reduce the chance of to-space overflow. + self.immix_space.scheduler().work_buckets[WorkBucketStage::Unconstrained] + .add(ReleaseLOSNursery); + } + self.common + .prepare(tls, pause == Pause::Full || pause == Pause::InitialMark); + if super::MATURE_EVACUATION && (pause == Pause::FinalMark || pause == Pause::Full) { + self.process_mature_evacuation_remset(); + } + if super::MATURE_EVACUATION && (pause == Pause::InitialMark || pause == Pause::Full) { + // Select mature evacuation set + self.schedule_defrag_selection_packets(); + } + self.num_clean_blocks_released_lazy + .store(0, Ordering::SeqCst); + self.immix_space.prepare_rc(pause); + self.block_allocation + .reset_block_mark_for_mutator_reused_blocks(pause); + } + + fn release(&mut self, tls: VMWorkerThread) { + let _new_ratio = super::SURVIVAL_RATIO_PREDICTOR.update_ratio(); + let pause = self.current_pause().unwrap(); + if pause == Pause::FinalMark || pause == Pause::Full { + VM::VMCollection::update_weak_processor(false); + } + ::VMCollection::vm_release(); + self.common.los.is_end_of_satb_or_full_gc = false; + self.common + .release(tls, pause == Pause::Full || pause == Pause::FinalMark); + self.block_allocation + .sweep_nursery_blocks(self.immix_space.scheduler(), pause); + self.block_allocation.sweep_mutator_reused_blocks(pause); + // Check if we want to do all decs and sweeping in the pause + if super::disable_lasy_dec_for_current_gc() { + self.immix_space + .scheduler() + .process_concurrent_packets_in_pause(); + } else { + debug_assert_ne!(pause, Pause::Full); + } + self.immix_space.release_rc(); + self.schedule_mature_sweeping(pause); + // swap roots + let mut prev_roots = self.prev_roots.write().unwrap(); + let mut curr_roots = self.curr_roots.write().unwrap(); + std::mem::swap::>(&mut prev_roots, &mut curr_roots); + debug_assert!(curr_roots.is_empty()); + } + + fn get_collection_reserved_pages(&self) -> usize { + let survival = { + let predicted_survival = (self.block_allocation.clean_nursery_mb() as f64 + * super::SURVIVAL_RATIO_PREDICTOR.ratio()) + as usize; + predicted_survival << LOG_CONSERVATIVE_SURVIVAL_RATIO_MULTIPLER + }; + survival + self.immix_space.defrag_headroom_pages() + } + + fn get_used_pages(&self) -> usize { + self.immix_space.reserved_pages() + self.common.get_used_pages() + } + + fn base(&self) -> &BasePlan { + &self.common.base + } + + fn base_mut(&mut self) -> &mut BasePlan { + &mut self.common.base + } + + fn common(&self) -> &CommonPlan { + &self.common + } + + /// Get a mutable reference to the common plan. See [`Self::common`]. + fn common_mut(&mut self) -> &mut CommonPlan { + &mut self.common + } + + fn on_pause_start(&self, mmtk: &'static MMTK) { + super::NO_EVAC.store(false, Ordering::SeqCst); + let pause = self.current_pause().unwrap(); + + // Individual RC pauses that don't overlap with concurrent tracing consist of a GC cycle. + // Concurrent tracing, including RC pauses in between, counts as one GC cycle. + // A Full GC counts as a GC cycle. + if pause == Pause::RefCount && !self.concurrent_work_in_progress() + || pause == Pause::InitialMark + || pause == Pause::Full + { + mmtk.gc_trigger.policy.on_gc_start(mmtk); + } + + super::SURVIVAL_RATIO_PREDICTOR + .set_alloc_size(self.block_allocation.total_young_allocation_in_bytes()); + + if pause == Pause::Full || pause == Pause::InitialMark { + // Reset block mark and object mark table. + let work_packets = self.generate_full_trace_prepare_tasks(); + self.immix_space.scheduler().work_buckets[WorkBucketStage::RCProcessIncs] + .bulk_add(work_packets); + } + + for mutator in ::VMActivePlan::mutators() { + mutator.flush(); + } + + if pause == Pause::FinalMark { + self.set_concurrent_marking_state(false); + } + } + + fn on_pause_end(&mut self, mmtk: &'static MMTK, tls: VMWorkerThread) { + super::DISABLE_LASY_DEC_FOR_CURRENT_GC.store(false, Ordering::SeqCst); + // self.immix_space.flush_page_resource(); + let pause = self.current_pause().unwrap(); + if pause == Pause::InitialMark { + self.set_concurrent_marking_state(true); + } + self.previous_pause.store(Some(pause), Ordering::SeqCst); + self.current_pause.store(None, Ordering::SeqCst); + LAZY_SWEEPING_JOBS.write().swap(); + if super::LAZY_DECREMENTS { + let perform_cycle_collection = + self.get_available_pages() < super::CYCLE_TRIGGER_THRESHOLD; + self.hint_cycle_gc + .store(perform_cycle_collection, Ordering::SeqCst); + self.hint_emergency_gc.store(false, Ordering::SeqCst); + self.perform_cycle_collection.store(false, Ordering::SeqCst); + } + self.avail_pages_at_end_of_last_gc + .store(self.get_available_pages(), Ordering::SeqCst); + HEAP_AFTER_GC.store(self.get_reserved_pages(), Ordering::SeqCst); + + self.common_mut().on_pause_end(tls); + + // Individual RC pauses that don't overlap with concurrent tracing consist of a GC cycle. + // Concurrent tracing, including RC pauses in between, counts as one GC cycle. + // A Full GC counts as a GC cycle. + if pause == Pause::RefCount && !self.concurrent_work_in_progress() + || pause == Pause::FinalMark + || pause == Pause::Full + { + mmtk.gc_trigger.policy.on_gc_end(mmtk); + } + } + + fn root_scanning_stage(&self) -> WorkBucketStage { + WorkBucketStage::RCProcessIncs + } + + fn concurrent(&self) -> Option<&dyn ConcurrentPlan> { + Some(self) + } +} + +impl ConcurrentPlan for LXR { + fn current_pause(&self) -> Option { + self.current_pause.load(Ordering::SeqCst) + } + + fn concurrent_work_in_progress(&self) -> bool { + self.in_concurrent_marking.load(Ordering::Acquire) + } +} + +impl LXR { + /// Creates a new LXR plan: registers the RC and unlogged-bit side metadata, + /// constructs the Immix mature/nursery space and common plan, and runs + /// GC-specific initialization. + pub fn new(args: CreateGeneralPlanArgs) -> Box { + let num_workers = args.scheduler.num_workers(); + // Note: `Block::DEFRAG_STATE_TABLE` doesn't need to be listed here; it's already + // registered unconditionally by `SideMetadataContext::new_global_specs` since every + // Immix-family plan (not just LXR) requires it. + let immix_specs = metadata::extract_side_metadata(&[ + MetadataSpec::OnSide(RC_TABLE), + MetadataSpec::OnSide( + *VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC + .as_spec() + .extract_side_spec(), + ), + ]); + let global_side_metadata_specs = SideMetadataContext::new_global_specs(&immix_specs); + let mut plan_args = CreateSpecificPlanArgs { + global_args: args, + constraints: &LXR_CONSTRAINTS, + global_side_metadata_specs, + }; + let immix_space = ImmixSpace::new( + plan_args.get_mature_space_args("immix", true, false, VMRequest::discontiguous()), + ImmixSpaceArgs { + never_move_objects: false, + mixed_age: false, + }, + ); + let mut lxr = Box::new(LXR { + immix_space, + common: CommonPlan::new(plan_args), + perform_cycle_collection: AtomicBool::new(false), + hint_cycle_gc: AtomicBool::new(false), + hint_emergency_gc: AtomicBool::new(false), + current_pause: Atomic::new(None), + previous_pause: Atomic::new(None), + avail_pages_at_end_of_last_gc: AtomicUsize::new(0), + zeroing_packets_scheduled: AtomicBool::new(false), + decide_cycle_collection: (Mutex::new(true), Condvar::new()), + in_concurrent_marking: AtomicBool::new(false), + prev_roots: Default::default(), + curr_roots: Default::default(), + rc: RefCountHelper::NEW, + block_allocation: BlockAllocation::new(), + evac_set: MatureEvacuationSet::default(), + mature_evac_remset: MatureEvecRemSet::new(num_workers), + possibly_dead_mature_blocks: Default::default(), + num_clean_blocks_released_lazy: Default::default(), + }); + + lxr.gc_init(); + + // Note: `verify_side_metadata_sanity` is invoked later by `MMTK::new`, after the dynamic + // side metadata base address has been initialized. It must not be called here during plan + // construction, as the side metadata layout is not yet registered at this point. + + lxr + } + + /// Returns whether concurrent marking (used for cycle collection) is enabled + /// in this build, i.e. the `lxr_no_cm` feature is not set. + pub fn cm_enabled(&self) -> bool { + !cfg!(feature = "lxr_no_cm") + } + + fn schedule_defrag_selection_packets(&self) { + self.evac_set + .schedule_defrag_selection_packets(&self.immix_space) + } + + /// Generate chunk sweep work packets. + fn generate_dead_cycle_sweep_tasks(&self) -> Vec>> { + self.immix_space.chunk_map.generate_tasks_batched(|chunks| { + Box::new(SweepDeadCycles::new( + chunks, + LazySweepingJobsCounter::new_decs(), + )) + }) + } + + fn schedule_mature_sweeping(&self, pause: Pause) { + if pause == Pause::Full || pause == Pause::FinalMark { + self.evac_set + .sweep_mature_evac_candidates(&self.immix_space); + let disable_lasy_dec_for_current_gc = + crate::plan::lxr::disable_lasy_dec_for_current_gc(); + let dead_cycle_sweep_packets = self.generate_dead_cycle_sweep_tasks(); + let sweep_los = RCSweepMatureAfterSATBLOS::new(LazySweepingJobsCounter::new_decs()); + if super::LAZY_DECREMENTS && !disable_lasy_dec_for_current_gc { + debug_assert_ne!(pause, Pause::Full); + let concurrent_bucket = + &self.immix_space.scheduler().work_buckets[WorkBucketStage::Concurrent]; + concurrent_bucket.bulk_add_deferred(dead_cycle_sweep_packets); + concurrent_bucket.add_deferred(Box::new(sweep_los)); + } else { + self.immix_space.scheduler().work_buckets[WorkBucketStage::STWRCDecsAndSweep] + .bulk_add(dead_cycle_sweep_packets); + self.immix_space.scheduler().work_buckets[WorkBucketStage::STWRCDecsAndSweep] + .add(sweep_los); + } + } + } + + /// Generate chunk sweep work packets. + fn generate_full_trace_prepare_tasks(&self) -> Vec>> { + self.immix_space + .chunk_map + .generate_tasks_batched(|chunks| Box::new(PrepareChunksForFullGC { chunks })) + } + + fn schedule_rc_block_sweeping_tasks(&self, counter: LazySweepingJobsCounter) { + // while let Some(x) = self.last_mutator_recycled_blocks.pop() { + // x.set_state(BlockState::Marked); + // } + // This may happen either within a pause, or in concurrent. + let size = self.possibly_dead_mature_blocks.len(); + let num_bins = self.immix_space.scheduler().num_workers(); + let bin_cap = size / num_bins + if size % num_bins == 0 { 0 } else { 1 }; + let mut bins = (0..num_bins) + .map(|_| Vec::with_capacity(bin_cap)) + .collect::>>(); + 'out: for bin in bins.iter_mut() { + for _ in 0..bin_cap { + if let Some(block) = self.possibly_dead_mature_blocks.pop() { + bin.push(block); + } else { + break 'out; + } + } + } + let packets = bins + .into_iter() + .map::>, _>(|blocks| { + Box::new(SweepBlocksAfterDecs::new(blocks, counter.clone())) + }) + .collect(); + self.immix_space.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets); + } + + pub(super) fn process_mature_evacuation_remset(&self) { + self.mature_evac_remset.flush_all(); + let packets = self.mature_evac_remset.take_global_packets(); + self.immix_space.scheduler().work_buckets[WorkBucketStage::RCEvacuateMature] + .bulk_add(packets); + } + + pub(super) fn add_to_possibly_dead_mature_blocks(&self, block: Block, is_defrag_source: bool) { + if block.log() { + self.possibly_dead_mature_blocks + .push((block, is_defrag_source)); + } + } + + fn next_gc_is_emergency_gc( + &self, + total_pages: usize, + mature_space_pages: usize, + emergency_threshold: usize, + ) -> bool { + let min_avail_pages = usize::min(total_pages * emergency_threshold / 100, 1 << 30 >> 12); + total_pages < min_avail_pages + mature_space_pages + } + + fn next_gc_is_cycle_gc(&self, mature_space_pages: usize, pause: Pause) -> bool { + if pause == Pause::FinalMark || pause == Pause::Full { + super::MATURE_LIVE_PREDICTOR.update(mature_space_pages); + } + let live_mature_pages = super::MATURE_LIVE_PREDICTOR.live_pages() as usize; + let garbage = mature_space_pages.saturating_sub(live_mature_pages); + let total_pages = self.get_total_pages(); + !self.concurrent_work_in_progress() + && (self.cm_enabled() && garbage * 100 >= super::TRACE_THRESHOLD * total_pages) + } + + fn decide_next_gc_may_perform_cycle_collection(&self, pause: Pause) { + let (lock, cvar) = &self.decide_cycle_collection; + let notify = || { + let mut decide_cycle_collection = lock.lock().unwrap(); + *decide_cycle_collection = true; + cvar.notify_one(); + }; + // Reset states + self.hint_cycle_gc.store(false, Ordering::SeqCst); + self.hint_emergency_gc.store(false, Ordering::SeqCst); + let emergency_threshold = super::RC_STOP_PERCENT; + // Calculate mature space size + let total_pages = self.get_total_pages(); + let mature_space_pages = { + let released_los_pages = self.los().num_pages_released_lazy.load(Ordering::SeqCst); + HEAP_AFTER_GC + .load(Ordering::SeqCst) + .saturating_sub( + self.num_clean_blocks_released_lazy.load(Ordering::SeqCst) << Block::LOG_PAGES, + ) + .saturating_sub(released_los_pages) + }; + // Decide next GC kind + let hint_cycle_gc = self.next_gc_is_cycle_gc(mature_space_pages, pause); + let hint_emergency_gc = + self.next_gc_is_emergency_gc(total_pages, mature_space_pages, emergency_threshold); + // Update states + self.hint_cycle_gc.store(hint_cycle_gc, Ordering::SeqCst); + self.hint_emergency_gc + .store(hint_emergency_gc, Ordering::SeqCst); + // Eager mark-table zeroing + if !cfg!(feature = "sanity") && hint_cycle_gc { + self.schedule_mark_table_zeroing_tasks(None); + } + notify(); + } + + fn schedule_mark_table_zeroing_tasks(&self, pause: Option) { + if let Some(pause) = pause { + assert!(pause == Pause::InitialMark || pause == Pause::Full); + if self.zeroing_packets_scheduled.load(Ordering::SeqCst) { + return; + } + } + let work_packets = self + .immix_space + .chunk_map + .generate_tasks_batched(|chunks| Box::new(ConcurrentChunkMetadataZeroing { chunks })); + self.immix_space.scheduler().work_buckets[WorkBucketStage::Unconstrained] + .bulk_add(work_packets); + self.zeroing_packets_scheduled.store(true, Ordering::SeqCst); + } + + fn wait_for_decide_cycle_collection(&self) { + let (lock, cvar) = &self.decide_cycle_collection; + let mut decide_cycle_collection = lock.lock().unwrap(); + while !*decide_cycle_collection { + decide_cycle_collection = cvar.wait(decide_cycle_collection).unwrap(); + } + *decide_cycle_collection = false; + } + + fn select_collection_kind(&self) -> Pause { + self.wait_for_decide_cycle_collection(); + + let emergency = self.base().global_state.is_emergency_collection(); + let user_triggered = self.base().global_state.is_user_triggered_collection(); + let cm_in_progress = self.concurrent_work_in_progress(); + let cm_packets_drained = super::concurrent_marking_packets_drained(); + let hint_cycle_gc = self.hint_cycle_gc.load(Ordering::SeqCst); + let hint_emergency_gc = self.hint_emergency_gc.load(Ordering::SeqCst); + // If CM is finished, do a final mark pause + if cm_in_progress && cm_packets_drained { + return Pause::FinalMark; + } + + // Either final mark pause or full pause for emergency GC + if emergency || user_triggered || hint_emergency_gc { + return if cm_in_progress { + Pause::FinalMark + } else { + Pause::Full + }; + } + + // Should trigger CM? + if hint_cycle_gc && !cm_in_progress { + if self.cm_enabled() { + Pause::InitialMark + } else { + Pause::Full + } + } else { + Pause::RefCount + } + } + + fn disable_unnecessary_buckets(&'static self, scheduler: &GCWorkScheduler, pause: Pause) { + // Set conditional buckets + scheduler.work_buckets[WorkBucketStage::RCProcessIncs].set_enabled(true); + scheduler.work_buckets[WorkBucketStage::Prepare].set_enabled(pause != Pause::RefCount); + let final_mark_or_full = pause == Pause::FinalMark || pause == Pause::Full; + scheduler.work_buckets[WorkBucketStage::Closure].set_enabled(final_mark_or_full); + scheduler.work_buckets[WorkBucketStage::WeakRefClosure].set_enabled(final_mark_or_full); + scheduler.work_buckets[WorkBucketStage::FinalRefClosure].set_enabled(final_mark_or_full); + scheduler.work_buckets[WorkBucketStage::PhantomRefClosure].set_enabled(final_mark_or_full); + scheduler.work_buckets[WorkBucketStage::STWRCDecsAndSweep] + .set_enabled(!(super::LAZY_DECREMENTS && pause != Pause::Full)); + // Always enabled + scheduler.work_buckets[WorkBucketStage::Concurrent].set_enabled(true); + scheduler.work_buckets[WorkBucketStage::ConcurrentResumable].set_enabled(true); + // Always disabled + scheduler.work_buckets[WorkBucketStage::TPinningClosure].set_enabled(false); + scheduler.work_buckets[WorkBucketStage::PinningRootsTrace].set_enabled(false); + scheduler.work_buckets[WorkBucketStage::VMRefClosure].set_enabled(false); + scheduler.work_buckets[WorkBucketStage::VMRefForwarding].set_enabled(false); + scheduler.work_buckets[WorkBucketStage::SoftRefClosure].set_enabled(false); + scheduler.work_buckets[WorkBucketStage::CalculateForwarding].set_enabled(false); + scheduler.work_buckets[WorkBucketStage::SecondRoots].set_enabled(false); + scheduler.work_buckets[WorkBucketStage::RefForwarding].set_enabled(false); + scheduler.work_buckets[WorkBucketStage::FinalizableForwarding].set_enabled(false); + scheduler.work_buckets[WorkBucketStage::Compact].set_enabled(false); + } + + fn schedule_rc_collection(&'static self, scheduler: &GCWorkScheduler) { + log::info!("Scheduling RC collection..."); + self.disable_unnecessary_buckets(scheduler, Pause::RefCount); + // Before start yielding, wrap all the roots from the previous GC with work-packets. + self.process_prev_roots(scheduler); + // Stop & scan mutators (mutator scanning can happen before STW) + scheduler.work_buckets[WorkBucketStage::Unconstrained] + .add(StopMutators::>::new_with_flush()); + // Prepare global/collectors/mutators + scheduler.work_buckets[WorkBucketStage::RCProcessIncs].add(FastRCPrepare); + // Release global/collectors/mutators + scheduler.work_buckets[WorkBucketStage::Release] + .add(Release::>::new(self)); + } + + fn schedule_concurrent_marking_initial_pause(&'static self, scheduler: &GCWorkScheduler) { + log::info!("Scheduling concurrent marking initial pause..."); + self.disable_unnecessary_buckets(scheduler, Pause::InitialMark); + self.process_prev_roots(scheduler); + scheduler.work_buckets[WorkBucketStage::Unconstrained] + .add(StopMutators::>::new_with_flush()); + scheduler.work_buckets[WorkBucketStage::Prepare] + .add(Prepare::>::new(self)); + scheduler.work_buckets[WorkBucketStage::Release] + .add(Release::>::new(self)); + } + + fn schedule_concurrent_marking_final_pause(&'static self, scheduler: &GCWorkScheduler) { + log::info!("Scheduling concurrent marking final pause..."); + self.disable_unnecessary_buckets(scheduler, Pause::FinalMark); + self.process_prev_roots(scheduler); + scheduler.work_buckets[WorkBucketStage::Unconstrained] + .add(StopMutators::>::new_with_flush()); + + scheduler.work_buckets[WorkBucketStage::Prepare] + .add(Prepare::>::new(self)); + scheduler.work_buckets[WorkBucketStage::Release] + .add(Release::>::new(self)); + } + + fn schedule_emergency_full_heap_collection(&'static self, scheduler: &GCWorkScheduler) { + log::info!("Scheduling emergency full-heap collection..."); + super::DISABLE_LASY_DEC_FOR_CURRENT_GC.store(true, Ordering::SeqCst); + self.disable_unnecessary_buckets(scheduler, Pause::Full); + // Before start yielding, wrap all the roots from the previous GC with work-packets. + self.process_prev_roots(scheduler); + // Stop & scan mutators (mutator scanning can happen before STW) + scheduler.work_buckets[WorkBucketStage::Unconstrained] + .add(StopMutators::>::new_with_flush()); + // Prepare global/collectors/mutators + scheduler.work_buckets[WorkBucketStage::Prepare] + .add(Prepare::>::new(self)); + // Release global/collectors/mutators + scheduler.work_buckets[WorkBucketStage::Release] + .add(Release::>::new(self)); + } + + fn process_prev_roots(&self, scheduler: &GCWorkScheduler) { + let prev_roots = self.prev_roots.read().unwrap(); + let mut work_packets: Vec>> = Vec::with_capacity(prev_roots.len()); + while let Some(decs) = prev_roots.pop() { + work_packets.push(Box::new(ProcessDecs::new( + decs, + LazySweepingJobsCounter::new_decs(), + ))) + } + if work_packets.is_empty() { + work_packets.push(Box::new(ProcessDecs::new( + vec![], + LazySweepingJobsCounter::new_decs(), + ))); + } + if super::LAZY_DECREMENTS { + scheduler.work_buckets[WorkBucketStage::Concurrent].bulk_add_deferred(work_packets); + } else { + scheduler.work_buckets[WorkBucketStage::STWRCDecsAndSweep].bulk_add(work_packets); + } + } + + /// Returns whether the current GC pause performs cycle collection (tracing), + /// as opposed to being a pure reference-counting pause. + pub fn perform_cycle_collection(&self) -> bool { + self.perform_cycle_collection.load(Ordering::SeqCst) + } + + /// Returns the kind of GC pause currently in progress, or `None` if no pause + /// is currently executing. + pub fn current_pause(&self) -> Option { + self.current_pause.load(Ordering::SeqCst) + } + + /// Returns the kind of the most recently completed GC pause. + pub fn previous_pause(&self) -> Option { + self.previous_pause.load(Ordering::SeqCst) + } + + /// Returns whether the given object is in a block that was selected for + /// defragmentation (evacuation) in the current collection. + pub fn in_defrag(&self, o: ObjectReference) -> bool { + Block::in_defrag_block(o) + } + + /// Returns whether the given address is in the Immix space and in a block + /// that was selected for defragmentation (evacuation). + pub fn address_in_defrag(&self, a: Address) -> bool { + self.immix_space.address_in_space(a) && Block::address_in_defrag_block(a) + } + + /// Attempts to mark the object as live, in whichever space (Immix or large + /// object space) it belongs to. Returns `true` if this call performed the + /// marking (i.e. the object was previously unmarked). + pub fn mark(&self, o: ObjectReference) -> bool { + if self.immix_space.in_space(o) { + self.immix_space.attempt_mark(o) + } else { + self.common.los.attempt_mark(o) + } + } + + /// Like [`Self::mark`], but takes an explicit `los` flag indicating whether + /// the object is in the large object space, avoiding a space lookup. + pub fn mark2(&self, o: ObjectReference, los: bool) -> bool { + if !los { + self.immix_space.attempt_mark(o) + } else { + self.common.los.attempt_mark(o) + } + } + + /// Returns whether the object has already been marked as live in whichever + /// space (Immix or large object space) it belongs to. + pub fn is_marked(&self, o: ObjectReference) -> bool { + if self.immix_space.in_space(o) { + self.immix_space.is_marked(o) + } else { + self.common.los.is_marked(o) + } + } + + /// Returns a reference to the large object space, shared with the common plan. + pub const fn los(&self) -> &LargeObjectSpace { + &self.common.los + } + + fn on_lazy_decs_finished(&self, c: LazySweepingJobsCounter) { + self.schedule_rc_block_sweeping_tasks(c); + } + + fn on_lazy_sweeping_finished(&self) { + self.immix_space.flush_page_resource(); + // Update counters + if !super::LAZY_DECREMENTS { + HEAP_AFTER_GC.store(self.get_used_pages(), Ordering::SeqCst); + } + let pause = match self.current_pause() { + Some(p) => p, + None => self.previous_pause().unwrap(), + }; + self.decide_next_gc_may_perform_cycle_collection(pause); + } + + fn gc_init(&mut self) { + self.immix_space.rc_enabled = true; + self.common.los.rc_enabled = true; + unsafe { + let me: &'static Self = &*(self as *const Self); + me.block_allocation.init(&me.immix_space, me); + me.immix_space.install_hooks(&me.block_allocation); + self.common.los.lxr = Some(me); + } + let mut lazy_sweeping_jobs = LAZY_SWEEPING_JOBS.write(); + lazy_sweeping_jobs.swap(); + let lxr_ptr = self as *const Self as usize; + lazy_sweeping_jobs.end_of_decs = Some(Box::new(move |c| { + let lxr = unsafe { &*(lxr_ptr as *const Self) }; + lxr.on_lazy_decs_finished(c); + })); + lazy_sweeping_jobs.end_of_lazy = Some(Box::new(move || { + let lxr = unsafe { &*(lxr_ptr as *const Self) }; + lxr.on_lazy_sweeping_finished(); + })); + } + + fn set_concurrent_marking_state(&self, active: bool) { + self.in_concurrent_marking.store(active, Ordering::SeqCst); + } + + /// Returns the global MMTk options. + pub fn options(&self) -> &Options { + &self.common.base.options + } +} diff --git a/src/plan/lxr/mature_evac.rs b/src/plan/lxr/mature_evac.rs new file mode 100644 index 00000000000..7df39b3dbc4 --- /dev/null +++ b/src/plan/lxr/mature_evac.rs @@ -0,0 +1,227 @@ +use std::sync::Mutex; +use std::{cell::UnsafeCell, marker::PhantomData}; + +use crate::plan::concurrent::Pause; +use crate::plan::global::Plan; +use crate::plan::lxr::gc_work::mature_evac::SelectDefragBlocks; +use crate::plan::lxr::gc_work::mature_evac::SELECT_DEFRAG_BLOCK_JOB_COUNTER; +use crate::policy::immix::block::{Block, BlockState}; +use crate::policy::immix::line::Line; +use crate::policy::immix::ImmixSpace; +use crate::policy::space::Space; +use crate::scheduler::WorkBucketStage; +use crate::util::linear_scan::Region; +use crate::util::metadata::side_metadata::spec_defs::{IX_LINE_REUSE_COUNT, LOS_PAGE_REUSE_COUNT}; +use crate::util::ObjectReference; +use crate::{ + plan::lxr::LXR, + scheduler::GCWork, + vm::{slot::Slot, VMBinding}, +}; + +use super::gc_work::mature_evac::EvacuateMatureObjects; +use crate::util::constants::LOG_BYTES_IN_PAGE; +use atomic::Ordering; +use crossbeam::queue::SegQueue; +use std::sync::atomic::AtomicUsize; + +#[repr(C)] +pub struct RemSetEntry(VM::VMSlot, u8); + +impl RemSetEntry { + fn encode(slot: VM::VMSlot, ix: bool) -> Self { + let reuse = if ix { + IX_LINE_REUSE_COUNT.load_atomic::(slot.to_address(), Ordering::SeqCst) + } else { + LOS_PAGE_REUSE_COUNT.load_atomic::(slot.to_address(), Ordering::SeqCst) + }; + Self(slot, reuse) + } + + pub fn decode(&self) -> (VM::VMSlot, u8) { + (self.0, self.1) + } +} + +pub struct MatureEvecRemSet { + pub gc_buffers: Vec>>>, + pub global_packets: Mutex>>>, + local_packets: Vec>>>>, + _p: PhantomData, + size: AtomicUsize, +} + +unsafe impl Send for MatureEvecRemSet {} +unsafe impl Sync for MatureEvecRemSet {} + +impl MatureEvecRemSet { + pub fn new(workers: usize) -> Self { + let mut rs = Self { + gc_buffers: vec![], + global_packets: Mutex::new(vec![]), + local_packets: vec![], + _p: PhantomData, + size: AtomicUsize::new(0), + }; + rs.gc_buffers + .resize_with(workers, || UnsafeCell::new(vec![])); + rs.local_packets + .resize_with(workers, || UnsafeCell::new(vec![])); + rs + } + + #[allow(clippy::mut_from_ref)] + fn gc_buffer(&self, id: usize) -> &mut Vec> { + unsafe { &mut *self.gc_buffers[id].get() } + } + + pub fn flush_all(&self) { + let mut mature_evac_remsets = self.global_packets.lock().unwrap(); + self.size.store(0, Ordering::SeqCst); + for id in 0..self.gc_buffers.len() { + if !self.gc_buffer(id).is_empty() { + let remset = std::mem::take(self.gc_buffer(id)); + mature_evac_remsets.push(Box::new(EvacuateMatureObjects::new(remset))); + } + } + for id in 0..self.local_packets.len() { + let buf = unsafe { &mut *self.local_packets[id].get() }; + if !buf.is_empty() { + let packets = std::mem::take(buf); + for p in packets { + mature_evac_remsets.push(p); + } + } + } + } + + pub fn take_global_packets(&self) -> Vec>> { + let mut mature_evac_remsets = self.global_packets.lock().unwrap(); + std::mem::take(&mut *mature_evac_remsets) + } + + #[cold] + fn flush(&self, id: usize) { + if !self.gc_buffer(id).is_empty() { + let remset = std::mem::take(self.gc_buffer(id)); + self.size.fetch_add(remset.len(), Ordering::SeqCst); + let w = EvacuateMatureObjects::new(remset); + let packet_buffer = unsafe { &mut *self.local_packets[id].get() }; + packet_buffer.push(Box::new(w)); + } + } + + pub fn record(&self, s: VM::VMSlot, _o: ObjectReference, lxr: &LXR) { + let id = crate::scheduler::current_worker_ordinal().unwrap(); + let ix = lxr.immix_space.address_in_space(s.to_address()); + self.gc_buffer(id).push(RemSetEntry::::encode(s, ix)); + if self.gc_buffer(id).len() >= EvacuateMatureObjects::::CAPACITY { + self.flush(id) + } + } +} + +#[derive(Default)] +pub struct MatureEvacuationSet { + pub fragmented_blocks: SegQueue>, + pub fragmented_blocks_size: AtomicUsize, + pub blocks_in_fragmented_chunks: SegQueue>, + pub blocks_in_fragmented_chunks_size: AtomicUsize, + pub defrag_blocks: Mutex>, + pub num_defrag_blocks: AtomicUsize, +} + +impl MatureEvacuationSet { + /// Release all the mature defrag source blocks + pub fn sweep_mature_evac_candidates(&self, space: &ImmixSpace) { + let mut defrag_blocks: Vec = + std::mem::take(&mut *self.defrag_blocks.lock().unwrap()); + if defrag_blocks.is_empty() { + return; + } + while let Some(block) = defrag_blocks.pop() { + if !block.is_defrag_source() || block.get_state() == BlockState::Unallocated { + // This block has been eagerly released (probably be reused again). Skip it. + continue; + } + block.clear_rc_table(); + block.clear_striddle_table(); + block.rc_sweep_mature::(space, true); + assert!(!block.is_defrag_source()); + } + } + + pub fn schedule_defrag_selection_packets(&self, space: &ImmixSpace) { + let tasks = space.chunk_map.generate_tasks_batched(|chunks| { + Box::new(SelectDefragBlocks { + chunks, + defrag_threshold: 1, + }) + }); + self.fragmented_blocks_size.store(0, Ordering::SeqCst); + SELECT_DEFRAG_BLOCK_JOB_COUNTER.store(tasks.len(), Ordering::SeqCst); + space.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(tasks); + } + + pub fn skip_block(b: Block) -> bool { + let s = b.get_state(); + b.is_defrag_source() || s == BlockState::Unallocated || s == BlockState::Nursery + } + + fn select_fragmented_blocks( + &self, + selected_blocks: &mut Vec, + copy_bytes: &mut usize, + max_copy_bytes: usize, + ) { + let mut blocks = Vec::with_capacity(self.fragmented_blocks_size.load(Ordering::SeqCst)); + while let Some(mut x) = self.fragmented_blocks.pop() { + blocks.append(&mut x); + } + blocks.sort_by_key(|x| x.1); + while let Some((block, _dead_bytes)) = blocks.pop() { + if Self::skip_block(block) { + continue; + } + block.set_as_defrag_source(true); + selected_blocks.push(block); + *copy_bytes += (Block::BYTES - (block.calc_dead_lines() << Line::LOG_BYTES)) >> 1; + if *copy_bytes >= max_copy_bytes { + break; + } + } + } + + #[allow(clippy::assertions_on_constants)] + pub fn select_mature_evacuation_candidates(&self, lxr: &LXR) { + debug_assert!(crate::plan::lxr::MATURE_EVACUATION); + if lxr.current_pause().unwrap() == Pause::Full { + // Make sure LOS sweeping finishes before evac selection begin + // FIXME: This can be done in parallel with SelectDefragBlocksInChunk packets + let los = lxr.common().get_los(); + los.release_rc_nursery_objects(); + } + // Select mature defrag blocks + let available_clean_pages_for_defrag = if lxr.current_pause().unwrap() == Pause::Full { + lxr.get_total_pages() + .saturating_sub(lxr.get_used_pages()) + .max(lxr.immix_space.defrag_headroom_pages()) + } else { + lxr.immix_space.defrag_headroom_pages() + }; + let max_copy_bytes = available_clean_pages_for_defrag << LOG_BYTES_IN_PAGE; + let mut copy_bytes = 0usize; + let mut selected_blocks = vec![]; + self.select_fragmented_blocks(&mut selected_blocks, &mut copy_bytes, max_copy_bytes); + self.num_defrag_blocks + .store(selected_blocks.len(), Ordering::SeqCst); + let mut defrag_blocks = self.defrag_blocks.lock().unwrap(); + *defrag_blocks = selected_blocks; + // cleanup + assert!(self.fragmented_blocks.is_empty()); + assert!(self.blocks_in_fragmented_chunks.is_empty()); + self.fragmented_blocks_size.store(0, Ordering::SeqCst); + self.blocks_in_fragmented_chunks_size + .store(0, Ordering::SeqCst); + } +} diff --git a/src/plan/lxr/mod.rs b/src/plan/lxr/mod.rs new file mode 100644 index 00000000000..54ecda953fb --- /dev/null +++ b/src/plan/lxr/mod.rs @@ -0,0 +1,248 @@ +mod barrier; +mod block_allocation; +mod gc_work; +pub(super) mod global; +mod mature_evac; +pub(super) mod mutator; + +use std::sync::atomic::{AtomicBool, AtomicUsize}; +use std::sync::Arc; + +pub use self::global::LXR; +pub use self::global::LXR_CONSTRAINTS; + +use atomic::Atomic; +use atomic::Ordering; +use spin::Lazy; +type RwLock = spin::rwlock::RwLock; + +// --- LXR-specific global state --- + +static NUM_CONCURRENT_TRACING_PACKETS: AtomicUsize = AtomicUsize::new(0); +static DISABLE_LASY_DEC_FOR_CURRENT_GC: AtomicBool = AtomicBool::new(false); +static NO_EVAC: AtomicBool = AtomicBool::new(false); + +// --- LXR-specific global constants/flags --- + +/// Enable Lazy Decrements +const LAZY_DECREMENTS: bool = !cfg!(feature = "lxr_no_lazy"); + +/// Enable Nursery Evacuation +const NURSERY_EVACUATION: bool = !cfg!(feature = "lxr_no_nursery_evac"); + +/// Enable Mature Evacuation +pub(crate) const MATURE_EVACUATION: bool = !cfg!(feature = "lxr_no_mature_evac"); + +/// Stop triggering CM or RC pauses, and trigger Full GCs instead if the available heap after a RC pause is still small. +const RC_STOP_PERCENT: usize = 15; + +/// Trigger an RC pause when the predicted max survival size is larger than this threshold. +const MAX_SURVIVAL_MB: usize = 128; + +/// Trigger a concurrent marking cycle when the predicted mature size is larger than this threshold. +const TRACE_THRESHOLD: usize = 20; + +/// Start a concurrent marking cycle when the available pages in the previous pause is smaller than this threshold. +const CYCLE_TRIGGER_THRESHOLD: usize = 1024; + +fn concurrent_marking_packets_drained() -> bool { + NUM_CONCURRENT_TRACING_PACKETS.load(Ordering::SeqCst) == 0 +} + +fn disable_lasy_dec_for_current_gc() -> bool { + DISABLE_LASY_DEC_FOR_CURRENT_GC.load(Ordering::SeqCst) +} + +// --- Lazy sweeping job counters --- + +struct LazySweepingJobsCounter { + decs_counter: Option>, + counter: Arc, +} +impl LazySweepingJobsCounter { + pub fn new_decs() -> Self { + let lazy_sweeping_jobs = LAZY_SWEEPING_JOBS.read(); + let decs_counter = lazy_sweeping_jobs.curr_decs_counter.as_ref().unwrap(); + decs_counter.fetch_add(1, Ordering::SeqCst); + let counter = lazy_sweeping_jobs.curr_counter.as_ref().unwrap(); + counter.fetch_add(1, Ordering::SeqCst); + Self { + decs_counter: Some(decs_counter.clone()), + counter: counter.clone(), + } + } + + #[allow(clippy::should_implement_trait)] + pub fn clone(&self) -> Self { + self.counter.fetch_add(1, Ordering::SeqCst); + Self { + decs_counter: None, + counter: self.counter.clone(), + } + } + + pub fn clone_with_decs(&self) -> Self { + self.decs_counter + .as_ref() + .unwrap() + .fetch_add(1, Ordering::SeqCst); + self.counter.fetch_add(1, Ordering::SeqCst); + Self { + decs_counter: self.decs_counter.clone(), + counter: self.counter.clone(), + } + } +} + +impl Drop for LazySweepingJobsCounter { + fn drop(&mut self) { + let lazy_sweeping_jobs = LAZY_SWEEPING_JOBS.read(); + if let Some(decs) = self.decs_counter.as_ref() { + if decs.fetch_sub(1, Ordering::SeqCst) == 1 { + let f = lazy_sweeping_jobs.end_of_decs.as_ref().unwrap(); + f(self.clone()) + } + } + if self.counter.fetch_sub(1, Ordering::SeqCst) == 1 { + if let Some(f) = lazy_sweeping_jobs.end_of_lazy.as_ref() { + f() + } + } + } +} + +struct LazySweepingJobs { + prev_decs_counter: Option>, + curr_decs_counter: Option>, + prev_counter: Option>, + curr_counter: Option>, + pub end_of_decs: Option>, + pub end_of_lazy: Option>, +} + +impl LazySweepingJobs { + fn new() -> Self { + Self { + prev_decs_counter: None, + curr_decs_counter: None, + prev_counter: None, + curr_counter: None, + end_of_decs: None, + end_of_lazy: None, + } + } + + pub fn all_finished() -> bool { + LAZY_SWEEPING_JOBS + .read() + .prev_counter + .as_ref() + .map(|c| c.load(Ordering::SeqCst)) + .unwrap_or(0) + == 0 + } + + pub fn swap(&mut self) { + self.prev_decs_counter = self.curr_decs_counter.take(); + self.curr_decs_counter = Some(Arc::new(AtomicUsize::new(0))); + self.prev_counter = self.curr_counter.take(); + self.curr_counter = Some(Arc::new(AtomicUsize::new(0))); + } +} + +static LAZY_SWEEPING_JOBS: Lazy> = + Lazy::new(|| RwLock::new(LazySweepingJobs::new())); + +static SURVIVAL_RATIO_PREDICTOR: SurvivalRatioPredictor = SurvivalRatioPredictor { + prev_ratio: Atomic::new(0.01), + alloc_vol: AtomicUsize::new(0), + copy_promote_vol: AtomicUsize::new(0), +}; + +struct SurvivalRatioPredictor { + prev_ratio: Atomic, + alloc_vol: AtomicUsize, + copy_promote_vol: AtomicUsize, +} + +impl SurvivalRatioPredictor { + pub fn set_alloc_size(&self, size: usize) { + assert_eq!(self.alloc_vol.load(Ordering::SeqCst), 0); + self.alloc_vol.store(size, Ordering::SeqCst); + } + + pub fn ratio(&self) -> f64 { + self.prev_ratio.load(Ordering::Relaxed) + } + + pub fn update_ratio(&self) -> f64 { + if self.alloc_vol.load(Ordering::SeqCst) == 0 { + self.copy_promote_vol.store(0, Ordering::SeqCst); + return self.ratio(); + } + let prev = self.prev_ratio.load(Ordering::SeqCst); + let curr = self.copy_promote_vol.load(Ordering::SeqCst) as f64 + / self.alloc_vol.load(Ordering::SeqCst) as f64; + let curr = f64::min(curr, 1.0); + let ratio = (curr * 3f64 + prev) / 4f64; + let ratio = f64::min(ratio, 1.0); + self.prev_ratio.store(ratio, Ordering::SeqCst); + self.alloc_vol.store(0, Ordering::SeqCst); + self.copy_promote_vol.store(0, Ordering::SeqCst); + ratio + } +} + +struct SurvivalRatioPredictorLocal { + copy_promote_vol: AtomicUsize, +} + +impl Default for SurvivalRatioPredictorLocal { + fn default() -> Self { + Self { + copy_promote_vol: AtomicUsize::new(0), + } + } +} + +impl SurvivalRatioPredictorLocal { + pub fn record_copied_promotion(&self, size: usize) { + self.copy_promote_vol.store( + self.copy_promote_vol.load(Ordering::Relaxed) + size, + Ordering::Relaxed, + ); + } + + pub fn sync(&self) { + SURVIVAL_RATIO_PREDICTOR.copy_promote_vol.fetch_add( + self.copy_promote_vol.load(Ordering::Relaxed), + Ordering::Relaxed, + ); + } +} + +static MATURE_LIVE_PREDICTOR: MatureLivePredictor = MatureLivePredictor { + live_pages: Atomic::new(0f64), +}; + +struct MatureLivePredictor { + live_pages: Atomic, +} + +impl MatureLivePredictor { + pub fn live_pages(&self) -> f64 { + self.live_pages.load(Ordering::Relaxed) + } + + pub fn update(&self, live_pages: usize) -> f64 { + // println!("live_pages {}", live_pages); + let prev = self.live_pages.load(Ordering::Relaxed); + let curr = live_pages as f64; + let weight = 3f64; + let next = (weight * curr + prev) / (weight + 1f64); + // println!("predict {}", next); + // crate::add_mature_reclaim(live_pages, prev); + self.live_pages.store(next, Ordering::Relaxed); + next + } +} diff --git a/src/plan/lxr/mutator.rs b/src/plan/lxr/mutator.rs new file mode 100644 index 00000000000..0bb4a325c57 --- /dev/null +++ b/src/plan/lxr/mutator.rs @@ -0,0 +1,78 @@ +use super::barrier::LXRFieldBarrierSemantics; +use super::LXR; +use crate::plan::barriers::FieldBarrier; +use crate::plan::mutator_context::create_allocator_mapping; +use crate::plan::mutator_context::create_space_mapping; +use crate::plan::mutator_context::Mutator; +use crate::plan::mutator_context::MutatorConfig; +use crate::plan::mutator_context::ReservedAllocators; +use crate::plan::AllocationSemantics; +use crate::util::alloc::allocators::{AllocatorSelector, Allocators}; +use crate::util::alloc::ImmixAllocator; +use crate::util::opaque_pointer::{VMMutatorThread, VMWorkerThread}; +use crate::vm::VMBinding; +use crate::MMTK; +use enum_map::EnumMap; + +pub fn lxr_mutator_prepare(mutator: &mut Mutator, _tls: VMWorkerThread) { + let immix_allocator = unsafe { + mutator + .allocators + .get_allocator_mut(mutator.config.allocator_mapping[AllocationSemantics::Default]) + } + .downcast_mut::>() + .unwrap(); + immix_allocator.reset(); +} + +pub fn lxr_mutator_release(mutator: &mut Mutator, _tls: VMWorkerThread) { + let immix_allocator = unsafe { + mutator + .allocators + .get_allocator_mut(mutator.config.allocator_mapping[AllocationSemantics::Default]) + } + .downcast_mut::>() + .unwrap(); + immix_allocator.reset(); +} + +const RESERVED_ALLOCATORS: ReservedAllocators = ReservedAllocators { + n_immix: 1, + ..ReservedAllocators::DEFAULT +}; + +lazy_static! { + pub static ref ALLOCATOR_MAPPING: EnumMap = { + let mut map = create_allocator_mapping(RESERVED_ALLOCATORS, true); + map[AllocationSemantics::Default] = AllocatorSelector::Immix(0); + map + }; +} + +pub fn create_lxr_mutator( + mutator_tls: VMMutatorThread, + mmtk: &'static MMTK, +) -> Mutator { + let lxr = mmtk.get_plan().downcast_ref::>().unwrap(); + let config = MutatorConfig { + allocator_mapping: &ALLOCATOR_MAPPING, + space_mapping: Box::new({ + let mut vec = create_space_mapping(RESERVED_ALLOCATORS, true, mmtk.get_plan()); + vec.push((AllocatorSelector::Immix(0), &lxr.immix_space)); + vec + }), + prepare_func: &lxr_mutator_prepare, + release_func: &lxr_mutator_release, + }; + + Mutator { + allocators: Allocators::::new(mutator_tls, mmtk, &config.space_mapping), + barrier: Box::new(FieldBarrier::new(LXRFieldBarrierSemantics::new( + mmtk, + mutator_tls, + ))), + mutator_tls, + config, + plan: mmtk.get_plan(), + } +} diff --git a/src/plan/marksweep/global.rs b/src/plan/marksweep/global.rs index 382b1198b1a..4637b8333c8 100644 --- a/src/plan/marksweep/global.rs +++ b/src/plan/marksweep/global.rs @@ -67,9 +67,9 @@ impl Plan for MarkSweep { self.common.release(tls, true); } - fn end_of_pause(&mut self, mmtk: &'static MMTK, tls: VMWorkerThread) { + fn on_pause_end(&mut self, mmtk: &'static MMTK, tls: VMWorkerThread) { self.ms.end_of_gc(); - self.common.end_of_pause(tls); + self.common.on_pause_end(tls); mmtk.gc_trigger.policy.on_gc_end(mmtk); } diff --git a/src/plan/mod.rs b/src/plan/mod.rs index 92dc2c9faeb..337ba118bd0 100644 --- a/src/plan/mod.rs +++ b/src/plan/mod.rs @@ -13,7 +13,7 @@ //! //! For more about implementing a plan, it is recommended to read the [MMTk tutorial](/docs/tutorial/Tutorial.md). -mod barriers; +pub mod barriers; pub use barriers::BarrierSelector; mod gc_work; @@ -44,8 +44,12 @@ mod generational; /// Sticky plans (using sticky marks for generational behaviors without a copying nursery) mod sticky; -mod concurrent; -mod immix; +/// Concurrent GC plans, which perform (parts of) tracing concurrently with mutators. +pub mod concurrent; +/// The Immix plan, a mostly-copying mark-region GC. +pub mod immix; +/// The LXR plan, a low-latency reference-counted GC. +pub mod lxr; mod markcompact; mod marksweep; mod nogc; diff --git a/src/plan/mutator_context.rs b/src/plan/mutator_context.rs index 3c433f4c8fd..6426fdb36a2 100644 --- a/src/plan/mutator_context.rs +++ b/src/plan/mutator_context.rs @@ -47,7 +47,7 @@ pub(crate) fn unreachable_release_func( _mutator: &mut Mutator, _tls: VMWorkerThread, ) { - unreachable!("`MutatorConfig::release_func` must not be called for the current plan.") + // unreachable!("`MutatorConfig::release_func` must not be called for the current plan.") } /// An mutator release implementation for plans that use [`crate::plan::global::CommonPlan`]. diff --git a/src/plan/nogc/global.rs b/src/plan/nogc/global.rs index f739fba70d0..811ae619b43 100644 --- a/src/plan/nogc/global.rs +++ b/src/plan/nogc/global.rs @@ -67,7 +67,7 @@ impl Plan for NoGC { unreachable!() } - fn end_of_pause(&mut self, _mmtk: &'static MMTK, _tls: VMWorkerThread) { + fn on_pause_end(&mut self, _mmtk: &'static MMTK, _tls: VMWorkerThread) { unreachable!() } diff --git a/src/plan/plan_constraints.rs b/src/plan/plan_constraints.rs index 18d13dba78e..d9b420c5833 100644 --- a/src/plan/plan_constraints.rs +++ b/src/plan/plan_constraints.rs @@ -7,6 +7,7 @@ use crate::util::constants::*; /// Most of the constraints are constants. Each plan should declare a constant of this struct, /// and use the constant wherever possible. However, for plan-neutral implementations, /// these constraints are not constant. +#[derive(Clone, Debug)] pub struct PlanConstraints { /// Does the plan collect garbage? Obviously most plans do, but NoGC does not collect. pub collects_garbage: bool, @@ -21,6 +22,9 @@ pub struct PlanConstraints { pub max_non_los_copy_bytes: usize, /// Does this plan use the log bit? See vm::ObjectModel::GLOBAL_LOG_BIT_SPEC. pub needs_log_bit: bool, + /// Does this plan use the field-level (rather than object-level) unlogged bit for its write barrier? + /// See vm::ObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC. + pub needs_field_log_bit: bool, /// Some plans may allow benign race for testing mark bit, and this will lead to trace the same /// edge multiple times. If a plan allows tracing duplicated edges, we will not run duplicate /// edge check in extreme_assertions. @@ -41,6 +45,8 @@ pub struct PlanConstraints { /// Some policies do object forwarding after the first liveness transitive closure, such as mark compact. /// For plans that use those policies, they should set this as true. pub needs_forward_after_liveness: bool, + /// True if this plan is reference-counting based (e.g. LXR), rather than tracing-only. + pub rc_enabled: bool, /// Some (in fact, most) plans do nothing when preparing mutators before tracing (i.e. in /// `MutatorConfig::prepare_func`). Those plans can set this to `false` so that the /// `PrepareMutator` work packets will not be created at all. @@ -68,7 +74,9 @@ impl PlanConstraints { may_trace_duplicate_edges: cfg!(feature = "marksweep_as_nonmoving"), needs_forward_after_liveness: false, needs_log_bit: false, + needs_field_log_bit: false, barrier: BarrierSelector::NoBarrier, + rc_enabled: false, // If we use mark sweep as non moving space, we need to prepare mutator. See [`common_prepare_func`]. needs_prepare_mutator: cfg!(feature = "marksweep_as_nonmoving"), generational: false, diff --git a/src/plan/sticky/immix/global.rs b/src/plan/sticky/immix/global.rs index 9f395b96e97..e0d237403c2 100644 --- a/src/plan/sticky/immix/global.rs +++ b/src/plan/sticky/immix/global.rs @@ -155,7 +155,7 @@ impl Plan for StickyImmix { } } - fn end_of_pause( + fn on_pause_end( &mut self, mmtk: &'static MMTK, tls: crate::util::opaque_pointer::VMWorkerThread, @@ -169,7 +169,7 @@ impl Plan for StickyImmix { self.immix .set_last_gc_was_defrag(was_defrag, Ordering::Relaxed); - self.immix.common.end_of_pause(tls); + self.immix.common.on_pause_end(tls); mmtk.gc_trigger.policy.on_gc_end(mmtk); } diff --git a/src/plan/tracing/gc_work/root.rs b/src/plan/tracing/gc_work/root.rs index cc46d6a02e3..df5de0ba705 100644 --- a/src/plan/tracing/gc_work/root.rs +++ b/src/plan/tracing/gc_work/root.rs @@ -8,7 +8,7 @@ use crate::{ }, VectorObjectQueue, }, - scheduler::{GCWork, GCWorker, WorkBucketStage}, + scheduler::{gc_work::RootKind, GCWork, GCWorker, WorkBucketStage}, util::ObjectReference, vm::{RootsKind, RootsWorkFactory, VMBinding}, MMTK, @@ -42,7 +42,11 @@ impl, PT: Trace> Clone impl, PT: Trace> RootsWorkFactory for DefaultRootsWorkFactory { - fn create_process_roots_work(&mut self, slots: Vec) { + fn create_process_roots_work_with_root_kind( + &mut self, + slots: Vec, + _kind: RootKind, + ) { // Note: We should use the same USDT name "mmtk:roots" for all the three kinds of roots. A // VM binding may not call all of the three methods in this impl. For example, the OpenJDK // binding only calls `create_process_roots_work`, and the Ruby binding only calls diff --git a/src/plan/tracing/mod.rs b/src/plan/tracing/mod.rs index e68e19f8755..212fd1ee770 100644 --- a/src/plan/tracing/mod.rs +++ b/src/plan/tracing/mod.rs @@ -6,8 +6,8 @@ use std::marker::PhantomData; use crate::plan::PlanTraceObject; use crate::policy::gc_work::TraceKind; use crate::scheduler::{GCWorker, EDGES_WORK_BUFFER_SIZE}; -use crate::util::{ObjectReference, VMThread, VMWorkerThread}; -use crate::vm::{Scanning, VMBinding}; +use crate::util::ObjectReference; +use crate::vm::VMBinding; use crate::{Plan, MMTK}; pub(crate) mod gc_work; @@ -342,6 +342,11 @@ impl VectorQueue { pub fn clear(&mut self) { self.buffer.clear() } + + /// Swap the underlying buffer with the given vector. + pub fn swap(&mut self, new_buffer: &mut Vec) { + std::mem::swap(&mut self.buffer, new_buffer) + } } impl Default for VectorQueue { @@ -355,24 +360,3 @@ impl ObjectQueue for VectorQueue { self.push(v); } } - -/// For iterating over the slots of an object. -// FIXME: This type iterates slots, but all of its current use cases only care about the values in the slots. -// And it currently only works if the object supports slot enqueuing (i.e. `Scanning::scan_object` is implemented). -// We may refactor the interface according to -pub(crate) struct SlotIterator { - _p: PhantomData, -} - -impl SlotIterator { - /// Iterate over the slots of an object by applying a function to each slot. - pub fn iterate_fields(object: ObjectReference, _tls: VMThread, mut f: F) { - // FIXME: We should use tls from the arguments. - // See https://github.com/mmtk/mmtk-core/issues/1375 - let fake_tls = VMWorkerThread(VMThread::UNINITIALIZED); - if !>::support_slot_enqueuing(fake_tls, object) { - panic!("SlotIterator::iterate_fields cannot be used on objects that don't support slot-enqueuing"); - } - >::scan_object(fake_tls, object, &mut f); - } -} diff --git a/src/policy/immix/block.rs b/src/policy/immix/block.rs index 6d1efd96fbd..b57e67d8bfa 100644 --- a/src/policy/immix/block.rs +++ b/src/policy/immix/block.rs @@ -1,18 +1,19 @@ use super::defrag::Histogram; -use super::line::Line; +use super::line::{Line, RCArray}; use super::ImmixSpace; use crate::util::constants::*; use crate::util::heap::blockpageresource::BlockPool; use crate::util::heap::chunk_map::Chunk; -use crate::util::linear_scan::{Region, RegionIterator}; -use crate::util::metadata::side_metadata::{MetadataByteArrayRef, SideMetadataSpec}; +use crate::util::linear_scan::{Region, RegionIterator, UnstraddlableRegion}; +use crate::util::metadata::side_metadata::*; #[cfg(feature = "vo_bit")] use crate::util::metadata::vo_bit; #[cfg(feature = "object_pinning")] use crate::util::metadata::MetadataSpec; use crate::util::object_enum::BlockMayHaveObjects; -use crate::util::Address; +use crate::util::{Address, ObjectReference}; use crate::vm::*; +use bytemuck::NoUninit; use std::sync::atomic::Ordering; /// The block allocation state. @@ -20,10 +21,14 @@ use std::sync::atomic::Ordering; pub enum BlockState { /// the block is not allocated. Unallocated, + /// the block is a young block. + Nursery, /// the block is allocated but not marked. Unmarked, /// the block is allocated and marked. Marked, + /// RC mutator recycled blocks. + Reusing, /// the block is marked as reusable. Reusable { unavailable_lines: u8 }, } @@ -35,6 +40,8 @@ impl BlockState { const MARK_UNMARKED: u8 = u8::MAX; /// Private constant const MARK_MARKED: u8 = u8::MAX - 1; + const MARK_NURSERY: u8 = u8::MAX - 2; + const MARK_REUSING: u8 = u8::MAX - 3; } impl From for BlockState { @@ -43,6 +50,8 @@ impl From for BlockState { Self::MARK_UNALLOCATED => BlockState::Unallocated, Self::MARK_UNMARKED => BlockState::Unmarked, Self::MARK_MARKED => BlockState::Marked, + Self::MARK_NURSERY => BlockState::Nursery, + Self::MARK_REUSING => BlockState::Reusing, unavailable_lines => BlockState::Reusable { unavailable_lines }, } } @@ -54,7 +63,12 @@ impl From for u8 { BlockState::Unallocated => BlockState::MARK_UNALLOCATED, BlockState::Unmarked => BlockState::MARK_UNMARKED, BlockState::Marked => BlockState::MARK_MARKED, - BlockState::Reusable { unavailable_lines } => unavailable_lines, + BlockState::Nursery => BlockState::MARK_NURSERY, + BlockState::Reusing => BlockState::MARK_REUSING, + BlockState::Reusable { unavailable_lines } => { + assert_ne!(unavailable_lines, 0); + u8::min(unavailable_lines, u8::MAX - 4) + } } } } @@ -68,7 +82,7 @@ impl BlockState { /// Data structure to reference an immix block. #[repr(transparent)] -#[derive(Debug, Clone, Copy, PartialOrd, PartialEq)] +#[derive(Debug, Clone, Copy, PartialOrd, PartialEq, NoUninit)] pub struct Block(Address); impl Region for Block { @@ -87,6 +101,9 @@ impl Region for Block { } } +/// An objects cannot straddle multiple Immix blocks. +impl UnstraddlableRegion for Block {} + impl BlockMayHaveObjects for Block { fn may_have_objects(&self) -> bool { self.get_state() != BlockState::Unallocated @@ -110,6 +127,27 @@ impl Block { /// Block mark table (side) pub const MARK_TABLE: SideMetadataSpec = crate::util::metadata::side_metadata::spec_defs::IX_BLOCK_MARK; + pub const LOG_TABLE: SideMetadataSpec = + crate::util::metadata::side_metadata::spec_defs::IX_BLOCK_LOG; + pub const NURSERY_PROMOTION_STATE_TABLE: SideMetadataSpec = + crate::util::metadata::side_metadata::spec_defs::NURSERY_PROMOTION_STATE; + + pub fn calc_dead_lines(&self) -> usize { + let mut dead_lines = 0; + let rc_array = RCArray::of(*self); + for i in 0..Self::LINES { + if rc_array.is_dead(i) { + dead_lines += 1; + } + } + dead_lines + } + + pub const ZERO: Self = Self(Address::ZERO); + + pub fn is_zero(&self) -> bool { + self.0.is_zero() + } /// Get the chunk containing the block. pub fn chunk(&self) -> Chunk { @@ -135,6 +173,30 @@ impl Block { Self::MARK_TABLE.store_atomic::(self.start(), state, Ordering::SeqCst); } + /// Set block mark state. + pub fn fetch_update_state( + &self, + mut f: impl FnMut(BlockState) -> Option, + ) -> Result { + Self::MARK_TABLE + .fetch_update_atomic::(self.start(), Ordering::SeqCst, Ordering::SeqCst, |s| { + f(s.into()).map(u8::from) + }) + .map(|x| x.into()) + .map_err(|x| x.into()) + } + + pub fn attempt_dealloc(&self, ignore_reusing_blocks: bool) -> bool { + self.fetch_update_state(|s| { + if (ignore_reusing_blocks && s == BlockState::Reusing) || s == BlockState::Unallocated { + None + } else { + Some(BlockState::Unallocated) + } + }) + .is_ok() + } + // Defrag byte const DEFRAG_SOURCE_STATE: u8 = u8::MAX; @@ -147,6 +209,14 @@ impl Block { byte == Self::DEFRAG_SOURCE_STATE } + pub fn in_defrag_block(o: ObjectReference) -> bool { + Block::containing(o).is_defrag_source() + } + + pub fn address_in_defrag_block(a: Address) -> bool { + Block::from_unaligned_address(a).is_defrag_source() + } + /// Mark the block for defragmentation. pub fn set_as_defrag_source(&self, defrag: bool) { let byte = if defrag { Self::DEFRAG_SOURCE_STATE } else { 0 }; @@ -166,18 +236,43 @@ impl Block { } /// Initialize a clean block after acquired from page-resource. - pub fn init(&self, copy: bool) { - self.set_state(if copy { - BlockState::Marked + pub fn init(&self, copy: bool, reuse: bool, space: &ImmixSpace) { + if space.rc_enabled { + if !reuse { + debug_assert_eq!(self.get_state(), BlockState::Unallocated); + } + self.clear_in_place_promoted(); + if !copy && reuse { + self.set_state(BlockState::Reusing); + debug_assert!(!self.is_defrag_source()); + } else if copy { + if reuse { + debug_assert!(!self.is_defrag_source()); + } + self.set_state(BlockState::Unmarked); + self.set_as_defrag_source(false); + } else { + self.set_state(BlockState::Nursery); + self.set_as_defrag_source(false); + } } else { - BlockState::Unmarked - }); - Self::DEFRAG_STATE_TABLE.store_atomic::(self.start(), 0, Ordering::SeqCst); + self.set_state(if copy { + BlockState::Marked + } else { + BlockState::Unmarked + }); + if !reuse { + Self::DEFRAG_STATE_TABLE.store_atomic::(self.start(), 0, Ordering::SeqCst); + } + } } /// Deinitalize a block before releasing. - pub fn deinit(&self) { + pub fn deinit(&self, space: &ImmixSpace) { self.set_state(BlockState::Unallocated); + if space.rc_enabled { + self.set_as_defrag_source(false); + } } pub fn start_line(&self) -> Line { @@ -195,6 +290,107 @@ impl Block { RegionIterator::::new(self.start_line(), self.end_line()) } + pub fn clear_rc_table(&self) { + crate::util::rc::RC_TABLE.bzero_metadata(self.start(), Block::BYTES); + } + + pub fn clear_striddle_table(&self) { + crate::util::rc::RC_STRADDLE_LINES.bzero_metadata(self.start(), Block::BYTES); + } + + #[allow(unused)] + pub(crate) fn clear_mark_table(&self) { + VM::VMObjectModel::LOCAL_MARK_BIT_SPEC + .extract_side_spec() + .bzero_metadata(self.start(), Self::BYTES); + } + + pub(crate) fn initialize_mark_table_as_marked(&self) { + let meta = VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.extract_side_spec(); + let start: *mut u8 = address_to_meta_address(meta, self.start()).to_mut_ptr(); + let limit: *mut u8 = address_to_meta_address(meta, self.end()).to_mut_ptr(); + unsafe { + let bytes = limit.offset_from(start) as usize; + std::ptr::write_bytes(start, 0xffu8, bytes); + } + } + + pub fn log(&self) -> bool { + loop { + let old_value: u8 = Self::LOG_TABLE.load_atomic(self.start(), Ordering::Relaxed); + if old_value == 1 { + return false; + } + if Self::LOG_TABLE + .compare_exchange_atomic(self.start(), 0u8, 1u8, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + return true; + } + } + } + + pub fn set_as_in_place_promoted(&self) { + if self.is_in_place_promoted() { + return; + } + unsafe { Self::NURSERY_PROMOTION_STATE_TABLE.store(self.start(), 1u8) }; + } + + pub fn is_in_place_promoted(&self) -> bool { + Self::NURSERY_PROMOTION_STATE_TABLE.load_atomic::(self.start(), Ordering::Relaxed) != 0 + } + + pub fn clear_in_place_promoted(&self) { + unsafe { Self::NURSERY_PROMOTION_STATE_TABLE.store(self.start(), 0u8) }; + } + + pub fn unlog(&self) { + Self::LOG_TABLE.store_atomic(self.start(), 0u8, Ordering::Relaxed); + } + + pub fn clear_field_unlog_table(&self) { + VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC + .as_spec() + .extract_side_spec() + .bzero_metadata(self.start(), Block::BYTES); + } + + pub fn initialize_field_unlog_table_as_unlogged(&self) { + let meta = *VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC + .as_spec() + .extract_side_spec(); + let start: *mut u8 = address_to_meta_address(&meta, self.start()).to_mut_ptr(); + let limit: *mut u8 = address_to_meta_address(&meta, self.end()).to_mut_ptr(); + unsafe { + let bytes = limit.offset_from(start) as usize; + std::ptr::write_bytes(start, 0xffu8, bytes); + } + } + + #[allow(clippy::assertions_on_constants)] + pub fn rc_dead(&self) -> bool { + type UInt = u128; + const LOG_BITS_IN_UINT: usize = + (std::mem::size_of::() << 3).trailing_zeros() as usize; + debug_assert!( + Self::LOG_BYTES - crate::util::rc::LOG_MIN_OBJECT_SIZE + + crate::util::rc::LOG_REF_COUNT_BITS + >= LOG_BITS_IN_UINT + ); + let start = + address_to_meta_address(&crate::util::rc::RC_TABLE, self.start()).to_ptr::(); + let limit = + address_to_meta_address(&crate::util::rc::RC_TABLE, self.end()).to_ptr::(); + let rc_table = unsafe { std::slice::from_raw_parts(start, limit.offset_from(start) as _) }; + for x in rc_table { + if *x != 0 { + return false; + } + } + true + } + /// Sweep this block. pub fn sweep( &self, @@ -202,6 +398,10 @@ impl Block { mark_histogram: &mut Histogram, line_mark_state: Option, ) -> BlockSweepResult { + // This method is not called when using RC. + assert!(!space.rc_enabled); + + self.set_as_defrag_source(false); if super::BLOCK_ONLY { match self.get_state() { BlockState::Unallocated => unreachable!("Must not sweep unallocated block."), @@ -219,7 +419,7 @@ impl Block { } // Release the block if it is allocated but not marked by the current GC. - space.release_block(*self); + space.release_block(*self, false); BlockSweepResult::Swept } BlockState::Marked => { @@ -269,7 +469,7 @@ impl Block { vo_bit::helper::on_region_swept::(self, false); // Release the block if non of its lines are marked. - space.release_block(*self); + space.release_block(*self, false); BlockSweepResult::Swept } else { // There are some marked lines. Keep the block live. @@ -277,7 +477,7 @@ impl Block { if is_reusable { // There are holes. Mark the block as reusable. self.set_state(BlockState::Reusable { - unavailable_lines: marked_lines as _, + unavailable_lines: usize::min(marked_lines, u8::MAX as usize) as _, }); space.reusable_blocks.push(*self) } else { @@ -301,6 +501,141 @@ impl Block { } } + pub fn rc_sweep_nursery(&self, space: &ImmixSpace) -> bool { + let is_in_place_promoted = self.is_in_place_promoted(); + self.clear_in_place_promoted(); + if is_in_place_promoted { + self.set_state(BlockState::Reusable { + unavailable_lines: 1 as _, + }); + + // Bulk clear the VO bits of reusable (unmarked) lines. + // Lines that are not marked may contain nursery objects that have never received any inc, + // and their VO bits need to be cleared before the lines can be reused. + #[cfg(feature = "vo_bit")] + { + let rc_array = RCArray::of(*self); + + for (i, line) in self.lines().enumerate() { + if rc_array.is_dead(i) { + crate::util::metadata::vo_bit::bzero_vo_bit(line.start(), Line::BYTES); + } + } + } + + space.reusable_blocks.push(*self); + false + } else { + debug_assert!(self.rc_dead(), "{:?} has non-zero rc value", self); + debug_assert_ne!(self.get_state(), super::block::BlockState::Unallocated); + + // Bulk clear the VO bits of the entire block. + // This block may contain nursery objects that have never received any inc, + // and their VO bits need to be cleared before the block can be reused. + #[cfg(feature = "vo_bit")] + crate::util::metadata::vo_bit::bzero_vo_bit(self.start(), Self::BYTES); + + space.release_block(*self, false); + true + } + } + + pub fn attempt_mutator_reuse(&self) -> bool { + self.fetch_update_state(|s| { + if s.is_reusable() { + Some(BlockState::Reusing) + } else { + None + } + }) + .is_ok() + } + + pub fn rc_sweep_mature(&self, space: &ImmixSpace, defrag: bool) -> bool { + if self.get_state() == BlockState::Unallocated || self.get_state() == BlockState::Nursery { + return false; + } + if defrag || self.rc_dead() { + if self.attempt_dealloc(true) { + // Bulk clear the VO bits of the entire block. + // Dec operations may reduce some object's RC to 0, + // at which time their VO bits are cleared, too. + // But some lines may also contain objects that have never received any inc, + // and their VO bits need to be cleared before the block can be reused. + #[cfg(feature = "vo_bit")] + crate::util::metadata::vo_bit::bzero_vo_bit(self.start(), Self::BYTES); + + space.release_block(*self, true); + return true; + } + } else if !super::BLOCK_ONLY { + // See the caller of this function. + // At least one object is dead in the block. + let add_as_reusable = { + let has_holes = self.has_holes(); + self.fetch_update_state(|s| { + if s == BlockState::Reusing + || s == BlockState::Unallocated + || s.is_reusable() + || !has_holes + { + None + } else { + Some(BlockState::Reusable { + unavailable_lines: 1 as _, + }) + } + }) + .is_ok() + }; + if add_as_reusable { + // Bulk clear the VO bits of reusable (unmarked) lines. + // Dec operations may reduce some object's RC to 0, + // at which time their VO bits are cleared, too. + // But some lines may also contain objects that have never received any inc, + // and their VO bits need to be cleared before the block can be reused. + #[cfg(feature = "vo_bit")] + { + let rc_array = RCArray::of(*self); + + for (i, line) in self.lines().enumerate() { + if rc_array.is_dead(i) { + crate::util::metadata::vo_bit::bzero_vo_bit(line.start(), Line::BYTES); + } + } + } + space.reusable_blocks.push(*self); + } + } + false + } + + pub fn rc_table_start(&self) -> Address { + address_to_meta_address(&crate::util::rc::RC_TABLE, self.start()) + } + + pub fn has_holes(&self) -> bool { + let rc_array = RCArray::of(*self); + let mut found_free_line = false; + let mut free_lines = 0; + for i in 0..Self::LINES { + if rc_array.is_dead(i) { + if i == 0 || found_free_line { + free_lines += 1 + } else if !found_free_line { + found_free_line = true; + } + if free_lines > 0 { + return true; + } + } else { + free_lines = 0; + found_free_line = false; + } + } + false + } + /// Clear VO bits metadata for unmarked regions. /// This is useful for clearing VO bits during nursery GC for StickyImmix /// at which time young objects (allocated in unmarked regions) may die diff --git a/src/policy/immix/defrag.rs b/src/policy/immix/defrag.rs index 7be6711adca..48501ac0638 100644 --- a/src/policy/immix/defrag.rs +++ b/src/policy/immix/defrag.rs @@ -72,6 +72,7 @@ impl Defrag { user_triggered: bool, exhausted_reusable_space: bool, full_heap_system_gc: bool, + rc_enabled: bool, stress_defrag: bool, ) { let in_defrag = defrag_enabled @@ -79,7 +80,8 @@ impl Defrag { || (collection_attempts > 1) || !exhausted_reusable_space || stress_defrag - || (collect_whole_heap && user_triggered && full_heap_system_gc)); + || (collect_whole_heap && user_triggered && full_heap_system_gc)) + && !rc_enabled; { // These details are useful for debugging why a debug GC is triggered or not triggered. diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index f822f17564d..891899bb7ff 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -1,26 +1,30 @@ use super::defrag::StatsForDefrag; use super::line::*; use super::{block::*, defrag::Defrag}; +use crate::plan::concurrent::Pause; use crate::plan::tracing::OptionObjectQueue; use crate::policy::gc_work::{TraceKind, DEFAULT_TRACE, TRACE_KIND_TRANSITIVE_PIN}; use crate::policy::sft::GCWorkerMutRef; use crate::policy::sft::SFT; use crate::policy::sft_map::SFTMap; use crate::policy::space::{CommonSpace, Space}; +use crate::scheduler::gc_work::PrepareCollector; use crate::util::alloc::allocator::AllocationOptions; use crate::util::alloc::allocator::AllocatorContext; use crate::util::constants::LOG_BYTES_IN_PAGE; use crate::util::heap::chunk_map::*; use crate::util::heap::BlockPageResource; use crate::util::heap::PageResource; -use crate::util::linear_scan::{Region, RegionIterator}; +use crate::util::linear_scan::{Region, RegionIterator, UnstraddlableRegion}; use crate::util::metadata::log_bit::UnlogBitsOperation; -use crate::util::metadata::side_metadata::SideMetadataSpec; +use crate::util::metadata::side_metadata::spec_defs::IX_LINE_REUSE_COUNT; +use crate::util::metadata::side_metadata::*; #[cfg(feature = "vo_bit")] use crate::util::metadata::vo_bit; use crate::util::metadata::{self, MetadataSpec}; use crate::util::object_enum::ObjectEnumerator; use crate::util::object_forwarding; +use crate::util::rc::RefCountHelper; use crate::util::{copy::*, epilogue, object_enum}; use crate::util::{Address, ObjectReference}; use crate::vm::*; @@ -31,14 +35,32 @@ use crate::{ MMTK, }; use atomic::Ordering; -use std::sync::{atomic::AtomicU8, atomic::AtomicUsize, Arc}; +use std::sync::atomic::AtomicUsize; +use std::sync::OnceLock; +use std::sync::{atomic::AtomicU8, Arc}; pub(crate) const TRACE_KIND_FAST: TraceKind = 0; pub(crate) const TRACE_KIND_DEFRAG: TraceKind = 1; +/// Plan-level hooks invoked by ImmixSpace during mutator allocation. +/// Default impls are no-ops; LXR provides the concrete implementation. +pub trait ImmixHooks: Send + Sync { + /// Called after a fresh clean block is acquired. `copy` distinguishes + /// mutator vs. GC-copy allocation. The hook owns any plan-specific + /// per-block bookkeeping (e.g. nursery list, mark-table init). + fn on_clean_block_acquired(&self, _block: Block, _copy: bool) {} + /// Called after a reusable block is handed out to a mutator. + fn on_reusable_block_acquired(&self, _block: Block, _copy: bool) {} + /// Whether tracing is in progress; consulted on the mutator + /// reused-line fast path so newly handed-out lines can be marked. + fn cm_in_progress_or_final_mark(&self) -> bool { + false + } +} + pub struct ImmixSpace { common: CommonSpace, - pr: BlockPageResource, + pub pr: BlockPageResource, /// Allocation status for all chunks in immix space pub chunk_map: ChunkMap, /// Current line mark state @@ -51,12 +73,17 @@ pub struct ImmixSpace { pub(super) defrag: Defrag, /// How many lines have been consumed since last GC? lines_consumed: AtomicUsize, + reused_lines_consumed: AtomicUsize, /// Object mark state mark_state: u8, /// Work packet scheduler scheduler: Arc>, /// Some settings for this space space_args: ImmixSpaceArgs, + hooks: OnceLock<&'static dyn ImmixHooks>, + pub rc_enabled: bool, + pub is_end_of_satb_or_full_gc: bool, + pub rc: RefCountHelper, } /// Some arguments for Immix Space. @@ -92,6 +119,35 @@ impl SFT for ImmixSpace { } fn is_live(&self, object: ObjectReference) -> bool { + if self.rc_enabled { + if self.is_end_of_satb_or_full_gc { + if self.is_marked(object) { + let block = Block::containing(object); + if block.is_defrag_source() { + if object_forwarding::is_forwarded::(object) { + let forwarded = + object_forwarding::read_forwarding_pointer::(object); + return self.is_marked(forwarded) && self.rc.count(forwarded) > 0; + } else { + return false; + } + } + return self.rc.count(object) > 0; + } else if object_forwarding::is_forwarded::(object) { + let forwarded = object_forwarding::read_forwarding_pointer::(object); + debug_assert!( + forwarded.to_raw_address().is_mapped(), + "Invalid forwarded object: {:?} -> {:?}", + object, + forwarded + ); + return self.is_marked(forwarded) && self.rc.count(forwarded) > 0; + } else { + return false; + } + } + return self.rc.count(object) > 0 || object_forwarding::is_forwarded::(object); + } // If the mark bit is set, it is live. if self.is_marked(object) { return true; @@ -105,6 +161,18 @@ impl SFT for ImmixSpace { // If the object is forwarded, it is live, too. object_forwarding::is_forwarded::(object) } + + fn is_reachable(&self, object: ObjectReference) -> bool { + if self.rc_enabled { + if object_forwarding::is_forwarded::(object) { + let forwarded = object_forwarding::read_forwarding_pointer::(object); + return self.is_marked(forwarded) && self.rc.count(forwarded) > 0; + } + self.is_marked(object) && self.rc.count(object) > 0 + } else { + self.is_live(object) + } + } #[cfg(feature = "object_pinning")] fn pin_object(&self, object: ObjectReference) -> bool { if self.space_args.never_move_objects { @@ -166,10 +234,14 @@ impl SFT for ImmixSpace { fn debug_print_object_info(&self, object: ObjectReference) { println!("marked = {}", self.is_marked(object)); - println!( - "line marked = {}", - Line::from_unaligned_address(object.to_raw_address()).is_marked(self.mark_state) - ); + // The line mark table isn't mapped when RC is enabled (LXR tracks liveness via + // block state and reference counts instead), so skip it in that case. + if !self.rc_enabled { + println!( + "line marked = {}", + Line::from_unaligned_address(object.to_raw_address()).is_marked(self.mark_state) + ); + } println!( "block state = {:?}", Block::from_unaligned_address(object.to_raw_address()).get_state() @@ -281,8 +353,8 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace } else if KIND == DEFAULT_TRACE { // FIXME: This is hacky. When we do a default trace, this should be a nonmoving space. // The only exception is the nursery GC for sticky immix, for which, we use default trace. - // This function is only used for PlanTrace, and for sticky immix nursery GC, we use - // GenNurseryTrace. So it still works. But this is quite hacky anyway. + // This function is only used for PlanProcessEdges, and for sticky immix nursery GC, we use + // GenNurseryProcessEdges. So it still works. But this is quite hacky anyway. // See https://github.com/mmtk/mmtk-core/issues/1314 for details. false } else { @@ -297,10 +369,20 @@ impl ImmixSpace { const MARKED_STATE: u8 = 1; /// Get side metadata specs - fn side_metadata_specs() -> Vec { + fn side_metadata_specs(rc_enabled: bool) -> Vec { + if rc_enabled { + let meta = vec![ + MetadataSpec::OnSide(Block::MARK_TABLE), + *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, + MetadataSpec::OnSide(crate::util::rc::RC_STRADDLE_LINES), + MetadataSpec::OnSide(Block::LOG_TABLE), + MetadataSpec::OnSide(Block::NURSERY_PROMOTION_STATE_TABLE), + MetadataSpec::OnSide(IX_LINE_REUSE_COUNT), + ]; + return metadata::extract_side_metadata(&meta); + } metadata::extract_side_metadata(&if super::BLOCK_ONLY { vec![ - MetadataSpec::OnSide(Block::DEFRAG_STATE_TABLE), MetadataSpec::OnSide(Block::MARK_TABLE), *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, *VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, @@ -311,7 +393,6 @@ impl ImmixSpace { } else { vec![ MetadataSpec::OnSide(Line::MARK_TABLE), - MetadataSpec::OnSide(Block::DEFRAG_STATE_TABLE), MetadataSpec::OnSide(Block::MARK_TABLE), *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC, *VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC, @@ -355,12 +436,21 @@ impl ImmixSpace { "Number of lines in a block should not exceed BlockState::MARK_MARKED" ); + // TODO: The VO bit strategy is only relevant to tracing GC. + // LXR currently ignores the strategy. #[cfg(feature = "vo_bit")] - vo_bit::helper::validate_config::(); + if !args.constraints.rc_enabled { + vo_bit::helper::validate_config::(); + } + let vm_map = args.vm_map; let scheduler = args.scheduler.clone(); - let common = - CommonSpace::new(args.into_policy_args(true, false, Self::side_metadata_specs())); + let rc_enabled = args.constraints.rc_enabled; + let common = CommonSpace::new(args.into_policy_args( + true, + false, + Self::side_metadata_specs(rc_enabled), + )); let space_index = common.descriptor.get_index(); ImmixSpace { pr: if common.vmrequest.is_discontiguous() { @@ -383,18 +473,26 @@ impl ImmixSpace { line_mark_state: AtomicU8::new(Line::RESET_MARK_STATE), line_unavail_state: AtomicU8::new(Line::RESET_MARK_STATE), lines_consumed: AtomicUsize::new(0), + reused_lines_consumed: AtomicUsize::new(0), reusable_blocks: ReusableBlockPool::new(scheduler.num_workers()), defrag: Defrag::default(), // Set to the correct mark state when inititialized. We cannot rely on prepare to set it (prepare may get skipped in nursery GCs). mark_state: Self::MARKED_STATE, - scheduler: scheduler.clone(), + scheduler, space_args, + hooks: OnceLock::new(), + rc_enabled, + is_end_of_satb_or_full_gc: false, + rc: RefCountHelper::NEW, } } /// Flush the thread-local queues in BlockPageResource pub fn flush_page_resource(&self) { - self.reusable_blocks.flush_all(); + // FIXME: Do we need this for LXR? We observed this to cause fails on conix. + if !self.rc_enabled { + self.reusable_blocks.flush_all(); + } #[cfg(target_pointer_width = "64")] self.pr.flush_all() } @@ -426,22 +524,70 @@ impl ImmixSpace { user_triggered_collection, self.reusable_blocks.len() == 0, full_heap_system_gc, + self.rc_enabled, *self.common.options.immix_always_defrag, ); self.defrag.in_defrag() } /// Get work packet scheduler - fn scheduler(&self) -> &GCWorkScheduler { + pub fn scheduler(&self) -> &GCWorkScheduler { &self.scheduler } + /// Install the plan-level hooks. Called once by the owning plan during `gc_init`. + pub fn install_hooks(&self, hooks: &'static dyn ImmixHooks) { + self.hooks + .set(hooks) + .unwrap_or_else(|_| panic!("ImmixSpace::install_hooks called more than once")); + } + + fn hooks(&self) -> Option<&'static dyn ImmixHooks> { + self.hooks.get().copied() + } + + pub fn prepare_rc(&mut self, pause: Pause) { + // Initialize mark state for tracing + if pause == Pause::Full || pause == Pause::InitialMark { + // Update mark_state + if VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.is_on_side() { + self.mark_state = Self::MARKED_STATE; + } else { + // For header metadata, we use cyclic mark bits. + unimplemented!("cyclic mark bits is not supported at the moment"); + } + } + // Release nursery blocks + if pause != Pause::RefCount { + if pause == Pause::Full { + // Reset worker TLABs. + // The block of the current worker TLAB may be selected as part of the mature evacuation set. + for w in &self.scheduler().worker_group.workers_shared { + let result = w.designated_work.push(Box::new(PrepareCollector)); + debug_assert!(result.is_ok()); + } + } + self.flush_page_resource(); + } + if pause == Pause::FinalMark || pause == Pause::Full { + self.is_end_of_satb_or_full_gc = true; + } + } + + pub fn release_rc(&mut self) { + self.flush_page_resource(); + self.rc.reset_inc_buffer_size(); + self.is_end_of_satb_or_full_gc = false; + self.reused_lines_consumed.store(0, Ordering::Relaxed); + } + pub(crate) fn prepare( &mut self, major_gc: bool, plan_stats: Option, unlog_bits_op: UnlogBitsOperation, ) { + debug_assert!(!self.rc_enabled); if major_gc { // Update mark_state if VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.is_on_side() { @@ -483,8 +629,9 @@ impl ImmixSpace { } } + // TODO: The VO bit strategy is currently not applicable to RC. #[cfg(feature = "vo_bit")] - if vo_bit::helper::need_to_clear_vo_bits_before_tracing::() { + if !self.rc_enabled && vo_bit::helper::need_to_clear_vo_bits_before_tracing::() { let maybe_scope = if major_gc { // If it is major GC, we always clear all VO bits because we are doing full-heap // tracing. @@ -522,6 +669,7 @@ impl ImmixSpace { /// Release for the immix space. pub(crate) fn release(&mut self, major_gc: bool, unlog_bits_op: UnlogBitsOperation) { + debug_assert!(!self.rc_enabled); if major_gc { // Update line_unavail_state for hole searching after this GC. if !super::BLOCK_ONLY { @@ -574,8 +722,11 @@ impl ImmixSpace { } /// Release a block. - pub fn release_block(&self, block: Block) { - block.deinit(); + pub fn release_block(&self, block: Block, zero_unlog_table: bool) { + if zero_unlog_table { + block.clear_field_unlog_table::(); + } + block.deinit(self); self.pr.release_block(block); } @@ -590,12 +741,19 @@ impl ImmixSpace { if block_address.is_zero() { return None; } - self.defrag.notify_new_clean_block(copy); let block = Block::from_aligned_address(block_address); - block.init(copy); + if !self.rc_enabled || self.defrag.in_defrag() { + self.defrag.notify_new_clean_block(copy); + } + if let Some(hooks) = self.hooks() { + hooks.on_clean_block_acquired(block, copy); + } + block.init(copy, false, self); self.chunk_map.set_allocated(block.chunk(), true); - self.lines_consumed - .fetch_add(Block::LINES, Ordering::SeqCst); + if !self.rc_enabled { + self.lines_consumed + .fetch_add(Block::LINES, Ordering::SeqCst); + } Some(block) } @@ -606,44 +764,94 @@ impl ImmixSpace { } loop { let block = self.reusable_blocks.pop()?; - // Skip blocks that should be evacuated. if copy && block.is_defrag_source() { continue; } - - // Get available lines. Do this before block.init which will reset block state. - let lines_delta = match block.get_state() { - BlockState::Reusable { unavailable_lines } => { - Block::LINES - unavailable_lines as usize + if self.rc_enabled { + if crate::plan::lxr::MATURE_EVACUATION && block.is_defrag_source() { + continue; } - BlockState::Unmarked => Block::LINES, - _ => unreachable!("{:?} {:?}", block, block.get_state()), - }; - self.lines_consumed.fetch_add(lines_delta, Ordering::SeqCst); + // Blocks in the `reusable_blocks` queue can be released after some RC collections. + // These blocks can either have `Unallocated` state, or be reallocated again. + // Skip these cases and only return the truly reusable blocks. + if !block.get_state().is_reusable() { + continue; + } + if !block.attempt_mutator_reuse() { + continue; + } + if let Some(hooks) = self.hooks() { + hooks.on_reusable_block_acquired(block, copy); + } + } else { + // Get available lines. Do this before block.init which will reset block state. + let lines_delta = match block.get_state() { + BlockState::Reusable { unavailable_lines } => { + Block::LINES - unavailable_lines as usize + } + BlockState::Unmarked => Block::LINES, + _ => unreachable!("{:?} {:?}", block, block.get_state()), + }; + self.lines_consumed.fetch_add(lines_delta, Ordering::SeqCst); + } - block.init(copy); + block.init(copy, true, self); return Some(block); } } + pub fn trace_object_without_moving_rc( + &self, + queue: &mut impl ObjectQueue, + object: ObjectReference, + ) -> ObjectReference { + if self.attempt_mark(object) { + let addr = object.to_raw_address().as_usize(); + let straddle = if (addr & 0b11110000) == 0 { + self.rc.is_straddle_line(Line::containing_obj_ref(object)) + } else { + false + }; + if !straddle { + queue.enqueue(object); + } + } + object + } + /// Trace and mark objects without evacuation. pub fn trace_object_without_moving( &self, queue: &mut impl ObjectQueue, object: ObjectReference, ) -> ObjectReference { + // This function should not be called during RC. + // Otherwise the VO bit handling will be incorrect. + debug_assert!(!self.rc_enabled); + #[cfg(feature = "vo_bit")] vo_bit::helper::on_trace_object::(object); - if self.attempt_mark(object, self.mark_state) { - // Mark block and lines - if !super::BLOCK_ONLY { - if !super::MARK_LINE_AT_SCAN_TIME { - self.mark_lines(object); + if self.attempt_mark(object) { + if self.rc_enabled { + let straddle = self.rc.is_straddle_line(Line::containing_obj_ref(object)); + if straddle { + return object; } } else { - Block::containing(object).set_state(BlockState::Marked); + // Mark block and lines + if !super::BLOCK_ONLY { + if !super::MARK_LINE_AT_SCAN_TIME { + self.mark_lines(object); + } + } else { + let block = Block::containing(object); + let state = block.get_state(); + if state != BlockState::Nursery && state != BlockState::Marked { + block.set_state(BlockState::Marked); + } + } } #[cfg(feature = "vo_bit")] @@ -651,7 +859,9 @@ impl ImmixSpace { // Visit node queue.enqueue(object); - self.unlog_object_if_needed(object); + if !self.rc_enabled { + self.unlog_object_if_needed(object); + } return object; } object @@ -667,6 +877,10 @@ impl ImmixSpace { worker: &mut GCWorker, nursery_collection: bool, ) -> ObjectReference { + // This function should not be called when RC is enabled. + // Otherwise the VO bit handling will be incorrect. + debug_assert!(!self.rc_enabled); + let copy_context = worker.get_copy_context_mut(); debug_assert!(!super::BLOCK_ONLY); @@ -708,10 +922,11 @@ impl ImmixSpace { } else { // We won the forwarding race; actually forward and copy the object if it is not pinned // and we have sufficient space in our copy allocator + debug_assert!(!nursery_collection || !self.rc_enabled); let new_object = if self.is_pinned(object) || (!nursery_collection && self.defrag.space_exhausted()) { - self.attempt_mark(object, self.mark_state); + self.attempt_mark(object); object_forwarding::clear_forwarding_bits::(object); Block::containing(object).set_state(BlockState::Marked); @@ -728,8 +943,9 @@ impl ImmixSpace { } else { // We are forwarding objects. When the copy allocator allocates the block, it should // mark the block. So we do not need to explicitly mark it here. - - object_forwarding::forward_object::( + // Clippy complains if the "vo_bit" feature is not enabled. + #[allow(clippy::let_and_return)] + let new_object = object_forwarding::try_forward_object::( object, semantics, copy_context, @@ -745,11 +961,14 @@ impl ImmixSpace { vo_bit::helper::on_object_forwarded::(new_object); }, ) + .expect("to-space overflow"); + + new_object }; - debug_assert_eq!( - Block::containing(new_object).get_state(), - BlockState::Marked - ); + debug_assert!({ + let state = Block::containing(new_object).get_state(); + state == BlockState::Marked || state == BlockState::Nursery + }); queue.enqueue(new_object); debug_assert!(new_object.is_live()); @@ -757,7 +976,90 @@ impl ImmixSpace { } } + pub fn rc_trace_object( + &self, + queue: &mut Q, + object: ObjectReference, + semantics: CopySemantics, + pause: Pause, + mark: bool, + worker: &mut GCWorker, + ) -> ObjectReference { + debug_assert!(self.rc_enabled); + if crate::plan::lxr::MATURE_EVACUATION && Block::containing(object).is_defrag_source() { + self.trace_forward_rc_mature_object(queue, object, semantics, pause, worker) + } else if crate::plan::lxr::MATURE_EVACUATION { + self.trace_mark_rc_mature_object(queue, object, pause, mark) + } else { + self.trace_object_without_moving(queue, object) + } + } + + pub fn trace_mark_rc_mature_object( + &self, + queue: &mut impl ObjectQueue, + object: ObjectReference, + _pause: Pause, + mark: bool, + ) -> ObjectReference { + debug_assert!( + !object_forwarding::is_forwarded::(object), + "object {:?} is forwarded", + object + ); + if mark && self.attempt_mark(object) { + queue.enqueue(object); + } + object + } + + #[allow(clippy::assertions_on_constants)] + pub fn trace_forward_rc_mature_object( + &self, + queue: &mut Q, + object: ObjectReference, + _semantics: CopySemantics, + _pause: Pause, + worker: &mut GCWorker, + ) -> ObjectReference { + let copy_context = worker.get_copy_context_mut(); + let forwarding_status = object_forwarding::attempt_to_forward::(object); + if object_forwarding::state_is_forwarded_or_being_forwarded(forwarding_status) { + object_forwarding::spin_and_get_forwarded_object::(object, forwarding_status) + } else { + // Evacuate the mature object + let new = object_forwarding::try_forward_object::( + object, + CopySemantics::DefaultCopy, + copy_context, + |_new_object| { + // When using RC, we set the VO bit of the forwarded object. + #[cfg(feature = "vo_bit")] + vo_bit::set_vo_bit(_new_object); + }, + ) + .expect("to-space overflow"); + // Transfer RC count + if new.get_size::() > Line::BYTES { + self.rc.mark_straddle_object(new); + } + self.rc.set(new, self.rc.count(object)); + self.attempt_mark(new); + self.unmark(object); + queue.enqueue(new); + debug_assert_ne!( + self.rc.count(new), + 0, + "ERROR Invalid {:?} rc={}", + new, + self.rc.count(new) + ); + new + } + } + fn unlog_object_if_needed(&self, object: ObjectReference) { + debug_assert!(!self.rc_enabled); if self.common.unlog_traced_object { // Make sure the side metadata for the line can fit into one byte. For smaller line size, we should // use `mark_as_unlogged` instead to mark the bit. @@ -784,18 +1086,21 @@ impl ImmixSpace { #[allow(clippy::assertions_on_constants)] pub fn mark_lines(&self, object: ObjectReference) { debug_assert!(!super::BLOCK_ONLY); + if self.rc_enabled { + return; + } Line::mark_lines_for_object::(object, self.line_mark_state.load(Ordering::Acquire)); } /// Atomically mark an object. - fn attempt_mark(&self, object: ObjectReference, mark_state: u8) -> bool { + pub fn attempt_mark(&self, object: ObjectReference) -> bool { loop { let old_value = VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.load_atomic::( object, None, Ordering::SeqCst, ); - if old_value == mark_state { + if old_value == self.mark_state { return false; } @@ -803,7 +1108,7 @@ impl ImmixSpace { .compare_exchange_metadata::( object, old_value, - mark_state, + self.mark_state, None, Ordering::SeqCst, Ordering::SeqCst, @@ -816,7 +1121,22 @@ impl ImmixSpace { true } - /// Check if an object is marked. + /// Atomically mark an object. + pub fn unmark(&self, object: ObjectReference) -> bool { + let result = VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.fetch_update_metadata::( + object, + Ordering::Relaxed, + Ordering::Relaxed, + |v| { + if v != 1 { + return None; + } + Some(0) + }, + ); + result.is_ok() + } + fn is_marked_with(&self, object: ObjectReference, mark_state: u8) -> bool { let old_value = VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.load_atomic::( object, @@ -847,8 +1167,94 @@ impl ImmixSpace { /// /// Returns None if the search could not find any more holes. #[allow(clippy::assertions_on_constants)] - pub fn get_next_available_lines(&self, search_start: Line) -> Option<(Line, Line)> { + pub fn get_next_available_lines(&self, copy: bool, search_start: Line) -> Option<(Line, Line)> { debug_assert!(!super::BLOCK_ONLY); + if self.rc_enabled { + self.rc_get_next_available_lines(copy, search_start) + } else { + self.normal_get_next_available_lines(search_start) + } + } + + /// Search holes by ref-counts instead of line marks + #[allow(clippy::assertions_on_constants)] + pub fn rc_get_next_available_lines( + &self, + copy: bool, + search_start: Line, + ) -> Option<(Line, Line)> { + debug_assert!(!super::BLOCK_ONLY); + debug_assert!(self.rc_enabled); + let block = search_start.block(); + let rc_array = RCArray::of(block); + let limit = Block::LINES; + // Find start + let first_free_cursor = { + let start_cursor = search_start.get_index_within_block(); + let mut first_free_cursor = None; + let mut find_free_line = false; + for i in start_cursor..limit { + if rc_array.is_dead(i) { + if i == 0 { + first_free_cursor = Some(i); + break; + } else if !find_free_line { + // This skips the first line of a hole + // because `mark_straddle_object_with_size` may or may not set the RC + // of the last line an object straddles. + find_free_line = true; + } else { + first_free_cursor = Some(i); + break; + } + } else { + find_free_line = false; + } + } + first_free_cursor + }; + let start = match first_free_cursor { + Some(c) => c, + _ => return None, + }; + // Find limit + let end = { + let mut cursor = start + 1; + while cursor < limit { + if !rc_array.is_dead(cursor) { + break; + } + cursor += 1; + } + cursor + }; + let start = Line::from_aligned_address(block.start()).next_nth(start); + let end = Line::from_aligned_address(block.start()).next_nth(end); + if self.common.needs_log_bit { + if !copy { + Line::clear_field_unlog_table::(start..end); + } else { + Line::initialize_field_unlog_table_as_unlogged::(start..end); + } + } + let num_lines = Line::steps_between(&start, &end).unwrap(); + if !copy { + self.reused_lines_consumed + .fetch_add(num_lines, Ordering::Relaxed); + } + if self + .hooks() + .is_some_and(|h| h.cm_in_progress_or_final_mark()) + { + Line::initialize_mark_table_as_marked::(start..end); + Line::inc_reuse_counts(start..end); + } + Some((start, end)) + } + + #[allow(clippy::assertions_on_constants)] + pub fn normal_get_next_available_lines(&self, search_start: Line) -> Option<(Line, Line)> { + debug_assert!(!self.rc_enabled); let unavail_state = self.line_unavail_state.load(Ordering::Acquire); let current_state = self.line_mark_state.load(Ordering::Acquire); let block = search_start.block(); @@ -890,12 +1296,22 @@ impl ImmixSpace { } } + pub(crate) fn get_mutator_recycled_lines_in_pages(&self) -> usize { + debug_assert!(self.rc_enabled); + self.reused_lines_consumed.load(Ordering::Relaxed) + >> (LOG_BYTES_IN_PAGE - Line::LOG_BYTES as u8) + } + pub(crate) fn get_pages_allocated(&self) -> usize { - self.lines_consumed.load(Ordering::SeqCst) >> (LOG_BYTES_IN_PAGE - Line::LOG_BYTES as u8) + debug_assert!(!self.rc_enabled); + self.lines_consumed.load(Ordering::Relaxed) >> (LOG_BYTES_IN_PAGE - Line::LOG_BYTES as u8) } /// Post copy routine for Immix copy contexts fn post_copy(&self, object: ObjectReference, _bytes: usize) { + if self.rc_enabled { + return; + } // Mark the object VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.store_atomic::( object, @@ -1191,7 +1607,7 @@ impl ImmixHybridCopyContext { space: &'static ImmixSpace, ) -> Self { ImmixHybridCopyContext { - copy_allocator: ImmixAllocator::new(tls.0, Some(space), context.clone(), false), + copy_allocator: ImmixAllocator::new(tls.0, Some(space), context.clone(), true), defrag_allocator: ImmixAllocator::new(tls.0, Some(space), context, true), } } diff --git a/src/policy/immix/line.rs b/src/policy/immix/line.rs index a8d2e9686ed..6151a48b12c 100644 --- a/src/policy/immix/line.rs +++ b/src/policy/immix/line.rs @@ -1,12 +1,16 @@ use std::ops::Range; use super::block::Block; +use crate::util::constants::{LOG_BITS_IN_BYTE, LOG_BYTES_IN_WORD}; use crate::util::linear_scan::{Region, RegionIterator}; -use crate::util::metadata::side_metadata::SideMetadataSpec; +use crate::util::metadata::side_metadata::spec_defs::IX_LINE_REUSE_COUNT; +use crate::util::metadata::side_metadata::*; +use crate::util::rc; use crate::{ util::{Address, ObjectReference}, vm::*, }; +use atomic::Ordering; /// Data structure to reference a line within an immix block. #[repr(transparent)] @@ -37,6 +41,16 @@ impl Line { pub const MARK_TABLE: SideMetadataSpec = crate::util::metadata::side_metadata::spec_defs::IX_LINE_MARK; + /// Return the line that contains the starting address of an object. + pub fn containing_obj_start(object: ObjectReference) -> Self { + Self::from_unaligned_address(VM::VMObjectModel::ref_to_object_start(object)) + } + + /// Return the line that contains the raw address of the object reference. + pub fn containing_obj_ref(object: ObjectReference) -> Self { + Self(object.to_raw_address()) + } + /// Get the block containing the line. pub fn block(&self) -> Block { debug_assert!(!super::BLOCK_ONLY); @@ -91,10 +105,23 @@ impl Line { /// doesn't need to explicitly mark bump-allocated objects in the fast path. pub fn initialize_mark_table_as_marked(lines: Range) { let meta = VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.extract_side_spec(); - let start = lines.start.start(); - let limit = lines.end.start(); - let size = limit - start; - meta.bset_metadata(start, size); + let start: *mut u8 = address_to_meta_address(meta, lines.start.start()).to_mut_ptr(); + let limit: *mut u8 = address_to_meta_address(meta, lines.end.start()).to_mut_ptr(); + unsafe { + let bytes = limit.offset_from(start) as usize; + std::ptr::write_bytes(start, 0xffu8, bytes); + } + } + + pub fn inc_reuse_counts(lines: Range) { + let mut l = lines.start; + while l < lines.end { + let addr = l.start(); + let count = IX_LINE_REUSE_COUNT.load_atomic::(addr, Ordering::SeqCst); + let new_count = if count == u8::MAX { 0 } else { count + 1 }; + IX_LINE_REUSE_COUNT.store_atomic::(addr, new_count, Ordering::SeqCst); + l = l.next(); + } } /// Bulk set line mark states. @@ -111,4 +138,156 @@ impl Line { Self::bulk_set_line_mark_states(line_mark_state, lines.clone()); Self::initialize_mark_table_as_marked::(lines); } + + pub fn clear_field_unlog_table(lines: Range) { + let unlog_bit = *VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC + .as_spec() + .extract_side_spec(); + let log_meta_bits_per_line = Line::LOG_BYTES - LOG_BYTES_IN_WORD as usize + + if !VM::VMObjectModel::COMPRESSED_PTR_ENABLED { + 0 + } else { + 1 + }; + debug_assert!((1 << log_meta_bits_per_line) >= 8); + let log_meta_bytes_per_line = log_meta_bits_per_line - LOG_BITS_IN_BYTE as usize; + // FIXME: Performance + let start = lines.start.start(); + let meta_start = address_to_meta_address(&unlog_bit, start); + let meta_bytes = + Line::steps_between(&lines.start, &lines.end).unwrap() << log_meta_bytes_per_line; + crate::util::memory::zero(meta_start, meta_bytes) + } + + pub fn initialize_field_unlog_table_as_unlogged(lines: Range) { + let unlog_bit = *VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC + .as_spec() + .extract_side_spec(); + let log_meta_bits_per_line = Line::LOG_BYTES - LOG_BYTES_IN_WORD as usize + + if !VM::VMObjectModel::COMPRESSED_PTR_ENABLED { + 0 + } else { + 1 + }; + debug_assert!((1 << log_meta_bits_per_line) >= 8); + let log_meta_bytes_per_line = log_meta_bits_per_line - LOG_BITS_IN_BYTE as usize; + // FIXME: Performance + let start = lines.start.start(); + let meta_start = address_to_meta_address(&unlog_bit, start); + let meta_bytes = + Line::steps_between(&lines.start, &lines.end).unwrap() << log_meta_bytes_per_line; + unsafe { + std::ptr::write_bytes::(meta_start.to_mut_ptr(), 0xffu8, meta_bytes); + } + } +} + +// type UInt = + +pub trait UintType: 'static + Sized { + type Type: 'static + Sized + Copy + Eq + PartialEq; + fn is_zero(v: Self::Type) -> bool; +} + +pub struct Uint {} + +impl UintType for Uint<8> { + type Type = u8; + fn is_zero(v: Self::Type) -> bool { + v == 0 + } +} + +impl UintType for Uint<16> { + type Type = u16; + fn is_zero(v: Self::Type) -> bool { + v == 0 + } +} + +impl UintType for Uint<32> { + type Type = u32; + fn is_zero(v: Self::Type) -> bool { + v == 0 + } +} + +impl UintType for Uint<64> { + type Type = u64; + fn is_zero(v: Self::Type) -> bool { + v == 0 + } +} + +impl UintType for Uint<128> { + type Type = u128; + fn is_zero(v: Self::Type) -> bool { + v == 0 + } +} + +#[repr(transparent)] +#[derive(Clone, Copy, Eq, PartialEq)] +pub struct UInt256([u8; 256 / 8]); + +impl UintType for Uint<256> { + type Type = UInt256; + fn is_zero(v: Self::Type) -> bool { + v == UInt256([0; 256 / 8]) + } +} + +#[repr(transparent)] +#[derive(Clone, Copy, Eq, PartialEq)] +pub struct UInt512([u8; 512 / 8]); + +impl UintType for Uint<512> { + type Type = UInt512; + fn is_zero(v: Self::Type) -> bool { + v == UInt512([0; 512 / 8]) + } +} + +#[repr(transparent)] +#[derive(Clone, Copy, Eq, PartialEq)] +pub struct UInt1024([u8; 1024 / 8]); + +impl UintType for Uint<1024> { + type Type = UInt1024; + fn is_zero(v: Self::Type) -> bool { + v == UInt1024([0; 1024 / 8]) + } +} + +#[repr(transparent)] +#[derive(Clone, Copy, Eq, PartialEq)] +pub struct UInt2048([u8; 2048 / 8]); + +impl UintType for Uint<2048> { + type Type = UInt2048; + fn is_zero(v: Self::Type) -> bool { + v == UInt2048([0; 2048 / 8]) + } +} + +const LOG_BITS_PER_LINE: usize = Line::LOG_BYTES - rc::LOG_MIN_OBJECT_SIZE + rc::LOG_REF_COUNT_BITS; +const BITS_PER_LINE: usize = 1 << LOG_BITS_PER_LINE; +const LOG_BITS_PER_BLOCK: usize = + Block::LOG_BYTES - rc::LOG_MIN_OBJECT_SIZE + rc::LOG_REF_COUNT_BITS; +const BITS_PER_BLOCK: usize = 1 << LOG_BITS_PER_BLOCK; + +pub struct RCArray { + table: &'static [ as UintType>::Type; BITS_PER_BLOCK / BITS_PER_LINE], +} + +impl RCArray { + pub fn of(block: Block) -> Self { + Self { + table: unsafe { &*block.rc_table_start().to_ptr() }, + } + } + + pub fn is_dead(&self, i: usize) -> bool { + as UintType>::is_zero(self.table[i]) + } } diff --git a/src/policy/immix/mod.rs b/src/policy/immix/mod.rs index d5895e94700..7c0ca8c04bf 100644 --- a/src/policy/immix/mod.rs +++ b/src/policy/immix/mod.rs @@ -5,8 +5,7 @@ pub mod line; pub use immixspace::*; -use crate::policy::immix::block::Block; -use crate::util::linear_scan::Region; +use crate::{policy::immix::block::Block, util::linear_scan::Region}; /// The max object size for immix: half of a block pub const MAX_IMMIX_OBJECT_SIZE: usize = Block::BYTES >> 1; diff --git a/src/policy/largeobjectspace.rs b/src/policy/largeobjectspace.rs index cb5399e6abc..844cc527970 100644 --- a/src/policy/largeobjectspace.rs +++ b/src/policy/largeobjectspace.rs @@ -1,5 +1,6 @@ use atomic::Ordering; +use crate::plan::concurrent::global::ConcurrentPlan; use crate::plan::tracing::{ObjectQueue, OptionObjectQueue}; use crate::policy::sft::GCWorkerMutRef; use crate::policy::sft::SFT; @@ -8,13 +9,17 @@ use crate::util::alloc::allocator::AllocationOptions; use crate::util::constants::BYTES_IN_PAGE; use crate::util::heap::{FreeListPageResource, PageResource}; use crate::util::metadata; +use crate::util::metadata::side_metadata::spec_defs::LOS_PAGE_REUSE_COUNT; +use crate::util::metadata::MetadataSpec; use crate::util::object_enum::ClosureObjectEnumerator; use crate::util::object_enum::ObjectEnumerator; use crate::util::opaque_pointer::*; +use crate::util::rc::RefCountHelper; use crate::util::treadmill::TreadMill; use crate::util::{Address, ObjectReference}; use crate::vm::ObjectModel; use crate::vm::VMBinding; +use std::sync::atomic::AtomicUsize; #[allow(unused)] const PAGE_MASK: usize = !(BYTES_IN_PAGE - 1); @@ -31,6 +36,12 @@ pub struct LargeObjectSpace { in_nursery_gc: bool, treadmill: TreadMill, clear_log_bit_on_sweep: bool, + trace_in_progress: bool, + pub num_pages_released_lazy: AtomicUsize, + pub rc_enabled: bool, + pub(crate) rc: RefCountHelper, + pub is_end_of_satb_or_full_gc: bool, + pub(crate) lxr: Option<&'static crate::plan::lxr::LXR>, } impl SFT for LargeObjectSpace { @@ -38,8 +49,24 @@ impl SFT for LargeObjectSpace { self.get_name() } fn is_live(&self, object: ObjectReference) -> bool { + if self.rc_enabled { + if self.is_end_of_satb_or_full_gc { + return self.is_marked(object) && self.rc.count(object) > 0; + } + return self.rc.count(object) > 0; + } + if self.trace_in_progress { + return true; + } self.test_mark_bit(object, self.mark_state) } + fn is_reachable(&self, object: ObjectReference) -> bool { + if self.rc_enabled { + self.test_mark_bit(object, self.mark_state) && self.rc.count(object) > 0 + } else { + self.is_live(object) + } + } #[cfg(feature = "object_pinning")] fn pin_object(&self, _object: ObjectReference) -> bool { false @@ -60,8 +87,7 @@ impl SFT for LargeObjectSpace { true } - fn initialize_object_metadata(&self, object: ObjectReference, _bytes: usize) { - // VO bit: Set for all objects. + fn initialize_object_metadata(&self, object: ObjectReference, bytes: usize) { #[cfg(feature = "vo_bit")] crate::util::metadata::vo_bit::set_vo_bit(object); #[cfg(all(feature = "vo_bit", debug_assertions))] @@ -75,6 +101,25 @@ impl SFT for LargeObjectSpace { ); } + // VO bit: Set for all objects. + if self.rc_enabled { + // Add to treadmill nursery + self.treadmill.add_to_treadmill(object, true); + // Initialize mark bit + self.test_and_mark(object, self.mark_state); + // Initialize metadata + let lxr = self.lxr.unwrap(); + if lxr.concurrent_work_in_progress() { + for off in (0..bytes).step_by(BYTES_IN_PAGE) { + let a = object.to_raw_address() + off; + let count = LOS_PAGE_REUSE_COUNT.load_atomic::(a, Ordering::SeqCst); + let new_count = if count == u8::MAX { 0 } else { count + 1 }; + LOS_PAGE_REUSE_COUNT.store_atomic::(a, new_count, Ordering::SeqCst); + } + } + return; + } + let allocate_as_live = self.should_allocate_as_live(); let into_nursery = !allocate_as_live; @@ -280,11 +325,18 @@ impl LargeObjectSpace { ) -> Self { let is_discontiguous = args.vmrequest.is_discontiguous(); let vm_map = args.vm_map; - let common = CommonSpace::new(args.into_policy_args( - false, - false, - metadata::extract_side_metadata(&[*VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC]), - )); + let rc_enabled = args.constraints.rc_enabled; + let specs = if rc_enabled { + vec![ + *VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC, + MetadataSpec::OnSide(LOS_PAGE_REUSE_COUNT), + ] + } else { + vec![*VM::VMObjectModel::LOCAL_LOS_MARK_NURSERY_SPEC] + }; + let policy_args = + args.into_policy_args(false, false, metadata::extract_side_metadata(&specs)); + let common = CommonSpace::new(policy_args); let mut pr = if is_discontiguous { FreeListPageResource::new_discontiguous(vm_map) } else { @@ -302,18 +354,64 @@ impl LargeObjectSpace { in_nursery_gc: false, treadmill: TreadMill::new(), clear_log_bit_on_sweep, + trace_in_progress: false, + num_pages_released_lazy: Default::default(), + rc_enabled: false, + rc: RefCountHelper::NEW, + is_end_of_satb_or_full_gc: false, + lxr: None, + } + } + + fn release_object(&self, object: ObjectReference) -> usize { + let start = get_super_page(object.to_object_start::()); + #[cfg(feature = "vo_bit")] + crate::util::metadata::vo_bit::unset_vo_bit(object); + if self.rc_enabled { + debug_assert_eq!(self.rc.count(object), 0); + let pages = self.pr.get_pages(start); + // TODO: Currently this code path assumes the collector is LXR and it uses field log bit. + // When we can use object log bit for LXR, we should merge with `sweep_large_pages` + // and clear object log bit instead. + VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC + .as_spec() + .extract_side_spec() + .bzero_metadata(start, pages * BYTES_IN_PAGE); + } + self.pr.release_pages(start) + } + + pub fn release_rc_nursery_objects(&self) { + debug_assert!(self.rc_enabled); + // promote nursery objects or release dead nursery + for o in self.treadmill.collect_alloc_nursery() { + if self.rc.count(o) == 0 { + self.release_object(o); + } else { + self.treadmill.add_to_treadmill(o, false); + } } } pub fn prepare(&mut self, full_heap: bool) { + self.trace_in_progress = true; if full_heap { self.mark_state = MARK_BIT - self.mark_state; } + self.num_pages_released_lazy.store(0, Ordering::Relaxed); + if self.rc_enabled { + return; + } self.treadmill.flip(full_heap); self.in_nursery_gc = !full_heap; } pub fn release(&mut self, full_heap: bool) { + self.trace_in_progress = false; + if self.rc_enabled { + self.release_rc_nursery_objects(); + return; + } // We swapped the allocation nursery and the collection nursery when GC starts, and we don't // add objects to the allocation nursery during GC. It should have remained empty during // the whole GC. @@ -341,6 +439,12 @@ impl LargeObjectSpace { "{:x}: VO bit not set", object ); + if self.rc_enabled { + if self.test_and_mark(object, self.mark_state) { + queue.enqueue(object); + } + return object; + } let nursery_object = self.is_in_nursery(object); trace!( "LOS object {} {} a nursery object", @@ -373,14 +477,14 @@ impl LargeObjectSpace { fn sweep_large_pages(&mut self, sweep_nursery: bool) { let sweep = |object: ObjectReference| { - #[cfg(feature = "vo_bit")] - crate::util::metadata::vo_bit::unset_vo_bit(object); // Clear log bits for dead objects to prevent a new nursery object having the unlog bit set + // TODO: This code path assumes the log bit is object log bit instead of field log bit. + // When generational plans and ConcurrentImmix support field log bit, + // we can push this clean-up operation into `Self::release_object`. if self.clear_log_bit_on_sweep { VM::VMObjectModel::GLOBAL_LOG_BIT_SPEC.clear::(object, Ordering::SeqCst); } - self.pr - .release_pages(get_super_page(object.to_object_start::())); + self.release_object(object); }; if sweep_nursery { for object in self.treadmill.collect_nursery() { @@ -413,13 +517,27 @@ impl LargeObjectSpace { self.acquire(tls, pages, alloc_options) } + pub fn attempt_mark(&self, object: ObjectReference) -> bool { + self.test_and_mark(object, self.mark_state) + } + + pub fn rc_free(&self, o: ObjectReference) { + if self.treadmill.remove_mature(o) { + let pages = self.release_object(o); + self.num_pages_released_lazy + .fetch_add(pages, Ordering::Relaxed); + } + } + /// Test if the object's mark bit is the same as the given value. If it is not the same, /// the method will attemp to mark the object and clear its nursery bit. If the attempt /// succeeds, the method will return true, meaning the object is marked by this invocation. /// Otherwise, it returns false. fn test_and_mark(&self, object: ObjectReference, value: u8) -> bool { loop { - let mask = if self.in_nursery_gc { + let mask = if self.rc_enabled { + MARK_BIT + } else if self.in_nursery_gc { LOS_BIT_MASK } else { MARK_BIT @@ -470,6 +588,20 @@ impl LargeObjectSpace { == NURSERY_BIT } + pub fn sweep_rc_mature_objects_after_satb(&self, is_live: &impl Fn(ObjectReference) -> bool) { + self.treadmill.retain_mature(|o| { + if !is_live(*o) { + self.rc.set(*o, 0); + let pages = self.release_object(*o); + self.num_pages_released_lazy + .fetch_add(pages, Ordering::Relaxed); + false + } else { + true + } + }); + } + pub fn is_marked(&self, object: ObjectReference) -> bool { self.test_mark_bit(object, self.mark_state) } diff --git a/src/policy/marksweepspace/native_ms/block.rs b/src/policy/marksweepspace/native_ms/block.rs index 5bef9a3e529..96f8117cf1c 100644 --- a/src/policy/marksweepspace/native_ms/block.rs +++ b/src/policy/marksweepspace/native_ms/block.rs @@ -7,6 +7,7 @@ use super::MarkSweepSpace; use crate::util::constants::LOG_BYTES_IN_PAGE; use crate::util::heap::chunk_map::*; use crate::util::linear_scan::Region; +use crate::util::linear_scan::UnstraddlableRegion; use crate::util::object_enum::BlockMayHaveObjects; use crate::vm::ObjectModel; use crate::{ @@ -49,6 +50,9 @@ impl Region for Block { } } +/// An objects cannot straddle multiple native blocks. +impl UnstraddlableRegion for Block {} + impl BlockMayHaveObjects for Block { fn may_have_objects(&self) -> bool { self.get_state() != BlockState::Unallocated diff --git a/src/policy/marksweepspace/native_ms/global.rs b/src/policy/marksweepspace/native_ms/global.rs index c7b25e865da..d4ecec7760c 100644 --- a/src/policy/marksweepspace/native_ms/global.rs +++ b/src/policy/marksweepspace/native_ms/global.rs @@ -10,6 +10,7 @@ use crate::{ copy::CopySemantics, epilogue, heap::{BlockPageResource, PageResource}, + linear_scan::UnstraddlableRegion, metadata::{self, side_metadata::SideMetadataSpec, MetadataSpec}, object_enum::{self, ObjectEnumerator}, ObjectReference, diff --git a/src/policy/sft.rs b/src/policy/sft.rs index 9befbe13189..f15ac12a659 100644 --- a/src/policy/sft.rs +++ b/src/policy/sft.rs @@ -22,7 +22,7 @@ use std::marker::PhantomData; /// /// We use the SFT trait to simplify typing for Rust, so our table is a /// table of SFT rather than Space. -pub trait SFT { +pub trait SFT: Sync + 'static { /// The space name fn name(&self) -> &'static str; @@ -150,11 +150,8 @@ impl SFT for EmptySpaceSFT { fn name(&self) -> &'static str { EMPTY_SFT_NAME } - fn is_live(&self, object: ObjectReference) -> bool { - panic!( - "Called is_live() on {:x}, which maps to an empty space", - object - ) + fn is_live(&self, _object: ObjectReference) -> bool { + false } #[cfg(feature = "sanity")] fn is_sane(&self) -> bool { diff --git a/src/policy/sft_map.rs b/src/policy/sft_map.rs index 42ed3f3db74..20d1c8a5380 100644 --- a/src/policy/sft_map.rs +++ b/src/policy/sft_map.rs @@ -471,6 +471,9 @@ mod sparse_chunk_map { impl SFTMap for SFTSparseChunkMap { fn has_sft_entry(&self, addr: Address) -> bool { + if addr < vm_layout().heap_start || addr >= vm_layout().heap_end { + return false; + } addr.chunk_index() < vm_layout().max_chunks() } diff --git a/src/policy/space.rs b/src/policy/space.rs index 31f2058a094..de27fe1d61b 100644 --- a/src/policy/space.rs +++ b/src/policy/space.rs @@ -531,6 +531,7 @@ pub struct CommonSpace { /// This field equals to needs_log_bit in the plan constraints. // TODO: This should be a constant for performance. pub needs_log_bit: bool, + pub needs_field_log_bit: bool, pub unlog_allocated_object: bool, pub unlog_traced_object: bool, @@ -607,6 +608,7 @@ impl CommonSpace { vm_map: args.plan_args.vm_map, mmapper: args.plan_args.mmapper, needs_log_bit: args.plan_args.constraints.needs_log_bit, + needs_field_log_bit: args.plan_args.constraints.needs_field_log_bit, unlog_allocated_object: args.plan_args.unlog_allocated_object, unlog_traced_object: args.plan_args.unlog_traced_object, gc_trigger: args.plan_args.gc_trigger.clone(), diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 7e7d4101b34..fbf9e40ddf9 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -5,6 +5,41 @@ use crate::*; use std::marker::PhantomData; use std::sync::atomic::Ordering; +/// The kind of a set of roots. Used by LXR to decide how roots should be processed +/// (e.g. reference counting and remembered-set recording). +#[repr(u8)] +#[derive(Debug, Eq, PartialEq, Clone, Copy)] +pub enum RootKind { + /// Ordinary strong roots, e.g. mutator stacks and globals. These are reference-counted and + /// marked like any other strong reference. + Strong, + /// Roots held by recently JIT-compiled ("young") code-cache entries. These are recorded into + /// the remembered set (instead of being reference-counted) so the code cache can be + /// re-scanned on a later GC. + YoungCodeCacheRoots, + /// Roots that hold weak references. These must not keep their referents alive and are not + /// reference-counted. + Weak, +} + +impl RootKind { + /// Whether roots of this kind should be recorded into the remembered set rather than being + /// processed like normal roots. + pub fn should_record_remset(&self) -> bool { + matches!(self, RootKind::YoungCodeCacheRoots) + } + + /// Whether roots of this kind should skip marking and reference-count decrements. + pub fn should_skip_mark_and_decs(&self) -> bool { + matches!(self, RootKind::YoungCodeCacheRoots) || matches!(self, RootKind::Weak) + } + + /// Whether roots of this kind should skip reference-count decrements. + pub fn should_skip_decs(&self) -> bool { + matches!(self, RootKind::YoungCodeCacheRoots) || matches!(self, RootKind::Weak) + } +} + pub struct ScheduleCollection; impl GCWork for ScheduleCollection { @@ -187,9 +222,12 @@ impl GCWork for ReleaseCollector { /// TODO: Smaller work granularity #[derive(Default)] pub struct StopMutators { - /// If this is true, we skip creating root-scanning work packets. + /// If this is true, we skip creating [`ScanMutatorRoots`] work packets for mutators. /// By default, this is false. - skip_roots: bool, + skip_mutator_roots: bool, + /// If this is true, we skip scanning VM-specific roots. + /// By default, this is false. + skip_vm_roots: bool, /// Flush mutators once they are stopped. By default this is false. [`ScanMutatorRoots`] will flush mutators. flush_mutator: bool, phantom: PhantomData, @@ -198,16 +236,24 @@ pub struct StopMutators { impl StopMutators { pub fn new() -> Self { Self { - skip_roots: false, + skip_mutator_roots: false, + skip_vm_roots: false, flush_mutator: false, phantom: PhantomData, } } + pub fn new_with_flush() -> Self { + let mut me = Self::new(); + me.flush_mutator = true; + me + } + /// Create a `StopMutators` work packet that does not create any root-scanning work packets, and will simply flush mutators. pub fn new_no_scan_roots() -> Self { Self { - skip_roots: true, + skip_mutator_roots: true, + skip_vm_roots: true, flush_mutator: true, phantom: PhantomData, } @@ -225,21 +271,20 @@ impl GCWork for StopMutators { if self.flush_mutator { mutator.flush(); } - if !self.skip_roots { - mmtk.scheduler.work_buckets[WorkBucketStage::Prepare] + if !self.skip_mutator_roots { + mmtk.scheduler.work_buckets[mmtk.get_plan().root_scanning_stage()] .add(ScanMutatorRoots::(mutator)); } }); trace!("stop_all_mutators end"); - // This also tells the GC trigger whether a new GC cycle has started (see - // `Plan::notify_mutators_paused`). - mmtk.get_plan().notify_mutators_paused(mmtk); + // This also tells the GC trigger whether a new GC cycle has started (see `Plan::gc_pause_start`). + mmtk.get_plan().on_pause_start(mmtk); mmtk.scheduler.notify_mutators_paused(mmtk); // Tell GC trigger that the pause started. mmtk.gc_trigger.policy.on_pause_start(mmtk); - if !self.skip_roots { - mmtk.scheduler.work_buckets[WorkBucketStage::Prepare] - .add(ScanVMSpecificRoots::::new()); + if !self.skip_vm_roots { + let factory = C::make_roots_work_factory(mmtk); + ::VMScanning::scan_vm_specific_roots(worker.tls, factory); } } } diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index 4af8b7bf31c..9a7bfcbfece 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -24,7 +24,9 @@ pub(crate) use work::GCWorkContext; mod work_bucket; pub use work_bucket::WorkBucketStage; -mod worker; +/// GC worker threads: the [`worker::GCWorker`] struct representing a single worker, the +/// (crate-private) `WorkerGroup` that manages all workers, and per-worker shared state. +pub mod worker; mod worker_goals; mod worker_monitor; pub(crate) use worker::current_worker_ordinal; @@ -32,3 +34,4 @@ pub use worker::GCWorker; pub(crate) use worker::GCWorkerShared; pub(crate) mod gc_work; +pub use gc_work::RootKind; diff --git a/src/scheduler/scheduler.rs b/src/scheduler/scheduler.rs index 4f6ef19afb2..295ae92426c 100644 --- a/src/scheduler/scheduler.rs +++ b/src/scheduler/scheduler.rs @@ -8,9 +8,9 @@ use super::worker_goals::{WorkerGoal, WorkerGoals}; use super::worker_monitor::{LastParkedResult, WorkerMonitor}; use super::*; use crate::mmtk::MMTK; -use crate::plan::tracing::gc_work::weakref::{ - VMForwardWeakRefs, VMPostForwarding, VMProcessWeakRefs, -}; +use crate::plan::concurrent::Pause; +use crate::plan::lxr::LXR; +use crate::plan::tracing::gc_work::weakref::VMForwardWeakRefs; use crate::util::opaque_pointer::*; use crate::util::options::AffinityKind; use crate::vm::Collection; @@ -40,6 +40,7 @@ unsafe impl Sync for GCWorkScheduler {} impl GCWorkScheduler { pub fn new(num_workers: usize, affinity: AffinityKind) -> Arc { + assert!(num_workers > 0); let worker_monitor: Arc = Arc::new(WorkerMonitor::new(num_workers)); let worker_group = WorkerGroup::new(num_workers); @@ -79,6 +80,25 @@ impl GCWorkScheduler { }) } + pub fn process_concurrent_packets_in_pause(&self) { + let mut packets = vec![]; + // Buggy + let bucket = &self.work_buckets[WorkBucketStage::Concurrent]; + loop { + if bucket.is_empty() { + break; + } + match bucket.get_queue().steal() { + Steal::Success(w) => packets.push(w), + Steal::Empty => break, + Steal::Retry => {} + } + } + if !packets.is_empty() { + self.work_buckets[WorkBucketStage::STWRCDecsAndSweep].bulk_add(packets); + } + } + pub fn num_workers(&self) -> usize { self.worker_group.as_ref().worker_count() } @@ -239,8 +259,8 @@ impl GCWorkScheduler { // `VMProcessWeakRefs` packet can be an ordinary packet (doesn't have to be a sentinel) // because there are no other packets in the bucket. We set it as sentinel for // consistency. - self.work_buckets[WorkBucketStage::VMRefClosure] - .set_sentinel(Box::new(VMProcessWeakRefs::::new())); + // self.work_buckets[WorkBucketStage::VMRefClosure] + // .set_sentinel(Box::new(VMProcessWeakRefs::::new())); if plan.constraints().needs_forward_after_liveness { // VM-specific weak ref forwarding @@ -248,7 +268,7 @@ impl GCWorkScheduler { .add(VMForwardWeakRefs::::new()); } - self.work_buckets[WorkBucketStage::Release].add(VMPostForwarding::::default()); + // self.work_buckets[WorkBucketStage::Release].add(VMPostForwarding::::default()); } fn are_buckets_drained(&self, buckets: &[WorkBucketStage]) -> bool { @@ -360,6 +380,10 @@ impl GCWorkScheduler { pub(crate) fn assert_all_open_buckets_are_empty(&self) { let mut error_example = None; for (id, bucket) in self.work_buckets.iter() { + if id == WorkBucketStage::ConcurrentResumable { + // Concurrent resumable bucket is a special case. It can be non-empty. + continue; + } if bucket.is_enabled() && bucket.is_open() && !bucket.is_empty() { error!("Work bucket {:?} is not drained!", id); error!("Queue: {:?}", bucket.get_queue().debug_dump_packets()); @@ -383,7 +407,15 @@ impl GCWorkScheduler { return Steal::Success(w); } // Try get a packet from a work bucket. - for work_bucket in self.work_buckets.values() { + let plan = worker.mmtk.get_plan(); + let in_concurrent_or_final_mark = plan + .concurrent() + .map(|c| c.current_pause().is_none() || c.current_pause() == Some(Pause::FinalMark)) + .unwrap_or(false); + for (stage, work_bucket) in self.work_buckets.iter() { + if !in_concurrent_or_final_mark && stage == WorkBucketStage::ConcurrentResumable { + continue; + } match work_bucket.poll(&worker.local_work_buffer) { Steal::Success(w) => return Steal::Success(w), Steal::Retry => should_retry = true, @@ -566,6 +598,12 @@ impl GCWorkScheduler { false } + fn do_vm_release(&self, mmtk: &MMTK) { + if mmtk.get_plan().downcast_ref::>().is_none() { + ::VMCollection::vm_release(); + } + } + /// Called when GC has finished, i.e. when all work packets have been executed. /// /// Return `true` if any concurrent work packets have been scheduled. @@ -578,6 +616,8 @@ impl GCWorkScheduler { self.close_all_stw_buckets(); self.debug_assert_all_stw_buckets_closed(); + self.do_vm_release(worker.mmtk); + let mmtk = worker.mmtk; // Tell GC trigger that GC ended - this happens before we resume mutators. @@ -586,9 +626,8 @@ impl GCWorkScheduler { // All other workers are parked, so it is safe to access the Plan instance mutably. probe!(mmtk, plan_end_of_gc_begin); let plan_mut: &mut dyn Plan = unsafe { mmtk.get_plan_mut() }; - // This also tells the GC trigger whether the GC cycle has ended (see - // `Plan::end_of_pause`). - plan_mut.end_of_pause(mmtk, worker.tls); + // This also tells the GC trigger whether the GC cycle has ended (see `Plan::on_pause_end`). + plan_mut.on_pause_end(mmtk, worker.tls); probe!(mmtk, plan_end_of_gc_end); // Compute the elapsed time of the GC. @@ -639,7 +678,7 @@ impl GCWorkScheduler { // Reset the triggering information. mmtk.state.reset_collection_trigger(); - let concurrent_work_scheduled = self.schedule_concurrent_packets(); + let concurrent_work_scheduled = self.schedule_concurrent_packets(mmtk); self.debug_assert_all_stw_buckets_closed(); // Set to NotInGC after everything, and right before resuming mutators. @@ -701,16 +740,33 @@ impl GCWorkScheduler { self.worker_monitor.notify_work_available(true); } - pub(super) fn schedule_concurrent_packets(&self) -> bool { - let concurrent_bucket = &self.work_buckets[WorkBucketStage::Concurrent]; - if !concurrent_bucket.is_empty() { - concurrent_bucket.set_enabled(true); - concurrent_bucket.open(); - true - } else { - concurrent_bucket.set_enabled(false); - concurrent_bucket.close(); - false - } + pub(super) fn schedule_concurrent_packets(&self, mmtk: &MMTK) -> bool { + // Only LXR defers work into the inactive queue (via `add_deferred`/`bulk_add_deferred`), + // so only LXR needs the `Concurrent` bucket's queue flipped here. For every other plan + // (e.g. the generic `ConcurrentImmix`), flipping would silently orphan any packets left + // in the currently-active queue when a STW pause interrupts an in-progress concurrent + // phase: `is_empty()` would check the other (empty) queue and the bucket would be closed + // as if drained, even though unprocessed concurrent-marking work is still sitting in the + // now-inactive queue. Keep this the same enable/disable-only mechanism as master unless + // the plan is LXR. + let is_lxr = mmtk.get_plan().downcast_ref::>().is_some(); + let enable_bucket = |stage: WorkBucketStage, flip: bool| { + let bucket = &self.work_buckets[stage]; + if flip && is_lxr { + bucket.flip(); + } + if !bucket.is_empty() { + bucket.set_enabled(true); + bucket.open(); + true + } else { + bucket.set_enabled(false); + bucket.close(); + false + } + }; + let a = enable_bucket(WorkBucketStage::Concurrent, true); + let b = enable_bucket(WorkBucketStage::ConcurrentResumable, false); + a || b } } diff --git a/src/scheduler/work_bucket.rs b/src/scheduler/work_bucket.rs index ad609afe48c..22dee98c529 100644 --- a/src/scheduler/work_bucket.rs +++ b/src/scheduler/work_bucket.rs @@ -7,34 +7,68 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; pub(super) struct BucketQueue { - queue: Injector>>, + flag: AtomicBool, + queue0: Injector>>, + queue1: Injector>>, } impl BucketQueue { fn new() -> Self { Self { - queue: Injector::new(), + flag: AtomicBool::new(false), + queue0: Injector::new(), + queue1: Injector::new(), + } + } + + fn active_queue(&self) -> &Injector>> { + if self.flag.load(Ordering::Relaxed) { + &self.queue1 + } else { + &self.queue0 + } + } + + fn inactive_queue(&self) -> &Injector>> { + if self.flag.load(Ordering::Relaxed) { + &self.queue0 + } else { + &self.queue1 } } fn is_empty(&self) -> bool { - self.queue.is_empty() + self.active_queue().is_empty() + } + + pub(super) fn steal(&self) -> Steal>> { + self.active_queue().steal() } fn steal_batch_and_pop( &self, dest: &Worker>>, ) -> Steal>> { - self.queue.steal_batch_and_pop(dest) + self.active_queue().steal_batch_and_pop(dest) } fn push(&self, w: Box>) { - self.queue.push(w); + self.active_queue().push(w); } fn push_all(&self, ws: Vec>>) { for w in ws { - self.queue.push(w); + self.active_queue().push(w); + } + } + + fn push_inactive(&self, w: Box>) { + self.inactive_queue().push(w); + } + + fn push_all_inactive(&self, ws: Vec>>) { + for w in ws { + self.inactive_queue().push(w); } } @@ -43,11 +77,12 @@ impl BucketQueue { /// (e.g. when the execution has failed already and the system is going to panic). pub fn debug_dump_packets(&self) -> Vec { let mut items = Vec::new(); + let queue = self.active_queue(); { // Drain queue by stealing until empty loop { - match self.queue.steal() { + match queue.steal() { crossbeam::deque::Steal::Success(work) => { items.push(work); } @@ -66,7 +101,7 @@ impl BucketQueue { // Push items back into the queue { for work in items { - self.queue.push(work); + queue.push(work); } } @@ -87,7 +122,6 @@ pub struct WorkBucket { /// The stage name of this bucket. stage: WorkBucketStage, queue: BucketQueue, - prioritized_queue: Option>, monitor: Arc, /// The open condition for a bucket. If this is `Some`, the bucket will be open /// when the condition is met. If this is `None`, the bucket needs to be open manually. @@ -113,7 +147,6 @@ impl WorkBucket { enabled: AtomicBool::new(stage.is_enabled_by_default()), stage, queue: BucketQueue::new(), - prioritized_queue: None, monitor, can_open: None, sentinel: Mutex::new(None), @@ -128,8 +161,8 @@ impl WorkBucket { self.enabled.load(Ordering::Relaxed) } - pub fn enable_prioritized_queue(&mut self) { - self.prioritized_queue = Some(BucketQueue::new()); + pub fn flip(&self) { + self.queue.flag.fetch_xor(true, Ordering::SeqCst); } fn notify_one_worker(&self) { @@ -162,11 +195,6 @@ impl WorkBucket { /// Test if the bucket is drained pub fn is_empty(&self) -> bool { self.queue.is_empty() - && self - .prioritized_queue - .as_ref() - .map(|q| q.is_empty()) - .unwrap_or(true) } pub fn is_drained(&self) -> bool { @@ -183,25 +211,30 @@ impl WorkBucket { self.open.store(false, Ordering::Relaxed); } - /// Add a work packet to this bucket - /// Panic if this bucket cannot receive prioritized packets. - pub fn add_prioritized(&self, work: Box>) { - self.prioritized_queue.as_ref().unwrap().push(work); - self.notify_one_worker(); - } - /// Add a work packet to this bucket pub fn add>(&self, work: W) { + debug_assert!(self.is_enabled()); self.queue.push(Box::new(work)); self.notify_one_worker(); } /// Add a work packet to this bucket pub fn add_boxed(&self, work: Box>) { + debug_assert!(self.is_enabled()); self.queue.push(work); self.notify_one_worker(); } + pub fn add_deferred(&self, work: Box>) { + debug_assert!(self.is_enabled()); + self.queue.push_inactive(work); + } + + pub fn bulk_add_deferred(&self, work_vec: Vec>>) { + debug_assert!(self.is_enabled()); + self.queue.push_all_inactive(work_vec); + } + /// Add a work packet to this bucket, but do not notify any workers. /// This is useful when the current thread is holding the mutex of `WorkerMonitor` which is /// used for notifying workers. This usually happens if the current thread is the last worker @@ -215,20 +248,21 @@ impl WorkBucket { self.queue.push(work); } - /// Add multiple packets with a higher priority. - /// Panic if this bucket cannot receive prioritized packets. - pub fn bulk_add_prioritized(&self, work_vec: Vec>>) { - self.prioritized_queue.as_ref().unwrap().push_all(work_vec); - self.notify_all_workers(); - } - /// Add multiple packets pub fn bulk_add(&self, work_vec: Vec>>) { + debug_assert!(self.is_enabled()); if work_vec.is_empty() { return; } + let len = work_vec.len(); self.queue.push_all(work_vec); - self.notify_all_workers(); + if self.is_open() { + if len == 1 { + self.notify_one_worker(); + } else { + self.notify_all_workers(); + } + } } /// Get a work packet from this bucket @@ -236,13 +270,7 @@ impl WorkBucket { if !self.is_enabled() || !self.is_open() || self.is_empty() { return Steal::Empty; } - if let Some(prioritized_queue) = self.prioritized_queue.as_ref() { - prioritized_queue - .steal_batch_and_pop(worker) - .or_else(|| self.queue.steal_batch_and_pop(worker)) - } else { - self.queue.steal_batch_and_pop(worker) - } + self.queue.steal_batch_and_pop(worker) } pub fn set_open_condition( @@ -304,7 +332,7 @@ impl WorkBucket { /// This enum defines all the work bucket types. The scheduler /// will instantiate a work bucket for each stage defined here. -#[derive(Debug, Enum, Copy, Clone, Eq, PartialEq)] +#[derive(Debug, Enum, Copy, Clone, Eq, PartialEq, Hash)] pub enum WorkBucketStage { /// This bucket is always open. Unconstrained, @@ -312,6 +340,18 @@ pub enum WorkBucketStage { /// work in the unconstrained bucket will always be consumed during STW. Users can disable this bucket /// and cache some concurrent work during STW, and only enable this bucket and allow concurrent execution once a STW is done. Concurrent, + /// Concurrent work that may be resumed across a stop-the-world pause (LXR-specific), such as + /// concurrent marking packets discovered while processing reference-count increments during + /// `InitialMark`. Unlike other stages, this bucket is allowed to remain non-empty when a STW + /// pause ends. + ConcurrentResumable, + /// The first stop-the-world stage (see [`WorkBucketStage::FIRST_STW_STAGE`]). Used to join + /// outstanding concurrent work, e.g. flushing SATB mod-buffer packets recorded by the LXR + /// barrier, before the rest of the STW stages proceed. + FinishConcurrentWork, + /// Process reference-count increments recorded by the LXR barrier. LXR also uses this stage + /// to scan roots (see `root_scanning_stage`). + RCProcessIncs, /// Preparation work. Plans, spaces, GC workers, mutators, etc. should be prepared for GC at /// this stage. Prepare, @@ -364,14 +404,20 @@ pub enum WorkBucketStage { /// Work packets that should be done just before GC shall go here. This includes releasing /// resources and setting states in plans, spaces, GC workers, mutators, etc. Release, + /// Process reference-count decrements recorded by the LXR barrier, and sweep objects whose + /// reference count has dropped to zero, during a stop-the-world pause. This is used when + /// lazy (concurrent) decrements are disabled for the current GC. + STWRCDecsAndSweep, /// Resume mutators and end GC. Final, } +// Alias +#[allow(non_upper_case_globals)] impl WorkBucketStage { /// The first stop-the-world stage. This stage has no open condition, and will be opened manually /// once all the mutators threads are stopped. - pub const FIRST_STW_STAGE: Self = WorkBucketStage::Prepare; + pub const FIRST_STW_STAGE: Self = WorkBucketStage::FinishConcurrentWork; /// Is this the first stop-the-world stage? See [`Self::FIRST_STW_STAGE`]. pub const fn is_first_stw_stage(&self) -> bool { @@ -387,13 +433,16 @@ impl WorkBucketStage { pub const fn is_open_by_default(&self) -> bool { matches!( self, - WorkBucketStage::Unconstrained | WorkBucketStage::Concurrent + WorkBucketStage::Unconstrained + | WorkBucketStage::Concurrent + | WorkBucketStage::ConcurrentResumable ) } /// Is this stage enabled by default? pub const fn is_enabled_by_default(&self) -> bool { !matches!(self, WorkBucketStage::Concurrent) + && !matches!(self, WorkBucketStage::ConcurrentResumable) } /// Is this stage sequentially opened? All the stop-the-world stages, except the first one, are sequentially opened. @@ -410,7 +459,14 @@ impl WorkBucketStage { pub const fn is_concurrent(&self) -> bool { matches!( self, - WorkBucketStage::Unconstrained | WorkBucketStage::Concurrent + WorkBucketStage::Unconstrained + | WorkBucketStage::Concurrent + | WorkBucketStage::ConcurrentResumable ) } + + /// Alias for [`WorkBucketStage::Closure`], used by LXR when scheduling mature-space + /// evacuation remset packets so that they are processed as part of the transitive closure + /// stage. + pub const RCEvacuateMature: Self = Self::Closure; } diff --git a/src/scheduler/worker.rs b/src/scheduler/worker.rs index af9905666ef..2cc9e7f6917 100644 --- a/src/scheduler/worker.rs +++ b/src/scheduler/worker.rs @@ -20,17 +20,33 @@ pub type ThreadId = usize; thread_local! { /// Current worker's ordinal static WORKER_ORDINAL: Atomic = const { Atomic::new(ThreadId::MAX) }; + static _WORKER: Atomic = const { Atomic::new(0) }; +} + +lazy_static! { + static ref _WORKERS: Mutex> = Mutex::new(Vec::new()); +} + +/// Release the copy context of every currently registered GC worker. +/// This is used to reset per-worker copying-GC state, e.g. before workers are respawned. +pub fn reset_workers() { + let workers = _WORKERS.lock().unwrap(); + for w in workers.iter() { + let w = w.as_mut_ptr::>(); + unsafe { + (*w).get_copy_context_mut().release(); + } + } } /// Get current worker ordinal. Return `None` if the current thread is not a worker. -pub fn current_worker_ordinal() -> ThreadId { +pub fn current_worker_ordinal() -> Option { let ordinal = WORKER_ORDINAL.with(|x| x.load(Ordering::Relaxed)); - debug_assert_ne!( - ordinal, - ThreadId::MAX, - "Thread-local variable WORKER_ORDINAL not set yet." - ); - ordinal + if ordinal == ThreadId::MAX { + None + } else { + Some(ordinal) + } } /// The struct has one instance per worker, but is shared between workers via the scheduler @@ -51,6 +67,8 @@ pub struct GCWorkerShared { } impl GCWorkerShared { + /// Create a new `GCWorkerShared` instance, optionally with a `stealer` handle that other + /// workers can use to steal work packets from this worker's local queue. pub fn new(stealer: Option>>>) -> Self { Self { stat: Default::default(), @@ -111,10 +129,12 @@ const STAT_BORROWED_MSG: &str = "GCWorkerShared.stat is already borrowed. This the mutator calls harness_begin or harness_end while the GC is running."; impl GCWorkerShared { + /// Immutably borrow this worker's local statistics. pub fn borrow_stat(&self) -> AtomicRef<'_, WorkerLocalStat> { self.stat.try_borrow().expect(STAT_BORROWED_MSG) } + /// Mutably borrow this worker's local statistics. pub fn borrow_stat_mut(&self) -> AtomicRefMut<'_, WorkerLocalStat> { self.stat.try_borrow_mut().expect(STAT_BORROWED_MSG) } @@ -151,19 +171,24 @@ impl GCWorker { } } + /// Get current worker. + pub fn current() -> &'static mut Self { + let ptr = _WORKER.with(|x| x.load(Ordering::Relaxed)) as *mut Self; + unsafe { &mut *ptr } + } + const LOCALLY_CACHED_WORK_PACKETS: usize = 16; - /// Add a work packet to the work queue and mark it with a higher priority. - /// If the bucket is open, the packet will be pushed to the local queue, otherwise it will be - /// pushed to the global bucket with a higher priority. - pub fn add_work_prioritized(&mut self, bucket: WorkBucketStage, work: impl GCWork) { + /// Add a boxed work packet to the work queue, in the given bucket. + /// Like [`GCWorker::add_work`], but the work packet is already boxed. + pub fn add_boxed_work(&mut self, bucket: WorkBucketStage, work: Box>) { if !self.scheduler().work_buckets[bucket].is_open() || self.local_work_buffer.len() >= Self::LOCALLY_CACHED_WORK_PACKETS { - self.scheduler.work_buckets[bucket].add_prioritized(Box::new(work)); + self.scheduler.work_buckets[bucket].add_boxed(work); return; } - self.local_work_buffer.push(Box::new(work)); + self.local_work_buffer.push(work); } /// Add a work packet to the work queue. @@ -226,6 +251,17 @@ impl GCWorker { crate::util::rust_util::debug_process_thread_id(), ); WORKER_ORDINAL.with(|x| x.store(self.ordinal, Ordering::SeqCst)); + let worker = (&mut *self as &mut Self) as *mut Self; + _WORKER.with(|x| { + x.store( + (&mut *self as &mut Self) as *mut Self as usize, + Ordering::SeqCst, + ) + }); + _WORKERS + .lock() + .unwrap() + .push(OpaquePointer::from_mut_ptr(worker)); self.scheduler.resolve_affinity(self.ordinal); self.tls = tls; self.copy = crate::plan::create_gc_worker_context(tls, mmtk); @@ -261,6 +297,7 @@ impl GCWorker { typename ); work.do_work_with_stat(&mut self, mmtk); + std::mem::drop(work); } debug!( "Worker exiting. ordinal: {}, {}", diff --git a/src/util/address.rs b/src/util/address.rs index 0e974452966..7e1d498f94a 100644 --- a/src/util/address.rs +++ b/src/util/address.rs @@ -8,6 +8,10 @@ use std::ops::*; use std::sync::atomic::Ordering; use crate::mmtk::{MMAPPER, SFT_MAP}; +use crate::util::metadata::log_bit::LOGGED_VALUE; +use crate::util::VMThread; +use crate::util::VMWorkerThread; +use crate::vm::ObjectModel; /// size in bytes pub type ByteSize = usize; @@ -347,6 +351,44 @@ impl Address { } } + /// Check whether the field at this address is logged, i.e. whether the field-level write + /// barrier has already recorded a write to it and can skip its slow path. + pub fn is_field_logged(self) -> bool { + debug_assert!(!self.is_zero()); + unsafe { + VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC + .as_spec() + .extract_side_spec() + .load::(self) + == LOGGED_VALUE + } + } + + /// Mark the field(s) covered by this address as unlogged (using a relaxed, non-atomic store), + /// so that a subsequent write to them will be caught by the field-level write barrier's slow path again. + pub fn unlog_field_relaxed(self) { + debug_assert!(!self.is_zero()); + let heap_bytes_per_unlog_byte = if VM::VMObjectModel::COMPRESSED_PTR_ENABLED { + 32usize + } else { + 64 + }; + let a = self.align_down(heap_bytes_per_unlog_byte); + unsafe { + VM::VMObjectModel::GLOBAL_FIELD_UNLOG_BIT_SPEC + .as_spec() + .extract_side_spec() + .store_byte_relaxed(a, 0xffu8) + } + } + + /// Converts the address to an [`ObjectReference`]. The address must be non-zero and must be + /// the raw address of a valid object as defined by the VM's object model. + pub fn to_object_reference(self) -> ObjectReference { + debug_assert!(!self.is_zero()); + unsafe { ObjectReference::from_raw_address_unchecked(self) } + } + /// Returns the intersection of the two address ranges. The returned range could /// be empty if there is no intersection between the ranges. pub fn range_intersection(r1: &Range
, r2: &Range
) -> Range
{ @@ -470,6 +512,7 @@ mod tests { } } +use crate::vm::Scanning; use crate::vm::VMBinding; /// `ObjectReference` represents address for an object. Compared with `Address`, operations allowed @@ -704,6 +747,23 @@ impl ObjectReference { pub fn is_sane(self) -> bool { unsafe { SFT_MAP.get_unchecked(self.to_raw_address()) }.is_sane() } + + /// Get the current size (in bytes) of the object, as determined by the VM's object model. + pub fn get_size(self) -> usize { + VM::VMObjectModel::get_current_size(self) + } + + /// Iterate over the slots (fields) of the object, calling `f` for each slot the VM's scanning + /// implementation reports for this object. + pub fn iterate_fields(self, _tls: VMThread, mut f: F) { + // FIXME: We should use tls from the arguments. + // See https://github.com/mmtk/mmtk-core/issues/1375 + let fake_tls = VMWorkerThread(VMThread::UNINITIALIZED); + if !>::support_slot_enqueuing(fake_tls, self) { + panic!("SlotIterator::iterate_fields cannot be used on objects that don't support slot-enqueuing"); + } + >::scan_object(fake_tls, self, &mut f); + } } /// allows print Address as upper-case hex value diff --git a/src/util/alloc/immix_allocator.rs b/src/util/alloc/immix_allocator.rs index eb2e5235fac..b356b1ca310 100644 --- a/src/util/alloc/immix_allocator.rs +++ b/src/util/alloc/immix_allocator.rs @@ -63,13 +63,6 @@ impl Allocator for ImmixAllocator { } fn alloc(&mut self, size: usize, align: usize, offset: usize) -> Address { - debug_assert!( - size <= crate::policy::immix::MAX_IMMIX_OBJECT_SIZE, - "Trying to allocate a {} bytes object, which is larger than MAX_IMMIX_OBJECT_SIZE {}", - size, - crate::policy::immix::MAX_IMMIX_OBJECT_SIZE - ); - let result = align_allocation_no_fill::(self.bump_pointer.cursor, align, offset); let new_cursor = result + size; @@ -237,7 +230,8 @@ impl ImmixAllocator { fn acquire_recyclable_lines(&mut self, size: usize, align: usize, offset: usize) -> bool { while self.line.is_some() || self.acquire_recyclable_block() { let line = self.line.unwrap(); - if let Some((start_line, end_line)) = self.immix_space().get_next_available_lines(line) + if let Some((start_line, end_line)) = + self.immix_space().get_next_available_lines(self.copy, line) { // Find recyclable lines. Update the bump allocation cursor and limit. self.bump_pointer.cursor = start_line.start(); @@ -308,13 +302,16 @@ impl ImmixAllocator { block.start(), block.end() ); - // Bulk clear stale line mark state - Line::MARK_TABLE - .bzero_metadata(block.start(), crate::policy::immix::block::Block::BYTES); - // mark objects if concurrent marking is active - if self.immix_space().should_allocate_as_live() { - let state = self.space.line_mark_state.load(Ordering::Acquire); - Line::eager_mark_lines::(state, block.start_line()..block.end_line()); + // FIXME: Why don't we need this for LXR? Conix needs this. + if !self.immix_space().rc_enabled { + // Bulk clear stale line mark state + Line::MARK_TABLE + .bzero_metadata(block.start(), crate::policy::immix::block::Block::BYTES); + // mark objects if concurrent marking is active + if self.immix_space().should_allocate_as_live() { + let state = self.space.line_mark_state.load(Ordering::Acquire); + Line::eager_mark_lines::(state, block.start_line()..block.end_line()); + } } if self.request_for_large { self.large_bump_pointer.cursor = block.start(); diff --git a/src/util/conversions.rs b/src/util/conversions.rs index f8077ec6895..55ea33c5a54 100644 --- a/src/util/conversions.rs +++ b/src/util/conversions.rs @@ -34,12 +34,12 @@ pub fn bytes_to_chunks_up(bytes: usize) -> usize { /// Convert an address to the chunk index (aligned down). pub fn address_to_chunk_index(addr: Address) -> usize { - addr >> LOG_BYTES_IN_CHUNK + (addr - vm_layout().heap_start) >> LOG_BYTES_IN_CHUNK } /// Convert a chunk index to the start address of the chunk. pub fn chunk_index_to_address(chunk: usize) -> Address { - unsafe { Address::from_usize(chunk << LOG_BYTES_IN_CHUNK) } + vm_layout().heap_start + (chunk << LOG_BYTES_IN_CHUNK) } /// Align up an integer to the given alignment. `align` must be a power of two. diff --git a/src/util/heap/blockpageresource.rs b/src/util/heap/blockpageresource.rs index 7e40a252193..0cf241e38a9 100644 --- a/src/util/heap/blockpageresource.rs +++ b/src/util/heap/blockpageresource.rs @@ -327,7 +327,7 @@ impl BlockPool { /// Push a block to the thread-local queue pub fn push(&self, block: B) { self.count.fetch_add(1, Ordering::SeqCst); - let id = crate::scheduler::current_worker_ordinal(); + let id = crate::scheduler::current_worker_ordinal().unwrap(); let failed = unsafe { self.worker_local_freed_blocks[id] .push_relaxed(block) diff --git a/src/util/heap/chunk_map.rs b/src/util/heap/chunk_map.rs index 69caf12b127..47a5944d369 100644 --- a/src/util/heap/chunk_map.rs +++ b/src/util/heap/chunk_map.rs @@ -1,4 +1,5 @@ use crate::scheduler::GCWork; +use crate::scheduler::GCWorker; use crate::util::linear_scan::Region; use crate::util::linear_scan::RegionIterator; use crate::util::metadata::side_metadata::SideMetadataSpec; @@ -162,6 +163,12 @@ impl ChunkMap { } } + pub fn is_allocated(&self, chunk: Chunk) -> bool { + self.get(chunk) + .map(|state| state.is_allocated()) + .unwrap_or_default() + } + /// Get chunk state. Return None if the chunk does not belong to the space. pub fn get(&self, chunk: Chunk) -> Option { let state = self.get_internal(chunk); @@ -192,4 +199,26 @@ impl ChunkMap { } work_packets } + + pub fn generate_tasks_batched( + &self, + func: impl Fn(Range) -> Box>, + ) -> Vec>> { + let mut work_packets: Vec>> = vec![]; + let chunk_range = self.chunk_range.lock(); + let chunks = (chunk_range.end.start() - chunk_range.start.start()) >> Chunk::LOG_BYTES; + let num_bins = GCWorker::::current().mmtk.scheduler.num_workers() * 8; + let bin_size = chunks.div_ceil(num_bins); + for i in (0..chunks).step_by(bin_size) { + let start = chunk_range.start.next_nth(i); + let end = chunk_range.start.next_nth(i + bin_size); + let end = if end > chunk_range.end { + chunk_range.end + } else { + end + }; + work_packets.push(func(start..end)); + } + work_packets + } } diff --git a/src/util/heap/freelistpageresource.rs b/src/util/heap/freelistpageresource.rs index b80b522ddaf..fe75c8b626f 100644 --- a/src/util/heap/freelistpageresource.rs +++ b/src/util/heap/freelistpageresource.rs @@ -326,6 +326,13 @@ impl FreeListPageResource { self.common.release_discontiguous_chunks(chunk); } + pub fn get_pages(&self, start: Address) -> usize { + debug_assert!(conversions::is_page_aligned(start)); + let sync = self.sync.lock().unwrap(); + let page_offset = conversions::bytes_to_pages_up(start - sync.start); + sync.free_list.size(page_offset as _) as _ + } + /// Release pages previously allocated by `alloc_pages`. /// /// Warning: This method acquires the mutex `self.sync`. If multiple threads release pages @@ -334,7 +341,7 @@ impl FreeListPageResource { /// large object space are recommended to use [`BlockPageResource`] whenever possible. /// /// [`BlockPageResource`]: crate::util::heap::blockpageresource::BlockPageResource - pub fn release_pages(&self, first: Address) { + pub fn release_pages(&self, first: Address) -> usize { debug_assert!(conversions::is_page_aligned(first)); let mut sync = self.sync.lock().unwrap(); let page_offset = conversions::bytes_to_pages_up(first - sync.start); @@ -354,6 +361,7 @@ impl FreeListPageResource { // only discontiguous spaces use chunks self.release_free_chunks(first, freed as _, &mut sync); } + pages as _ } fn release_free_chunks( diff --git a/src/util/heap/layout/map32.rs b/src/util/heap/layout/map32.rs index 7ad2b14fd64..1f208beccbf 100644 --- a/src/util/heap/layout/map32.rs +++ b/src/util/heap/layout/map32.rs @@ -37,8 +37,8 @@ impl Map32 { let max_chunks = vm_layout().max_chunks(); Map32 { inner: UnsafeCell::new(Map32Inner { - prev_link: vec![0; max_chunks], - next_link: vec![0; max_chunks], + prev_link: vec![-1; max_chunks], + next_link: vec![-1; max_chunks], region_map: IntArrayFreeList::new(max_chunks, max_chunks as _, 1), global_page_map: IntArrayFreeList::new(1, 1, MAX_SPACES), shared_discontig_fl_count: 0, @@ -115,7 +115,6 @@ impl VMMap for Map32 { ) -> Address { let (_sync, self_mut) = self.mut_self_with_sync(); let chunk = self_mut.region_map.alloc(chunks as _); - debug_assert!(chunk != 0); if chunk == -1 { return Address::zero(); } @@ -123,19 +122,22 @@ impl VMMap for Map32 { let rtn = conversions::chunk_index_to_address(chunk as _); self.insert(rtn, chunks << LOG_BYTES_IN_CHUNK, descriptor); if head.is_zero() { - debug_assert!(self.next_link[chunk as usize] == 0); + debug_assert!(self.next_link[chunk as usize] == -1); } else { self_mut.next_link[chunk as usize] = head.chunk_index() as _; self_mut.prev_link[head.chunk_index()] = chunk; } - debug_assert!(self.prev_link[chunk as usize] == 0); + debug_assert!(self.prev_link[chunk as usize] == -1); rtn } fn get_next_contiguous_region(&self, start: Address) -> Address { + if start.is_zero() { + return Address::ZERO; + } debug_assert!(start == conversions::chunk_align_down(start)); let chunk = start.chunk_index(); - if chunk == 0 || self.next_link[chunk] == 0 { + if self.next_link[chunk] == -1 { unsafe { Address::zero() } } else { let a = self.next_link[chunk]; @@ -167,11 +169,11 @@ impl VMMap for Map32 { debug_assert!(any_chunk == conversions::chunk_align_down(any_chunk)); if !any_chunk.is_zero() { let chunk = any_chunk.chunk_index(); - while self_mut.next_link[chunk] != 0 { + while self_mut.next_link[chunk] != -1 { let x = self_mut.next_link[chunk]; self.free_contiguous_chunks_no_lock(x); } - while self_mut.prev_link[chunk] != 0 { + while self_mut.prev_link[chunk] != -1 { let x = self_mut.prev_link[chunk]; self.free_contiguous_chunks_no_lock(x); } @@ -285,14 +287,14 @@ impl Map32 { self.mut_self().total_available_discontiguous_chunks += chunks as usize; let next = self.next_link[chunk as usize]; let prev = self.prev_link[chunk as usize]; - if next != 0 { + if next != -1 { self.mut_self().prev_link[next as usize] = prev }; - if prev != 0 { + if prev != -1 { self.mut_self().next_link[prev as usize] = next }; - self.mut_self().prev_link[chunk as usize] = 0; - self.mut_self().next_link[chunk as usize] = 0; + self.mut_self().prev_link[chunk as usize] = -1; + self.mut_self().next_link[chunk as usize] = -1; for offset in 0..chunks { let index = (chunk + offset) as usize; let chunk_start = conversions::chunk_index_to_address(index); diff --git a/src/util/linear_scan.rs b/src/util/linear_scan.rs index 4386b6794b5..37f05132527 100644 --- a/src/util/linear_scan.rs +++ b/src/util/linear_scan.rs @@ -125,9 +125,12 @@ pub trait Region: Copy + PartialEq + PartialOrd { debug_assert!(self.start().as_usize() < usize::MAX - (n << Self::LOG_BYTES)); Self::from_aligned_address(self.start() + (n << Self::LOG_BYTES)) } - /// Return the region that contains the object. - fn containing(object: ObjectReference) -> Self { - Self::from_unaligned_address(object.to_raw_address()) + /// Get the number of lines between the given two lines. + fn steps_between(start: &Self, end: &Self) -> Option { + if start.start() > end.start() { + return None; + } + Some((end.start() - start.start()) >> Self::LOG_BYTES) } /// Check if the given address is in the region. fn includes_address(&self, addr: Address) -> bool { @@ -135,6 +138,30 @@ pub trait Region: Copy + PartialEq + PartialOrd { } } +/// An unstraddlable region. No object can straddle (i.e. span over, overrlap with) more than one +/// [`UnstraddlableRegion`]. In other words, any object is either in the region or not in the +/// region. +/// +/// For example, in [`crate::policy::immix::ImmixSpace`], a [`crate::policy::immix::block::Block`] +/// is an unstraddlable region because objects cannot straddle multiple blocks. In contrast a +/// [`crate::policy::immix::line::Line`] is not an unstraddlable region because an object can +/// straddle multiple lines. +/// +/// Because the raw address of a [`ObjectReference`] must be inside an object, an object is in an +/// [`UnstraddlableRegion`] if an only if the raw address of its [`ObjectReference`] is in the +/// [`UnstraddlableRegion`]. +pub trait UnstraddlableRegion: Region { + /// Return the region that contains the object. + fn containing(object: ObjectReference) -> Self { + Self::from_unaligned_address(object.to_raw_address()) + } + + /// Reeturn whether a region contains an object. + fn contains(&self, object: ObjectReference) -> bool { + self.includes_address(object.to_raw_address()) + } +} + /// An iterator for contiguous regions. pub struct RegionIterator { current: R, diff --git a/src/util/metadata/log_bit.rs b/src/util/metadata/log_bit.rs index 6ea012acbdd..1ccae6a4fcf 100644 --- a/src/util/metadata/log_bit.rs +++ b/src/util/metadata/log_bit.rs @@ -7,15 +7,23 @@ use std::sync::atomic::Ordering; use super::MetadataSpec; +/// The value stored in the log bit/byte indicating that the object or field is unlogged, i.e. it +/// has not yet been recorded in the remembered set and the write barrier should still take its slow path. +pub const UNLOGGED_VALUE: u8 = 0b1; + +/// The value stored in the log bit/byte indicating that the object or field is logged, i.e. it +/// has already been recorded in the remembered set and the write barrier can skip its slow path. +pub const LOGGED_VALUE: u8 = 0b0; + impl VMGlobalLogBitSpec { /// Clear the unlog bit to log object (0 means logged) pub fn clear(&self, object: ObjectReference, order: Ordering) { - self.store_atomic::(object, 0, None, order) + self.store_atomic::(object, LOGGED_VALUE, None, order) } /// Mark the log bit as unlogged (1 means unlogged) pub fn mark_as_unlogged(&self, object: ObjectReference, order: Ordering) { - self.store_atomic::(object, 1, None, order) + self.store_atomic::(object, UNLOGGED_VALUE, None, order) } /// Mark the entire byte as unlogged if the log bit is in the side metadata. As it marks the entire byte, @@ -35,9 +43,16 @@ impl VMGlobalLogBitSpec { } } - /// Check if the log bit represents the unlogged state (the bit is 1). + /// Check if the log bit represents the unlogged state. pub fn is_unlogged(&self, object: ObjectReference, order: Ordering) -> bool { - self.load_atomic::(object, None, order) == 1 + self.load_atomic::(object, None, order) == UNLOGGED_VALUE + } +} + +impl MetadataSpec { + /// Mark the log bit as unlogged (1 means unlogged) + pub fn mark_as_unlogged(&self, object: ObjectReference, order: Ordering) { + self.store_atomic::(object, UNLOGGED_VALUE, None, order) } } diff --git a/src/util/metadata/side_metadata/global.rs b/src/util/metadata/side_metadata/global.rs index 4b9bac989d6..937cee64dde 100644 --- a/src/util/metadata/side_metadata/global.rs +++ b/src/util/metadata/side_metadata/global.rs @@ -537,6 +537,27 @@ impl SideMetadataSpec { ) } + /// Non-atomically load a raw byte from the side metadata byte that is mapped to the data address. + /// Unlike [`SideMetadataSpec::load`], this always reads a whole byte regardless of the number of + /// bits used by this spec, and does not mask/shift out unrelated bits sharing that byte. + pub fn load_byte(&self, data_addr: Address) -> u8 { + let meta_addr = address_to_meta_address(self, data_addr); + unsafe { meta_addr.load::() } + } + + /// Non-atomically store a raw byte to the side metadata byte that is mapped to the data address. + /// + /// # Safety + /// + /// This is unsafe because: + /// + /// 1. Concurrent access to this operation is undefined behaviour. + /// 2. Interleaving non-atomic and atomic operations is undefined behaviour. + pub unsafe fn store_byte_relaxed(&self, data_addr: Address, byte: u8) { + let meta_addr = address_to_meta_address(self, data_addr); + meta_addr.store::(byte); + } + /// Loads a value from the side metadata for the given address. /// This method has similar semantics to `store` in Rust atomics. pub fn load_atomic(&self, data_addr: Address, order: Ordering) -> T { @@ -927,13 +948,21 @@ impl SideMetadataSpec { /// Fetches the value for this side metadata for the given address, and applies a function to it that returns an optional new value. /// This method has similar semantics to `fetch_update` in Rust atomics. /// Returns a Result of Ok(previous_value) if the function returned Some(_), else Err(previous_value). - pub fn fetch_update_atomic Option + Copy>( + pub fn fetch_update_atomic Option>( &self, data_addr: Address, set_order: Ordering, fetch_order: Ordering, mut f: F, ) -> std::result::Result { + // `f` may have side effects (e.g. it may capture and mutate local state), so under + // `extreme_assertions` we must not call it a second time just to recompute the new + // value for the sanity check. Instead, stash the new value computed during the actual + // update here, and have the verify closure read it back. + #[cfg(feature = "extreme_assertions")] + let last_new_val: std::cell::Cell> = std::cell::Cell::new(None); + #[cfg(feature = "extreme_assertions")] + let last_new_val_ref = &last_new_val; self.side_metadata_access::( data_addr, None, @@ -950,7 +979,10 @@ impl SideMetadataSpec { fetch_order, |raw_byte: u8| { let old_val = (raw_byte & mask) >> lshift; - f(FromPrimitive::from_u8(old_val).unwrap()).map(|new_val| { + let new_val = f(FromPrimitive::from_u8(old_val).unwrap()); + #[cfg(feature = "extreme_assertions")] + last_new_val_ref.set(new_val); + new_val.map(|new_val| { (raw_byte & !mask) | ((new_val.to_u8().unwrap() << lshift) & mask) }) @@ -960,13 +992,25 @@ impl SideMetadataSpec { .map(|x| FromPrimitive::from_u8((x & mask) >> lshift).unwrap()) .map_err(|x| FromPrimitive::from_u8((x & mask) >> lshift).unwrap()) } else { - unsafe { T::fetch_update(meta_addr, set_order, fetch_order, f) } + unsafe { + T::fetch_update(meta_addr, set_order, fetch_order, |old_val| { + let new_val = f(old_val); + #[cfg(feature = "extreme_assertions")] + last_new_val_ref.set(new_val); + new_val + }) + } } }, |_result| { #[cfg(feature = "extreme_assertions")] if let Ok(old_val) = _result { - sanity::verify_update::(self, data_addr, old_val, f(old_val).unwrap()) + sanity::verify_update::( + self, + data_addr, + old_val, + last_new_val_ref.get().unwrap(), + ) } }, ) @@ -1463,6 +1507,12 @@ impl SideMetadataContext { // and both policies use the chunk map, we just add the chunk map table globally. ret.push(crate::util::heap::chunk_map::ChunkMap::ALLOC_TABLE); + // `Block::init()` unconditionally records per-block defrag state (e.g. whether a block + // is a defrag source) for every Immix-family plan, not just plans with reference + // counting enabled. So this table must always be mapped, regardless of which plan + // requests it. + ret.push(crate::policy::immix::block::Block::DEFRAG_STATE_TABLE); + ret.extend_from_slice(specs); ret } diff --git a/src/util/metadata/side_metadata/layout.rs b/src/util/metadata/side_metadata/layout.rs index 279e584c7e8..d7e19409211 100644 --- a/src/util/metadata/side_metadata/layout.rs +++ b/src/util/metadata/side_metadata/layout.rs @@ -191,7 +191,8 @@ pub(crate) fn global_side_metadata_bytes() -> usize { /// A value of 2 means the space required for global side metadata must be less than 1/4th of the source data. /// So, a value of `n` means this ratio must be less than $2^-n$. #[cfg(target_pointer_width = "32")] -pub(super) const LOG_GLOBAL_SIDE_METADATA_WORST_CASE_RATIO: usize = 3; +// FIXME: Increased from 3 to 2 to allow LXR with VO bit (which uses slightly more than 1/8th of the address space) +pub(super) const LOG_GLOBAL_SIDE_METADATA_WORST_CASE_RATIO: usize = 2; #[cfg(target_pointer_width = "64")] pub(super) const LOG_GLOBAL_SIDE_METADATA_WORST_CASE_RATIO: usize = 1; diff --git a/src/util/metadata/side_metadata/sanity.rs b/src/util/metadata/side_metadata/sanity.rs index c5ed4bf20a4..e162ba9d783 100644 --- a/src/util/metadata/side_metadata/sanity.rs +++ b/src/util/metadata/side_metadata/sanity.rs @@ -695,10 +695,10 @@ mod tests { }; assert!(verify_global_specs_total_size(&[spec_1]).is_ok()); - #[cfg(target_pointer_width = "64")] assert!(verify_global_specs_total_size(&[spec_1, spec_2]).is_ok()); + // On 32-bit, the budget is 1/4 of the address space, so 2 specs (each 1/8) fit but 3 don't. #[cfg(target_pointer_width = "32")] - assert!(verify_global_specs_total_size(&[spec_1, spec_2]).is_err()); + assert!(verify_global_specs_total_size(&[spec_1, spec_2, spec_1]).is_err()); let spec_2 = SideMetadataSpec { name: "spec_2", @@ -732,7 +732,13 @@ mod tests { }; assert!(verify_global_specs_total_size(&[spec_1, spec_2]).is_ok()); + // 3 copies exceed the budget on 64-bit, but not on 32-bit (which has a larger budget + // relative to these spec sizes); 5 copies exceed it on both. + #[cfg(target_pointer_width = "64")] assert!(verify_global_specs_total_size(&[spec_1, spec_2, spec_1]).is_err()); + #[cfg(target_pointer_width = "32")] + assert!(verify_global_specs_total_size(&[spec_1, spec_2, spec_1]).is_ok()); + assert!(verify_global_specs_total_size(&[spec_1, spec_2, spec_1, spec_2, spec_1]).is_err()); } #[test] diff --git a/src/util/metadata/side_metadata/spec_defs.rs b/src/util/metadata/side_metadata/spec_defs.rs index 07341f90c02..140649ded49 100644 --- a/src/util/metadata/side_metadata/spec_defs.rs +++ b/src/util/metadata/side_metadata/spec_defs.rs @@ -59,6 +59,10 @@ define_side_metadata_specs!( VO_BIT = (global: true, log_num_of_bits: 0, log_bytes_in_region: LOG_MIN_OBJECT_SIZE as usize), // Track the index in SFT map for a chunk (only used for SFT sparse chunk map) SFT_DENSE_CHUNK_MAP_INDEX = (global: true, log_num_of_bits: 3, log_bytes_in_region: LOG_BYTES_IN_CHUNK), + // Reference counts + RC_TABLE = (global: true, log_num_of_bits: crate::util::rc::LOG_REF_COUNT_BITS, log_bytes_in_region: crate::util::rc::LOG_MIN_OBJECT_SIZE), + // Record defrag state for immix blocks + IX_BLOCK_DEFRAG = (global: true, log_num_of_bits: 3, log_bytes_in_region: crate::policy::immix::block::Block::LOG_BYTES), // Mark chunks (any plan that uses the chunk map should include this spec in their global sidemetadata specs) CHUNK_MARK = (global: true, log_num_of_bits: 3, log_bytes_in_region: crate::util::heap::chunk_map::Chunk::LOG_BYTES), ); @@ -73,10 +77,16 @@ define_side_metadata_specs!( MS_OFFSET_MALLOC = (global: false, log_num_of_bits: 0, log_bytes_in_region: LOG_MIN_OBJECT_SIZE as usize), // Mark lines by immix IX_LINE_MARK = (global: false, log_num_of_bits: 3, log_bytes_in_region: crate::policy::immix::line::Line::LOG_BYTES), - // Record defrag state for immix blocks - IX_BLOCK_DEFRAG = (global: false, log_num_of_bits: 3, log_bytes_in_region: crate::policy::immix::block::Block::LOG_BYTES), // Mark blocks by immix IX_BLOCK_MARK = (global: false, log_num_of_bits: 3, log_bytes_in_region: crate::policy::immix::block::Block::LOG_BYTES), + // Straddle line marks + RC_STRADDLE_LINES = (global: false, log_num_of_bits: 3, log_bytes_in_region: crate::policy::immix::line::Line::LOG_BYTES), + // LXR Block logging bits + IX_BLOCK_LOG = (global: false, log_num_of_bits: 0, log_bytes_in_region: crate::policy::immix::block::Block::LOG_BYTES), + NURSERY_PROMOTION_STATE = (global: false, log_num_of_bits: 3, log_bytes_in_region: crate::policy::immix::block::Block::LOG_BYTES), + PHASE_EPOCH = (global: false, log_num_of_bits: 3, log_bytes_in_region: crate::policy::immix::block::Block::LOG_BYTES), + IX_LINE_REUSE_COUNT = (global: false, log_num_of_bits: 3, log_bytes_in_region: crate::policy::immix::line::Line::LOG_BYTES), + LOS_PAGE_REUSE_COUNT = (global: false, log_num_of_bits: 3, log_bytes_in_region: LOG_BYTES_IN_PAGE as usize), // Mark blocks by (native mimalloc) marksweep MS_BLOCK_MARK = (global: false, log_num_of_bits: 3, log_bytes_in_region: crate::policy::marksweepspace::native_ms::Block::LOG_BYTES), // Next block in list for native mimalloc diff --git a/src/util/mod.rs b/src/util/mod.rs index 8b7c7185815..35327495d80 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -38,6 +38,8 @@ pub mod opaque_pointer; pub mod options; /// Operating system abstractions. pub mod os; +/// Reference counting support. +pub mod rc; #[cfg(feature = "test_private")] pub mod test_private; /// Test utilities. We need this module for `MockVM` in criterion benches, which does not include code with `cfg(test)`. diff --git a/src/util/object_forwarding.rs b/src/util/object_forwarding.rs index 76c0e937156..5a4b0919de8 100644 --- a/src/util/object_forwarding.rs +++ b/src/util/object_forwarding.rs @@ -74,6 +74,33 @@ pub fn spin_and_get_forwarded_object( } } +pub fn try_forward_object( + object: ObjectReference, + semantics: CopySemantics, + copy_context: &mut GCWorkerCopyContext, + on_after_forwarding: impl FnOnce(ObjectReference), +) -> Option { + let new_object = VM::VMObjectModel::try_copy(object, semantics, copy_context)?; + on_after_forwarding(new_object); + if let Some(shift) = forwarding_bits_offset_in_forwarding_pointer::() { + VM::VMObjectModel::LOCAL_FORWARDING_POINTER_SPEC.store_atomic::( + object, + new_object.to_raw_address().as_usize() | ((FORWARDED as usize) << shift), + None, + Ordering::SeqCst, + ) + } else { + write_forwarding_pointer::(object, new_object); + VM::VMObjectModel::LOCAL_FORWARDING_BITS_SPEC.store_atomic::( + object, + FORWARDED, + None, + Ordering::SeqCst, + ); + } + Some(new_object) +} + /// Copy an object and set the forwarding state. /// /// The caller can use `on_after_forwarding` to set extra metadata (including VO bits, mark bits, @@ -130,7 +157,7 @@ pub fn is_forwarded(object: ObjectReference) -> bool { get_forwarding_status::(object) == FORWARDED } -fn is_being_forwarded(object: ObjectReference) -> bool { +pub fn is_being_forwarded(object: ObjectReference) -> bool { get_forwarding_status::(object) == BEING_FORWARDED } diff --git a/src/util/opaque_pointer.rs b/src/util/opaque_pointer.rs index 1a16b98c043..9c7c5e92bed 100644 --- a/src/util/opaque_pointer.rs +++ b/src/util/opaque_pointer.rs @@ -27,6 +27,16 @@ impl OpaquePointer { OpaquePointer(addr.to_mut_ptr::()) } + /// Cast a raw mutable pointer to an [`OpaquePointer`]. + pub fn from_mut_ptr(ptr: *mut T) -> Self { + OpaquePointer(ptr as *mut c_void) + } + + /// Cast the opaque pointer to a raw mutable pointer. + pub fn as_mut_ptr(self) -> *mut T { + self.0 as *mut T + } + /// Cast the opaque pointer to an [`Address`] type. pub fn to_address(self) -> Address { Address::from_mut_ptr(self.0) diff --git a/src/util/options.rs b/src/util/options.rs index b048bb27c76..f94699866ae 100644 --- a/src/util/options.rs +++ b/src/util/options.rs @@ -50,6 +50,8 @@ pub enum PlanSelector { OVC, /// An Immix collector that uses a sticky mark bit to allow generational behaviors without a copying nursery. StickyImmix, + /// LXR GC + LXR, /// Concurrent non-moving immix using SATB ConcurrentImmix, } @@ -988,6 +990,8 @@ options! { /// there is no core with (perceived) ID 12. // XXX: This option is currently only supported on Linux. thread_affinity: AffinityKind [|v: &AffinityKind| v.validate()] = AffinityKind::OsDefault, + /// Verbosity level for GC statistics and logging output, from 0 (quiet) to 10 (most verbose). + verbose: usize [|v: &usize| *v <= 10] = 0, /// Set the GC trigger. This defines the heap size and how MMTk triggers a GC. /// Default to a fixed heap size of 0.5x physical memory. gc_trigger: GCTriggerSelector [|v: &GCTriggerSelector| v.validate()] = GCTriggerSelector::FixedHeapSize((OS::get_system_total_memory().unwrap_or(4 * 1024 * 1024 * 1024) as f64 * 0.5f64) as usize), @@ -1004,7 +1008,7 @@ options! { /// Percentage of heap size reserved for defragmentation. /// According to [this paper](https://doi.org/10.1145/1375581.1375586), Immix works well with /// headroom between 1% to 3% of the heap size. - immix_defrag_headroom_percent: usize [|v: &usize| *v <= 50] = 2, + immix_defrag_headroom_percent: usize [|v: &usize| *v <= 50] = 5, /// Disable concurrent marking in ConcurrentImmix. Setting this to true will make ConcurrentImmix behave exactly like full heap Immix. This option is only intended for debugging. concurrent_immix_disable_concurrent_marking: bool [always_valid] = false } diff --git a/src/util/rc.rs b/src/util/rc.rs new file mode 100644 index 00000000000..7e58a220220 --- /dev/null +++ b/src/util/rc.rs @@ -0,0 +1,299 @@ +use std::marker::PhantomData; +use std::sync::atomic::{AtomicU32, AtomicUsize}; + +use crate::util::linear_scan::Region; +use crate::util::{metadata::side_metadata::address_to_meta_address, Address}; +use crate::{ + policy::immix::{block::Block, line::Line}, + util::{metadata::side_metadata::SideMetadataSpec, ObjectReference}, + vm::*, +}; +use atomic::Ordering; + +/// Log2 of the number of bits used to store each object's reference count in the RC table. +pub const LOG_REF_COUNT_BITS: usize = 1; +/// Number of bits used to store each object's reference count in the RC table. +pub const REF_COUNT_BITS: u8 = 1 << LOG_REF_COUNT_BITS; +/// Bit mask covering the bits used to store a reference count. +pub const REF_COUNT_MASK: u8 = (((1u16 << REF_COUNT_BITS) - 1) & 0xff) as u8; +/// The maximum representable reference count. Once an object's count reaches this value it +/// is treated as saturated/sticky and is no longer incremented or decremented. +pub const MAX_REF_COUNT: u8 = REF_COUNT_MASK; + +/// Log2 of the minimum object size, i.e. the granularity at which reference counts are tracked. +pub const LOG_MIN_OBJECT_SIZE: usize = crate::util::constants::LOG_MIN_OBJECT_SIZE as _; +/// The minimum object size, i.e. the granularity at which reference counts are tracked. +pub const MIN_OBJECT_SIZE: usize = 1 << LOG_MIN_OBJECT_SIZE; + +/// Side metadata recording which Immix lines are "straddled" by an object that spans +/// multiple lines, so straddling objects can be identified without scanning their contents. +pub const RC_STRADDLE_LINES: SideMetadataSpec = + crate::util::metadata::side_metadata::spec_defs::RC_STRADDLE_LINES; + +/// Side metadata spec for the per-object reference count table. +pub const RC_TABLE: SideMetadataSpec = crate::util::metadata::side_metadata::spec_defs::RC_TABLE; + +static INC_BUFFER_SIZE: AtomicUsize = AtomicUsize::new(0); + +static TOTAL_INCS_PACKETS: AtomicU32 = AtomicU32::new(0); + +static TOTAL_INCS: AtomicU32 = AtomicU32::new(0); +static ROOT_INCS: AtomicU32 = AtomicU32::new(0); +static MATURE_INCS: AtomicU32 = AtomicU32::new(0); +static NURSERY_INCS: AtomicU32 = AtomicU32::new(0); +static FAST_NURSERY_INCS: AtomicU32 = AtomicU32::new(0); +static LOS_INCS: AtomicU32 = AtomicU32::new(0); + +static PROMOTED_OBJECTS: AtomicU32 = AtomicU32::new(0); +static PROMOTED_SCALARS: [AtomicU32; 3] = [AtomicU32::new(0), AtomicU32::new(0), AtomicU32::new(0)]; +static PROMOTED_PRIM_ARRAYS: [AtomicU32; 3] = + [AtomicU32::new(0), AtomicU32::new(0), AtomicU32::new(0)]; +static PROMOTED_OBJECT_ARRAYS: [AtomicU32; 3] = + [AtomicU32::new(0), AtomicU32::new(0), AtomicU32::new(0)]; + +/// A zero-sized helper type providing methods to read and update per-object reference count +/// metadata for LXR's reference counting plan. +#[repr(transparent)] +#[derive(Debug, Copy)] +pub struct RefCountHelper(PhantomData); + +impl RefCountHelper { + /// A singleton instance of `RefCountHelper` (the type is zero-sized, so it can be freely copied/cloned). + pub const NEW: Self = Self(PhantomData); + /// Whether extra reference-counting sanity checks are enabled (debug builds or the `sanity` feature). + pub const SANITY: bool = cfg!(debug_assertions) || cfg!(feature = "sanity"); + + /// Returns the current size of the global increment buffer, i.e. the number of pending + /// reference count increments that have been enqueued but not yet processed. + pub fn inc_buffer_size(&self) -> usize { + INC_BUFFER_SIZE.load(Ordering::Relaxed) + } + + /// Increases the global increment buffer size counter by `delta`. + pub fn increase_inc_buffer_size(&self, delta: usize) { + INC_BUFFER_SIZE.store( + INC_BUFFER_SIZE + .load(Ordering::Relaxed) + .saturating_add(delta), + Ordering::Relaxed, + ); + } + + /// Resets the global increment buffer size counter to zero. + pub fn reset_inc_buffer_size(&self) { + INC_BUFFER_SIZE.store(0, Ordering::Relaxed) + } + + /// Atomically updates the reference count of object `o` by applying `f` to its current + /// value, following the same semantics as `AtomicU8::fetch_update`. + pub fn fetch_update( + &self, + o: ObjectReference, + f: impl FnMut(u8) -> Option, + ) -> Result { + RC_TABLE.fetch_update_atomic(o.to_raw_address(), Ordering::Relaxed, Ordering::Relaxed, f) + } + + /// Returns `true` if object `o`'s reference count has saturated at `MAX_REF_COUNT` (sticky). + pub fn is_stuck(&self, o: ObjectReference) -> bool { + self.count(o) == MAX_REF_COUNT + } + + /// Forces object `o`'s reference count to `MAX_REF_COUNT`, permanently marking it as sticky + /// so it is never reclaimed by reference counting. + pub fn stick(&self, o: ObjectReference) -> Result { + self.fetch_update(o, |x| { + debug_assert!(x <= MAX_REF_COUNT); + if x == MAX_REF_COUNT { + None + } else { + Some(MAX_REF_COUNT) + } + }) + } + + /// Increments object `o`'s reference count by one, leaving it unchanged (saturating) once + /// it has reached `MAX_REF_COUNT`. + pub fn inc(&self, o: ObjectReference) -> Result { + #[cfg(feature = "vo_bit")] + debug_assert!( + crate::util::metadata::vo_bit::is_vo_bit_set(o), + "{o}: VO bit not set", + ); + + self.fetch_update(o, |x| { + debug_assert!(x <= MAX_REF_COUNT); + if x == MAX_REF_COUNT { + None + } else { + Some(x + 1) + } + }) + } + + /// Decrements object `o`'s reference count by one, unless it is already zero or has + /// saturated at `MAX_REF_COUNT` (sticky), in which case it is left unchanged. + pub fn dec(&self, o: ObjectReference) -> Result { + #[cfg(feature = "vo_bit")] + debug_assert!( + crate::util::metadata::vo_bit::is_vo_bit_set(o), + "{o}: VO bit not set", + ); + + self.fetch_update(o, |x| { + debug_assert!(x <= MAX_REF_COUNT); + if x == 0 || x == MAX_REF_COUNT + /* sticky */ + { + None + } else { + Some(x - 1) + } + }) + } + + /// Atomically sets object `o`'s reference count to `count`. + pub fn set(&self, o: ObjectReference, count: u8) { + RC_TABLE.store_atomic(o.to_raw_address(), count, Ordering::Relaxed) + } + + /// Sets object `o`'s reference count to `count` using a non-atomic store, for use where the + /// caller can guarantee there is no concurrent access. + pub fn set_relaxed(&self, o: ObjectReference, count: u8) { + unsafe { RC_TABLE.store(o.to_raw_address(), count) } + } + + /// Returns object `o`'s current reference count. + pub fn count(&self, o: ObjectReference) -> u8 { + RC_TABLE.load_atomic(o.to_raw_address(), Ordering::Relaxed) + } + + /// Returns `true` if the RC table entry at `o`'s address is zero. Used for both individual + /// objects and line-granularity entries (e.g. straddle line markers), which share the same table. + pub fn object_or_line_is_dead(&self, o: ObjectReference) -> bool { + RC_TABLE.load_byte(o.to_raw_address()) == 0 + } + + /// Returns a slice view over the raw RC table memory covering block `b`, reinterpreted as an + /// array of `UInt`, allowing the block's reference counts to be scanned in bulk. + pub fn rc_table_range(&self, b: Block) -> &'static [UInt] { + debug_assert!({ + let log_bits_in_uint: usize = + (std::mem::size_of::() << 3).trailing_zeros() as usize; + Block::LOG_BYTES - super::rc::LOG_MIN_OBJECT_SIZE + super::rc::LOG_REF_COUNT_BITS + >= log_bits_in_uint + }); + let start = address_to_meta_address(&super::rc::RC_TABLE, b.start()).to_ptr::(); + let limit = address_to_meta_address(&super::rc::RC_TABLE, b.end()).to_ptr::(); + let rc_table = unsafe { std::slice::from_raw_parts(start, limit.offset_from(start) as _) }; + rc_table + } + + /// Returns `true` if object `o`'s reference count is zero. + #[allow(unused)] + pub fn is_dead(&self, o: ObjectReference) -> bool { + let v: u8 = RC_TABLE.load_atomic(o.to_raw_address(), Ordering::Relaxed); + v == 0 + } + + /// Returns `true` if object `o`'s reference count is zero (dead) or has saturated at + /// `MAX_REF_COUNT` (sticky). + pub fn is_dead_or_stuck(&self, o: ObjectReference) -> bool { + let v: u8 = RC_TABLE.load_atomic(o.to_raw_address(), Ordering::Relaxed); + v == 0 || v == MAX_REF_COUNT + } + + /// Returns `true` if `line` is marked as being straddled by an object that spans multiple lines. + pub fn is_straddle_line(&self, line: Line) -> bool { + let v: u8 = unsafe { RC_STRADDLE_LINES.load::(line.start()) }; + v != 0 + } + + /// Returns `true` if address `a` falls within a live object whose containing line is marked + /// as a straddle line. + pub fn address_is_in_straddle_line(&self, a: Address) -> bool { + let line = Line::from_unaligned_address(a); + self.count(a.to_object_reference::()) != 0 && self.is_straddle_line(line) + } + + fn mark_straddle_object_with_size(&self, o: ObjectReference, size: usize) { + debug_assert!(size > Line::BYTES); + let start = o.to_object_start::(); + let end = start + size; + let start_line = Line::from_unaligned_address(start).next(); + let end_line = Line::from_unaligned_address(end); + // Note that `end_line` may be the last line overlapping with `o`. + // In that case, `end_line` will not be marked. + // It is OK because when searching for available lines (`rc_get_next_available_lines`), + // it always skips the first line in a hole. + let mut line = start_line; + while line != end_line { + unsafe { RC_STRADDLE_LINES.store(line.start(), 1u8) }; + self.set_relaxed(line.start().to_object_reference::(), 1); + line = line.next(); + } + } + + /// Marks every line (other than the first) spanned by object `o` as a straddle line, so the + /// object can be identified from any of the lines it straddles. + pub fn mark_straddle_object(&self, o: ObjectReference) { + let size = VM::VMObjectModel::get_current_size(o); + self.mark_straddle_object_with_size(o, size) + } + + /// Clears the straddle-line and reference-count markers set by `mark_straddle_object` for + /// every line (other than the first) spanned by object `o`. + pub fn unmark_straddle_object(&self, o: ObjectReference) { + // debug_assert!(crate::args::RC_NURSERY_EVACUATION); + let size = VM::VMObjectModel::get_current_size(o); + if size > Line::BYTES { + let start = o.to_object_start::(); + let end = start + size; + let start_line = Line::from_unaligned_address(start).next(); + let end_line = Line::from_unaligned_address(end); + // Note that `end_line` may be the last line overlapping with `o`. + // In that case, `end_line` will not be marked. + // It is OK because when searching for available lines (`rc_get_next_available_lines`), + // it always skips the first line in a hole. + let mut line = start_line; + while line != end_line { + self.set_relaxed(line.start().to_object_reference::(), 0); + unsafe { RC_STRADDLE_LINES.store(line.start(), 0u8) }; + line = line.next(); + } + } + } + + /// Debug assertion that every `MIN_OBJECT_SIZE` granule within object `o` has a reference + /// count of zero, used to verify that a reclaimed object has been fully cleared. + pub fn assert_zero_ref_count(&self, o: ObjectReference) { + let size = VM::VMObjectModel::get_current_size(o); + for i in (0..size).step_by(MIN_OBJECT_SIZE) { + let a = o.to_raw_address() + i; + assert_eq!(0, self.count(a.to_object_reference::())); + } + } + + /// Called when object `o` is promoted to mature space; marks it as a straddle object if it + /// spans more than one line, deriving its size from the VM binding. + pub fn promote(&self, o: ObjectReference) { + let size = o.get_size::(); + if size > Line::BYTES { + self.mark_straddle_object_with_size(o, size); + } + } + + /// Same as `promote`, but with the object's size supplied by the caller instead of being + /// queried from the VM binding. + pub fn promote_with_size(&self, o: ObjectReference, size: usize) { + if size > Line::BYTES { + self.mark_straddle_object_with_size(o, size); + } + } +} + +impl Clone for RefCountHelper { + fn clone(&self) -> Self { + Self(PhantomData) + } +} diff --git a/src/util/test_util/mock_vm.rs b/src/util/test_util/mock_vm.rs index 9087d22c713..aaf0701ecb9 100644 --- a/src/util/test_util/mock_vm.rs +++ b/src/util/test_util/mock_vm.rs @@ -464,11 +464,15 @@ impl crate::vm::Collection for MockVM { impl crate::vm::ObjectModel for MockVM { const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::in_header(0); + const GLOBAL_FIELD_UNLOG_BIT_SPEC: VMGlobalFieldUnlogBitSpec = + VMGlobalFieldUnlogBitSpec::side_first(); const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = VMLocalForwardingPointerSpec::in_header(0); const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec = VMLocalForwardingBitsSpec::in_header(0); - const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = VMLocalMarkBitSpec::in_header(0); + // LXR clears mark bits in bulk over side metadata, so this must be a side spec (unlike most + // of the other local specs here, which can stay in the header). + const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = VMLocalMarkBitSpec::side_first(); const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = VMLocalLOSMarkNurserySpec::in_header(0); @@ -549,10 +553,10 @@ impl crate::vm::Scanning for MockVM { fn support_slot_enqueuing(tls: VMWorkerThread, object: ObjectReference) -> bool { mock!(support_slot_enqueuing(tls, object)) } - fn scan_object::VMSlot>>( + fn scan_object( tls: VMWorkerThread, object: ObjectReference, - slot_visitor: &mut SV, + slot_visitor: &mut impl SlotVisitor<::VMSlot>, ) { mock!(scan_object( tls, diff --git a/src/util/treadmill.rs b/src/util/treadmill.rs index 8ae409a006a..bde3c7e4672 100644 --- a/src/util/treadmill.rs +++ b/src/util/treadmill.rs @@ -69,6 +69,12 @@ impl TreadMill { std::mem::take(&mut sync.collect_nursery) } + /// Take all objects from the `alloc_nursery`. + pub fn collect_alloc_nursery(&self) -> impl IntoIterator { + let mut sync = self.sync.lock().unwrap(); + std::mem::take(&mut sync.alloc_nursery) + } + /// Take all objects from the `from_space`. This is called during sweeping at which time all /// unreachable old objects are in the from-space. pub fn collect_mature(&self) -> impl IntoIterator { @@ -76,6 +82,20 @@ impl TreadMill { std::mem::take(&mut sync.from_space) } + /// Retain objects in the to-space that satisfy the given predicate. This is called during LXR's SATB sweeping + pub fn retain_mature(&self, f: impl FnMut(&ObjectReference) -> bool) { + let mut sync = self.sync.lock().unwrap(); + sync.to_space.retain(f); + } + + /// Remove an object from whichever set contains it. Returns true if the object was found. + /// Called by `rc_free` when an object's reference count reaches zero. + pub fn remove_mature(&self, object: ObjectReference) -> bool { + let mut sync = self.sync.lock().unwrap(); + assert!(sync.from_space.is_empty()); + sync.to_space.remove(&object) + } + /// Move an object to `to_space`. Called when an object is determined to be reachable. pub fn copy(&self, object: ObjectReference, is_in_nursery: bool) { let mut sync = self.sync.lock().unwrap(); diff --git a/src/vm/collection.rs b/src/vm/collection.rs index db0e32d6a92..4f409027e2c 100644 --- a/src/vm/collection.rs +++ b/src/vm/collection.rs @@ -122,6 +122,14 @@ pub trait Collection { /// * `tls`: The thread pointer for the current GC thread. fn schedule_finalization(_tls: VMWorkerThread) {} + /// A hook for the VM to update the state of its weak reference processor at the end of the + /// release phase of a GC. This is currently called by the LXR plan after a full or final-mark + /// pause, before the reachability of weak references has changed further. + /// + /// Arguments: + /// * `_lxr`: Whether the current GC is being driven by the LXR plan. + fn update_weak_processor(_lxr: bool) {} + /// A hook for the VM to do work after forwarding objects. /// /// This function is called after all of the following have finished: @@ -145,6 +153,9 @@ pub trait Collection { /// * `tls_worker`: The thread pointer for the worker thread performing this call. fn post_forwarding(_tls: VMWorkerThread) {} + /// Inform the VM to do its VM-specific release work at the end of a GC. + fn vm_release() {} + /// Return the amount of memory (in bytes) which the VM allocated outside the MMTk heap but /// wants to include into the current MMTk heap size. MMTk core will consider the reported /// memory as part of MMTk heap for the purpose of heap size accounting. diff --git a/src/vm/object_model.rs b/src/vm/object_model.rs index 8ecca1ac1bb..42e96d24ab9 100644 --- a/src/vm/object_model.rs +++ b/src/vm/object_model.rs @@ -94,6 +94,12 @@ pub trait ObjectModel { /// This bit is also referred to as unlogged bit in Java MMTk for this reason. const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec; + /// A global per-field 1-bit metadata used by LXR's field barrier to record whether a field has + /// already been logged (recorded) for the current GC. It is generally located in side metadata, + /// with one bit per field-sized slot rather than one bit per object as with + /// [`GLOBAL_LOG_BIT_SPEC`](crate::vm::ObjectModel::GLOBAL_LOG_BIT_SPEC). + const GLOBAL_FIELD_UNLOG_BIT_SPEC: VMGlobalFieldUnlogBitSpec; + /// A local word-size metadata for the forwarding pointer, used by copying plans. It is almost always /// located in the object header as it is fine to destroy an object header in order to copy it. const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec; @@ -123,6 +129,11 @@ pub trait ObjectModel { // TODO: Cleanup and place the LOS mark and nursery bits in the header. See here: https://github.com/mmtk/mmtk-core/issues/847 const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec; + /// Set this to true if the VM binding uses compressed (narrow) object pointers, e.g. compressed + /// oops on a 64-bit heap. When enabled, MMTk adjusts the size of certain per-object and per-field + /// side metadata (such as the field unlog bits) to match the narrower pointer/field width. + const COMPRESSED_PTR_ENABLED: bool = false; + /// Set this to true if the VM binding requires the valid object (VO) bits to be available /// during tracing. If this constant is set to `false`, it is undefined behavior if the binding /// attempts to access VO bits during tracing. @@ -366,6 +377,24 @@ pub trait ObjectModel { copy_context: &mut GCWorkerCopyContext, ) -> ObjectReference; + /// Attempt to copy an object, allowing the copy to fail (e.g. under concurrent copying, where + /// another GC worker may already be copying the same object). Returns the address of the new + /// object on success, or `None` if the copy could not be performed. The default implementation + /// is unimplemented; bindings/plans that support failable copying (such as LXR) should override + /// this method. + /// + /// Arguments: + /// * `from`: The address of the object to be copied. + /// * `semantics`: The copy semantic to use. + /// * `copy_context`: The `GCWorkerCopyContext` for the GC thread. + fn try_copy( + _from: ObjectReference, + _semantics: CopySemantics, + _copy_context: &mut GCWorkerCopyContext, + ) -> Option { + unimplemented!() + } + /// Copy an object. This is required /// for delayed-copy collectors such as compacting collectors. During the /// collection, MMTk reserves a region in the heap for an object as per @@ -480,6 +509,7 @@ pub trait ObjectModel { pub mod specs { use crate::util::constants::LOG_BITS_IN_WORD; + use crate::util::constants::LOG_BYTES_IN_ADDRESS; use crate::util::constants::LOG_BYTES_IN_PAGE; use crate::util::constants::LOG_MIN_OBJECT_SIZE; use crate::util::metadata::side_metadata::*; @@ -494,6 +524,10 @@ pub mod specs { macro_rules! define_vm_metadata_spec { ($(#[$outer:meta])*$spec_name: ident, $is_global: expr, $log_num_bits: expr, $side_min_obj_size: expr) => { $(#[$outer])* + /// A newtype wrapper around [`MetadataSpec`] that identifies this particular per-object + /// metadata (e.g. whether it is in the header or on the side, and where). Generated by the + /// `define_vm_metadata_spec` macro for each metadata kind declared on + /// [`crate::vm::ObjectModel`]. pub struct $spec_name(MetadataSpec); impl $spec_name { /// The number of bits (in log2) that are needed for the spec. @@ -541,6 +575,29 @@ pub mod specs { })) } } + /// Like [`side_first`](Self::side_first), but for use with compressed pointers + /// ([`crate::vm::ObjectModel::COMPRESSED_PTR_ENABLED`]): the region size is one bit + /// narrower to match the reduced field/pointer width, so this declares the first side + /// metadata of its kind (global or local) sized for a compressed-pointer heap. + pub const fn side_first_compressed() -> Self { + if Self::IS_GLOBAL { + Self(MetadataSpec::OnSide(SideMetadataSpec { + name: stringify!($spec_name), + is_global: Self::IS_GLOBAL, + offset: GLOBAL_SIDE_METADATA_VM_BASE_OFFSET, + log_num_of_bits: Self::LOG_NUM_BITS, + log_bytes_in_region: $side_min_obj_size as usize - 1, + })) + } else { + Self(MetadataSpec::OnSide(SideMetadataSpec { + name: stringify!($spec_name), + is_global: Self::IS_GLOBAL, + offset: LOCAL_SIDE_METADATA_VM_BASE_OFFSET, + log_num_of_bits: Self::LOG_NUM_BITS, + log_bytes_in_region: $side_min_obj_size as usize - 1, + })) + } + } /// Declare that the VM uses side metadata for this metadata type, /// and the side metadata should be laid out after the given side metadata spec. @@ -588,6 +645,7 @@ pub mod specs { 0, LOG_MIN_OBJECT_SIZE ); + define_vm_metadata_spec!(VMGlobalFieldUnlogBitSpec, true, 0, LOG_BYTES_IN_ADDRESS); // Forwarding pointer: word size per object, local define_vm_metadata_spec!( /// 1-word local metadata for spaces that may copy objects. diff --git a/src/vm/scanning.rs b/src/vm/scanning.rs index 6b3c72282dc..b51472ee554 100644 --- a/src/vm/scanning.rs +++ b/src/vm/scanning.rs @@ -1,4 +1,5 @@ use crate::plan::Mutator; +use crate::scheduler::gc_work::RootKind; use crate::scheduler::GCWorker; use crate::util::ObjectReference; use crate::util::VMWorkerThread; @@ -109,9 +110,18 @@ pub trait RootsWorkFactory: Clone + Send + 'static { /// /// The work packet may update the slots. /// + /// Equivalent to `self.create_process_roots_work_experimental(slots, RootKind::Strong)`. + /// /// Arguments: /// * `slots`: A vector of slots. - fn create_process_roots_work(&mut self, slots: Vec); + fn create_process_roots_work(&mut self, slots: Vec) { + self.create_process_roots_work_with_root_kind(slots, RootKind::Strong); + } + + /// An experimental API to support weak and young code cache roots. + /// + /// Currently only used by the LXR plan and the OpenJDK binding. + fn create_process_roots_work_with_root_kind(&mut self, slots: Vec, kind: RootKind); /// Create work packets to handle non-transitively pinning roots. /// @@ -197,10 +207,10 @@ pub trait Scanning { /// * `tls`: The VM-specific thread-local storage for the current worker. /// * `object`: The object to be scanned. /// * `slot_visitor`: Called back for each field. - fn scan_object>( + fn scan_object( tls: VMWorkerThread, object: ObjectReference, - slot_visitor: &mut SV, + slot_visitor: &mut impl SlotVisitor, ); /// Delegated scanning of a object, visiting each reference field encountered, and tracing the diff --git a/src/vm/slot.rs b/src/vm/slot.rs index 035592f20fc..12fd46fe64e 100644 --- a/src/vm/slot.rs +++ b/src/vm/slot.rs @@ -154,6 +154,14 @@ pub trait Slot: Copy + Send + Sync + Debug + PartialEq + Eq + Hash { fn prefetch_store(&self) { // no-op by default } + + /// Return the raw memory address of this slot. This is used, for example, by LXR's field + /// barrier and remembered-set code to access per-field side metadata (such as the field unlog + /// bit) that is indexed by the slot's address. The default implementation is unimplemented; + /// slot types that are used with such features must override this method. + fn to_address(&self) -> Address { + unimplemented!() + } } /// A simple slot implementation that represents a word-sized slot which holds the raw address of @@ -218,6 +226,10 @@ impl Slot for Address { fn store(&self, object: ObjectReference) { unsafe { Address::store(*self, object) } } + + fn to_address(&self) -> Address { + *self + } } #[test] diff --git a/src/vm/tests/mock_tests/mock_test_allocator_info.rs b/src/vm/tests/mock_tests/mock_test_allocator_info.rs index 99db9918474..6e63216cbf4 100644 --- a/src/vm/tests/mock_tests/mock_test_allocator_info.rs +++ b/src/vm/tests/mock_tests/mock_test_allocator_info.rs @@ -30,7 +30,8 @@ pub fn test_allocator_info() { | PlanSelector::Lisp2 | PlanSelector::OVC | PlanSelector::ConcurrentImmix - | PlanSelector::StickyImmix => { + | PlanSelector::StickyImmix + | PlanSelector::LXR => { // These plans all use bump pointer allocator. let AllocatorInfo::BumpPointer { bump_pointer_offset, diff --git a/src/vm/tests/mock_tests/mock_test_doc_weakref_code_example.rs b/src/vm/tests/mock_tests/mock_test_doc_weakref_code_example.rs index 218c461b720..296cb55330d 100644 --- a/src/vm/tests/mock_tests/mock_test_doc_weakref_code_example.rs +++ b/src/vm/tests/mock_tests/mock_test_doc_weakref_code_example.rs @@ -65,10 +65,10 @@ impl Scanning for VMScanning { // Methods after this are placeholders. We only ensure they compile. - fn scan_object::VMSlot>>( + fn scan_object( _tls: crate::util::VMWorkerThread, _object: ObjectReference, - _slot_visitor: &mut SV, + _slot_visitor: &mut impl crate::vm::SlotVisitor<::VMSlot>, ) { unimplemented!() } diff --git a/src/vm/tests/mock_tests/mock_test_wait_for_no_collection_in_progress.rs b/src/vm/tests/mock_tests/mock_test_wait_for_no_collection_in_progress.rs index afe0eb83fc9..e0f6e63765d 100644 --- a/src/vm/tests/mock_tests/mock_test_wait_for_no_collection_in_progress.rs +++ b/src/vm/tests/mock_tests/mock_test_wait_for_no_collection_in_progress.rs @@ -2,7 +2,7 @@ use super::mock_test_prelude::*; use crate::global_state::GcStatus; -use crate::util::{OpaquePointer, VMMutatorThread, VMThread}; +use crate::util::{Address, OpaquePointer, VMMutatorThread, VMThread}; use std::sync::{Condvar, Mutex}; use std::time::Duration; @@ -59,8 +59,11 @@ pub fn disable_collection_fails_while_gc_in_progress() { // Thread A: trigger a GC. `handle_user_collection_request` blocks the calling // thread in (our mocked) `block_for_gc` until the GC finishes. let thread_to_trigger_gc = std::thread::spawn(move || { - let tls = VMMutatorThread(VMThread(OpaquePointer::UNINITIALIZED)); - memory_manager::handle_user_collection_request(mmtk, tls); + // FIXME: We should use a null address as tls. Use 1 to work around a strange impl in handle_user_collection_request. + let tls = VMMutatorThread(VMThread(OpaquePointer::from_address(unsafe { + Address::from_usize(1) + }))); + memory_manager::handle_user_collection_request(mmtk, tls, true); { let mut sync = SHARED_DATA.mutex.lock().unwrap(); sync.thread_a_finished = true; diff --git a/tests/test_roots_work_factory.rs b/tests/test_roots_work_factory.rs index baf3adbf96a..697b3280143 100644 --- a/tests/test_roots_work_factory.rs +++ b/tests/test_roots_work_factory.rs @@ -6,6 +6,7 @@ use std::sync::{Arc, Mutex}; use mmtk::{ + scheduler::RootKind, util::{Address, ObjectReference}, vm::RootsWorkFactory, }; @@ -21,7 +22,7 @@ impl MockScanning { } fn mock_scan_roots(&self, mut factory: impl mmtk::vm::RootsWorkFactory
) { - factory.create_process_roots_work(self.roots.clone()); + factory.create_process_roots_work(self.roots.clone(), RootKind::Strong); } } @@ -42,7 +43,7 @@ struct MockFactory { } impl RootsWorkFactory
for MockFactory { - fn create_process_roots_work(&mut self, slots: Vec
) { + fn create_process_roots_work(&mut self, slots: Vec
, _kind: RootKind) { assert_eq!(slots, SLOTS); match self.round { 1 => {