From 5905662df0e070122355148cbd810bed85a39712 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 22 Feb 2021 22:11:54 +1100 Subject: [PATCH 001/494] Add immix allocator --- openjdk/mmtk.h | 1 + openjdk/mmtkBarrierSetAssembler_x86.cpp | 2 +- openjdk/mmtkBarrierSetC2.cpp | 2 +- openjdk/mmtkMutator.hpp | 21 +++++++++++++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 047ae650..3279d08d 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -48,6 +48,7 @@ struct AllocatorSelector { }; #define TAG_BUMP_POINTER 0 #define TAG_LARGE_OBJECT 1 +#define TAG_IMMIX 2 extern AllocatorSelector get_allocator_mapping(int allocator); diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index ace63230..3494b0a0 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -45,7 +45,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa AllocatorSelector selector = get_allocator_mapping(AllocatorDefault); // Only bump pointer allocator is implemented. - if (selector.tag != TAG_BUMP_POINTER) { + if (selector.tag != TAG_BUMP_POINTER && selector.tag != TAG_IMMIX) { fatal("unimplemented allocator fastpath\n"); } diff --git a/openjdk/mmtkBarrierSetC2.cpp b/openjdk/mmtkBarrierSetC2.cpp index 1eb76d02..847097f9 100644 --- a/openjdk/mmtkBarrierSetC2.cpp +++ b/openjdk/mmtkBarrierSetC2.cpp @@ -125,7 +125,7 @@ void MMTkBarrierSetC2::expand_allocate( AllocatorSelector selector = get_allocator_mapping(AllocatorDefault); // Only bump pointer allocator is implemented. - if (selector.tag != TAG_BUMP_POINTER) { + if (selector.tag != TAG_BUMP_POINTER && selector.tag != TAG_IMMIX) { fatal("unimplemented allocator fastpath\n"); } diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp index ac957247..7e812f68 100644 --- a/openjdk/mmtkMutator.hpp +++ b/openjdk/mmtkMutator.hpp @@ -21,6 +21,7 @@ struct RustDynPtr { // These constants should match the constants defind in mmtk::util::alloc::allocators const int MAX_BUMP_ALLOCATORS = 5; const int MAX_LARGE_OBJECT_ALLOCATORS = 1; +const int MAX_IMMIX_ALLOCATORS = 1; // The following types should have the same layout as the types with the same name in MMTk core (Rust) @@ -38,9 +39,29 @@ struct LargeObjectAllocator { RustDynPtr plan; }; +struct ImmixAllocator { + void* tls; + void* cursor; + void* limit; + RustDynPtr space; + RustDynPtr plan; + uint8_t hot; + uint8_t copy; + void* large_cursor; + void* large_limit; + uint8_t request_for_large; + uint8_t straddle; + uint32_t line_use_count; + void* mark_table; + void* recyclable_block; + uint32_t line; + uint8_t recyclable_exhausted; +}; + struct Allocators { BumpAllocator bump_pointer[MAX_BUMP_ALLOCATORS]; LargeObjectAllocator large_object[MAX_LARGE_OBJECT_ALLOCATORS]; + ImmixAllocator immix[MAX_IMMIX_ALLOCATORS]; }; struct MutatorConfig { From 9cc7b2a349572eb594b3fd56c8025c117c33fd62 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 28 Feb 2021 13:07:06 +1100 Subject: [PATCH 002/494] Reuse lines --- openjdk/mmtkMutator.hpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp index 7e812f68..f20cbbc2 100644 --- a/openjdk/mmtkMutator.hpp +++ b/openjdk/mmtkMutator.hpp @@ -51,10 +51,11 @@ struct ImmixAllocator { void* large_limit; uint8_t request_for_large; uint8_t straddle; - uint32_t line_use_count; + uintptr_t line_use_count; void* mark_table; + uint8_t recyclable_block_tag; void* recyclable_block; - uint32_t line; + uintptr_t line; uint8_t recyclable_exhausted; }; From 0a1a8a4ee03851949807e197b4efb62d0320589e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 28 Feb 2021 13:22:55 +1100 Subject: [PATCH 003/494] Refactor --- openjdk/mmtkMutator.hpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp index f20cbbc2..c9d6cb13 100644 --- a/openjdk/mmtkMutator.hpp +++ b/openjdk/mmtkMutator.hpp @@ -53,9 +53,10 @@ struct ImmixAllocator { uint8_t straddle; uintptr_t line_use_count; void* mark_table; - uint8_t recyclable_block_tag; - void* recyclable_block; - uintptr_t line; + uint8_t recyclable_block_opt_tag; + void* recyclable_block_opt; + uint8_t line_opt_tag; + uintptr_t line_opt; uint8_t recyclable_exhausted; }; From 95b62365dbdc206c7f4f2b4f71968e1767ef5da9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 28 Feb 2021 14:05:41 +1100 Subject: [PATCH 004/494] Fix allocation fastpath --- openjdk/mmtkBarrierSetAssembler_x86.cpp | 25 +++++++++++++------ openjdk/mmtkBarrierSetC2.cpp | 33 +++++++++++++++++-------- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index 3494b0a0..e7180b77 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -48,15 +48,26 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa if (selector.tag != TAG_BUMP_POINTER && selector.tag != TAG_IMMIX) { fatal("unimplemented allocator fastpath\n"); } - + // Calculat offsets of top and end. We now assume we are using bump pointer. - int allocator_base_offset = in_bytes(JavaThread::third_party_heap_mutator_offset()) - + in_bytes(byte_offset_of(MMTkMutatorContext, allocators)) - + in_bytes(byte_offset_of(Allocators, bump_pointer)) - + selector.index * sizeof(BumpAllocator); + int allocator_base_offset; + Address cursor, limit; - Address cursor = Address(r15_thread, allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, cursor))); - Address limit = Address(r15_thread, allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, limit))); + if (selector.tag == TAG_IMMIX) { + allocator_base_offset = in_bytes(JavaThread::third_party_heap_mutator_offset()) + + in_bytes(byte_offset_of(MMTkMutatorContext, allocators)) + + in_bytes(byte_offset_of(Allocators, immix)) + + selector.index * sizeof(ImmixAllocator); + cursor = Address(r15_thread, allocator_base_offset + in_bytes(byte_offset_of(ImmixAllocator, cursor))); + limit = Address(r15_thread, allocator_base_offset + in_bytes(byte_offset_of(ImmixAllocator, limit))); + } else { + allocator_base_offset = in_bytes(JavaThread::third_party_heap_mutator_offset()) + + in_bytes(byte_offset_of(MMTkMutatorContext, allocators)) + + in_bytes(byte_offset_of(Allocators, bump_pointer)) + + selector.index * sizeof(BumpAllocator); + cursor = Address(r15_thread, allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, cursor))); + limit = Address(r15_thread, allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, limit))); + } // obj = load lab.cursor __ movptr(obj, cursor); // end = obj + size diff --git a/openjdk/mmtkBarrierSetC2.cpp b/openjdk/mmtkBarrierSetC2.cpp index 847097f9..ac60b0fb 100644 --- a/openjdk/mmtkBarrierSetC2.cpp +++ b/openjdk/mmtkBarrierSetC2.cpp @@ -130,16 +130,29 @@ void MMTkBarrierSetC2::expand_allocate( } // Calculat offsets of top and end. We now assume we are using bump pointer. - int allocator_base_offset = in_bytes(JavaThread::third_party_heap_mutator_offset()) - + in_bytes(byte_offset_of(MMTkMutatorContext, allocators)) - + in_bytes(byte_offset_of(Allocators, bump_pointer)) - + selector.index * sizeof(BumpAllocator); - - Node* thread = x->transform_later(new ThreadLocalNode()); - int tlab_top_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, cursor)); - int tlab_end_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, limit)); - eden_top_adr = x->basic_plus_adr(x->top()/*not oop*/, thread, tlab_top_offset); - eden_end_adr = x->basic_plus_adr(x->top()/*not oop*/, thread, tlab_end_offset); + if (selector.tag == TAG_IMMIX) { + int allocator_base_offset = in_bytes(JavaThread::third_party_heap_mutator_offset()) + + in_bytes(byte_offset_of(MMTkMutatorContext, allocators)) + + in_bytes(byte_offset_of(Allocators, immix)) + + selector.index * sizeof(ImmixAllocator); + + Node* thread = x->transform_later(new ThreadLocalNode()); + int tlab_top_offset = allocator_base_offset + in_bytes(byte_offset_of(ImmixAllocator, cursor)); + int tlab_end_offset = allocator_base_offset + in_bytes(byte_offset_of(ImmixAllocator, limit)); + eden_top_adr = x->basic_plus_adr(x->top()/*not oop*/, thread, tlab_top_offset); + eden_end_adr = x->basic_plus_adr(x->top()/*not oop*/, thread, tlab_end_offset); + } else { + int allocator_base_offset = in_bytes(JavaThread::third_party_heap_mutator_offset()) + + in_bytes(byte_offset_of(MMTkMutatorContext, allocators)) + + in_bytes(byte_offset_of(Allocators, bump_pointer)) + + selector.index * sizeof(BumpAllocator); + + Node* thread = x->transform_later(new ThreadLocalNode()); + int tlab_top_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, cursor)); + int tlab_end_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, limit)); + eden_top_adr = x->basic_plus_adr(x->top()/*not oop*/, thread, tlab_top_offset); + eden_end_adr = x->basic_plus_adr(x->top()/*not oop*/, thread, tlab_end_offset); + } } // set_eden_pointers(eden_top_adr, eden_end_adr); From 577b4ebb208c2868f542ae4c058d5085910ab0b7 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 1 Mar 2021 10:44:50 +1100 Subject: [PATCH 005/494] update mmtk-core --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 8f9aec23..14f58272 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -21,7 +21,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "ssh://git@github.com/mmtk/mmtk-core.git", rev = "1c7d7abb8e2adc5a9f64326af2f5af6a5d335497" } +mmtk = { git = "ssh://git@github.com/mmtk/mmtk-core.git", rev = "bd75773a498f78330499899fac6484b223aecb92" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From b6e9d49e0fab10a265ea2a2db99fd108b7735c37 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 1 Mar 2021 11:12:42 +1100 Subject: [PATCH 006/494] Update mmtk-core --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 14f58272..05698e24 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -21,7 +21,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "ssh://git@github.com/mmtk/mmtk-core.git", rev = "bd75773a498f78330499899fac6484b223aecb92" } +mmtk = { git = "ssh://git@github.com/mmtk/mmtk-core.git", rev = "9cc0376156e4dc5ea748649eb962239422bb9c46" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From 552368eca461b38f02d3e1bf671b2036f30d342a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 10 Mar 2021 10:38:50 +1100 Subject: [PATCH 007/494] inline hot methods --- mmtk/src/object_model.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 099edbd4..fecd0322 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -52,10 +52,12 @@ impl ObjectModel for VMObjectModel { unimplemented!() } + #[inline(always)] fn object_start_ref(object: ObjectReference) -> Address { object.to_address() } + #[inline(always)] fn ref_to_address(object: ObjectReference) -> Address { object.to_address() } From 7f9d18275d4dc4b8a8641b647c4fd8be3517237a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 25 Mar 2021 18:55:27 +1100 Subject: [PATCH 008/494] Update --- openjdk/mmtkMutator.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp index c9d6cb13..321d46ca 100644 --- a/openjdk/mmtkMutator.hpp +++ b/openjdk/mmtkMutator.hpp @@ -43,7 +43,7 @@ struct ImmixAllocator { void* tls; void* cursor; void* limit; - RustDynPtr space; + void* immix_space; RustDynPtr plan; uint8_t hot; uint8_t copy; From 2336b1dfa7c4e2ffb59cdd14086e9f1f71b75b1e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 28 Mar 2021 12:14:58 +1100 Subject: [PATCH 009/494] Update --- mmtk/src/api.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index b292c583..5f2ca80a 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -1,5 +1,5 @@ use libc::{c_char, c_void}; -use mmtk::memory_manager; +use mmtk::{memory_manager, policy::immix::block::Block}; use mmtk::scheduler::GCWorker; use mmtk::util::alloc::allocators::AllocatorSelector; use mmtk::util::alloc::is_alloced_by_malloc; @@ -78,7 +78,13 @@ pub extern "C" fn alloc( offset: isize, allocator: AllocationSemantics, ) -> Address { - memory_manager::alloc::(unsafe { &mut *mutator }, size, align, offset, allocator) + if size > Block::BYTES { + let address = memory_manager::alloc::(unsafe { &mut *mutator }, size, align, offset, AllocationSemantics::Los); + memory_manager::post_alloc::(unsafe { &mut *mutator }, unsafe { address.to_object_reference() }, size, AllocationSemantics::Los); + address + } else { + memory_manager::alloc::(unsafe { &mut *mutator }, size, align, offset, allocator) + } } #[no_mangle] From e6c557071af3c91d26319d4663f634dca8c8d11f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 11 May 2021 11:06:03 +1000 Subject: [PATCH 010/494] Move post_alloc to rust --- mmtk/src/api.rs | 14 +++++++------- openjdk/CompileThirdPartyHeap.gmk | 4 ---- openjdk/mmtkMutator.cpp | 3 --- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 5f2ca80a..efa2bfae 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -78,13 +78,13 @@ pub extern "C" fn alloc( offset: isize, allocator: AllocationSemantics, ) -> Address { - if size > Block::BYTES { - let address = memory_manager::alloc::(unsafe { &mut *mutator }, size, align, offset, AllocationSemantics::Los); - memory_manager::post_alloc::(unsafe { &mut *mutator }, unsafe { address.to_object_reference() }, size, AllocationSemantics::Los); - address - } else { - memory_manager::alloc::(unsafe { &mut *mutator }, size, align, offset, allocator) - } + let mutator = unsafe { &mut *mutator }; + let allocator = mutator.check_allocator(size, align, allocator); + let address = memory_manager::alloc::(mutator, size, align, offset, allocator); + // Post allococation. Currently we are only calling post_alloc in slowpath here. + // TODO: We also need to call them in the fastpath. + post_alloc(mutator, unsafe { address.to_object_reference() }, size, allocator); + address } #[no_mangle] diff --git a/openjdk/CompileThirdPartyHeap.gmk b/openjdk/CompileThirdPartyHeap.gmk index f6cdbde8..a310dbd7 100644 --- a/openjdk/CompileThirdPartyHeap.gmk +++ b/openjdk/CompileThirdPartyHeap.gmk @@ -2,10 +2,6 @@ MMTK_RUST_ROOT = $(TOPDIR)/../../mmtk MMTK_CPP_ROOT = $(TOPDIR)/../../openjdk -ifdef MMTK_PLAN - GC_FEATURES=--features $(MMTK_PLAN) -endif - LIB_MMTK := $(JVM_LIB_OUTPUTDIR)/libmmtk_openjdk.so ifeq ($(DEBUG_LEVEL), release) diff --git a/openjdk/mmtkMutator.cpp b/openjdk/mmtkMutator.cpp index b2fedaec..05c799ff 100644 --- a/openjdk/mmtkMutator.cpp +++ b/openjdk/mmtkMutator.cpp @@ -9,9 +9,6 @@ MMTkMutatorContext MMTkMutatorContext::bind(::Thread* current) { HeapWord* MMTkMutatorContext::alloc(size_t bytes, Allocator allocator) { // FIXME: Proper use of slow-path api HeapWord* o = (HeapWord*) ::alloc((MMTk_Mutator) this, bytes, HeapWordSize, 0, allocator); - // Post allococation. Currently we are only calling post_alloc in slowpath here. - // TODO: We also need to call them in the fastpath. - ::post_alloc((MMTk_Mutator) this, o, bytes, allocator); return o; } From 549a09911cc0912a4ad8156a6c4caedecfc0a9ec Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 13 May 2021 13:17:47 +1000 Subject: [PATCH 011/494] Cleanup --- mmtk/src/api.rs | 2 +- mmtk/src/object_model.rs | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 41d9196c..852619d3 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -3,7 +3,7 @@ use crate::OpenJDK_Upcalls; use crate::SINGLETON; use crate::UPCALLS; use libc::{c_char, c_void}; -use mmtk::{memory_manager, policy::immix::block::Block}; +use mmtk::memory_manager; use mmtk::plan::barriers::BarrierSelector; use mmtk::scheduler::GCWorker; use mmtk::util::alloc::allocators::AllocatorSelector; diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 95e4b861..428b7a4f 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -48,12 +48,10 @@ impl ObjectModel for VMObjectModel { unimplemented!() } - #[inline(always)] fn object_start_ref(object: ObjectReference) -> Address { object.to_address() } - #[inline(always)] fn ref_to_address(object: ObjectReference) -> Address { object.to_address() } From 15d961ea2024dda312e4436ad1db9a9c5032c7b9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 26 May 2021 10:36:15 +1000 Subject: [PATCH 012/494] Fix --- mmtk/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 6124b0ea..d3ce5f05 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -1,7 +1,6 @@ // specialization is considered as an incomplete feature. #![allow(incomplete_features)] #![feature(specialization)] -#![feature(const_fn)] #![feature(box_syntax)] #![feature(vec_into_raw_parts)] #![feature(once_cell)] From 877e884051b42ffd7be471466df4f84cd9ea941f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 28 Jun 2021 15:04:36 +1000 Subject: [PATCH 013/494] Update mmtk-core --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index e0b1d7b3..98071865 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -20,7 +20,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "a8499062f9df749a6cf7acf1e626d642913206bb" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "0a4e5cd6fd81688c44c1f2e189da8e53c4075acd" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From e1bcc9299b9e0a8736de7e076253801e385e85e2 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 9 Jul 2021 12:09:46 +1000 Subject: [PATCH 014/494] Update mmtk-core --- mmtk/Cargo.toml | 2 +- mmtk/src/vm_metadata/constants.rs | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 1737fb5c..d6eb5aec 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -20,7 +20,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "1e7964656eb0988e3fa4f216c5f8dfb1dba7b31f" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "76c2fa091f18f842680c32a6140c19b1cbe02d4d" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } diff --git a/mmtk/src/vm_metadata/constants.rs b/mmtk/src/vm_metadata/constants.rs index e24a199a..902cc13a 100644 --- a/mmtk/src/vm_metadata/constants.rs +++ b/mmtk/src/vm_metadata/constants.rs @@ -1,6 +1,3 @@ -use mmtk::util::metadata::side_metadata::{ - GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS, LOCAL_SIDE_METADATA_VM_BASE_ADDRESS, -}; use mmtk::vm::*; #[cfg(target_pointer_width = "64")] @@ -15,7 +12,7 @@ pub(crate) const FORWARDING_POINTER_OFFSET: isize = 0; /// Global logging bit metadata spec /// 1 bit per object pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = - VMGlobalLogBitSpec::side(GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize()); + VMGlobalLogBitSpec::side_first(); // Global MetadataSpecs - End @@ -39,10 +36,6 @@ pub(crate) const MARKING_METADATA_SPEC: VMLocalMarkBitSpec = /// PolicySpecific mark-and-nursery bits metadata spec /// 2-bits per object pub(crate) const LOS_METADATA_SPEC: VMLocalLOSMarkNurserySpec = - VMLocalLOSMarkNurserySpec::side(if cfg!(target_pointer_width = "64") { - LOCAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize() - } else { - 0 - }); + VMLocalLOSMarkNurserySpec::side_first(); // PolicySpecific MetadataSpecs - End From 8290738025a992503c41f7ede228d534b7ffacdd Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 9 Jul 2021 14:42:37 +1000 Subject: [PATCH 015/494] cargo fmt --- mmtk/src/vm_metadata/constants.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mmtk/src/vm_metadata/constants.rs b/mmtk/src/vm_metadata/constants.rs index 902cc13a..61ee794d 100644 --- a/mmtk/src/vm_metadata/constants.rs +++ b/mmtk/src/vm_metadata/constants.rs @@ -11,8 +11,7 @@ pub(crate) const FORWARDING_POINTER_OFFSET: isize = 0; /// Global logging bit metadata spec /// 1 bit per object -pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = - VMGlobalLogBitSpec::side_first(); +pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); // Global MetadataSpecs - End From ba8ee0624b2ad6adf238c329e98cd309d51bad29 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 16 Jul 2021 11:15:48 +1000 Subject: [PATCH 016/494] Fix after rebase --- mmtk/src/api.rs | 8 +------- openjdk/mmtkMutator.cpp | 3 +++ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index e5eabe71..8ac51d64 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -83,13 +83,7 @@ pub extern "C" fn alloc( offset: isize, allocator: AllocationSemantics, ) -> Address { - let mutator = unsafe { &mut *mutator }; - let allocator = mutator.check_allocator(size, align, allocator); - let address = memory_manager::alloc::(mutator, size, align, offset, allocator); - // Post allococation. Currently we are only calling post_alloc in slowpath here. - // TODO: We also need to call them in the fastpath. - post_alloc(mutator, unsafe { address.to_object_reference() }, size, allocator); - address + memory_manager::alloc::(unsafe { &mut *mutator }, size, align, offset, allocator) } #[no_mangle] diff --git a/openjdk/mmtkMutator.cpp b/openjdk/mmtkMutator.cpp index 3fc15160..9997322a 100644 --- a/openjdk/mmtkMutator.cpp +++ b/openjdk/mmtkMutator.cpp @@ -18,6 +18,9 @@ HeapWord* MMTkMutatorContext::alloc(size_t bytes, Allocator allocator) { // FIXME: Proper use of slow-path api HeapWord* o = (HeapWord*) ::alloc((MMTk_Mutator) this, bytes, HeapWordSize, 0, allocator); + // Post allococation. Currently we are only calling post_alloc in slowpath here. + // TODO: We also need to call them in the fastpath. + ::post_alloc((MMTk_Mutator) this, o, bytes, allocator); return o; } From 41c8b45f5d204221b8a8e966bf50bd09dbe9cfc5 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 16 Jul 2021 12:39:11 +1000 Subject: [PATCH 017/494] extract common code --- openjdk/mmtkBarrierSetC2.cpp | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/openjdk/mmtkBarrierSetC2.cpp b/openjdk/mmtkBarrierSetC2.cpp index e0204f1d..d5418864 100644 --- a/openjdk/mmtkBarrierSetC2.cpp +++ b/openjdk/mmtkBarrierSetC2.cpp @@ -182,29 +182,25 @@ void MMTkBarrierSetC2::expand_allocate( } // Calculat offsets of top and end. We now assume we are using bump pointer. + int allocators_base_offset = in_bytes(JavaThread::third_party_heap_mutator_offset()) + + in_bytes(byte_offset_of(MMTkMutatorContext, allocators)); + int tlab_top_offset, tlab_end_offset; if (selector.tag == TAG_IMMIX) { - int allocator_base_offset = in_bytes(JavaThread::third_party_heap_mutator_offset()) - + in_bytes(byte_offset_of(MMTkMutatorContext, allocators)) + int allocator_base_offset = allocators_base_offset + in_bytes(byte_offset_of(Allocators, immix)) + selector.index * sizeof(ImmixAllocator); - - Node* thread = x->transform_later(new ThreadLocalNode()); - int tlab_top_offset = allocator_base_offset + in_bytes(byte_offset_of(ImmixAllocator, cursor)); - int tlab_end_offset = allocator_base_offset + in_bytes(byte_offset_of(ImmixAllocator, limit)); - eden_top_adr = x->basic_plus_adr(x->top()/*not oop*/, thread, tlab_top_offset); - eden_end_adr = x->basic_plus_adr(x->top()/*not oop*/, thread, tlab_end_offset); + tlab_top_offset = allocator_base_offset + in_bytes(byte_offset_of(ImmixAllocator, cursor)); + tlab_end_offset = allocator_base_offset + in_bytes(byte_offset_of(ImmixAllocator, limit)); } else { - int allocator_base_offset = in_bytes(JavaThread::third_party_heap_mutator_offset()) - + in_bytes(byte_offset_of(MMTkMutatorContext, allocators)) + int allocator_base_offset = allocators_base_offset + in_bytes(byte_offset_of(Allocators, bump_pointer)) + selector.index * sizeof(BumpAllocator); - - Node* thread = x->transform_later(new ThreadLocalNode()); - int tlab_top_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, cursor)); - int tlab_end_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, limit)); - eden_top_adr = x->basic_plus_adr(x->top()/*not oop*/, thread, tlab_top_offset); - eden_end_adr = x->basic_plus_adr(x->top()/*not oop*/, thread, tlab_end_offset); + tlab_top_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, cursor)); + tlab_end_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, limit)); } + Node* thread = x->transform_later(new ThreadLocalNode()); + eden_top_adr = x->basic_plus_adr(x->top()/*not oop*/, thread, tlab_top_offset); + eden_end_adr = x->basic_plus_adr(x->top()/*not oop*/, thread, tlab_end_offset); } // set_eden_pointers(eden_top_adr, eden_end_adr); From bfa0e904e804aeafbe1e9bf1e77365a06bf1deb0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 16 Jul 2021 12:41:09 +1000 Subject: [PATCH 018/494] Update mmtk --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 0a120c33..07b2ecb2 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -20,7 +20,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "0875c9084fc11114f122f4f4d59d27f2ceeae07b" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "d8e120b191e3ca7a473a08eeefa9d6050ffc8a35" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From 198f0af97cd5870d3cd2e3e3e7565c8a19731dff Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 18 Jul 2021 17:38:42 +1000 Subject: [PATCH 019/494] barrier slow call --- mmtk/src/api.rs | 21 ++++++++++++--------- openjdk/CompileThirdPartyHeap.gmk | 2 +- openjdk/barriers/mmtkObjectBarrier.cpp | 24 ++++++++++++++---------- openjdk/barriers/mmtkObjectBarrier.hpp | 8 ++++---- openjdk/mmtk.h | 2 +- openjdk/mmtkBarrierSet.cpp | 6 +++--- openjdk/mmtkBarrierSet.hpp | 14 ++++++++------ 7 files changed, 43 insertions(+), 34 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 8ac51d64..e61fdc66 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -22,13 +22,14 @@ static OBJECT_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("Object #[no_mangle] pub extern "C" fn mmtk_active_barrier() -> *const c_char { - match SINGLETON.get_plan().constraints().barrier { - BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), - BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), - // In case we have more barriers in mmtk-core. - #[allow(unreachable_patterns)] - _ => unimplemented!(), - } + // match SINGLETON.get_plan().constraints().barrier { + // BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), + // BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), + // // In case we have more barriers in mmtk-core. + // #[allow(unreachable_patterns)] + // _ => unimplemented!(), + // } + OBJECT_BARRIER.as_ptr() } /// # Safety @@ -240,9 +241,11 @@ pub extern "C" fn executable() -> bool { #[no_mangle] pub extern "C" fn record_modified_node( mutator: &'static mut Mutator, - obj: ObjectReference, + src: ObjectReference, + slot: Address, + val: ObjectReference, ) { - mutator.record_modified_node(obj); + mutator.record_modified_node(src, slot, val); } // finalization diff --git a/openjdk/CompileThirdPartyHeap.gmk b/openjdk/CompileThirdPartyHeap.gmk index a310dbd7..d3bc9e11 100644 --- a/openjdk/CompileThirdPartyHeap.gmk +++ b/openjdk/CompileThirdPartyHeap.gmk @@ -18,7 +18,7 @@ else endif $(LIB_MMTK): FORCE - cargo $(CARGO_VERSION) build --manifest-path=$(MMTK_RUST_ROOT)/Cargo.toml $(CARGO_PROFILE_FLAG) $(GC_FEATURES) + cargo build --manifest-path=$(MMTK_RUST_ROOT)/Cargo.toml $(CARGO_PROFILE_FLAG) $(GC_FEATURES) cp $(MMTK_RUST_ROOT)/target/$(CARGO_PROFILE)/libmmtk_openjdk.so $(LIB_MMTK) JVM_LIBS += -L$(JVM_LIB_OUTPUTDIR) -lmmtk_openjdk diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 81c09350..f0f22dab 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -1,11 +1,11 @@ #include "mmtkObjectBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" -void MMTkObjectBarrierSetRuntime::record_modified_node_slow(void* obj) { - ::record_modified_node((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) obj); +void MMTkObjectBarrierSetRuntime::record_modified_node_slow(void* src, void* slot, void* val) { + ::record_modified_node((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, val); } -void MMTkObjectBarrierSetRuntime::record_modified_node(oop src) { +void MMTkObjectBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset, oop val) { #if MMTK_ENABLE_OBJECT_BARRIER_FASTPATH intptr_t addr = (intptr_t) (void*) src; uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); @@ -15,7 +15,7 @@ void MMTkObjectBarrierSetRuntime::record_modified_node(oop src) { record_modified_node_slow((void*) src); } #else - record_modified_node_slow((void*) src); + record_modified_node_slow((void*) src, (void*) (((intptr_t) (void*) src) + offset), (void*) val); #endif } @@ -31,12 +31,12 @@ void MMTkObjectBarrierSetAssembler::oop_store_at(MacroAssembler* masm, Decorator return; } - BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); + record_modified_node(masm, dst, val, tmp1, tmp2); - record_modified_node(masm, dst.base(), tmp1, tmp2); + BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); } -void MMTkObjectBarrierSetAssembler::record_modified_node(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) { +void MMTkObjectBarrierSetAssembler::record_modified_node(MacroAssembler* masm, Address dst, Register val, Register tmp1, Register tmp2) { #if MMTK_ENABLE_OBJECT_BARRIER_FASTPATH Label done; @@ -70,9 +70,13 @@ void MMTkObjectBarrierSetAssembler::record_modified_node(MacroAssembler* masm, R __ bind(done); #else - assert_different_registers(c_rarg0, obj); - __ movptr(c_rarg0, obj); - __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), 1); + // assert_different_registers(c_rarg0, val); + __ pusha(); + __ movptr(c_rarg0, dst.base()); + __ lea(c_rarg1, dst); + __ movptr(c_rarg2, val); + __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), 3); + __ popa(); #endif } diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index e3346b3b..2ea97ca7 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -12,7 +12,7 @@ #include "../mmtkBarrierSetC1.hpp" #include "../mmtkBarrierSetC2.hpp" -#define MMTK_ENABLE_OBJECT_BARRIER_FASTPATH true +#define MMTK_ENABLE_OBJECT_BARRIER_FASTPATH false #define SIDE_METADATA_WORST_CASE_RATIO_LOG 1 #define LOG_BYTES_IN_CHUNK 22 @@ -22,13 +22,13 @@ const intptr_t SIDE_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_B class MMTkObjectBarrierSetRuntime: public MMTkBarrierSetRuntime { public: - static void record_modified_node_slow(void* src); + static void record_modified_node_slow(void* src, void* slot, void* val); virtual bool is_slow_path_call(address call) { return call == CAST_FROM_FN_PTR(address, record_modified_node_slow); } - virtual void record_modified_node(oop src); + virtual void record_modified_node(oop src, ptrdiff_t offset, oop val); }; class MMTkObjectBarrierSetC1; @@ -36,7 +36,7 @@ class MMTkObjectBarrierStub; class MMTkObjectBarrierSetAssembler: public MMTkBarrierSetAssembler { void oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2); - void record_modified_node(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2); + void record_modified_node(MacroAssembler* masm, Address dst, Register val, Register tmp1, Register tmp2); public: virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { if (type == T_OBJECT || type == T_ARRAY) { diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index e8ad38d5..467ecfe3 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -34,7 +34,7 @@ extern void* alloc_slow_largeobject(MMTk_Mutator mutator, size_t size, extern void post_alloc(MMTk_Mutator mutator, void* refer, int bytes, int allocator); -extern void record_modified_node(MMTk_Mutator mutator, void* obj); +extern void record_modified_node(MMTk_Mutator mutator, void* src, void* slot, void* val); extern void release_buffer(void** buffer, size_t len, size_t cap); diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index 6445352d..22ebae1b 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -68,15 +68,15 @@ void MMTkBarrierSet::resize_covered_region(MemRegion new_region) { void MMTkBarrierSet::on_thread_destroy(Thread* thread) { - thread->third_party_heap_mutator.flush(); + // thread->third_party_heap_mutator.flush(); } void MMTkBarrierSet::on_thread_attach(JavaThread* thread) { - thread->third_party_heap_mutator.flush(); + // thread->third_party_heap_mutator.flush(); } void MMTkBarrierSet::on_thread_detach(JavaThread* thread) { - thread->third_party_heap_mutator.flush(); + // thread->third_party_heap_mutator.flush(); } diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index a572a9c4..0511bc29 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -40,7 +40,7 @@ class MMTkBarrierSetRuntime: public CHeapObj { public: - virtual void record_modified_node(oop object) {}; + virtual void record_modified_node(oop src, ptrdiff_t offset, oop val) {}; virtual bool is_slow_path_call(address call) { return false; } @@ -128,8 +128,8 @@ class MMTkBarrierSet : public BarrierSet { } static void oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) { + runtime()->record_modified_node(base, offset, value); Raw::oop_store_at(base, offset, value); - runtime()->record_modified_node(base); } template @@ -139,8 +139,8 @@ class MMTkBarrierSet : public BarrierSet { } static oop oop_atomic_cmpxchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset, oop compare_value) { + runtime()->record_modified_node(base, offset, new_value); oop result = Raw::oop_atomic_cmpxchg_at(new_value, base, offset, compare_value); - runtime()->record_modified_node(base); return result; } @@ -151,8 +151,8 @@ class MMTkBarrierSet : public BarrierSet { } static oop oop_atomic_xchg_in_heap_at(oop new_value, oop base, ptrdiff_t offset) { + runtime()->record_modified_node(base, offset, new_value); oop result = Raw::oop_atomic_xchg_at(new_value, base, offset); - runtime()->record_modified_node(base); return result; } @@ -160,16 +160,18 @@ class MMTkBarrierSet : public BarrierSet { static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) { + // for () { + // runtime()->record_modified_node((oop) dst_obj, i, ); + // } bool result = Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); - runtime()->record_modified_node((oop) dst_obj); return result; } static void clone_in_heap(oop src, oop dst, size_t size) { Raw::clone(src, dst, size); - runtime()->record_modified_node(dst); + // runtime()->record_modified_node(dst); } }; From 38b22f04b81e26e280b8c9d72edc4ece2c5e5ad8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 21 Jul 2021 10:58:19 +1000 Subject: [PATCH 020/494] WIP: Can pass one concurrent GC --- mmtk/src/collection.rs | 7 +++++++ mmtk/src/lib.rs | 1 + mmtk/src/scanning.rs | 2 +- openjdk/mmtk.h | 1 + openjdk/mmtkBarrierSet.hpp | 9 +++++++-- openjdk/mmtkUpcalls.cpp | 7 +++++++ 6 files changed, 24 insertions(+), 3 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 81b63269..b6fbd962 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -33,6 +33,13 @@ impl Collection for VMCollection { } } + + fn stop_all_mutators2(tls: VMWorkerThread) { + unsafe { + ((*UPCALLS).stop_mutators)(tls); + } + } + fn resume_mutators(tls: VMWorkerThread) { unsafe { ((*UPCALLS).resume_mutators)(tls); diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index bc328de1..41ffacc7 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -81,6 +81,7 @@ pub struct OpenJDK_Upcalls { pub scan_vm_thread_roots: extern "C" fn(process_edges: ProcessEdgesFn), pub number_of_mutators: extern "C" fn() -> usize, pub schedule_finalizer: extern "C" fn(), + pub stop_mutators: extern "C" fn(tls: VMWorkerThread), } pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index be874a84..3aa7470f 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -21,7 +21,7 @@ pub(crate) extern "C" fn create_process_edges_work::from_raw_parts(ptr, length, capacity) }; memory_manager::add_work_packet( &SINGLETON, - WorkBucketStage::Closure, + WorkBucketStage::Prepare, W::new(buf, false, &SINGLETON), ); } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 467ecfe3..78cbdb94 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -131,6 +131,7 @@ typedef struct { void (*scan_vm_thread_roots) (ProcessEdgesFn process_edges); size_t (*number_of_mutators)(); void (*schedule_finalizer)(); + void (*mmtk_stop_mutators)(void *tls); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls, size_t heap_size); diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 0511bc29..a3b5cc93 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -160,9 +160,14 @@ class MMTkBarrierSet : public BarrierSet { static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) { - // for () { - // runtime()->record_modified_node((oop) dst_obj, i, ); + // T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); + // T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); + // char* base = reinterpret_cast((void*) src_obj); + // for (const T* const end = src + length; src < end; src++, dst++) { + // int offset = reinterpret_cast((void*) src) - base; + // runtime()->record_modified_node(dst_obj, offset, (oop) *dst); // } + bool result = Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index a11e186f..bbbca076 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -53,6 +53,12 @@ static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(voi SafepointSynchronize::begin(); } +static void mmtk_stop_mutators(void *tls) { + gcInProgress = true; + MMTkHeap::_create_stack_scan_work = NULL; + SafepointSynchronize::begin(); +} + static void mmtk_resume_mutators(void *tls) { MMTkHeap::_create_stack_scan_work = NULL; SafepointSynchronize::end(); @@ -291,4 +297,5 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_scan_vm_thread_roots, mmtk_number_of_mutators, mmtk_schedule_finalizer, + mmtk_stop_mutators, }; \ No newline at end of file From 946a6052bf0c5612a58ea28ff4ce76209390f752 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 21 Jul 2021 14:01:31 +1000 Subject: [PATCH 021/494] WIP --- openjdk/mmtkBarrierSet.cpp | 6 +++--- openjdk/mmtkBarrierSet.hpp | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index 22ebae1b..6445352d 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -68,15 +68,15 @@ void MMTkBarrierSet::resize_covered_region(MemRegion new_region) { void MMTkBarrierSet::on_thread_destroy(Thread* thread) { - // thread->third_party_heap_mutator.flush(); + thread->third_party_heap_mutator.flush(); } void MMTkBarrierSet::on_thread_attach(JavaThread* thread) { - // thread->third_party_heap_mutator.flush(); + thread->third_party_heap_mutator.flush(); } void MMTkBarrierSet::on_thread_detach(JavaThread* thread) { - // thread->third_party_heap_mutator.flush(); + thread->third_party_heap_mutator.flush(); } diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index a3b5cc93..d3f59faa 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -156,6 +156,23 @@ class MMTkBarrierSet : public BarrierSet { return result; } + static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, + arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, + size_t length) { + oop* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); + oop* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); + char* base = reinterpret_cast((void*) src_obj); + for (const oop* const end = src + length; src < end; src++, dst++) { + int offset = reinterpret_cast((void*) src) - base; + runtime()->record_modified_node(dst_obj, offset, (oop) *dst); + } + + bool result = Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, + dst_obj, dst_offset_in_bytes, dst_raw, + length); + return result; + } + template static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, From c57950172ee813268e05d4cd45e981fc2a6ed395 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 21 Jul 2021 14:35:02 +1000 Subject: [PATCH 022/494] Revert changes for makr bit in_header() --- mmtk/src/vm_metadata/constants.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/vm_metadata/constants.rs b/mmtk/src/vm_metadata/constants.rs index 18e13648..12c5742a 100644 --- a/mmtk/src/vm_metadata/constants.rs +++ b/mmtk/src/vm_metadata/constants.rs @@ -31,7 +31,7 @@ pub(crate) const FORWARDING_BITS_METADATA_SPEC: VMLocalForwardingBitsSpec = /// 1 bit per object #[cfg(feature = "mark_bit_in_header")] pub(crate) const MARKING_METADATA_SPEC: VMLocalMarkBitSpec = - VMLocalMarkBitSpec::in_header(); + VMLocalMarkBitSpec::in_header(FORWARDING_BITS_OFFSET); #[cfg(not(feature = "mark_bit_in_header"))] pub(crate) const MARKING_METADATA_SPEC: VMLocalMarkBitSpec = From 1c719316ae011576359ec9eff61a3e0e8f64522c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 21 Jul 2021 16:05:39 +1000 Subject: [PATCH 023/494] Use side mark bits --- mmtk/src/vm_metadata/constants.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mmtk/src/vm_metadata/constants.rs b/mmtk/src/vm_metadata/constants.rs index e24a199a..831aebb9 100644 --- a/mmtk/src/vm_metadata/constants.rs +++ b/mmtk/src/vm_metadata/constants.rs @@ -1,3 +1,4 @@ +use mmtk::util::metadata::MetadataSpec; use mmtk::util::metadata::side_metadata::{ GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS, LOCAL_SIDE_METADATA_VM_BASE_ADDRESS, }; @@ -34,7 +35,10 @@ pub(crate) const FORWARDING_BITS_METADATA_SPEC: VMLocalForwardingBitsSpec = /// PolicySpecific mark bit metadata spec /// 1 bit per object pub(crate) const MARKING_METADATA_SPEC: VMLocalMarkBitSpec = - VMLocalMarkBitSpec::in_header(FORWARDING_BITS_OFFSET); + VMLocalMarkBitSpec::side(match LOS_METADATA_SPEC.get_spec() { + MetadataSpec::OnSide(side) => side.accumulated_size(), + _ => LOCAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize() + }); /// PolicySpecific mark-and-nursery bits metadata spec /// 2-bits per object From 52fe54bf75bdda48f20b14f37689ae1d1d216bce Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 22 Jul 2021 16:16:47 +1200 Subject: [PATCH 024/494] Update Cargo.toml --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 9aa90d67..d255ffad 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -20,7 +20,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "40c0a8ea0bccd2a123c3025887e5c6820fa50ea4" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "f50ac55df1b4819a0e2a412c89038d17cfcdc666" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From c728a06565954a8a0715715d30e53a8c910740f7 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 22 Jul 2021 14:35:36 +1000 Subject: [PATCH 025/494] Update mmtk --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 8adaae7d..118ab277 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -20,7 +20,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "d8e120b191e3ca7a473a08eeefa9d6050ffc8a35" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "7b3672d45f8fe659992ab6cc8724f27aa2e5f0fa" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From c442877b84ca35d265e1061bcf54df82379c9677 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 23 Jul 2021 12:26:37 +1000 Subject: [PATCH 026/494] Update mmtk-core --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 118ab277..b9d961d6 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -20,7 +20,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "7b3672d45f8fe659992ab6cc8724f27aa2e5f0fa" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "b2106d2b4df0f4c62e9a7af24b3a5df20819159d" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From 73591374439f821ec3999270e4add4d682ac4769 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 25 Jul 2021 12:48:15 +1000 Subject: [PATCH 027/494] Fix barrier call --- openjdk/barriers/mmtkObjectBarrier.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index f0f22dab..0acbb617 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -26,7 +26,7 @@ void MMTkObjectBarrierSetAssembler::oop_store_at(MacroAssembler* masm, Decorator bool as_normal = (decorators & AS_NORMAL) != 0; assert((decorators & IS_DEST_UNINITIALIZED) == 0, "unsupported"); - if (!in_heap || val == noreg) { + if (!in_heap) { BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); return; } @@ -74,7 +74,11 @@ void MMTkObjectBarrierSetAssembler::record_modified_node(MacroAssembler* masm, A __ pusha(); __ movptr(c_rarg0, dst.base()); __ lea(c_rarg1, dst); - __ movptr(c_rarg2, val); + if (val == noreg) { + __ movptr(c_rarg2, (int32_t) NULL_WORD); + } else { + __ movptr(c_rarg2, val); + } __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), 3); __ popa(); #endif From fd1f1ffe8fe7989c27d13ae93284f019519150fe Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 28 Jul 2021 11:44:43 +1000 Subject: [PATCH 028/494] Add barriers ? --- openjdk/mmtkBarrierSet.hpp | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index d3f59faa..e72dec63 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -156,13 +156,14 @@ class MMTkBarrierSet : public BarrierSet { return result; } - static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, - arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, + template + static bool oop_arraycopy_in_heap_impl(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, + arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) { - oop* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); - oop* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); + T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); + T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); char* base = reinterpret_cast((void*) src_obj); - for (const oop* const end = src + length; src < end; src++, dst++) { + for (const T* const end = src + length; src < end; src++, dst++) { int offset = reinterpret_cast((void*) src) - base; runtime()->record_modified_node(dst_obj, offset, (oop) *dst); } @@ -173,6 +174,33 @@ class MMTkBarrierSet : public BarrierSet { return result; } + + static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, + arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, + size_t length) { + return oop_arraycopy_in_heap_impl(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); + } + static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, arrayOop* src_raw, + arrayOop dst_obj, size_t dst_offset_in_bytes, arrayOop* dst_raw, + size_t length) { + return oop_arraycopy_in_heap_impl(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); + } + static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, instanceOop* src_raw, + arrayOop dst_obj, size_t dst_offset_in_bytes, instanceOop* dst_raw, + size_t length) { + return oop_arraycopy_in_heap_impl(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); + } + static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, objArrayOop* src_raw, + arrayOop dst_obj, size_t dst_offset_in_bytes, objArrayOop* dst_raw, + size_t length) { + return oop_arraycopy_in_heap_impl(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); + } + static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, typeArrayOop* src_raw, + arrayOop dst_obj, size_t dst_offset_in_bytes, typeArrayOop* dst_raw, + size_t length) { + return oop_arraycopy_in_heap_impl(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); + } + template static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, From a178535cfdd5045b8cbf56ab74976c6856e174d4 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 28 Jul 2021 21:14:47 +1000 Subject: [PATCH 029/494] Remove unused fields --- mmtk/Cargo.toml | 2 +- openjdk/mmtkMutator.hpp | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index b9d961d6..d2d0bd56 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -20,7 +20,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "b2106d2b4df0f4c62e9a7af24b3a5df20819159d" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "2a9b8ca2c30a58c6d490e0defb1ed125d79e49e9" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp index b4afe3f7..410499e0 100644 --- a/openjdk/mmtkMutator.hpp +++ b/openjdk/mmtkMutator.hpp @@ -51,14 +51,9 @@ struct ImmixAllocator { void* large_cursor; void* large_limit; uint8_t request_for_large; - uint8_t straddle; - uintptr_t line_use_count; - void* mark_table; - uint8_t recyclable_block_opt_tag; - void* recyclable_block_opt; + uint8_t _align[7]; uint8_t line_opt_tag; uintptr_t line_opt; - uint8_t recyclable_exhausted; }; struct MallocAllocator { From 2ded73910ae7cc061d75f3f55d5801b0bb5a14ec Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 31 Jul 2021 22:35:02 +1000 Subject: [PATCH 030/494] Fix multiple worker threads error --- mmtk/src/scanning.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 3aa7470f..cf2742df 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -21,7 +21,7 @@ pub(crate) extern "C" fn create_process_edges_work::from_raw_parts(ptr, length, capacity) }; memory_manager::add_work_packet( &SINGLETON, - WorkBucketStage::Prepare, + WorkBucketStage::ForwardRoots, W::new(buf, false, &SINGLETON), ); } From ecb863e035070c947ded34ab0aa0d7b1e97e96b1 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 2 Aug 2021 09:48:50 +1000 Subject: [PATCH 031/494] Add ci tests --- .github/scripts/ci-test.sh | 18 ++++++++++++++++++ mmtk/Cargo.toml | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/scripts/ci-test.sh b/.github/scripts/ci-test.sh index 656b4c2c..f856db55 100755 --- a/.github/scripts/ci-test.sh +++ b/.github/scripts/ci-test.sh @@ -116,3 +116,21 @@ build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHea build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar benchmarks/dacapo-2006-10-MR2.jar fop build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar benchmarks/dacapo-2006-10-MR2.jar luindex # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar benchmarks/dacapo-2006-10-MR2.jar pmd + +# --- Immix --- +export MMTK_PLAN=Immix + +# Test - the benchmarks that are commented out do not work yet +# Note: the command line options are necessary for now to ensure the benchmarks work. We may later change the options if we do not have these many constraints. +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar antlr +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar bloat - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar fop +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar jython - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar luindex +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar lusearch - validation failed +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar pmd +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar xalan - mmtk-core gets stuck in slowdebug build + +# These benchmarks take 40s+ for slowdebug build, we may consider removing them from the CI +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar hsqldb +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar eclipse diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 85a7c26a..0c723f17 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -20,7 +20,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "d4fd042975f3278534997ff1652d63c235d32f29" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "f11745df2e79a52f6e0ab7b152692da61c827e7a" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From e5632d76de258c2f89ade6b87be0b41ab8f8fd43 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 3 Aug 2021 10:04:53 +1000 Subject: [PATCH 032/494] Fix MetadataSpec api --- mmtk/src/vm_metadata/constants.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mmtk/src/vm_metadata/constants.rs b/mmtk/src/vm_metadata/constants.rs index c1796205..12c5742a 100644 --- a/mmtk/src/vm_metadata/constants.rs +++ b/mmtk/src/vm_metadata/constants.rs @@ -31,10 +31,7 @@ pub(crate) const FORWARDING_BITS_METADATA_SPEC: VMLocalForwardingBitsSpec = /// 1 bit per object #[cfg(feature = "mark_bit_in_header")] pub(crate) const MARKING_METADATA_SPEC: VMLocalMarkBitSpec = - VMLocalMarkBitSpec::side(match LOS_METADATA_SPEC.get_spec() { - MetadataSpec::OnSide(side) => side.accumulated_size(), - _ => LOCAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize() - }); + VMLocalMarkBitSpec::in_header(FORWARDING_BITS_OFFSET); #[cfg(not(feature = "mark_bit_in_header"))] pub(crate) const MARKING_METADATA_SPEC: VMLocalMarkBitSpec = From 6e3da7c6c1bd88b774e22fc9162e575daafdd2ff Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 4 Aug 2021 09:51:08 +1000 Subject: [PATCH 033/494] Run immix tests before build with MARK_IN_HEADER=1 --- .github/scripts/ci-test.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/scripts/ci-test.sh b/.github/scripts/ci-test.sh index f856db55..21cae4c4 100755 --- a/.github/scripts/ci-test.sh +++ b/.github/scripts/ci-test.sh @@ -65,6 +65,24 @@ build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHea #build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar benchmarks/dacapo-2006-10-MR2.jar pmd - OOM #build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar benchmarks/dacapo-2006-10-MR2.jar xalan - OOM +# --- Immix --- +export MMTK_PLAN=Immix + +# Test - the benchmarks that are commented out do not work yet +# Note: the command line options are necessary for now to ensure the benchmarks work. We may later change the options if we do not have these many constraints. +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar antlr +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar bloat - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar fop +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar jython - does not work for stock build +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar luindex +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar lusearch - validation failed +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar pmd +#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar xalan - mmtk-core gets stuck in slowdebug build + +# These benchmarks take 40s+ for slowdebug build, we may consider removing them from the CI +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar hsqldb +build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar eclipse + # --- MarkSweep --- export MMTK_PLAN=MarkSweep @@ -116,21 +134,3 @@ build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHea build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar benchmarks/dacapo-2006-10-MR2.jar fop build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar benchmarks/dacapo-2006-10-MR2.jar luindex # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar benchmarks/dacapo-2006-10-MR2.jar pmd - -# --- Immix --- -export MMTK_PLAN=Immix - -# Test - the benchmarks that are commented out do not work yet -# Note: the command line options are necessary for now to ensure the benchmarks work. We may later change the options if we do not have these many constraints. -build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar antlr -#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar bloat - does not work for stock build -build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar fop -#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar jython - does not work for stock build -build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar luindex -#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar lusearch - validation failed -build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar pmd -#build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar xalan - mmtk-core gets stuck in slowdebug build - -# These benchmarks take 40s+ for slowdebug build, we may consider removing them from the CI -build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar hsqldb -build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar benchmarks/dacapo-2006-10-MR2.jar eclipse From e00a74306417ac1bb97ae9e97ed456f9a1655e43 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 8 Aug 2021 14:15:40 +1000 Subject: [PATCH 034/494] C2 barriers --- openjdk/barriers/mmtkObjectBarrier.cpp | 10 +++------- openjdk/barriers/mmtkObjectBarrier.hpp | 13 +++++++------ openjdk/mmtkBarrierSetC2.hpp | 7 ++++--- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 0acbb617..0ffbb989 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -175,11 +175,7 @@ const TypeFunc* record_modified_node_entry_Type() { return TypeFunc::make(domain, range); } -void MMTkObjectBarrierSetC2::record_modified_node(GraphKit* kit, Node* src, Node* val) const { - if (val != NULL && val->is_Con()) { - const Type* t = val->bottom_type(); - if (t == TypePtr::NULL_PTR) return; - } +void MMTkObjectBarrierSetC2::record_modified_node(GraphKit* kit, Node* src, Node* slot, Node* val) const { MMTkIdealKit ideal(kit, true); @@ -200,8 +196,8 @@ void MMTkObjectBarrierSetC2::record_modified_node(GraphKit* kit, Node* src, Node Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src); } __ end_if(); #else - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src, slot, val); #endif kit->final_sync(ideal); // Final sync IdealKit and GraphKit. diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index 2ea97ca7..0fbce3a4 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -152,31 +152,32 @@ class MMTkObjectBarrierSetC1: public MMTkBarrierSetC1 { #define __ ideal. class MMTkObjectBarrierSetC2: public MMTkBarrierSetC2 { - void record_modified_node(GraphKit* kit, Node* node, Node* val) const; + void record_modified_node(GraphKit* kit, Node* node, Node* slot, Node* val) const; public: virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const { + if (access.is_oop()) record_modified_node(access.kit(), access.base(), access.addr().node(), val.node()); Node* store = BarrierSetC2::store_at_resolved(access, val); - if (access.is_oop()) record_modified_node(access.kit(), access.base(), val.node()); return store; } virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { + if (access.is_oop()) record_modified_node(access.kit(), access.base(), access.addr().node(), new_val); Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); - if (access.is_oop()) record_modified_node(access.kit(), access.base(), new_val); return result; } virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { + if (access.is_oop()) record_modified_node(access.kit(), access.base(), access.addr().node(), new_val); Node* load_store = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); - if (access.is_oop()) record_modified_node(access.kit(), access.base(), new_val); return load_store; } virtual Node* atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const { + if (access.is_oop()) record_modified_node(access.kit(), access.base(), access.addr().node(), new_val); Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, value_type); - if (access.is_oop()) record_modified_node(access.kit(), access.base(), new_val); return result; } virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { BarrierSetC2::clone(kit, src, dst, size, is_array); - record_modified_node(kit, dst, NULL); + // FIXME: This is required for generational barriers. + // record_modified_node(kit, dst, NULL, NULL); } virtual bool is_gc_barrier_node(Node* node) const { if (node->Opcode() != Op_CallLeaf) return false; diff --git a/openjdk/mmtkBarrierSetC2.hpp b/openjdk/mmtkBarrierSetC2.hpp index 7c17e8c5..2f9f6a30 100644 --- a/openjdk/mmtkBarrierSetC2.hpp +++ b/openjdk/mmtkBarrierSetC2.hpp @@ -84,7 +84,7 @@ class MMTkIdealKit: public IdealKit { template inline void build_type_func_helper(const Type** fields, T t, Types... ts) { fields[0] = t; - build_type_func_helper(fields + 1); + build_type_func_helper(fields + 1, ts...); } public: using IdealKit::IdealKit; @@ -97,9 +97,10 @@ class MMTkIdealKit: public IdealKit { template inline const TypeFunc* func_type(Types... types) { - const Type** fields = TypeTuple::fields(sizeof...(types)); + const int num_types = sizeof...(types); + const Type** fields = TypeTuple::fields(num_types); build_type_func_helper(fields + TypeFunc::Parms, types...); - const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+num_types, fields); fields = TypeTuple::fields(0); const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields); return TypeFunc::make(domain, range); From 45269f469eb100be9c728a4375f3f9bd6346d472 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 13 Aug 2021 14:45:32 +1000 Subject: [PATCH 035/494] WIP: Add FieldLoggingBarrier --- mmtk/src/api.rs | 17 +- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 206 +++++++++++++++++++ openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 204 ++++++++++++++++++ openjdk/barriers/mmtkObjectBarrier.cpp | 10 +- openjdk/barriers/mmtkObjectBarrier.hpp | 4 - openjdk/mmtkBarrierSet.cpp | 2 + openjdk/mmtkBarrierSet.hpp | 3 + 7 files changed, 429 insertions(+), 17 deletions(-) create mode 100644 openjdk/barriers/mmtkFieldLoggingBarrier.cpp create mode 100644 openjdk/barriers/mmtkFieldLoggingBarrier.hpp diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index e61fdc66..049e31bf 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -19,17 +19,18 @@ use std::lazy::SyncLazy; // Supported barriers: static NO_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("NoBarrier").unwrap()); static OBJECT_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("ObjectBarrier").unwrap()); +static FIELD_LOGGING_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("FieldLoggingBarrier").unwrap()); #[no_mangle] pub extern "C" fn mmtk_active_barrier() -> *const c_char { - // match SINGLETON.get_plan().constraints().barrier { - // BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), - // BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), - // // In case we have more barriers in mmtk-core. - // #[allow(unreachable_patterns)] - // _ => unimplemented!(), - // } - OBJECT_BARRIER.as_ptr() + match SINGLETON.get_plan().constraints().barrier { + BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), + BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), + BarrierSelector::FieldLoggingBarrier => FIELD_LOGGING_BARRIER.as_ptr(), + // In case we have more barriers in mmtk-core. + #[allow(unreachable_patterns)] + _ => unimplemented!(), + } } /// # Safety diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp new file mode 100644 index 00000000..d365c1ce --- /dev/null +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -0,0 +1,206 @@ +#include "mmtkFieldLoggingBarrier.hpp" +#include "runtime/interfaceSupport.inline.hpp" + +void MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow(void* src, void* slot, void* val) { + ::record_modified_node((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, val); +} + +void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset, oop val) { +#if MMTK_ENABLE_BARRIER_FASTPATH + intptr_t addr = (intptr_t) (void*) src; + uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + intptr_t shift = (addr >> 3) & 0b111; + uint8_t byte_val = *meta_addr; + if (((byte_val >> shift) & 1) == 1) { + record_modified_node_slow((void*) src); + } +#else + record_modified_node_slow((void*) src, (void*) (((intptr_t) (void*) src) + offset), (void*) val); +#endif +} + +#define __ masm-> + +void MMTkFieldLoggingBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { + bool in_heap = (decorators & IN_HEAP) != 0; + bool as_normal = (decorators & AS_NORMAL) != 0; + assert((decorators & IS_DEST_UNINITIALIZED) == 0, "unsupported"); + + if (!in_heap) { + BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); + return; + } + + record_modified_node(masm, dst, val, tmp1, tmp2); + + BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); +} + +void MMTkFieldLoggingBarrierSetAssembler::record_modified_node(MacroAssembler* masm, Address dst, Register val, Register tmp1, Register tmp2) { +#if MMTK_ENABLE_BARRIER_FASTPATH + Label done; + + Register tmp3 = rscratch1; + Register tmp4 = rscratch2; + assert_different_registers(obj, tmp2, tmp3); + assert_different_registers(tmp4, rcx); + + // tmp2 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); + __ movptr(tmp3, obj); + __ shrptr(tmp3, 6); + __ movptr(tmp2, SIDE_METADATA_BASE_ADDRESS); + __ movb(tmp2, Address(tmp2, tmp3)); + // tmp3 = (obj >> 3) & 7 + __ movptr(tmp3, obj); + __ shrptr(tmp3, 3); + __ andptr(tmp3, 7); + // tmp2 = tmp2 >> tmp3 + __ movptr(tmp4, rcx); + __ movl(rcx, tmp3); + __ shrptr(tmp2); + __ movptr(rcx, tmp4); + // if ((tmp2 & 1) == 1) goto slowpath; + __ andptr(tmp2, 1); + __ cmpptr(tmp2, 1); + __ jcc(Assembler::notEqual, done); + + assert_different_registers(c_rarg0, obj); + __ movptr(c_rarg0, obj); + __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), 1); + + __ bind(done); +#else + // assert_different_registers(c_rarg0, val); + __ pusha(); + __ movptr(c_rarg0, dst.base()); + __ lea(c_rarg1, dst); + if (val == noreg) { + __ movptr(c_rarg2, (int32_t) NULL_WORD); + } else { + __ movptr(c_rarg2, val); + } + __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), 3); + __ popa(); +#endif +} + +#undef __ + +#ifdef ASSERT +#define __ gen->lir(__FILE__, __LINE__)-> +#else +#define __ gen->lir()-> +#endif + +void MMTkFieldLoggingBarrierSetC1::record_modified_node(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) { + LIRGenerator* gen = access.gen(); + DecoratorSet decorators = access.decorators(); + if ((decorators & IN_HEAP) == 0) return; + if (!src->is_register()) { + LIR_Opr reg = gen->new_pointer_register(); + if (src->is_constant()) { + __ move(src, reg); + } else { + __ leal(src, reg); + } + src = reg; + } + assert(src->is_register(), "must be a register at this point"); + if (!slot->is_register()) { + LIR_Opr reg = gen->new_pointer_register(); + if (slot->is_constant()) { + __ move(slot, reg); + } else { + __ leal(slot, reg); + } + slot = reg; + } + assert(slot->is_register(), "must be a register at this point"); + if (!new_val->is_register()) { + LIR_Opr new_val_reg = gen->new_register(T_OBJECT); + if (new_val->is_constant()) { + __ move(new_val, new_val_reg); + } else { + __ leal(new_val, new_val_reg); + } + new_val = new_val_reg; + } + assert(new_val->is_register(), "must be a register at this point"); + CodeStub* slow = new MMTkFieldLoggingBarrierStub(src, slot, new_val); + +#if MMTK_ENABLE_BARRIER_FASTPATH + LIR_Opr addr = src; + // uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + LIR_Opr offset = gen->new_pointer_register(); + __ move(addr, offset); + __ shift_right(offset, 6, offset); + LIR_Opr base = gen->new_pointer_register(); + __ move(LIR_OprFact::longConst(SIDE_METADATA_BASE_ADDRESS), base); + LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); + // intptr_t shift = (addr >> 3) & 0b111; + LIR_Opr shift_long = gen->new_pointer_register(); + __ move(addr, shift_long); + __ shift_right(shift_long, 3, shift_long); + __ logical_and(shift_long, LIR_OprFact::longConst(0b111), shift_long); + LIR_Opr shift_int = gen->new_register(T_INT); + __ convert(Bytecodes::_l2i, shift_long, shift_int); + LIR_Opr shift = LIRGenerator::shiftCountOpr(); + __ move(shift_int, shift); + // uint8_t byte_val = *meta_addr; + LIR_Opr byte_val = gen->new_register(T_INT); + __ move(meta_addr, byte_val); + // if (((byte_val >> shift) & 1) == 1) slow; + LIR_Opr result = byte_val; + __ shift_right(result, shift, result, LIR_OprFact::illegalOpr); + __ logical_and(result, LIR_OprFact::intConst(1), result); + __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(1)); + __ branch(lir_cond_equal, LP64_ONLY(T_LONG) NOT_LP64(T_INT), slow); +#else + __ jump(slow); +#endif + + __ branch_destination(slow->continuation()); +} + +#undef __ + +#define __ ideal. + +static const TypeFunc* record_modified_node_entry_Type() { + const Type **fields = TypeTuple::fields(1); + fields[TypeFunc::Parms+0] = TypeOopPtr::BOTTOM; // oop src + const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); + fields = TypeTuple::fields(0); + const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields); + return TypeFunc::make(domain, range); +} + +void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src, Node* slot, Node* val) const { + + MMTkIdealKit ideal(kit, true); + +#if MMTK_ENABLE_BARRIER_FASTPATH + Node* no_base = __ top(); + float unlikely = PROB_UNLIKELY(0.999); + + Node* zero = __ ConI(0); + Node* addr = __ CastPX(__ ctrl(), src); + Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); + Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); + Node* shift = __ URShiftX(addr, __ ConI(3)); + shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); + Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); + + __ if_then(result, BoolTest::ne, zero, unlikely); { + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src); + } __ end_if(); +#else + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src, slot, val); +#endif + + kit->final_sync(ideal); // Final sync IdealKit and GraphKit. +} + +#undef __ \ No newline at end of file diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp new file mode 100644 index 00000000..c2b4a389 --- /dev/null +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -0,0 +1,204 @@ +#ifndef MMTK_BARRIERS_FIELD_LOGGING_BARRIER +#define MMTK_BARRIERS_FIELD_LOGGING_BARRIER + +#include "opto/callnode.hpp" +#include "opto/idealKit.hpp" +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "gc/shared/barrierSet.hpp" +#include "../mmtk.h" +#include "../mmtkBarrierSet.hpp" +#include "../mmtkBarrierSetAssembler_x86.hpp" +#include "../mmtkBarrierSetC1.hpp" +#include "../mmtkBarrierSetC2.hpp" + +#define SIDE_METADATA_WORST_CASE_RATIO_LOG 1 +#define LOG_BYTES_IN_CHUNK 22 +#define CHUNK_MASK ((1L << LOG_BYTES_IN_CHUNK) - 1) + +class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { +public: + static void record_modified_node_slow(void* src, void* slot, void* val); + + virtual bool is_slow_path_call(address call) { + return call == CAST_FROM_FN_PTR(address, record_modified_node_slow); + } + + virtual void record_modified_node(oop src, ptrdiff_t offset, oop val); +}; + +class MMTkFieldLoggingBarrierSetC1; +class MMTkFieldLoggingBarrierStub; + +class MMTkFieldLoggingBarrierSetAssembler: public MMTkBarrierSetAssembler { + void oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2); + void record_modified_node(MacroAssembler* masm, Address dst, Register val, Register tmp1, Register tmp2); +public: + virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { + if (type == T_OBJECT || type == T_ARRAY) { + oop_store_at(masm, decorators, type, dst, val, tmp1, tmp2); + } else { + BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); + } + } + inline void gen_write_barrier_stub(LIR_Assembler* ce, MMTkFieldLoggingBarrierStub* stub); +#define __ sasm-> + void generate_c1_write_barrier_runtime_stub(StubAssembler* sasm) { + __ prologue("mmtk_write_barrier", false); + + Address store_addr(rbp, 3*BytesPerWord); + + Label done; + Label runtime; + + __ push(c_rarg0); + // __ push(c_rarg1); + // __ push(c_rarg2); + __ push(rax); + + __ load_parameter(0, c_rarg0); + // __ load_parameter(1, c_rarg1); + // __ load_parameter(2, c_rarg2); + + __ bind(runtime); + + __ save_live_registers_no_oop_map(true); + + __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), 1); + + __ restore_live_registers(true); + + __ bind(done); + __ pop(rax); + // __ pop(c_rarg2); + // __ pop(c_rarg1); + __ pop(c_rarg0); + + __ epilogue(); + } +#undef __ +}; + +#ifdef ASSERT +#define __ gen->lir(__FILE__, __LINE__)-> +#else +#define __ gen->lir()-> +#endif + +struct MMTkFieldLoggingBarrierStub: CodeStub { + LIR_Opr _src, _slot, _new_val; + MMTkFieldLoggingBarrierStub(LIR_Opr src, LIR_Opr slot, LIR_Opr new_val): _src(src), _slot(slot), _new_val(new_val) {} + virtual void emit_code(LIR_Assembler* ce) { + MMTkFieldLoggingBarrierSetAssembler* bs = (MMTkFieldLoggingBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); + bs->gen_write_barrier_stub(ce, this); + } + virtual void visit(LIR_OpVisitState* visitor) { + visitor->do_slow_case(); + if (_src != NULL) visitor->do_input(_src); + if (_slot != NULL) visitor->do_input(_slot); + if (_new_val != NULL) visitor->do_input(_new_val); + } + NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkWriteBarrierStub"); }); +}; + +class MMTkFieldLoggingBarrierSetC1: public MMTkBarrierSetC1 { +public: + class MMTkFieldLoggingBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { + virtual OopMapSet* generate_code(StubAssembler* sasm) { + MMTkFieldLoggingBarrierSetAssembler* bs = (MMTkFieldLoggingBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); + bs->generate_c1_write_barrier_runtime_stub(sasm); + return NULL; + } + }; + void record_modified_node(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val); +public: + CodeBlob* _write_barrier_c1_runtime_code_blob; + virtual void store_at_resolved(LIRAccess& access, LIR_Opr value) { + BarrierSetC1::store_at_resolved(access, value); + if (access.is_oop()) record_modified_node(access, access.base().opr(), access.resolved_addr(), value); + } + virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) { + LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); + if (access.is_oop()) record_modified_node(access, access.base().opr(), access.resolved_addr(), new_value.result()); + return result; + } + virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) { + LIR_Opr result = BarrierSetC1::atomic_xchg_at_resolved(access, value); + if (access.is_oop()) record_modified_node(access, access.base().opr(), access.resolved_addr(), value.result()); + return result; + } + virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob) { + MMTkFieldLoggingBarrierCodeGenClosure write_code_gen_cl; + _write_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "write_code_gen_cl", false, &write_code_gen_cl); + } + virtual LIR_Opr resolve_address(LIRAccess& access, bool resolve_in_register) { + DecoratorSet decorators = access.decorators(); + bool needs_patching = (decorators & C1_NEEDS_PATCHING) != 0; + bool is_write = (decorators & C1_WRITE_ACCESS) != 0; + bool is_array = (decorators & IS_ARRAY) != 0; + bool on_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; + bool precise = is_array || on_anonymous; + resolve_in_register |= !needs_patching && is_write && access.is_oop() && precise; + return BarrierSetC1::resolve_address(access, resolve_in_register); + } +}; + +#undef __ + +#define __ ideal. + +class MMTkFieldLoggingBarrierSetC2: public MMTkBarrierSetC2 { + void record_modified_node(GraphKit* kit, Node* node, Node* slot, Node* val) const; +public: + virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const { + if (access.is_oop()) record_modified_node(access.kit(), access.base(), access.addr().node(), val.node()); + Node* store = BarrierSetC2::store_at_resolved(access, val); + return store; + } + virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { + if (access.is_oop()) record_modified_node(access.kit(), access.base(), access.addr().node(), new_val); + Node* result = BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); + return result; + } + virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicAccess& access, Node* expected_val, Node* new_val, const Type* value_type) const { + if (access.is_oop()) record_modified_node(access.kit(), access.base(), access.addr().node(), new_val); + Node* load_store = BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); + return load_store; + } + virtual Node* atomic_xchg_at_resolved(C2AtomicAccess& access, Node* new_val, const Type* value_type) const { + if (access.is_oop()) record_modified_node(access.kit(), access.base(), access.addr().node(), new_val); + Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, new_val, value_type); + return result; + } + virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { + BarrierSetC2::clone(kit, src, dst, size, is_array); + // FIXME: This is required for generational barriers. + // record_modified_node(kit, dst, NULL, NULL); + } + virtual bool is_gc_barrier_node(Node* node) const { + if (node->Opcode() != Op_CallLeaf) return false; + CallLeafNode *call = node->as_CallLeaf(); + return call->_name != NULL && strcmp(call->_name, "record_modified_node") == 0; + } +}; + +#undef __ + +#define __ ce->masm()-> +inline void MMTkFieldLoggingBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, MMTkFieldLoggingBarrierStub* stub) { + MMTkFieldLoggingBarrierSetC1* bs = (MMTkFieldLoggingBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1(); + __ bind(*stub->entry()); + ce->store_parameter(stub->_src->as_pointer_register(), 0); + __ call(RuntimeAddress(bs->_write_barrier_c1_runtime_code_blob->code_begin())); + __ jmp(*stub->continuation()); +} +#undef __ + +struct MMTkFieldLoggingBarrier: MMTkBarrierImpl< + MMTkFieldLoggingBarrierSetRuntime, + MMTkFieldLoggingBarrierSetAssembler, + MMTkFieldLoggingBarrierSetC1, + MMTkFieldLoggingBarrierSetC2 +> {}; + +#endif diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 0ffbb989..8fa7314a 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -6,7 +6,7 @@ void MMTkObjectBarrierSetRuntime::record_modified_node_slow(void* src, void* slo } void MMTkObjectBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset, oop val) { -#if MMTK_ENABLE_OBJECT_BARRIER_FASTPATH +#if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = (intptr_t) (void*) src; uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); intptr_t shift = (addr >> 3) & 0b111; @@ -37,7 +37,7 @@ void MMTkObjectBarrierSetAssembler::oop_store_at(MacroAssembler* masm, Decorator } void MMTkObjectBarrierSetAssembler::record_modified_node(MacroAssembler* masm, Address dst, Register val, Register tmp1, Register tmp2) { -#if MMTK_ENABLE_OBJECT_BARRIER_FASTPATH +#if MMTK_ENABLE_BARRIER_FASTPATH Label done; Register tmp3 = rscratch1; @@ -128,7 +128,7 @@ void MMTkObjectBarrierSetC1::record_modified_node(LIRAccess& access, LIR_Opr src assert(new_val->is_register(), "must be a register at this point"); CodeStub* slow = new MMTkObjectBarrierStub(src, slot, new_val); -#if MMTK_ENABLE_OBJECT_BARRIER_FASTPATH +#if MMTK_ENABLE_BARRIER_FASTPATH LIR_Opr addr = src; // uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); LIR_Opr offset = gen->new_pointer_register(); @@ -166,7 +166,7 @@ void MMTkObjectBarrierSetC1::record_modified_node(LIRAccess& access, LIR_Opr src #define __ ideal. -const TypeFunc* record_modified_node_entry_Type() { +static const TypeFunc* record_modified_node_entry_Type() { const Type **fields = TypeTuple::fields(1); fields[TypeFunc::Parms+0] = TypeOopPtr::BOTTOM; // oop src const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); @@ -179,7 +179,7 @@ void MMTkObjectBarrierSetC2::record_modified_node(GraphKit* kit, Node* src, Node MMTkIdealKit ideal(kit, true); -#if MMTK_ENABLE_OBJECT_BARRIER_FASTPATH +#if MMTK_ENABLE_BARRIER_FASTPATH Node* no_base = __ top(); float unlikely = PROB_UNLIKELY(0.999); diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index 0fbce3a4..2d3201a0 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -12,14 +12,10 @@ #include "../mmtkBarrierSetC1.hpp" #include "../mmtkBarrierSetC2.hpp" -#define MMTK_ENABLE_OBJECT_BARRIER_FASTPATH false - #define SIDE_METADATA_WORST_CASE_RATIO_LOG 1 #define LOG_BYTES_IN_CHUNK 22 #define CHUNK_MASK ((1L << LOG_BYTES_IN_CHUNK) - 1) -const intptr_t SIDE_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; - class MMTkObjectBarrierSetRuntime: public MMTkBarrierSetRuntime { public: static void record_modified_node_slow(void* src, void* slot, void* val); diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index 6445352d..7e66bec1 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -34,6 +34,7 @@ #include "mmtkBarrierSetAssembler_x86.hpp" #include "barriers/mmtkNoBarrier.hpp" #include "barriers/mmtkObjectBarrier.hpp" +#include "barriers/mmtkFieldLoggingBarrier.hpp" MMTkBarrierBase* get_selected_barrier() { static MMTkBarrierBase* selected_barrier = NULL; @@ -41,6 +42,7 @@ MMTkBarrierBase* get_selected_barrier() { const char* barrier = mmtk_active_barrier(); if (strcmp(barrier, "NoBarrier") == 0) selected_barrier = new MMTkNoBarrier(); else if (strcmp(barrier, "ObjectBarrier") == 0) selected_barrier = new MMTkObjectBarrier(); + else if (strcmp(barrier, "FieldLoggingBarrier") == 0) selected_barrier = new MMTkFieldLoggingBarrier(); else guarantee(false, "Unimplemented"); return selected_barrier; } diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index e72dec63..22b4c1a9 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -37,6 +37,9 @@ #include "mmtkBarrierSetAssembler_x86.hpp" #define MMTK_ENABLE_ALLOCATION_FASTPATH true +#define MMTK_ENABLE_BARRIER_FASTPATH false + +const intptr_t SIDE_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; class MMTkBarrierSetRuntime: public CHeapObj { public: From 3939fc52df45e27b1804109528907872483b3e53 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 14 Aug 2021 19:10:21 +1000 Subject: [PATCH 036/494] FieldLoggingBarrier: interprepter and runtime fast-path --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 50 ++++++++++---------- openjdk/barriers/mmtkObjectBarrier.cpp | 14 ++---- openjdk/mmtkBarrierSet.hpp | 2 +- 3 files changed, 28 insertions(+), 38 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index d365c1ce..a9757425 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -11,8 +11,8 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); intptr_t shift = (addr >> 3) & 0b111; uint8_t byte_val = *meta_addr; - if (((byte_val >> shift) & 1) == 1) { - record_modified_node_slow((void*) src); + if (((byte_val >> shift) & 1) == 0) { + record_modified_node_slow((void*) src, (void*) (((intptr_t) (void*) src) + offset), (void*) val); } #else record_modified_node_slow((void*) src, (void*) (((intptr_t) (void*) src) + offset), (void*) val); @@ -42,31 +42,38 @@ void MMTkFieldLoggingBarrierSetAssembler::record_modified_node(MacroAssembler* m Register tmp3 = rscratch1; Register tmp4 = rscratch2; - assert_different_registers(obj, tmp2, tmp3); - assert_different_registers(tmp4, rcx); + Register tmp5 = tmp1 == dst.base() || tmp1 == dst.index() ? tmp2 : tmp1; - // tmp2 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); - __ movptr(tmp3, obj); + // tmp5 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); + __ lea(tmp3, dst); __ shrptr(tmp3, 6); - __ movptr(tmp2, SIDE_METADATA_BASE_ADDRESS); - __ movb(tmp2, Address(tmp2, tmp3)); + __ movptr(tmp5, SIDE_METADATA_BASE_ADDRESS); + __ movb(tmp5, Address(tmp5, tmp3)); // tmp3 = (obj >> 3) & 7 - __ movptr(tmp3, obj); + __ lea(tmp3, dst); __ shrptr(tmp3, 3); __ andptr(tmp3, 7); - // tmp2 = tmp2 >> tmp3 + // tmp5 = tmp5 >> tmp3 __ movptr(tmp4, rcx); __ movl(rcx, tmp3); - __ shrptr(tmp2); + __ shrptr(tmp5); __ movptr(rcx, tmp4); - // if ((tmp2 & 1) == 1) goto slowpath; - __ andptr(tmp2, 1); - __ cmpptr(tmp2, 1); + // if ((tmp5 & 1) == 1) goto slowpath; + __ andptr(tmp5, 1); + __ cmpptr(tmp5, 0); __ jcc(Assembler::notEqual, done); - assert_different_registers(c_rarg0, obj); - __ movptr(c_rarg0, obj); - __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), 1); + // TODO: Spill fewer registers + __ pusha(); + __ movptr(c_rarg0, dst.base()); + __ lea(c_rarg1, dst); + if (val == noreg) { + __ movptr(c_rarg2, (int32_t) NULL_WORD); + } else { + __ movptr(c_rarg2, val); + } + __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), 3); + __ popa(); __ bind(done); #else @@ -166,15 +173,6 @@ void MMTkFieldLoggingBarrierSetC1::record_modified_node(LIRAccess& access, LIR_O #define __ ideal. -static const TypeFunc* record_modified_node_entry_Type() { - const Type **fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = TypeOopPtr::BOTTOM; // oop src - const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); - fields = TypeTuple::fields(0); - const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields); - return TypeFunc::make(domain, range); -} - void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src, Node* slot, Node* val) const { MMTkIdealKit ideal(kit, true); diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 8fa7314a..a1b02898 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -12,7 +12,7 @@ void MMTkObjectBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset intptr_t shift = (addr >> 3) & 0b111; uint8_t byte_val = *meta_addr; if (((byte_val >> shift) & 1) == 1) { - record_modified_node_slow((void*) src); + record_modified_node_slow((void*) src, (void*) (((intptr_t) (void*) src) + offset), (void*) val); } #else record_modified_node_slow((void*) src, (void*) (((intptr_t) (void*) src) + offset), (void*) val); @@ -42,7 +42,8 @@ void MMTkObjectBarrierSetAssembler::record_modified_node(MacroAssembler* masm, A Register tmp3 = rscratch1; Register tmp4 = rscratch2; - assert_different_registers(obj, tmp2, tmp3); + Register obj = dst.base(); + // assert_different_registers(obj, tmp2, tmp3); assert_different_registers(tmp4, rcx); // tmp2 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); @@ -166,15 +167,6 @@ void MMTkObjectBarrierSetC1::record_modified_node(LIRAccess& access, LIR_Opr src #define __ ideal. -static const TypeFunc* record_modified_node_entry_Type() { - const Type **fields = TypeTuple::fields(1); - fields[TypeFunc::Parms+0] = TypeOopPtr::BOTTOM; // oop src - const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields); - fields = TypeTuple::fields(0); - const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields); - return TypeFunc::make(domain, range); -} - void MMTkObjectBarrierSetC2::record_modified_node(GraphKit* kit, Node* src, Node* slot, Node* val) const { MMTkIdealKit ideal(kit, true); diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 22b4c1a9..a9208458 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -37,7 +37,7 @@ #include "mmtkBarrierSetAssembler_x86.hpp" #define MMTK_ENABLE_ALLOCATION_FASTPATH true -#define MMTK_ENABLE_BARRIER_FASTPATH false +#define MMTK_ENABLE_BARRIER_FASTPATH true const intptr_t SIDE_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; From db61e6fed4b2df1154ebe3d8d4e9e84ec7ce294f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 14 Aug 2021 19:15:03 +1000 Subject: [PATCH 037/494] FieldLoggingBarrier: C2 fast path --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index a9757425..a4ad3fb5 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -182,16 +182,16 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src float unlikely = PROB_UNLIKELY(0.999); Node* zero = __ ConI(0); - Node* addr = __ CastPX(__ ctrl(), src); + Node* addr = __ CastPX(__ ctrl(), slot); Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); Node* shift = __ URShiftX(addr, __ ConI(3)); shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); - __ if_then(result, BoolTest::ne, zero, unlikely); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src); + __ if_then(result, BoolTest::eq, zero, unlikely); { + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src, slot, val); } __ end_if(); #else const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); From 2b9b3c6f987ebd5eef25ab8581b8c39dee0ecee4 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 19 Aug 2021 12:01:22 +1000 Subject: [PATCH 038/494] Refactor barrier --- mmtk/src/api.rs | 24 ++++++++++++++++++-- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 2 +- openjdk/barriers/mmtkObjectBarrier.cpp | 2 +- openjdk/mmtk.h | 4 +++- openjdk/mmtkBarrierSet.hpp | 20 +++------------- 5 files changed, 30 insertions(+), 22 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 049e31bf..ff264451 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -240,13 +240,33 @@ pub extern "C" fn executable() -> bool { } #[no_mangle] -pub extern "C" fn record_modified_node( +pub extern "C" fn mmtk_object_reference_write( mutator: &'static mut Mutator, src: ObjectReference, slot: Address, val: ObjectReference, ) { - mutator.record_modified_node(src, slot, val); + mutator.object_reference_write(src, slot, val); +} + +#[no_mangle] +pub extern "C" fn mmtk_object_reference_arraycopy( + mutator: &'static mut Mutator, + src: Address, + dst: Address, + len: usize, +) { + mutator.object_reference_arraycopy(src, dst, len); +} + +#[no_mangle] +pub extern "C" fn mmtk_object_reference_clone( + mutator: &'static mut Mutator, + src: ObjectReference, + dst: ObjectReference, + size: usize, +) { + mutator.object_reference_clone(src, dst, size); } // finalization diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index a4ad3fb5..ad18a1c5 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -2,7 +2,7 @@ #include "runtime/interfaceSupport.inline.hpp" void MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow(void* src, void* slot, void* val) { - ::record_modified_node((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, val); + ::mmtk_object_reference_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, val); } void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset, oop val) { diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index a1b02898..b76442da 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -2,7 +2,7 @@ #include "runtime/interfaceSupport.inline.hpp" void MMTkObjectBarrierSetRuntime::record_modified_node_slow(void* src, void* slot, void* val) { - ::record_modified_node((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, val); + ::mmtk_object_reference_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, val); } void MMTkObjectBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset, oop val) { diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 78cbdb94..a8a81927 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -34,7 +34,9 @@ extern void* alloc_slow_largeobject(MMTk_Mutator mutator, size_t size, extern void post_alloc(MMTk_Mutator mutator, void* refer, int bytes, int allocator); -extern void record_modified_node(MMTk_Mutator mutator, void* src, void* slot, void* val); +extern void mmtk_object_reference_write(MMTk_Mutator mutator, void* src, void* slot, void* val); +extern void mmtk_object_reference_arraycopy(MMTk_Mutator mutator, void* src, void* dst, size_t len); +extern void mmtk_object_reference_clone(MMTk_Mutator mutator, void* src, void* dst, size_t len); extern void release_buffer(void** buffer, size_t len, size_t cap); diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index a9208458..27f40b5c 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -165,12 +165,7 @@ class MMTkBarrierSet : public BarrierSet { size_t length) { T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); - char* base = reinterpret_cast((void*) src_obj); - for (const T* const end = src + length; src < end; src++, dst++) { - int offset = reinterpret_cast((void*) src) - base; - runtime()->record_modified_node(dst_obj, offset, (oop) *dst); - } - + ::mmtk_object_reference_arraycopy((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) dst, length); bool result = Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); @@ -208,23 +203,14 @@ class MMTkBarrierSet : public BarrierSet { static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) { - // T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); - // T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); - // char* base = reinterpret_cast((void*) src_obj); - // for (const T* const end = src + length; src < end; src++, dst++) { - // int offset = reinterpret_cast((void*) src) - base; - // runtime()->record_modified_node(dst_obj, offset, (oop) *dst); - // } - - bool result = Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, + return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); - return result; } static void clone_in_heap(oop src, oop dst, size_t size) { Raw::clone(src, dst, size); - // runtime()->record_modified_node(dst); + ::mmtk_object_reference_clone((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) dst, size); } }; From bb44e633587e9a9564d408a954834685a77fa0c0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 19 Aug 2021 12:21:40 +1000 Subject: [PATCH 039/494] cleanup --- mmtk/src/collection.rs | 1 - openjdk/barriers/mmtkObjectBarrier.cpp | 17 +++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index b6fbd962..f575e94d 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -33,7 +33,6 @@ impl Collection for VMCollection { } } - fn stop_all_mutators2(tls: VMWorkerThread) { unsafe { ((*UPCALLS).stop_mutators)(tls); diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index b76442da..73e253db 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -26,7 +26,7 @@ void MMTkObjectBarrierSetAssembler::oop_store_at(MacroAssembler* masm, Decorator bool as_normal = (decorators & AS_NORMAL) != 0; assert((decorators & IS_DEST_UNINITIALIZED) == 0, "unsupported"); - if (!in_heap) { + if (!in_heap || val == noreg) { BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); return; } @@ -43,7 +43,6 @@ void MMTkObjectBarrierSetAssembler::record_modified_node(MacroAssembler* masm, A Register tmp3 = rscratch1; Register tmp4 = rscratch2; Register obj = dst.base(); - // assert_different_registers(obj, tmp2, tmp3); assert_different_registers(tmp4, rcx); // tmp2 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); @@ -65,13 +64,19 @@ void MMTkObjectBarrierSetAssembler::record_modified_node(MacroAssembler* masm, A __ cmpptr(tmp2, 1); __ jcc(Assembler::notEqual, done); - assert_different_registers(c_rarg0, obj); - __ movptr(c_rarg0, obj); - __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), 1); + __ pusha(); + __ movptr(c_rarg0, dst.base()); + __ lea(c_rarg1, dst); + if (val == noreg) { + __ movptr(c_rarg2, (int32_t) NULL_WORD); + } else { + __ movptr(c_rarg2, val); + } + __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), 3); + __ popa(); __ bind(done); #else - // assert_different_registers(c_rarg0, val); __ pusha(); __ movptr(c_rarg0, dst.base()); __ lea(c_rarg1, dst); From 287f31dd777a3a524d08392615105848e1f1bf7e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 20 Aug 2021 11:03:43 +1000 Subject: [PATCH 040/494] Refactor barrier --- mmtk/src/api.rs | 10 +- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 4 + openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 1 + openjdk/barriers/mmtkObjectBarrier.cpp | 105 +++++++++++++++---- openjdk/barriers/mmtkObjectBarrier.hpp | 36 ++++--- openjdk/mmtk.h | 2 +- openjdk/mmtkBarrierSet.hpp | 17 ++- 7 files changed, 124 insertions(+), 51 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index ff264451..bae52e21 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -252,11 +252,13 @@ pub extern "C" fn mmtk_object_reference_write( #[no_mangle] pub extern "C" fn mmtk_object_reference_arraycopy( mutator: &'static mut Mutator, - src: Address, - dst: Address, + src: ObjectReference, + src_offset: usize, + dst: ObjectReference, + dst_offset: usize, len: usize, ) { - mutator.object_reference_arraycopy(src, dst, len); + mutator.object_reference_arraycopy(src, src_offset, dst, dst_offset, len); } #[no_mangle] @@ -266,7 +268,7 @@ pub extern "C" fn mmtk_object_reference_clone( dst: ObjectReference, size: usize, ) { - mutator.object_reference_clone(src, dst, size); + mutator.object_reference_clone(src, dst); } // finalization diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index ad18a1c5..1c516e48 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -19,6 +19,10 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t #endif } +void MMTkFieldLoggingBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) { + ::mmtk_object_reference_arraycopy((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src_obj, src_offset_in_bytes, (void*) dst_obj, dst_offset_in_bytes, length); +} + #define __ masm-> void MMTkFieldLoggingBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index c2b4a389..0f08b2b7 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -25,6 +25,7 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { } virtual void record_modified_node(oop src, ptrdiff_t offset, oop val); + virtual void record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length); }; class MMTkFieldLoggingBarrierSetC1; diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index 73e253db..d28a59fd 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -5,6 +5,10 @@ void MMTkObjectBarrierSetRuntime::record_modified_node_slow(void* src, void* slo ::mmtk_object_reference_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, val); } +void MMTkObjectBarrierSetRuntime::record_clone_slow(void* src, void* dst, size_t size) { + ::mmtk_object_reference_clone((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, size); +} + void MMTkObjectBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset, oop val) { #if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = (intptr_t) (void*) src; @@ -19,6 +23,34 @@ void MMTkObjectBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset #endif } +void MMTkObjectBarrierSetRuntime::record_clone(oop src, oop dst, size_t size) { +#if MMTK_ENABLE_BARRIER_FASTPATH + intptr_t addr = (intptr_t) (void*) dst; + uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + intptr_t shift = (addr >> 3) & 0b111; + uint8_t byte_val = *meta_addr; + if (((byte_val >> shift) & 1) == 1) { + record_clone_slow((void*) src, (void*) dst, size); + } +#else + record_clone_slow((void*) src, (void*) dst, size); +#endif +} + +void MMTkObjectBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) { +#if MMTK_ENABLE_BARRIER_FASTPATH + intptr_t addr = (intptr_t) (void*) dst_obj; + uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + intptr_t shift = (addr >> 3) & 0b111; + uint8_t byte_val = *meta_addr; + if (((byte_val >> shift) & 1) == 1) { + ::mmtk_object_reference_arraycopy((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src_obj, src_offset_in_bytes, (void*) dst_obj, dst_offset_in_bytes, length); + } +#else + ::mmtk_object_reference_arraycopy((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src_obj, src_offset_in_bytes, (void*) dst_obj, dst_offset_in_bytes, length); +#endif +} + #define __ masm-> void MMTkObjectBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { @@ -42,26 +74,25 @@ void MMTkObjectBarrierSetAssembler::record_modified_node(MacroAssembler* masm, A Register tmp3 = rscratch1; Register tmp4 = rscratch2; - Register obj = dst.base(); - assert_different_registers(tmp4, rcx); + Register tmp5 = tmp1 == dst.base() || tmp1 == dst.index() ? tmp2 : tmp1; - // tmp2 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); - __ movptr(tmp3, obj); + // tmp5 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); + __ movptr(tmp3, dst.base()); __ shrptr(tmp3, 6); - __ movptr(tmp2, SIDE_METADATA_BASE_ADDRESS); - __ movb(tmp2, Address(tmp2, tmp3)); + __ movptr(tmp5, SIDE_METADATA_BASE_ADDRESS); + __ movb(tmp5, Address(tmp5, tmp3)); // tmp3 = (obj >> 3) & 7 - __ movptr(tmp3, obj); + __ movptr(tmp3, dst.base()); __ shrptr(tmp3, 3); __ andptr(tmp3, 7); - // tmp2 = tmp2 >> tmp3 + // tmp5 = tmp5 >> tmp3 __ movptr(tmp4, rcx); __ movl(rcx, tmp3); - __ shrptr(tmp2); + __ shrptr(tmp5); __ movptr(rcx, tmp4); - // if ((tmp2 & 1) == 1) goto slowpath; - __ andptr(tmp2, 1); - __ cmpptr(tmp2, 1); + // if ((tmp5 & 1) == 0) goto slowpath; + __ andptr(tmp5, 1); + __ cmpptr(tmp5, 1); __ jcc(Assembler::notEqual, done); __ pusha(); @@ -113,13 +144,23 @@ void MMTkObjectBarrierSetC1::record_modified_node(LIRAccess& access, LIR_Opr src } assert(src->is_register(), "must be a register at this point"); if (!slot->is_register()) { - LIR_Opr reg = gen->new_pointer_register(); - if (slot->is_constant()) { - __ move(slot, reg); + // LIR_Opr reg = gen->new_pointer_register(); + // if (slot->is_constant()) { + // __ move(slot, reg); + // } else { + // __ leal(slot, reg); + // } + // slot = reg; + + LIR_Address* address = slot->as_address_ptr(); + LIR_Opr ptr = gen->new_pointer_register(); + if (!address->index()->is_valid() && address->disp() == 0) { + __ move(address->base(), ptr); } else { - __ leal(slot, reg); + assert(address->disp() != max_jint, "lea doesn't support patched addresses!"); + __ leal(slot, ptr); } - slot = reg; + slot = ptr; } assert(slot->is_register(), "must be a register at this point"); if (!new_val->is_register()) { @@ -189,8 +230,34 @@ void MMTkObjectBarrierSetC2::record_modified_node(GraphKit* kit, Node* src, Node Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); __ if_then(result, BoolTest::ne, zero, unlikely); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src, slot, val); + } __ end_if(); +#else + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src, slot, val); +#endif + + kit->final_sync(ideal); // Final sync IdealKit and GraphKit. +} + +void MMTkObjectBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const { + MMTkIdealKit ideal(kit, true); +#if MMTK_ENABLE_BARRIER_FASTPATH + Node* no_base = __ top(); + float unlikely = PROB_UNLIKELY(0.999); + + Node* zero = __ ConI(0); + Node* addr = __ CastPX(__ ctrl(), src); + Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); + Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); + Node* shift = __ URShiftX(addr, __ ConI(3)); + shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); + Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); + + __ if_then(result, BoolTest::ne, zero, unlikely); { + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); } __ end_if(); #else const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index 2d3201a0..b1994533 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -19,12 +19,15 @@ class MMTkObjectBarrierSetRuntime: public MMTkBarrierSetRuntime { public: static void record_modified_node_slow(void* src, void* slot, void* val); + static void record_clone_slow(void* src, void* dst, size_t size); virtual bool is_slow_path_call(address call) { return call == CAST_FROM_FN_PTR(address, record_modified_node_slow); } virtual void record_modified_node(oop src, ptrdiff_t offset, oop val); + virtual void record_clone(oop src, oop dst, size_t size); + virtual void record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length); }; class MMTkObjectBarrierSetC1; @@ -46,32 +49,33 @@ class MMTkObjectBarrierSetAssembler: public MMTkBarrierSetAssembler { void generate_c1_write_barrier_runtime_stub(StubAssembler* sasm) { __ prologue("mmtk_write_barrier", false); - Address store_addr(rbp, 3*BytesPerWord); + Address store_addr(rbp, 4*BytesPerWord); Label done; Label runtime; __ push(c_rarg0); - // __ push(c_rarg1); - // __ push(c_rarg2); + __ push(c_rarg1); + __ push(c_rarg2); __ push(rax); __ load_parameter(0, c_rarg0); - // __ load_parameter(1, c_rarg1); - // __ load_parameter(2, c_rarg2); + __ load_parameter(1, c_rarg1); + __ load_parameter(2, c_rarg2); __ bind(runtime); __ save_live_registers_no_oop_map(true); - __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), 1); + __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), 3); __ restore_live_registers(true); __ bind(done); + __ pop(rax); - // __ pop(c_rarg2); - // __ pop(c_rarg1); + __ pop(c_rarg2); + __ pop(c_rarg1); __ pop(c_rarg0); __ epilogue(); @@ -114,18 +118,16 @@ class MMTkObjectBarrierSetC1: public MMTkBarrierSetC1 { public: CodeBlob* _write_barrier_c1_runtime_code_blob; virtual void store_at_resolved(LIRAccess& access, LIR_Opr value) { - BarrierSetC1::store_at_resolved(access, value); if (access.is_oop()) record_modified_node(access, access.base().opr(), access.resolved_addr(), value); + BarrierSetC1::store_at_resolved(access, value); } virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) { - LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); if (access.is_oop()) record_modified_node(access, access.base().opr(), access.resolved_addr(), new_value.result()); - return result; + return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); } virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) { - LIR_Opr result = BarrierSetC1::atomic_xchg_at_resolved(access, value); if (access.is_oop()) record_modified_node(access, access.base().opr(), access.resolved_addr(), value.result()); - return result; + return BarrierSetC1::atomic_xchg_at_resolved(access, value); } virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob) { MMTkObjectBarrierCodeGenClosure write_code_gen_cl; @@ -149,6 +151,7 @@ class MMTkObjectBarrierSetC1: public MMTkBarrierSetC1 { class MMTkObjectBarrierSetC2: public MMTkBarrierSetC2 { void record_modified_node(GraphKit* kit, Node* node, Node* slot, Node* val) const; + void record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const; public: virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const { if (access.is_oop()) record_modified_node(access.kit(), access.base(), access.addr().node(), val.node()); @@ -171,14 +174,13 @@ class MMTkObjectBarrierSetC2: public MMTkBarrierSetC2 { return result; } virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { + record_clone(kit, src, dst, size); BarrierSetC2::clone(kit, src, dst, size, is_array); - // FIXME: This is required for generational barriers. - // record_modified_node(kit, dst, NULL, NULL); } virtual bool is_gc_barrier_node(Node* node) const { if (node->Opcode() != Op_CallLeaf) return false; CallLeafNode *call = node->as_CallLeaf(); - return call->_name != NULL && strcmp(call->_name, "record_modified_node") == 0; + return call->_name != NULL && (strcmp(call->_name, "record_modified_node") == 0 || strcmp(call->_name, "record_clone") == 0); } }; @@ -189,6 +191,8 @@ inline void MMTkObjectBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* MMTkObjectBarrierSetC1* bs = (MMTkObjectBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1(); __ bind(*stub->entry()); ce->store_parameter(stub->_src->as_pointer_register(), 0); + ce->store_parameter(stub->_slot->as_pointer_register(), 1); + ce->store_parameter(stub->_new_val->as_pointer_register(), 2); __ call(RuntimeAddress(bs->_write_barrier_c1_runtime_code_blob->code_begin())); __ jmp(*stub->continuation()); } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index a8a81927..c3e50ba5 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -35,7 +35,7 @@ extern void post_alloc(MMTk_Mutator mutator, void* refer, int bytes, int allocator); extern void mmtk_object_reference_write(MMTk_Mutator mutator, void* src, void* slot, void* val); -extern void mmtk_object_reference_arraycopy(MMTk_Mutator mutator, void* src, void* dst, size_t len); +extern void mmtk_object_reference_arraycopy(MMTk_Mutator mutator, void* src, size_t src_offset, void* dst, size_t dst_offset, size_t len); extern void mmtk_object_reference_clone(MMTk_Mutator mutator, void* src, void* dst, size_t len); extern void release_buffer(void** buffer, size_t len, size_t cap); diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 27f40b5c..c3662977 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -44,6 +44,8 @@ const intptr_t SIDE_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_B class MMTkBarrierSetRuntime: public CHeapObj { public: virtual void record_modified_node(oop src, ptrdiff_t offset, oop val) {}; + virtual void record_clone(oop src, oop dst, size_t size) {}; + virtual void record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) {}; virtual bool is_slow_path_call(address call) { return false; } @@ -163,13 +165,8 @@ class MMTkBarrierSet : public BarrierSet { static bool oop_arraycopy_in_heap_impl(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) { - T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); - T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); - ::mmtk_object_reference_arraycopy((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) dst, length); - bool result = Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, - dst_obj, dst_offset_in_bytes, dst_raw, - length); - return result; + runtime()->record_arraycopy(src_obj, src_offset_in_bytes, (oop*) src_raw, dst_obj, dst_offset_in_bytes, (oop*) dst_raw, length); + return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); } @@ -203,14 +200,12 @@ class MMTkBarrierSet : public BarrierSet { static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) { - return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, - dst_obj, dst_offset_in_bytes, dst_raw, - length); + return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); } static void clone_in_heap(oop src, oop dst, size_t size) { Raw::clone(src, dst, size); - ::mmtk_object_reference_clone((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) dst, size); + runtime()->record_clone(src, dst, size); } }; From bf963179580511a39fbbaba59b078c1071f3fddc Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 2 Sep 2021 05:07:25 +1000 Subject: [PATCH 041/494] Fix --- openjdk/barriers/mmtkObjectBarrier.cpp | 4 ++-- openjdk/mmtkBarrierSet.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index d28a59fd..e81c6271 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -260,8 +260,8 @@ void MMTkObjectBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, N Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); } __ end_if(); #else - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src, slot, val); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); #endif kit->final_sync(ideal); // Final sync IdealKit and GraphKit. diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index c3662977..9480a1cd 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -37,7 +37,7 @@ #include "mmtkBarrierSetAssembler_x86.hpp" #define MMTK_ENABLE_ALLOCATION_FASTPATH true -#define MMTK_ENABLE_BARRIER_FASTPATH true +#define MMTK_ENABLE_BARRIER_FASTPATH false const intptr_t SIDE_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; From 6bdd2b2dfc5907d2520839ba4d6db1ac12422c0c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 3 Sep 2021 13:54:14 +1000 Subject: [PATCH 042/494] WIP --- mmtk/src/vm_metadata/constants.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/vm_metadata/constants.rs b/mmtk/src/vm_metadata/constants.rs index 12c5742a..0efc2b53 100644 --- a/mmtk/src/vm_metadata/constants.rs +++ b/mmtk/src/vm_metadata/constants.rs @@ -11,7 +11,7 @@ pub(crate) const FORWARDING_POINTER_OFFSET: isize = 0; /// Global logging bit metadata spec /// 1 bit per object -pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); +pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_after(&mmtk::plan::immix::RC); // Global MetadataSpecs - End From d5a22cca33b19cd7c4756892a8af555944dbf215 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 5 Sep 2021 20:10:08 +1000 Subject: [PATCH 043/494] WIP: naive rc --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 48 ++++++++++++++++++++ openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 8 ++-- openjdk/mmtkBarrierSet.hpp | 2 +- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index 1c516e48..13a9d490 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -23,6 +23,26 @@ void MMTkFieldLoggingBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_ ::mmtk_object_reference_arraycopy((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src_obj, src_offset_in_bytes, (void*) dst_obj, dst_offset_in_bytes, length); } + + +void MMTkFieldLoggingBarrierSetRuntime::record_clone_slow(void* src, void* dst, size_t size) { + ::mmtk_object_reference_clone((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, size); +} + +void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t size) { +#if MMTK_ENABLE_BARRIER_FASTPATH + intptr_t addr = (intptr_t) (void*) dst; + uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + intptr_t shift = (addr >> 3) & 0b111; + uint8_t byte_val = *meta_addr; + if (((byte_val >> shift) & 1) == 1) { + record_clone_slow((void*) src, (void*) dst, size); + } +#else + record_clone_slow((void*) src, (void*) dst, size); +#endif +} + #define __ masm-> void MMTkFieldLoggingBarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { @@ -205,4 +225,32 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } + +void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const { + MMTkIdealKit ideal(kit, true); +#if MMTK_ENABLE_BARRIER_FASTPATH + Node* no_base = __ top(); + float unlikely = PROB_UNLIKELY(0.999); + + Node* zero = __ ConI(0); + Node* addr = __ CastPX(__ ctrl(), src); + Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); + Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); + Node* shift = __ URShiftX(addr, __ ConI(3)); + shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); + Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); + + __ if_then(result, BoolTest::ne, zero, unlikely); { + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); + } __ end_if(); +#else + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); +#endif + + kit->final_sync(ideal); // Final sync IdealKit and GraphKit. +} + + #undef __ \ No newline at end of file diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index 0f08b2b7..0617bfe4 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -19,6 +19,7 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { public: static void record_modified_node_slow(void* src, void* slot, void* val); + static void record_clone_slow(void* src, void* dst, size_t size); virtual bool is_slow_path_call(address call) { return call == CAST_FROM_FN_PTR(address, record_modified_node_slow); @@ -26,6 +27,7 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { virtual void record_modified_node(oop src, ptrdiff_t offset, oop val); virtual void record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length); + virtual void record_clone(oop src, oop dst, size_t size); }; class MMTkFieldLoggingBarrierSetC1; @@ -150,6 +152,7 @@ class MMTkFieldLoggingBarrierSetC1: public MMTkBarrierSetC1 { class MMTkFieldLoggingBarrierSetC2: public MMTkBarrierSetC2 { void record_modified_node(GraphKit* kit, Node* node, Node* slot, Node* val) const; + void record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const; public: virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const { if (access.is_oop()) record_modified_node(access.kit(), access.base(), access.addr().node(), val.node()); @@ -172,14 +175,13 @@ class MMTkFieldLoggingBarrierSetC2: public MMTkBarrierSetC2 { return result; } virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { + record_clone(kit, src, dst, size); BarrierSetC2::clone(kit, src, dst, size, is_array); - // FIXME: This is required for generational barriers. - // record_modified_node(kit, dst, NULL, NULL); } virtual bool is_gc_barrier_node(Node* node) const { if (node->Opcode() != Op_CallLeaf) return false; CallLeafNode *call = node->as_CallLeaf(); - return call->_name != NULL && strcmp(call->_name, "record_modified_node") == 0; + return call->_name != NULL && (strcmp(call->_name, "record_modified_node") == 0 || strcmp(call->_name, "record_clone") == 0); } }; diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 9480a1cd..397f1002 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -36,7 +36,7 @@ #include "mmtk.h" #include "mmtkBarrierSetAssembler_x86.hpp" -#define MMTK_ENABLE_ALLOCATION_FASTPATH true +#define MMTK_ENABLE_ALLOCATION_FASTPATH false #define MMTK_ENABLE_BARRIER_FASTPATH false const intptr_t SIDE_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; From ea0145a3cbd1d7d2e7bdb7aec1fff45e1bf606f7 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 8 Sep 2021 23:31:32 +1000 Subject: [PATCH 044/494] GenCopy field barrier --- mmtk/src/api.rs | 9 ++- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 58 ++++++++++++++++++-- openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 12 ++-- openjdk/barriers/mmtkObjectBarrier.cpp | 4 +- openjdk/barriers/mmtkObjectBarrier.hpp | 2 +- openjdk/mmtkBarrierSet.cpp | 5 ++ 6 files changed, 77 insertions(+), 13 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index bae52e21..c9e29e44 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -20,13 +20,20 @@ use std::lazy::SyncLazy; static NO_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("NoBarrier").unwrap()); static OBJECT_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("ObjectBarrier").unwrap()); static FIELD_LOGGING_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("FieldLoggingBarrier").unwrap()); +static FIELD_LOGGING_BARRIER_GEN: SyncLazy = SyncLazy::new(|| CString::new("FieldLoggingBarrier-GEN").unwrap()); #[no_mangle] pub extern "C" fn mmtk_active_barrier() -> *const c_char { match SINGLETON.get_plan().constraints().barrier { BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), - BarrierSelector::FieldLoggingBarrier => FIELD_LOGGING_BARRIER.as_ptr(), + BarrierSelector::FieldLoggingBarrier => { + if std::env::var("MMTK_PLAN") == Ok("GenCopy".to_owned()) { + FIELD_LOGGING_BARRIER_GEN.as_ptr() + } else { + FIELD_LOGGING_BARRIER.as_ptr() + } + } // In case we have more barriers in mmtk-core. #[allow(unreachable_patterns)] _ => unimplemented!(), diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index 1c516e48..46aa53f2 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -1,17 +1,23 @@ #include "mmtkFieldLoggingBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" +int MMTkFieldLoggingBarrierSetRuntime::unlogged_value = 0; + void MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow(void* src, void* slot, void* val) { ::mmtk_object_reference_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, val); } +void MMTkFieldLoggingBarrierSetRuntime::record_clone_slow(void* src, void* dst, size_t size) { + ::mmtk_object_reference_clone((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, size); +} + void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset, oop val) { #if MMTK_ENABLE_BARRIER_FASTPATH - intptr_t addr = (intptr_t) (void*) src; + intptr_t addr = ((intptr_t) (void*) src) + offset; uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); intptr_t shift = (addr >> 3) & 0b111; uint8_t byte_val = *meta_addr; - if (((byte_val >> shift) & 1) == 0) { + if (((byte_val >> shift) & 1) == MMTkFieldLoggingBarrierSetRuntime::unlogged_value) { record_modified_node_slow((void*) src, (void*) (((intptr_t) (void*) src) + offset), (void*) val); } #else @@ -19,6 +25,21 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t #endif } +void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t size) { + if (unlogged_value == 0) return; +#if MMTK_ENABLE_BARRIER_FASTPATH || true + intptr_t addr = (intptr_t) (void*) dst; + uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + intptr_t shift = (addr >> 3) & 0b111; + uint8_t byte_val = *meta_addr; + if (((byte_val >> shift) & 1) == 1) { + record_clone_slow((void*) src, (void*) dst, size); + } +#else + record_clone_slow((void*) src, (void*) dst, size); +#endif +} + void MMTkFieldLoggingBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) { ::mmtk_object_reference_arraycopy((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src_obj, src_offset_in_bytes, (void*) dst_obj, dst_offset_in_bytes, length); } @@ -64,7 +85,7 @@ void MMTkFieldLoggingBarrierSetAssembler::record_modified_node(MacroAssembler* m __ movptr(rcx, tmp4); // if ((tmp5 & 1) == 1) goto slowpath; __ andptr(tmp5, 1); - __ cmpptr(tmp5, 0); + __ cmpptr(tmp5, MMTkFieldLoggingBarrierSetRuntime::unlogged_value); __ jcc(Assembler::notEqual, done); // TODO: Spill fewer registers @@ -185,7 +206,7 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src Node* no_base = __ top(); float unlikely = PROB_UNLIKELY(0.999); - Node* zero = __ ConI(0); + Node* unlogged_value = __ ConI(MMTkFieldLoggingBarrierSetRuntime::unlogged_value); Node* addr = __ CastPX(__ ctrl(), slot); Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); @@ -193,7 +214,7 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); - __ if_then(result, BoolTest::eq, zero, unlikely); { + __ if_then(result, BoolTest::eq, unlogged_value, unlikely); { const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src, slot, val); } __ end_if(); @@ -205,4 +226,31 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } +void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const { + if (MMTkFieldLoggingBarrierSetRuntime::unlogged_value == 0) return; + MMTkIdealKit ideal(kit, true); +#if MMTK_ENABLE_BARRIER_FASTPATH + Node* no_base = __ top(); + float unlikely = PROB_UNLIKELY(0.999); + + Node* unlogged_value = __ ConI(MMTkFieldLoggingBarrierSetRuntime::unlogged_value); + Node* addr = __ CastPX(__ ctrl(), src); + Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); + Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); + Node* shift = __ URShiftX(addr, __ ConI(3)); + shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); + Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); + + __ if_then(result, BoolTest::eq, unlogged_value, unlikely); { + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); + } __ end_if(); +#else + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); +#endif + + kit->final_sync(ideal); // Final sync IdealKit and GraphKit. +} + #undef __ \ No newline at end of file diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index 0f08b2b7..9bdd8e47 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -18,13 +18,17 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { public: + static int unlogged_value; + static void record_modified_node_slow(void* src, void* slot, void* val); + static void record_clone_slow(void* src, void* dst, size_t size); virtual bool is_slow_path_call(address call) { - return call == CAST_FROM_FN_PTR(address, record_modified_node_slow); + return call == CAST_FROM_FN_PTR(address, record_modified_node_slow) || call == CAST_FROM_FN_PTR(address, record_clone_slow); } virtual void record_modified_node(oop src, ptrdiff_t offset, oop val); + virtual void record_clone(oop src, oop dst, size_t size); virtual void record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length); }; @@ -150,6 +154,7 @@ class MMTkFieldLoggingBarrierSetC1: public MMTkBarrierSetC1 { class MMTkFieldLoggingBarrierSetC2: public MMTkBarrierSetC2 { void record_modified_node(GraphKit* kit, Node* node, Node* slot, Node* val) const; + void record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const; public: virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const { if (access.is_oop()) record_modified_node(access.kit(), access.base(), access.addr().node(), val.node()); @@ -173,13 +178,12 @@ class MMTkFieldLoggingBarrierSetC2: public MMTkBarrierSetC2 { } virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { BarrierSetC2::clone(kit, src, dst, size, is_array); - // FIXME: This is required for generational barriers. - // record_modified_node(kit, dst, NULL, NULL); + // record_clone(kit, src, dst, size); } virtual bool is_gc_barrier_node(Node* node) const { if (node->Opcode() != Op_CallLeaf) return false; CallLeafNode *call = node->as_CallLeaf(); - return call->_name != NULL && strcmp(call->_name, "record_modified_node") == 0; + return call->_name != NULL && (strcmp(call->_name, "record_modified_node") == 0 || strcmp(call->_name, "record_clone") == 0); } }; diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index d28a59fd..e81c6271 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -260,8 +260,8 @@ void MMTkObjectBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, N Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); } __ end_if(); #else - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src, slot, val); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); #endif kit->final_sync(ideal); // Final sync IdealKit and GraphKit. diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index b1994533..7eb0132d 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -22,7 +22,7 @@ class MMTkObjectBarrierSetRuntime: public MMTkBarrierSetRuntime { static void record_clone_slow(void* src, void* dst, size_t size); virtual bool is_slow_path_call(address call) { - return call == CAST_FROM_FN_PTR(address, record_modified_node_slow); + return call == CAST_FROM_FN_PTR(address, record_modified_node_slow) || call == CAST_FROM_FN_PTR(address, record_clone_slow); } virtual void record_modified_node(oop src, ptrdiff_t offset, oop val); diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index 7e66bec1..d8dd05bf 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -40,9 +40,14 @@ MMTkBarrierBase* get_selected_barrier() { static MMTkBarrierBase* selected_barrier = NULL; if (selected_barrier) return selected_barrier; const char* barrier = mmtk_active_barrier(); + printf("%s\n", barrier); if (strcmp(barrier, "NoBarrier") == 0) selected_barrier = new MMTkNoBarrier(); else if (strcmp(barrier, "ObjectBarrier") == 0) selected_barrier = new MMTkObjectBarrier(); else if (strcmp(barrier, "FieldLoggingBarrier") == 0) selected_barrier = new MMTkFieldLoggingBarrier(); + else if (strcmp(barrier, "FieldLoggingBarrier-GEN") == 0) { + MMTkFieldLoggingBarrierSetRuntime::unlogged_value = 1; + selected_barrier = new MMTkFieldLoggingBarrier(); + } else guarantee(false, "Unimplemented"); return selected_barrier; } From a8789e096c93c772a5959132287c9abf6ca34a8d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 10 Sep 2021 12:48:07 +1000 Subject: [PATCH 045/494] Minor fix --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index 46aa53f2..63976c0e 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -27,7 +27,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t size) { if (unlogged_value == 0) return; -#if MMTK_ENABLE_BARRIER_FASTPATH || true +#if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = (intptr_t) (void*) dst; uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); intptr_t shift = (addr >> 3) & 0b111; From 8a028b22ce45cb34841d2c00f8199f7782b9d51e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 12 Sep 2021 16:53:42 +1000 Subject: [PATCH 046/494] WIP: RC --- mmtk/src/vm_metadata/constants.rs | 2 +- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 2 +- openjdk/mmtkBarrierSet.hpp | 2 +- openjdk/mmtkHeap.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mmtk/src/vm_metadata/constants.rs b/mmtk/src/vm_metadata/constants.rs index 0efc2b53..12c5742a 100644 --- a/mmtk/src/vm_metadata/constants.rs +++ b/mmtk/src/vm_metadata/constants.rs @@ -11,7 +11,7 @@ pub(crate) const FORWARDING_POINTER_OFFSET: isize = 0; /// Global logging bit metadata spec /// 1 bit per object -pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_after(&mmtk::plan::immix::RC); +pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); // Global MetadataSpecs - End diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index 63976c0e..3c343e58 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -26,7 +26,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t } void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t size) { - if (unlogged_value == 0) return; + // if (unlogged_value == 0) return; #if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = (intptr_t) (void*) dst; uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 53ee9a8b..d18e47ae 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -204,8 +204,8 @@ class MMTkBarrierSet : public BarrierSet { } static void clone_in_heap(oop src, oop dst, size_t size) { - Raw::clone(src, dst, size); runtime()->record_clone(src, dst, size); + Raw::clone(src, dst, size); } }; diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index b21775dd..f73db090 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -302,7 +302,7 @@ void MMTkHeap::initialize_serviceability() {//OK } // Print heap information on the given outputStream. -void MMTkHeap::print_on(outputStream* st) const {guarantee(false, "print on not supported");} +void MMTkHeap::print_on(outputStream* st) const {} // Print all GC threads (other than the VM thread) From cc543e52655eeee5c01b2d13feb2fe710139d1a4 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 14 Sep 2021 21:33:23 +1000 Subject: [PATCH 047/494] Use unlog bits --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 4 +--- openjdk/mmtkBarrierSet.cpp | 5 +---- openjdk/mmtkBarrierSet.hpp | 4 ++-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index 3c343e58..de5a160b 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -1,7 +1,7 @@ #include "mmtkFieldLoggingBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" -int MMTkFieldLoggingBarrierSetRuntime::unlogged_value = 0; +int MMTkFieldLoggingBarrierSetRuntime::unlogged_value = 1; void MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow(void* src, void* slot, void* val) { ::mmtk_object_reference_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, val); @@ -26,7 +26,6 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t } void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t size) { - // if (unlogged_value == 0) return; #if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = (intptr_t) (void*) dst; uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); @@ -227,7 +226,6 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src } void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const { - if (MMTkFieldLoggingBarrierSetRuntime::unlogged_value == 0) return; MMTkIdealKit ideal(kit, true); #if MMTK_ENABLE_BARRIER_FASTPATH Node* no_base = __ top(); diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index e2a24937..f24cae60 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -45,10 +45,7 @@ MMTkBarrierBase* get_selected_barrier() { if (strcmp(barrier, "NoBarrier") == 0) selected_barrier = new MMTkNoBarrier(); else if (strcmp(barrier, "ObjectBarrier") == 0) selected_barrier = new MMTkObjectBarrier(); else if (strcmp(barrier, "FieldLoggingBarrier") == 0) selected_barrier = new MMTkFieldLoggingBarrier(); - else if (strcmp(barrier, "FieldLoggingBarrier-GEN") == 0) { - MMTkFieldLoggingBarrierSetRuntime::unlogged_value = 1; - selected_barrier = new MMTkFieldLoggingBarrier(); - } + else if (strcmp(barrier, "FieldLoggingBarrier-GEN") == 0) selected_barrier = new MMTkFieldLoggingBarrier(); else guarantee(false, "Unimplemented"); return selected_barrier; } diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index d18e47ae..450fc262 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -36,8 +36,8 @@ #include "oops/oopsHierarchy.hpp" #include "utilities/fakeRttiSupport.hpp" -#define MMTK_ENABLE_ALLOCATION_FASTPATH false -#define MMTK_ENABLE_BARRIER_FASTPATH false +#define MMTK_ENABLE_ALLOCATION_FASTPATH true +#define MMTK_ENABLE_BARRIER_FASTPATH true const intptr_t SIDE_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; From 67faa1bfce161ba39b578324ba3c397245fb62bc Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 16 Sep 2021 14:02:50 +1000 Subject: [PATCH 048/494] Fix roots work packets --- mmtk/src/scanning.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index bbc425dc..b8bd8d06 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -22,7 +22,7 @@ pub(crate) extern "C" fn create_process_edges_work Date: Fri, 17 Sep 2021 14:18:52 +1000 Subject: [PATCH 049/494] Add "log-in-progress" state --- mmtk/Cargo.toml | 1 + openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 62 ++++++-------------- openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 2 - 3 files changed, 19 insertions(+), 46 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index af52c27f..92944ffa 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -10,6 +10,7 @@ crate-type = ["cdylib"] [profile.release] lto = true +codegen-units = 1 [dependencies] libc = "0.2" diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index de5a160b..ee302905 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -1,7 +1,7 @@ #include "mmtkFieldLoggingBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" -int MMTkFieldLoggingBarrierSetRuntime::unlogged_value = 1; +constexpr int kLoggedValue = 0; void MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow(void* src, void* slot, void* val) { ::mmtk_object_reference_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, val); @@ -14,10 +14,10 @@ void MMTkFieldLoggingBarrierSetRuntime::record_clone_slow(void* src, void* dst, void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset, oop val) { #if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = ((intptr_t) (void*) src) + offset; - uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); - intptr_t shift = (addr >> 3) & 0b111; + uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 5)); + intptr_t shift = ((addr >> 3) & 0b11) << 1; uint8_t byte_val = *meta_addr; - if (((byte_val >> shift) & 1) == MMTkFieldLoggingBarrierSetRuntime::unlogged_value) { + if (((byte_val >> shift) & 3) != kLoggedValue) { record_modified_node_slow((void*) src, (void*) (((intptr_t) (void*) src) + offset), (void*) val); } #else @@ -26,17 +26,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t } void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t size) { -#if MMTK_ENABLE_BARRIER_FASTPATH - intptr_t addr = (intptr_t) (void*) dst; - uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); - intptr_t shift = (addr >> 3) & 0b111; - uint8_t byte_val = *meta_addr; - if (((byte_val >> shift) & 1) == 1) { - record_clone_slow((void*) src, (void*) dst, size); - } -#else record_clone_slow((void*) src, (void*) dst, size); -#endif } void MMTkFieldLoggingBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) { @@ -70,22 +60,23 @@ void MMTkFieldLoggingBarrierSetAssembler::record_modified_node(MacroAssembler* m // tmp5 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); __ lea(tmp3, dst); - __ shrptr(tmp3, 6); + __ shrptr(tmp3, 5); __ movptr(tmp5, SIDE_METADATA_BASE_ADDRESS); __ movb(tmp5, Address(tmp5, tmp3)); - // tmp3 = (obj >> 3) & 7 + // tmp3 = ((obj >> 3) & 3) << 1 __ lea(tmp3, dst); __ shrptr(tmp3, 3); - __ andptr(tmp3, 7); + __ andptr(tmp3, 3); + __ shlptr(tmp3, 1); // tmp5 = tmp5 >> tmp3 __ movptr(tmp4, rcx); __ movl(rcx, tmp3); __ shrptr(tmp5); __ movptr(rcx, tmp4); - // if ((tmp5 & 1) == 1) goto slowpath; - __ andptr(tmp5, 1); - __ cmpptr(tmp5, MMTkFieldLoggingBarrierSetRuntime::unlogged_value); - __ jcc(Assembler::notEqual, done); + // if ((tmp5 & 3) == 1) goto slowpath; + __ andptr(tmp5, 3); + __ cmpptr(tmp5, kLoggedValue); + __ jcc(Assembler::equal, done); // TODO: Spill fewer registers __ pusha(); @@ -205,15 +196,16 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src Node* no_base = __ top(); float unlikely = PROB_UNLIKELY(0.999); - Node* unlogged_value = __ ConI(MMTkFieldLoggingBarrierSetRuntime::unlogged_value); + Node* logged_value = __ ConI(kLoggedValue); Node* addr = __ CastPX(__ ctrl(), slot); - Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); + Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(5))); Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); Node* shift = __ URShiftX(addr, __ ConI(3)); - shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); - Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); + shift = __ AndI(__ ConvL2I(shift), __ ConI(3)); + shift = __ LShiftI(shift, __ ConI(1)); + Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(3)); - __ if_then(result, BoolTest::eq, unlogged_value, unlikely); { + __ if_then(result, BoolTest::ne, logged_value, unlikely); { const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src, slot, val); } __ end_if(); @@ -227,26 +219,8 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const { MMTkIdealKit ideal(kit, true); -#if MMTK_ENABLE_BARRIER_FASTPATH - Node* no_base = __ top(); - float unlikely = PROB_UNLIKELY(0.999); - - Node* unlogged_value = __ ConI(MMTkFieldLoggingBarrierSetRuntime::unlogged_value); - Node* addr = __ CastPX(__ ctrl(), src); - Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); - Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); - Node* shift = __ URShiftX(addr, __ ConI(3)); - shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); - Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); - - __ if_then(result, BoolTest::eq, unlogged_value, unlikely); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); - Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); - } __ end_if(); -#else const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); -#endif kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index 4a0c9749..1840b4a0 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -18,8 +18,6 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { public: - static int unlogged_value; - static void record_modified_node_slow(void* src, void* slot, void* val); static void record_clone_slow(void* src, void* dst, size_t size); From 70123331b9f627a2b5c3143ff9bd8ec54f2e563f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 17 Sep 2021 22:36:38 +1000 Subject: [PATCH 050/494] Format --- mmtk/src/api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index c9e29e44..5b1de34a 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -273,7 +273,7 @@ pub extern "C" fn mmtk_object_reference_clone( mutator: &'static mut Mutator, src: ObjectReference, dst: ObjectReference, - size: usize, + _size: usize, ) { mutator.object_reference_clone(src, dst); } From fb934966bd6905e6ef30ff7857780ae6c7d1b195 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 18 Sep 2021 15:49:14 +1000 Subject: [PATCH 051/494] Remove C2 clone barrier (looks unnecessary) --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index ee302905..8bb6a417 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -217,12 +217,6 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } -void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const { - MMTkIdealKit ideal(kit, true); - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); - Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); - - kit->final_sync(ideal); // Final sync IdealKit and GraphKit. -} +void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const {} #undef __ \ No newline at end of file From 2103f27bc1e950920c3be862bd82dedc2b54158f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 18 Sep 2021 18:20:32 +1000 Subject: [PATCH 052/494] Port `oop::size()` to Rust --- mmtk/src/abi.rs | 71 +++++++++++++++++++++++++++++++++++++++- mmtk/src/lib.rs | 2 ++ mmtk/src/object_model.rs | 5 +-- openjdk/mmtkUpcalls.cpp | 3 +- 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 7d4ea47a..121cf793 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -1,4 +1,5 @@ use super::UPCALLS; +use mmtk::util::ObjectReference; use mmtk::util::constants::*; use mmtk::util::conversions; use mmtk::util::{Address, OpaquePointer}; @@ -64,9 +65,28 @@ pub struct Klass { } impl Klass { + pub const LH_NEUTRAL_VALUE: i32 = 0; + pub const LH_INSTANCE_SLOW_PATH_BIT: i32 = 0x01; + pub const LH_LOG2_ELEMENT_SIZE_SHIFT: i32 = BITS_IN_BYTE as i32 * 0; + pub const LH_LOG2_ELEMENT_SIZE_MASK: i32 = BITS_IN_LONG as i32 - 1; + pub const LH_HEADER_SIZE_SHIFT: i32 = BITS_IN_BYTE as i32 * 2; + pub const LH_HEADER_SIZE_MASK: i32 = (1 << BITS_IN_BYTE) - 1; + #[inline(always)] pub unsafe fn cast<'a, T>(&self) -> &'a T { &*(self as *const _ as usize as *const T) } + #[inline(always)] + pub const fn layout_helper_needs_slow_path(lh: i32) -> bool { + (lh & Self::LH_INSTANCE_SLOW_PATH_BIT) != 0 + } + #[inline(always)] + pub const fn layout_helper_log2_element_size(lh: i32) -> i32 { + (lh >> Self::LH_LOG2_ELEMENT_SIZE_SHIFT) & Self::LH_LOG2_ELEMENT_SIZE_MASK + } + #[inline(always)] + pub const fn layout_helper_header_size(lh: i32) -> i32 { + (lh >> Self::LH_HEADER_SIZE_SHIFT) & Self::LH_HEADER_SIZE_MASK + } } #[repr(C)] @@ -249,14 +269,63 @@ impl fmt::Debug for OopDesc { pub type Oop = &'static OopDesc; +impl const From for Oop { + #[inline(always)] + fn from(o: ObjectReference) -> Self { + unsafe { + mem::transmute(o) + } + } +} + +impl const Into for &OopDesc { + #[inline(always)] + fn into(self) -> ObjectReference { + unsafe { + mem::transmute::<&OopDesc, _>(self) + } + } +} + impl OopDesc { - pub unsafe fn as_array_oop(&self) -> ArrayOop { + #[inline(always)] + pub const unsafe fn as_array_oop(&self) -> ArrayOop { &*(self as *const OopDesc as *const ArrayOopDesc) } + #[inline(always)] pub fn get_field_address(&self, offset: i32) -> Address { Address::from_ref(self) + offset as isize } + + #[inline(always)] + unsafe fn size_slow(&self) -> usize { + ((*UPCALLS).get_object_size)(self.into()) + } + + #[inline(always)] + pub unsafe fn size(&self) -> usize { + let klass = self.klass; + let lh = klass.layout_helper; + if lh > Klass::LH_NEUTRAL_VALUE { + if !Klass::layout_helper_needs_slow_path(lh) { + lh as _ + } else { + self.size_slow() + } + } else if lh <= Klass::LH_NEUTRAL_VALUE { + if lh < Klass::LH_NEUTRAL_VALUE { + let array_length = self.as_array_oop::<()>().length(); + let mut size_in_bytes: usize = (array_length as usize) << Klass::layout_helper_log2_element_size(lh); + size_in_bytes += Klass::layout_helper_header_size(lh) as usize; + (size_in_bytes + 0b111) & !0b111 + } else { + self.size_slow() + } + } else { + unreachable!() + } + } } #[repr(C)] diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index c00427dd..8764aead 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -4,6 +4,8 @@ #![feature(box_syntax)] #![feature(vec_into_raw_parts)] #![feature(once_cell)] +#![feature(const_raw_ptr_deref)] +#![feature(const_trait_impl)] extern crate libc; extern crate mmtk; diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index fc27a959..7c610ae5 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -1,6 +1,7 @@ use std::sync::atomic::Ordering; use super::UPCALLS; +use crate::abi::Oop; use crate::{vm_metadata, OpenJDK}; use mmtk::util::metadata::header_metadata::HeaderMetadataSpec; use mmtk::util::{Address, ObjectReference}; @@ -89,7 +90,7 @@ impl ObjectModel for VMObjectModel { allocator: AllocationSemantics, copy_context: &mut impl CopyContext, ) -> ObjectReference { - let bytes = unsafe { ((*UPCALLS).get_object_size)(from) }; + let bytes = unsafe { Oop::from(from).size() }; let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::(), 0, allocator); // Copy @@ -111,7 +112,7 @@ impl ObjectModel for VMObjectModel { } fn get_current_size(object: ObjectReference) -> usize { - unsafe { ((*UPCALLS).get_object_size)(object) } + unsafe { Oop::from(object).size() } } fn get_type_descriptor(_reference: ObjectReference) -> &'static [i8] { diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index cae4b4fe..c711e662 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -180,7 +180,8 @@ static void mmtk_dump_object(void* object) { static size_t mmtk_get_object_size(void* object) { oop o = (oop) object; - return o->size() * HeapWordSize; + auto klass = o->klass(); + return klass->oop_size(o) << LogHeapWordSize; } static int mmtk_enter_vm() { From b174cb28e9f807be1226f30937061da7603be4d6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 19 Sep 2021 16:16:44 +1000 Subject: [PATCH 053/494] RC: Process incs before space.prepare --- mmtk/src/api.rs | 3 +++ mmtk/src/lib.rs | 9 +++++++++ mmtk/src/scanning.rs | 15 ++++++++++----- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 5b1de34a..f66d88e0 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -129,6 +129,9 @@ pub extern "C" fn will_never_move(object: ObjectReference) -> bool { // We trust the worker pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] pub extern "C" fn start_worker(tls: VMWorkerThread, worker: *mut GCWorker) { + unsafe { + crate::CURRENT_WORKER = Some(&mut *worker); + } memory_manager::start_worker::(tls, unsafe { worker.as_mut().unwrap() }, &SINGLETON) } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 8764aead..f56899e6 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -6,6 +6,7 @@ #![feature(once_cell)] #![feature(const_raw_ptr_deref)] #![feature(const_trait_impl)] +#![feature(thread_local)] extern crate libc; extern crate mmtk; @@ -86,6 +87,14 @@ pub struct OpenJDK_Upcalls { pub stop_mutators: extern "C" fn(tls: VMWorkerThread), } +#[thread_local] +pub static mut CURRENT_WORKER: Option<&'static mut GCWorker> = None; + +#[inline(always)] +pub fn current_worker() -> &'static mut GCWorker { + unsafe { CURRENT_WORKER.as_mut().unwrap() } +} + pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); #[no_mangle] diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index b8bd8d06..497d69d7 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -19,11 +19,16 @@ pub(crate) extern "C" fn create_process_edges_work NewBuffer { if !ptr.is_null() { let buf = unsafe { Vec::
::from_raw_parts(ptr, length, capacity) }; - memory_manager::add_work_packet( - &SINGLETON, - WorkBucketStage::ForwardRoots, - W::new(buf, true, &SINGLETON), - ); + let w = W::new(buf, true, &SINGLETON); + if W::RC_ROOTS { + crate::current_worker().add_work(WorkBucketStage::Unconstrained, w); + } else { + memory_manager::add_work_packet( + &SINGLETON, + WorkBucketStage::ProcessRoots, + w, + ); + } } let (ptr, _, capacity) = Vec::with_capacity(W::CAPACITY).into_raw_parts(); NewBuffer { ptr, capacity } From 32363ac1ffc36fde53f9d8edfb15f0b672d2f1e0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 19 Sep 2021 17:34:35 +1000 Subject: [PATCH 054/494] Minor fix --- mmtk/src/collection.rs | 3 ++- mmtk/src/scanning.rs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index f575e94d..881fdb1f 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -12,9 +12,10 @@ pub struct VMCollection {} extern "C" fn create_mutator_scan_work>( mutator: &'static mut Mutator, ) { + mutator.flush(); mmtk::memory_manager::add_work_packet( &SINGLETON, - WorkBucketStage::Prepare, + WorkBucketStage::Unconstrained, ScanStackRoot::(mutator), ); } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 497d69d7..481f1d86 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -79,7 +79,7 @@ impl Scanning for VMScanning { fn scan_vm_specific_roots>() { memory_manager::add_work_packets( &SINGLETON, - WorkBucketStage::Prepare, + WorkBucketStage::ScanGlobalRoots, vec![ box ScanUniverseRoots::::new(), box ScanJNIHandlesRoots::::new(), @@ -97,7 +97,7 @@ impl Scanning for VMScanning { if !(Self::SCAN_MUTATORS_IN_SAFEPOINT && Self::SINGLE_THREAD_MUTATOR_SCANNING) { memory_manager::add_work_packet( &SINGLETON, - WorkBucketStage::Prepare, + WorkBucketStage::ScanGlobalRoots, ScanVMThreadRoots::::new(), ); } From 08482713b87456e9efd45421478bef3904a3da2d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 22 Sep 2021 14:43:18 +1000 Subject: [PATCH 055/494] Fix GenCopy unlog bits --- mmtk/src/lib.rs | 4 ++++ openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 15 +++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index f56899e6..573b8897 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -105,6 +105,10 @@ pub static GLOBAL_SIDE_METADATA_BASE_ADDRESS: uintptr_t = pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS: uintptr_t = crate::mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize(); +#[no_mangle] +pub static GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS: uintptr_t = + crate::mmtk::util::metadata::side_metadata::RC_UNLOG_BIT_SIDE_METADATA_SPEC.get_absolute_offset().as_usize(); + #[no_mangle] pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t = crate::mmtk::util::metadata::side_metadata::ALLOC_SIDE_METADATA_ADDR.as_usize(); diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index 8bb6a417..e9b3a663 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -3,6 +3,9 @@ constexpr int kLoggedValue = 0; + +extern const intptr_t GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS; + void MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow(void* src, void* slot, void* val) { ::mmtk_object_reference_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, val); } @@ -14,7 +17,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_clone_slow(void* src, void* dst, void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset, oop val) { #if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = ((intptr_t) (void*) src) + offset; - uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 5)); + uint8_t* meta_addr = (uint8_t*) (GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS + (addr >> 5)); intptr_t shift = ((addr >> 3) & 0b11) << 1; uint8_t byte_val = *meta_addr; if (((byte_val >> shift) & 3) != kLoggedValue) { @@ -58,10 +61,10 @@ void MMTkFieldLoggingBarrierSetAssembler::record_modified_node(MacroAssembler* m Register tmp4 = rscratch2; Register tmp5 = tmp1 == dst.base() || tmp1 == dst.index() ? tmp2 : tmp1; - // tmp5 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); + // tmp5 = load-byte (GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS + (obj >> 6)); __ lea(tmp3, dst); __ shrptr(tmp3, 5); - __ movptr(tmp5, SIDE_METADATA_BASE_ADDRESS); + __ movptr(tmp5, GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS); __ movb(tmp5, Address(tmp5, tmp3)); // tmp3 = ((obj >> 3) & 3) << 1 __ lea(tmp3, dst); @@ -152,12 +155,12 @@ void MMTkFieldLoggingBarrierSetC1::record_modified_node(LIRAccess& access, LIR_O #if MMTK_ENABLE_BARRIER_FASTPATH LIR_Opr addr = src; - // uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + // uint8_t* meta_addr = (uint8_t*) (GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS + (addr >> 6)); LIR_Opr offset = gen->new_pointer_register(); __ move(addr, offset); __ shift_right(offset, 6, offset); LIR_Opr base = gen->new_pointer_register(); - __ move(LIR_OprFact::longConst(SIDE_METADATA_BASE_ADDRESS), base); + __ move(LIR_OprFact::longConst(GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS), base); LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); // intptr_t shift = (addr >> 3) & 0b111; LIR_Opr shift_long = gen->new_pointer_register(); @@ -198,7 +201,7 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src Node* logged_value = __ ConI(kLoggedValue); Node* addr = __ CastPX(__ ctrl(), slot); - Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(5))); + Node* meta_addr = __ AddP(no_base, __ ConP(GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS), __ URShiftX(addr, __ ConI(5))); Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); Node* shift = __ URShiftX(addr, __ ConI(3)); shift = __ AndI(__ ConvL2I(shift), __ ConI(3)); From d9e156f47af5aa75ec870a8f29a2a0c6efb7135b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 22 Sep 2021 17:01:56 +1000 Subject: [PATCH 056/494] ObjBarrier for SIM --- openjdk/barriers/mmtkObjectBarrier.cpp | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index cb564558..af648123 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -242,30 +242,6 @@ void MMTkObjectBarrierSetC2::record_modified_node(GraphKit* kit, Node* src, Node kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } -void MMTkObjectBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const { - MMTkIdealKit ideal(kit, true); -#if MMTK_ENABLE_BARRIER_FASTPATH - Node* no_base = __ top(); - float unlikely = PROB_UNLIKELY(0.999); - - Node* zero = __ ConI(0); - Node* addr = __ CastPX(__ ctrl(), src); - Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); - Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); - Node* shift = __ URShiftX(addr, __ ConI(3)); - shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); - Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); - - __ if_then(result, BoolTest::ne, zero, unlikely); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); - Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); - } __ end_if(); -#else - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); - Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkObjectBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); -#endif - - kit->final_sync(ideal); // Final sync IdealKit and GraphKit. -} +void MMTkObjectBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const {} #undef __ From b83111c45e16c6aa0398be48c81b82e597821db7 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 25 Sep 2021 14:05:25 +1000 Subject: [PATCH 057/494] Minor fix --- mmtk/src/object_model.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 7c610ae5..cc74cb30 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -95,8 +95,8 @@ impl ObjectModel for VMObjectModel { copy_context.alloc_copy(from, bytes, ::std::mem::size_of::(), 0, allocator); // Copy let src = from.to_address(); - for i in 0..bytes { - unsafe { (dst + i).store((src + i).load::()) }; + unsafe { + std::ptr::copy_nonoverlapping::(src.to_ptr(), dst.to_mut_ptr(), bytes) } let to_obj = unsafe { dst.to_object_reference() }; copy_context.post_copy(to_obj, unsafe { Address::zero() }, bytes, allocator); From e78a30f5fdeeca878082f33c244e8cf8a5b42972 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 28 Sep 2021 12:59:00 +1000 Subject: [PATCH 058/494] Separate edge lock bits --- mmtk/src/lib.rs | 4 -- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 55 +++++++++++--------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 573b8897..f56899e6 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -105,10 +105,6 @@ pub static GLOBAL_SIDE_METADATA_BASE_ADDRESS: uintptr_t = pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS: uintptr_t = crate::mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize(); -#[no_mangle] -pub static GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS: uintptr_t = - crate::mmtk::util::metadata::side_metadata::RC_UNLOG_BIT_SIDE_METADATA_SPEC.get_absolute_offset().as_usize(); - #[no_mangle] pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t = crate::mmtk::util::metadata::side_metadata::ALLOC_SIDE_METADATA_ADDR.as_usize(); diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index e9b3a663..b0306fe0 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -1,10 +1,7 @@ #include "mmtkFieldLoggingBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" -constexpr int kLoggedValue = 0; - - -extern const intptr_t GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS; +constexpr int kUnloggedValue = 1; void MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow(void* src, void* slot, void* val) { ::mmtk_object_reference_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, val); @@ -17,10 +14,10 @@ void MMTkFieldLoggingBarrierSetRuntime::record_clone_slow(void* src, void* dst, void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset, oop val) { #if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = ((intptr_t) (void*) src) + offset; - uint8_t* meta_addr = (uint8_t*) (GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS + (addr >> 5)); - intptr_t shift = ((addr >> 3) & 0b11) << 1; + uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + intptr_t shift = (addr >> 3) & 0b111; uint8_t byte_val = *meta_addr; - if (((byte_val >> shift) & 3) != kLoggedValue) { + if (((byte_val >> shift) & 1) == kUnloggedValue) { record_modified_node_slow((void*) src, (void*) (((intptr_t) (void*) src) + offset), (void*) val); } #else @@ -29,7 +26,17 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t } void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t size) { +#if MMTK_ENABLE_BARRIER_FASTPATH + intptr_t addr = (intptr_t) (void*) dst; + uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + intptr_t shift = (addr >> 3) & 0b111; + uint8_t byte_val = *meta_addr; + if (((byte_val >> shift) & 1) == 1) { + record_clone_slow((void*) src, (void*) dst, size); + } +#else record_clone_slow((void*) src, (void*) dst, size); +#endif } void MMTkFieldLoggingBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) { @@ -61,25 +68,24 @@ void MMTkFieldLoggingBarrierSetAssembler::record_modified_node(MacroAssembler* m Register tmp4 = rscratch2; Register tmp5 = tmp1 == dst.base() || tmp1 == dst.index() ? tmp2 : tmp1; - // tmp5 = load-byte (GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS + (obj >> 6)); + // tmp5 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); __ lea(tmp3, dst); - __ shrptr(tmp3, 5); - __ movptr(tmp5, GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS); + __ shrptr(tmp3, 6); + __ movptr(tmp5, SIDE_METADATA_BASE_ADDRESS); __ movb(tmp5, Address(tmp5, tmp3)); - // tmp3 = ((obj >> 3) & 3) << 1 + // tmp3 = (obj >> 3) & 7 __ lea(tmp3, dst); __ shrptr(tmp3, 3); - __ andptr(tmp3, 3); - __ shlptr(tmp3, 1); + __ andptr(tmp3, 7); // tmp5 = tmp5 >> tmp3 __ movptr(tmp4, rcx); __ movl(rcx, tmp3); __ shrptr(tmp5); __ movptr(rcx, tmp4); - // if ((tmp5 & 3) == 1) goto slowpath; - __ andptr(tmp5, 3); - __ cmpptr(tmp5, kLoggedValue); - __ jcc(Assembler::equal, done); + // if ((tmp5 & 1) == 1) goto slowpath; + __ andptr(tmp5, 1); + __ cmpptr(tmp5, kUnloggedValue); + __ jcc(Assembler::notEqual, done); // TODO: Spill fewer registers __ pusha(); @@ -155,12 +161,12 @@ void MMTkFieldLoggingBarrierSetC1::record_modified_node(LIRAccess& access, LIR_O #if MMTK_ENABLE_BARRIER_FASTPATH LIR_Opr addr = src; - // uint8_t* meta_addr = (uint8_t*) (GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS + (addr >> 6)); + // uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); LIR_Opr offset = gen->new_pointer_register(); __ move(addr, offset); __ shift_right(offset, 6, offset); LIR_Opr base = gen->new_pointer_register(); - __ move(LIR_OprFact::longConst(GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS), base); + __ move(LIR_OprFact::longConst(SIDE_METADATA_BASE_ADDRESS), base); LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); // intptr_t shift = (addr >> 3) & 0b111; LIR_Opr shift_long = gen->new_pointer_register(); @@ -199,16 +205,15 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src Node* no_base = __ top(); float unlikely = PROB_UNLIKELY(0.999); - Node* logged_value = __ ConI(kLoggedValue); + Node* unlogged_value = __ ConI(kUnloggedValue); Node* addr = __ CastPX(__ ctrl(), slot); - Node* meta_addr = __ AddP(no_base, __ ConP(GLOBAL_RC_UNLOG_BITS_BASE_ADDRESS), __ URShiftX(addr, __ ConI(5))); + Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); Node* shift = __ URShiftX(addr, __ ConI(3)); - shift = __ AndI(__ ConvL2I(shift), __ ConI(3)); - shift = __ LShiftI(shift, __ ConI(1)); - Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(3)); + shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); + Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); - __ if_then(result, BoolTest::ne, logged_value, unlikely); { + __ if_then(result, BoolTest::eq, unlogged_value, unlikely); { const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src, slot, val); } __ end_if(); From 7cc29636340c61dc6159391609bdbbae5dc38768 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 29 Sep 2021 19:12:49 +1000 Subject: [PATCH 059/494] Fix derived root pointers --- mmtk/src/lib.rs | 1 + mmtk/src/scanning.rs | 6 ++++++ openjdk/mmtk.h | 1 + openjdk/mmtkHeap.cpp | 2 ++ openjdk/mmtkUpcalls.cpp | 27 +++++++++++++++++++++++++++ 5 files changed, 37 insertions(+) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index f56899e6..dd20ca28 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -85,6 +85,7 @@ pub struct OpenJDK_Upcalls { pub number_of_mutators: extern "C" fn() -> usize, pub schedule_finalizer: extern "C" fn(), pub stop_mutators: extern "C" fn(tls: VMWorkerThread), + pub prepare_for_sanity_roots_scanning: extern "C" fn(), } #[thread_local] diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 481f1d86..365c26a7 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -106,4 +106,10 @@ impl Scanning for VMScanning { fn supports_return_barrier() -> bool { unimplemented!() } + + fn prepare_for_sanity_roots_scanning() { + unsafe { + ((*UPCALLS).prepare_for_sanity_roots_scanning)(); + } + } } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index dd6e9499..ac199bc9 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -136,6 +136,7 @@ typedef struct { size_t (*number_of_mutators)(); void (*schedule_finalizer)(); void (*mmtk_stop_mutators)(void *tls); + void (*prepare_for_sanity_roots_scanning)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls, size_t heap_size); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index f73db090..a36d0754 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -390,6 +390,7 @@ void MMTkHeap::scan_system_dictionary_roots(OopClosure& cl) { void MMTkHeap::scan_code_cache_roots(OopClosure& cl) { ResourceMark rm; MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); + CodeCache::gc_prologue(); CodeBlobToOopClosure cb_cl(&cl, true); { MutexLockerEx lock(CodeCache_lock, Mutex::_no_safepoint_check_flag); @@ -407,6 +408,7 @@ void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl) { MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); CLDToOopClosure cld_cl(&cl, false); ClassLoaderDataGraph::cld_do(&cld_cl); + ClassLoaderDataGraph::oops_do(&cl, false); } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { ResourceMark rm; diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index c711e662..44310d7d 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -44,6 +44,10 @@ static bool gcInProgress = false; static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(void* mutator)) { gcInProgress = true; MMTkHeap::_create_stack_scan_work = create_stack_scan_work; +#if COMPILER2_OR_JVMCI + DerivedPointerTable::clear(); +#endif + ClassLoaderDataGraph::clear_claimed_marks(); SafepointSynchronize::begin(); } @@ -54,6 +58,13 @@ static void mmtk_stop_mutators(void *tls) { } static void mmtk_resume_mutators(void *tls) { + ClassLoaderDataGraph::purge(); + CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); +#if COMPILER2_OR_JVMCI + DerivedPointerTable::update_pointers(); +#endif + MMTkHeap::_create_stack_scan_work = NULL; SafepointSynchronize::end(); MMTkHeap::heap()->gc_lock()->lock_without_safepoint_check(); @@ -253,6 +264,21 @@ static size_t mmtk_number_of_mutators() { return Threads::number_of_threads(); } +static void mmtk_prepare_for_sanity_roots_scanning() { + ClassLoaderDataGraph::purge(); + CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); +#if COMPILER2_OR_JVMCI + DerivedPointerTable::update_pointers(); +#endif + + +#if COMPILER2_OR_JVMCI + DerivedPointerTable::clear(); +#endif + ClassLoaderDataGraph::clear_claimed_marks(); +} + OpenJDK_Upcalls mmtk_upcalls = { mmtk_stop_all_mutators, mmtk_resume_mutators, @@ -293,4 +319,5 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_number_of_mutators, mmtk_schedule_finalizer, mmtk_stop_mutators, + mmtk_prepare_for_sanity_roots_scanning, }; From bab804dcee6f1e2b2558c6223862b56ae621ddb6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 29 Sep 2021 19:20:14 +1000 Subject: [PATCH 060/494] Fix GenCopy+FieldBarrier --- mmtk/src/api.rs | 9 +-------- openjdk/mmtkBarrierSet.cpp | 1 - 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index f66d88e0..24452855 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -20,20 +20,13 @@ use std::lazy::SyncLazy; static NO_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("NoBarrier").unwrap()); static OBJECT_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("ObjectBarrier").unwrap()); static FIELD_LOGGING_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("FieldLoggingBarrier").unwrap()); -static FIELD_LOGGING_BARRIER_GEN: SyncLazy = SyncLazy::new(|| CString::new("FieldLoggingBarrier-GEN").unwrap()); #[no_mangle] pub extern "C" fn mmtk_active_barrier() -> *const c_char { match SINGLETON.get_plan().constraints().barrier { BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), - BarrierSelector::FieldLoggingBarrier => { - if std::env::var("MMTK_PLAN") == Ok("GenCopy".to_owned()) { - FIELD_LOGGING_BARRIER_GEN.as_ptr() - } else { - FIELD_LOGGING_BARRIER.as_ptr() - } - } + BarrierSelector::FieldLoggingBarrier => FIELD_LOGGING_BARRIER.as_ptr(), // In case we have more barriers in mmtk-core. #[allow(unreachable_patterns)] _ => unimplemented!(), diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index f24cae60..2bdadb99 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -45,7 +45,6 @@ MMTkBarrierBase* get_selected_barrier() { if (strcmp(barrier, "NoBarrier") == 0) selected_barrier = new MMTkNoBarrier(); else if (strcmp(barrier, "ObjectBarrier") == 0) selected_barrier = new MMTkObjectBarrier(); else if (strcmp(barrier, "FieldLoggingBarrier") == 0) selected_barrier = new MMTkFieldLoggingBarrier(); - else if (strcmp(barrier, "FieldLoggingBarrier-GEN") == 0) selected_barrier = new MMTkFieldLoggingBarrier(); else guarantee(false, "Unimplemented"); return selected_barrier; } From e03305629ecdb0a798681f30a1790fe3137fc2ea Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 29 Sep 2021 19:36:44 +1000 Subject: [PATCH 061/494] Minor fix --- openjdk/mmtkHeap.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index a36d0754..d4870011 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -408,7 +408,6 @@ void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl) { MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); CLDToOopClosure cld_cl(&cl, false); ClassLoaderDataGraph::cld_do(&cld_cl); - ClassLoaderDataGraph::oops_do(&cl, false); } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { ResourceMark rm; From eb1e4c1acbf57451499f28e078d7225e0f838416 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 29 Sep 2021 21:35:06 +1000 Subject: [PATCH 062/494] Disable eager stack scanning temporarily --- mmtk/src/collection.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 881fdb1f..3baef0e3 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -15,7 +15,8 @@ extern "C" fn create_mutator_scan_work>( mutator.flush(); mmtk::memory_manager::add_work_packet( &SINGLETON, - WorkBucketStage::Unconstrained, + // FIXME: Should be Unconstrained! + WorkBucketStage::ScanGlobalRoots, ScanStackRoot::(mutator), ); } From 489d53c034e1da85bc40f2fa0f8a60d30667a647 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 2 Oct 2021 20:55:54 +1000 Subject: [PATCH 063/494] Cleanup --- mmtk/src/collection.rs | 6 ------ mmtk/src/lib.rs | 1 - openjdk/mmtk.h | 1 - openjdk/mmtkUpcalls.cpp | 7 ------- 4 files changed, 15 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 3baef0e3..fd98a1eb 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -35,12 +35,6 @@ impl Collection for VMCollection { } } - fn stop_all_mutators2(tls: VMWorkerThread) { - unsafe { - ((*UPCALLS).stop_mutators)(tls); - } - } - fn resume_mutators(tls: VMWorkerThread) { unsafe { ((*UPCALLS).resume_mutators)(tls); diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index dd20ca28..ba4c2b48 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -84,7 +84,6 @@ pub struct OpenJDK_Upcalls { pub scan_vm_thread_roots: extern "C" fn(process_edges: ProcessEdgesFn), pub number_of_mutators: extern "C" fn() -> usize, pub schedule_finalizer: extern "C" fn(), - pub stop_mutators: extern "C" fn(tls: VMWorkerThread), pub prepare_for_sanity_roots_scanning: extern "C" fn(), } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index ac199bc9..80be501c 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -135,7 +135,6 @@ typedef struct { void (*scan_vm_thread_roots) (ProcessEdgesFn process_edges); size_t (*number_of_mutators)(); void (*schedule_finalizer)(); - void (*mmtk_stop_mutators)(void *tls); void (*prepare_for_sanity_roots_scanning)(); } OpenJDK_Upcalls; diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 44310d7d..90e32588 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -51,12 +51,6 @@ static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(voi SafepointSynchronize::begin(); } -static void mmtk_stop_mutators(void *tls) { - gcInProgress = true; - MMTkHeap::_create_stack_scan_work = NULL; - SafepointSynchronize::begin(); -} - static void mmtk_resume_mutators(void *tls) { ClassLoaderDataGraph::purge(); CodeCache::gc_epilogue(); @@ -318,6 +312,5 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_scan_vm_thread_roots, mmtk_number_of_mutators, mmtk_schedule_finalizer, - mmtk_stop_mutators, mmtk_prepare_for_sanity_roots_scanning, }; From 0b9a3f048c3763595f8db8770f03e7832a0e76a9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 10 Oct 2021 21:32:28 +1100 Subject: [PATCH 064/494] Fix barrier (hopefully) --- mmtk/src/object_model.rs | 8 ++++ openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 50 +++++++++++++++----- openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 9 +++- openjdk/mmtkBarrierSetAssembler_x86.hpp | 1 + 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index cc74cb30..f7f11b23 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -1,3 +1,4 @@ +use std::ffi::CStr; use std::sync::atomic::Ordering; use super::UPCALLS; @@ -132,4 +133,11 @@ impl ObjectModel for VMObjectModel { ((*UPCALLS).dump_object)(object); } } + + fn dump_object_s(object: ObjectReference) -> String { + unsafe { + let c_str = ((*UPCALLS).dump_object_string)(object); + CStr::from_ptr(c_str).to_str().unwrap().to_owned() + } + } } diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index b0306fe0..75a08ba6 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -8,6 +8,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow(void* src, voi } void MMTkFieldLoggingBarrierSetRuntime::record_clone_slow(void* src, void* dst, size_t size) { + guarantee(false, "not implemented"); ::mmtk_object_reference_clone((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, size); } @@ -26,17 +27,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t } void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t size) { -#if MMTK_ENABLE_BARRIER_FASTPATH - intptr_t addr = (intptr_t) (void*) dst; - uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); - intptr_t shift = (addr >> 3) & 0b111; - uint8_t byte_val = *meta_addr; - if (((byte_val >> shift) & 1) == 1) { - record_clone_slow((void*) src, (void*) dst, size); - } -#else - record_clone_slow((void*) src, (void*) dst, size); -#endif + // record_clone_slow((void*) src, (void*) dst, size); } void MMTkFieldLoggingBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) { @@ -60,6 +51,36 @@ void MMTkFieldLoggingBarrierSetAssembler::oop_store_at(MacroAssembler* masm, Dec BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); } +void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { + if (type == T_OBJECT || type == T_ARRAY) { + __ pusha(); + assert_different_registers(c_rarg0, dst); + assert_different_registers(c_rarg0, count); + // guarantee(c_rarg0 != dst, "X"); + guarantee(rscratch1 != src, "X"); + guarantee(rscratch1 != dst, "X"); + guarantee(rscratch1 != count, "X"); + + guarantee(c_rarg0 != count, "X"); + + if (c_rarg0 == dst) { + __ movptr(rscratch1, dst); + } + __ movptr(c_rarg0, src); + if (c_rarg0 == dst) { + __ movptr(dst, rscratch1); + } + + assert_different_registers(c_rarg1, count); + guarantee(c_rarg1 != count, "X"); + + __ movptr(c_rarg1, dst); + __ movptr(c_rarg2, count); + __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_array_copy_slow), 3); + __ popa(); + } +} + void MMTkFieldLoggingBarrierSetAssembler::record_modified_node(MacroAssembler* masm, Address dst, Register val, Register tmp1, Register tmp2) { #if MMTK_ENABLE_BARRIER_FASTPATH Label done; @@ -225,6 +246,11 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } -void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const {} +void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const { + MMTkIdealKit ideal(kit, true); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); + kit->final_sync(ideal); // Final sync IdealKit and GraphKit. +} #undef __ \ No newline at end of file diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index 1840b4a0..8cd48ba1 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -20,9 +20,15 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { public: static void record_modified_node_slow(void* src, void* slot, void* val); static void record_clone_slow(void* src, void* dst, size_t size); + static void record_array_copy_slow(void* src, void* dst, size_t size) { + for (size_t i = 0; i < size; i++) { + auto d = (void*) (((intptr_t) dst) + (i << 3)); + record_modified_node_slow((oop) NULL, d, (oop) NULL); + } + } virtual bool is_slow_path_call(address call) { - return call == CAST_FROM_FN_PTR(address, record_modified_node_slow) || call == CAST_FROM_FN_PTR(address, record_clone_slow); + return call == CAST_FROM_FN_PTR(address, record_modified_node_slow) || call == CAST_FROM_FN_PTR(address, record_clone_slow) || call == CAST_FROM_FN_PTR(address, record_array_copy_slow); } virtual void record_modified_node(oop src, ptrdiff_t offset, oop val); @@ -44,6 +50,7 @@ class MMTkFieldLoggingBarrierSetAssembler: public MMTkBarrierSetAssembler { BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); } } + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; inline void gen_write_barrier_stub(LIR_Assembler* ce, MMTkFieldLoggingBarrierStub* stub); #define __ sasm-> void generate_c1_write_barrier_runtime_stub(StubAssembler* sasm) { diff --git a/openjdk/mmtkBarrierSetAssembler_x86.hpp b/openjdk/mmtkBarrierSetAssembler_x86.hpp index ab40ce2e..e8052c3c 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.hpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.hpp @@ -7,6 +7,7 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler { public: virtual void eden_allocate(MacroAssembler* masm, Register thread, Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Label& slow_case); + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override {} virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); } From 0d67fe98707972e379f9359cea3168168c4d3cef Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 11 Oct 2021 09:00:59 +1100 Subject: [PATCH 065/494] Fix barrier --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index 75a08ba6..ff219763 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -8,7 +8,6 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow(void* src, voi } void MMTkFieldLoggingBarrierSetRuntime::record_clone_slow(void* src, void* dst, size_t size) { - guarantee(false, "not implemented"); ::mmtk_object_reference_clone((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, size); } @@ -27,7 +26,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t } void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t size) { - // record_clone_slow((void*) src, (void*) dst, size); + record_clone_slow((void*) src, (void*) dst, size); } void MMTkFieldLoggingBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) { @@ -242,6 +241,9 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src, slot, val); #endif + kit->sync_kit(ideal); + kit->insert_mem_bar(Op_MemBarCPUOrder); + __ sync_kit(kit); kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } @@ -250,6 +252,11 @@ void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* MMTkIdealKit ideal(kit, true); const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); + + kit->sync_kit(ideal); + kit->insert_mem_bar(Op_MemBarCPUOrder); + __ sync_kit(kit); + kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } From 2c4fe58daa1829b301012e1f74b3610039b58ee5 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 12 Oct 2021 13:38:45 +1100 Subject: [PATCH 066/494] Fix barrier --- mmtk/Cargo.toml | 2 +- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index d5f55e36..e043313f 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -10,7 +10,7 @@ crate-type = ["cdylib"] [profile.release] lto = true -codegen-units = 1 +# codegen-units = 1 [dependencies] libc = "0.2" diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index ff219763..d09b4b3e 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -243,7 +243,7 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src #endif kit->sync_kit(ideal); kit->insert_mem_bar(Op_MemBarCPUOrder); - __ sync_kit(kit); + // __ sync_kit(kit); kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } @@ -255,7 +255,7 @@ void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* kit->sync_kit(ideal); kit->insert_mem_bar(Op_MemBarCPUOrder); - __ sync_kit(kit); + // __ sync_kit(kit); kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } From 811b2db3a6e3279c8ccce4dbc56829ba6cc205b5 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 12 Oct 2021 16:50:03 +1100 Subject: [PATCH 067/494] Fix release assertions --- mmtk/Cargo.toml | 2 ++ mmtk/src/abi.rs | 4 ++-- mmtk/src/scanning.rs | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index e043313f..6ee32ada 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -11,6 +11,7 @@ crate-type = ["cdylib"] [profile.release] lto = true # codegen-units = 1 +# debug-assertions = true [dependencies] libc = "0.2" @@ -27,6 +28,7 @@ mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "e581539f25d050bd3 [features] default = [] +release_debug_assertions = [] mmtk_extreme_assertions = ["mmtk/extreme_assertions"] nogc_lock_free = ["mmtk/nogc_lock_free"] nogc_no_zeroing = ["mmtk/nogc_no_zeroing"] diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 121cf793..6fd33524 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -39,7 +39,7 @@ pub enum KlassID { #[repr(C)] pub struct Klass { vptr: OpaquePointer, - #[cfg(debug_assertions)] + #[cfg(all(debug_assertions, not(feature="release_debug_assertions")))] valid: i32, pub layout_helper: i32, pub id: KlassID, @@ -133,7 +133,7 @@ pub struct InstanceKlass { // #if INCLUDE_JVMTI pub jvmti_cached_class_field_map: OpaquePointer, // JvmtiCachedClassFieldMap* // #endif - #[cfg(debug_assertions)] + #[cfg(all(debug_assertions, not(feature="release_debug_assertions")))] verify_count: i32, pub methods: OpaquePointer, // Array* pub default_methods: OpaquePointer, // Array* diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 365c26a7..90100f01 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -38,6 +38,7 @@ impl Scanning for VMScanning { const SCAN_MUTATORS_IN_SAFEPOINT: bool = false; const SINGLE_THREAD_MUTATOR_SCANNING: bool = false; + #[inline] fn scan_object( trace: &mut T, object: ObjectReference, From f509e81b1b9e6df1df4d5b6fc05ce56ac857e9f9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 19 Oct 2021 17:10:17 +1100 Subject: [PATCH 068/494] Minor update --- mmtk/Cargo.toml | 3 ++- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 6ee32ada..3c7f20bb 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -10,7 +10,8 @@ crate-type = ["cdylib"] [profile.release] lto = true -# codegen-units = 1 +# codegen-units = 24 +codegen-units = 1 # debug-assertions = true [dependencies] diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index d09b4b3e..a20c1f94 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -243,7 +243,6 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src #endif kit->sync_kit(ideal); kit->insert_mem_bar(Op_MemBarCPUOrder); - // __ sync_kit(kit); kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } @@ -255,7 +254,6 @@ void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* kit->sync_kit(ideal); kit->insert_mem_bar(Op_MemBarCPUOrder); - // __ sync_kit(kit); kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } From 5423354d8e1d1f39f05669eee05837bcae7f49bd Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 19 Oct 2021 21:21:22 +1100 Subject: [PATCH 069/494] Fix hanging in yieldpoint --- openjdk/mmtkUpcalls.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 3c21c920..6a31b622 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -107,9 +107,9 @@ static void mmtk_block_for_gc() { MMTkHeap::heap()->_last_gc_time = os::javaTimeNanos() / NANOSECS_PER_MILLISEC; log_debug(gc)("Thread (id=%d) will block waiting for GC to finish.", Thread::current()->osthread()->thread_id()); { - MutexLocker locker(MMTkHeap::heap()->gc_lock()); size_t my_count = mmtk_start_the_world_count; size_t next_count = my_count + 1; + MutexLocker locker(MMTkHeap::heap()->gc_lock()); while (mmtk_start_the_world_count < next_count) { MMTkHeap::heap()->gc_lock()->wait(); From cc107931dd87047cfbfa54999d1667c2420a0492 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 22 Oct 2021 10:55:10 +1100 Subject: [PATCH 070/494] Minor --- mmtk/Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 18ba41eb..878553f3 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -8,6 +8,10 @@ name = "mmtk_openjdk" # be careful - LTO is only allowed for certain crate types crate-type = ["cdylib"] +[profile.dev] +panic = "abort" +codegen-units = 24 + [profile.release] lto = true # codegen-units = 24 From d4e668c0724893dc7ac29e008797116dedfe6cad Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 22 Oct 2021 14:42:36 +1100 Subject: [PATCH 071/494] Disable clone barrier --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index a20c1f94..f377a0d8 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -248,14 +248,14 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src } void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const { - MMTkIdealKit ideal(kit, true); - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); - Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); + // MMTkIdealKit ideal(kit, true); + // const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); + // Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); - kit->sync_kit(ideal); - kit->insert_mem_bar(Op_MemBarCPUOrder); + // kit->sync_kit(ideal); + // kit->insert_mem_bar(Op_MemBarCPUOrder); - kit->final_sync(ideal); // Final sync IdealKit and GraphKit. + // kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } #undef __ \ No newline at end of file From 4d421c6d5a24f1db175ecd42bbe51a24d30976fd Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 26 Oct 2021 16:05:07 +1100 Subject: [PATCH 072/494] Disable clone barrier --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index f377a0d8..0d3f145f 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -8,7 +8,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow(void* src, voi } void MMTkFieldLoggingBarrierSetRuntime::record_clone_slow(void* src, void* dst, size_t size) { - ::mmtk_object_reference_clone((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, size); + // ::mmtk_object_reference_clone((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, size); } void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset, oop val) { @@ -26,7 +26,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t } void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t size) { - record_clone_slow((void*) src, (void*) dst, size); + // record_clone_slow((void*) src, (void*) dst, size); } void MMTkFieldLoggingBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) { From f8f4d795cd11253cf8f91f250d1c3a123a755283 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 28 Oct 2021 20:41:30 +1100 Subject: [PATCH 073/494] Fix arraycopy barrier --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 3 ++- openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 23 +++++++++++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index 0d3f145f..ec4850d9 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -30,7 +30,8 @@ void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t si } void MMTkFieldLoggingBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) { - ::mmtk_object_reference_arraycopy((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src_obj, src_offset_in_bytes, (void*) dst_obj, dst_offset_in_bytes, length); + // ::mmtk_object_reference_arraycopy((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src_obj, src_offset_in_bytes, (void*) dst_obj, dst_offset_in_bytes, length); + record_array_copy_inline((void*) (((intptr_t) (void*) src_obj) + src_offset_in_bytes), (void*) (((intptr_t) (void*) dst_obj) + dst_offset_in_bytes), length); } #define __ masm-> diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index 8cd48ba1..ff521f52 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -20,10 +20,27 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { public: static void record_modified_node_slow(void* src, void* slot, void* val); static void record_clone_slow(void* src, void* dst, size_t size); - static void record_array_copy_slow(void* src, void* dst, size_t size) { + static void record_array_copy_slow(const void* src, const void* dst, const size_t size) { + record_array_copy_inline(src, dst, size); + } + + inline static void record_array_copy_inline(const void* src, const void* dst, const size_t size) { for (size_t i = 0; i < size; i++) { - auto d = (void*) (((intptr_t) dst) + (i << 3)); - record_modified_node_slow((oop) NULL, d, (oop) NULL); + const auto offset = i << 3; + const auto addr = ((intptr_t) dst) + offset; + const uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + const uint8_t byte_val = *meta_addr; + const intptr_t shift = (addr >> 3) & 0b111; + if (((byte_val >> shift) & 1) != 0) { + ::mmtk_object_reference_arraycopy( + (MMTk_Mutator) &Thread::current()->third_party_heap_mutator, + (void*) (((intptr_t) src) + offset), + 0, + (void*) addr, + 0, + size - i + ); + } } } From 42fd99370d233bdcf7148e93d9eb366760fa61e6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 28 Oct 2021 21:24:20 +1100 Subject: [PATCH 074/494] Update array_copy --- openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 51 +++++++++++++++----- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index ff521f52..1c0596e8 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -20,27 +20,56 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { public: static void record_modified_node_slow(void* src, void* slot, void* val); static void record_clone_slow(void* src, void* dst, size_t size); - static void record_array_copy_slow(const void* src, const void* dst, const size_t size) { + static void record_array_copy_slow(void* src, void* dst, const size_t size) { record_array_copy_inline(src, dst, size); } - inline static void record_array_copy_inline(const void* src, const void* dst, const size_t size) { - for (size_t i = 0; i < size; i++) { - const auto offset = i << 3; - const auto addr = ((intptr_t) dst) + offset; - const uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); - const uint8_t byte_val = *meta_addr; - const intptr_t shift = (addr >> 3) & 0b111; - if (((byte_val >> shift) & 1) != 0) { + inline static void record_array_copy_inline(const void* src, void* dst, const size_t size) { + if (size == 0) return; + size_t i = 0; + { + const uint64_t* meta_addr = (uint64_t*) (SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)); + const uint64_t val = *meta_addr; + if (val != 0) { + while (i < size && (((intptr_t) dst) & 0b111111) != 0) { + const intptr_t shift = ((((intptr_t) dst) >> 3) & 0b111111) + i; + if (((val >> shift) & 1) != 0) { + ::mmtk_object_reference_arraycopy( + (MMTk_Mutator) &Thread::current()->third_party_heap_mutator, + (void*) (((intptr_t) src) + (i << 3)), + 0, + (void*) (((intptr_t) dst) + (i << 3)), + 0, + size - i + ); + return; + } + i++; + dst = (void*) (((intptr_t) dst) + 1); + } + } else { + const auto new_dst = (void*) ((((intptr_t) dst) + 0b111111L) & !0b111111L); + const auto dist = ((intptr_t) new_dst) - ((intptr_t) dst); + i = i + dist; + dst = new_dst; + } + } + + uint64_t* meta_addr = (uint64_t*) (SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)); + for (; i < size;) { + if (*meta_addr != 0) { ::mmtk_object_reference_arraycopy( (MMTk_Mutator) &Thread::current()->third_party_heap_mutator, - (void*) (((intptr_t) src) + offset), + (void*) (((intptr_t) src) + (i << 3)), 0, - (void*) addr, + (void*) (((intptr_t) dst) + (i << 3)), 0, size - i ); + return; } + i += 64; + meta_addr += 1; } } From 499ea3fe484d54cef958a7eb876751e53dcfc818 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 28 Oct 2021 22:20:01 +1100 Subject: [PATCH 075/494] Fix arraycopy --- openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 26 +++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index 1c0596e8..135bdd68 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -24,51 +24,53 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { record_array_copy_inline(src, dst, size); } - inline static void record_array_copy_inline(const void* src, void* dst, const size_t size) { + inline static void record_array_copy_inline(const void* src, const void* dst, const size_t size) { + const auto origin_dst = dst; if (size == 0) return; size_t i = 0; { - const uint64_t* meta_addr = (uint64_t*) (SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)); - const uint64_t val = *meta_addr; + const uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)); + const uint8_t val = *meta_addr; if (val != 0) { + const auto shift_base = ((((intptr_t) dst) >> 3) & 0b111); while (i < size && (((intptr_t) dst) & 0b111111) != 0) { - const intptr_t shift = ((((intptr_t) dst) >> 3) & 0b111111) + i; + const intptr_t shift = shift_base + i; if (((val >> shift) & 1) != 0) { ::mmtk_object_reference_arraycopy( (MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) (((intptr_t) src) + (i << 3)), 0, - (void*) (((intptr_t) dst) + (i << 3)), + (void*) (((intptr_t) origin_dst) + (i << 3)), 0, size - i ); return; } i++; - dst = (void*) (((intptr_t) dst) + 1); + dst = (void*) (((intptr_t) dst) + 8); } } else { - const auto new_dst = (void*) ((((intptr_t) dst) + 0b111111L) & !0b111111L); + const auto new_dst = (void*) ((((intptr_t) dst) + 0b111111L) >> 6 << 6); const auto dist = ((intptr_t) new_dst) - ((intptr_t) dst); - i = i + dist; + i = i + (dist >> 3); dst = new_dst; } } - - uint64_t* meta_addr = (uint64_t*) (SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)); + if (i >= size) return; + uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)); for (; i < size;) { if (*meta_addr != 0) { ::mmtk_object_reference_arraycopy( (MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) (((intptr_t) src) + (i << 3)), 0, - (void*) (((intptr_t) dst) + (i << 3)), + (void*) (((intptr_t) origin_dst) + (i << 3)), 0, size - i ); return; } - i += 64; + i += 8; meta_addr += 1; } } From 699e5e7b16bb700def38cbc80b8fb47c39a190aa Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 28 Oct 2021 22:31:41 +1100 Subject: [PATCH 076/494] arraycopy barrier: scan metadata in words instead of bytes --- openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index 135bdd68..307c98da 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -29,11 +29,11 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { if (size == 0) return; size_t i = 0; { - const uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)); - const uint8_t val = *meta_addr; + const uint64_t* meta_addr = (uint64_t*) ((SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)) >> 3 << 3); + const uint64_t val = *meta_addr; if (val != 0) { - const auto shift_base = ((((intptr_t) dst) >> 3) & 0b111); - while (i < size && (((intptr_t) dst) & 0b111111) != 0) { + const auto shift_base = ((((intptr_t) dst) >> 3) & 0b111111); + while (i < size && (((intptr_t) dst) & 0b111111111) != 0) { const intptr_t shift = shift_base + i; if (((val >> shift) & 1) != 0) { ::mmtk_object_reference_arraycopy( @@ -47,17 +47,17 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { return; } i++; - dst = (void*) (((intptr_t) dst) + 8); + dst = (void*) (((intptr_t) dst) + 64); } } else { - const auto new_dst = (void*) ((((intptr_t) dst) + 0b111111L) >> 6 << 6); + const auto new_dst = (void*) ((((intptr_t) dst) + 0b111111111L) >> 9 << 9); const auto dist = ((intptr_t) new_dst) - ((intptr_t) dst); i = i + (dist >> 3); dst = new_dst; } } if (i >= size) return; - uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)); + uint64_t* meta_addr = (uint64_t*) (SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)); for (; i < size;) { if (*meta_addr != 0) { ::mmtk_object_reference_arraycopy( @@ -70,7 +70,7 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { ); return; } - i += 8; + i += 64; meta_addr += 1; } } From 73856ff0d429b3aa695b62a8ba96b95e07617782 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 29 Oct 2021 01:24:20 +1100 Subject: [PATCH 077/494] Update array_copy barrier --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 14 ++ openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 145 +++++++++++++------ 2 files changed, 115 insertions(+), 44 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index ec4850d9..1d145316 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -53,6 +53,19 @@ void MMTkFieldLoggingBarrierSetAssembler::oop_store_at(MacroAssembler* masm, Dec void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { if (type == T_OBJECT || type == T_ARRAY) { + Label slow, done; + // Bailout if count is zero + // __ cmpptr(count, 0); + // __ jcc(Assembler::equal, done); + // Fast path if count is one + __ cmpptr(count, 1); + __ jcc(Assembler::notEqual, slow); + __ push(rax); + record_modified_node(masm, Address(dst, 0), src, rax, rax); + __ pop(rax); + __ jmp(done); + // Slow path + __ bind(slow); __ pusha(); assert_different_registers(c_rarg0, dst); assert_different_registers(c_rarg0, count); @@ -78,6 +91,7 @@ void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue(MacroAssembler* mas __ movptr(c_rarg2, count); __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_array_copy_slow), 3); __ popa(); + __ bind(done); } } diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index 307c98da..8ad98ef6 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -24,55 +24,112 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { record_array_copy_inline(src, dst, size); } - inline static void record_array_copy_inline(const void* src, const void* dst, const size_t size) { - const auto origin_dst = dst; + inline static void record_array_copy_mmtk_slow(const void* src, const void* dst, const size_t size) { + ::mmtk_object_reference_arraycopy( + (MMTk_Mutator) &Thread::current()->third_party_heap_mutator, + (void*) ((intptr_t) src), + 0, + (void*) ((intptr_t) dst), + 0, + size + ); + } + + inline static void record_array_copy_inline(const void* src_ptr, void* dst_ptr, const size_t size) { if (size == 0) return; - size_t i = 0; - { - const uint64_t* meta_addr = (uint64_t*) ((SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)) >> 3 << 3); - const uint64_t val = *meta_addr; - if (val != 0) { - const auto shift_base = ((((intptr_t) dst) >> 3) & 0b111111); - while (i < size && (((intptr_t) dst) & 0b111111111) != 0) { - const intptr_t shift = shift_base + i; - if (((val >> shift) & 1) != 0) { - ::mmtk_object_reference_arraycopy( - (MMTk_Mutator) &Thread::current()->third_party_heap_mutator, - (void*) (((intptr_t) src) + (i << 3)), - 0, - (void*) (((intptr_t) origin_dst) + (i << 3)), - 0, - size - i - ); - return; - } - i++; - dst = (void*) (((intptr_t) dst) + 64); - } - } else { - const auto new_dst = (void*) ((((intptr_t) dst) + 0b111111111L) >> 9 << 9); - const auto dist = ((intptr_t) new_dst) - ((intptr_t) dst); - i = i + (dist >> 3); - dst = new_dst; + if (size == 1) { + intptr_t addr = (intptr_t) dst_ptr; + uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + intptr_t shift = (addr >> 3) & 0b111; + uint8_t byte_val = *meta_addr; + if (((byte_val >> shift) & 1) != 0) { + record_modified_node_slow((void*) NULL, dst_ptr, (void*) NULL); } + return; } - if (i >= size) return; - uint64_t* meta_addr = (uint64_t*) (SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)); - for (; i < size;) { - if (*meta_addr != 0) { - ::mmtk_object_reference_arraycopy( - (MMTk_Mutator) &Thread::current()->third_party_heap_mutator, - (void*) (((intptr_t) src) + (i << 3)), - 0, - (void*) (((intptr_t) origin_dst) + (i << 3)), - 0, - size - i - ); - return; + + constexpr intptr_t kLogBitsInUInt = 6; + constexpr intptr_t kBitsInUInt = 1 << kLogBitsInUInt; + constexpr intptr_t kBytesInUInt = 8; + intptr_t origin_dst = (intptr_t) dst_ptr; + intptr_t dst = origin_dst; + intptr_t src = (intptr_t) src_ptr; + intptr_t limit = dst + (size << 3); + auto meta_addr = (uint64_t*) ((SIDE_METADATA_BASE_ADDRESS + (dst >> 6)) >> 3 << 3); + auto val = *meta_addr; + while (dst < limit) { + if ((dst & 0b111111111) == 0) { + auto val = *meta_addr; + for (auto j = 0; j < kBitsInUInt; j++) { + if (((val >> j) & 1) != 0) { + auto e = dst + (j << 3); + if (e < limit) { + auto offset = e - origin_dst; + record_array_copy_mmtk_slow((void*) (src + offset), (void*) e, size - (offset >> 3)); + return; + } + } + } + dst += 1 << (kLogBitsInUInt + 3); + meta_addr += kBytesInUInt; + } else { + auto shift = (dst >> 3) & (kBitsInUInt - 1); + if ((val >> shift) & 1 != 0) { + auto offset = dst - origin_dst; + record_array_copy_mmtk_slow((void*) (src + offset), (void*) dst, size - (offset >> 3)); + return; + } + dst += kBytesInUInt; } - i += 64; - meta_addr += 1; } + + // const auto origin_dst = dst; + // size_t i = 0; + // { + // const uint64_t* meta_addr = (uint64_t*) ((SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)) >> 3 << 3); + // const uint64_t val = *meta_addr; + // if (val != 0) { + // const auto shift_base = ((((intptr_t) dst) >> 3) & 0b111111); + // while (i < size && (((intptr_t) dst) & 0b111111111) != 0) { + // const intptr_t shift = shift_base + i; + // if (((val >> shift) & 1) != 0) { + // ::mmtk_object_reference_arraycopy( + // (MMTk_Mutator) &Thread::current()->third_party_heap_mutator, + // (void*) (((intptr_t) src) + (i << 3)), + // 0, + // (void*) (((intptr_t) origin_dst) + (i << 3)), + // 0, + // size - i + // ); + // return; + // } + // i++; + // dst = (void*) (((intptr_t) dst) + 8); + // } + // } else { + // const auto new_dst = (void*) ((((intptr_t) dst) + 0b111111111L) >> 9 << 9); + // const auto dist = ((intptr_t) new_dst) - ((intptr_t) dst); + // i = i + (dist >> 3); + // dst = new_dst; + // } + // } + // if (i >= size) return; + // uint64_t* meta_addr = (uint64_t*) (SIDE_METADATA_BASE_ADDRESS + (((intptr_t) dst) >> 6)); + // for (; i < size;) { + // if (*meta_addr != 0) { + // ::mmtk_object_reference_arraycopy( + // (MMTk_Mutator) &Thread::current()->third_party_heap_mutator, + // (void*) (((intptr_t) src) + (i << 3)), + // 0, + // (void*) (((intptr_t) origin_dst) + (i << 3)), + // 0, + // size - i + // ); + // return; + // } + // i += 64; + // meta_addr += 1; + // } } virtual bool is_slow_path_call(address call) { From 0ee53c6a8f3ab770472fd3a1a5c0e6f33e50f1e9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 29 Oct 2021 03:22:49 +1100 Subject: [PATCH 078/494] Fix arraycopy --- openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 70 ++++++++++---------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index 8ad98ef6..2dbbf3a2 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -48,40 +48,42 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { return; } - constexpr intptr_t kLogBitsInUInt = 6; - constexpr intptr_t kBitsInUInt = 1 << kLogBitsInUInt; - constexpr intptr_t kBytesInUInt = 8; - intptr_t origin_dst = (intptr_t) dst_ptr; - intptr_t dst = origin_dst; - intptr_t src = (intptr_t) src_ptr; - intptr_t limit = dst + (size << 3); - auto meta_addr = (uint64_t*) ((SIDE_METADATA_BASE_ADDRESS + (dst >> 6)) >> 3 << 3); - auto val = *meta_addr; - while (dst < limit) { - if ((dst & 0b111111111) == 0) { - auto val = *meta_addr; - for (auto j = 0; j < kBitsInUInt; j++) { - if (((val >> j) & 1) != 0) { - auto e = dst + (j << 3); - if (e < limit) { - auto offset = e - origin_dst; - record_array_copy_mmtk_slow((void*) (src + offset), (void*) e, size - (offset >> 3)); - return; - } - } - } - dst += 1 << (kLogBitsInUInt + 3); - meta_addr += kBytesInUInt; - } else { - auto shift = (dst >> 3) & (kBitsInUInt - 1); - if ((val >> shift) & 1 != 0) { - auto offset = dst - origin_dst; - record_array_copy_mmtk_slow((void*) (src + offset), (void*) dst, size - (offset >> 3)); - return; - } - dst += kBytesInUInt; - } - } + record_array_copy_mmtk_slow(src_ptr, dst_ptr, size); + + // constexpr intptr_t kLogBitsInUInt = 6; + // constexpr intptr_t kBitsInUInt = 1 << kLogBitsInUInt; + // constexpr intptr_t kBytesInUInt = 8; + // intptr_t origin_dst = (intptr_t) dst_ptr; + // intptr_t dst = origin_dst; + // intptr_t src = (intptr_t) src_ptr; + // intptr_t limit = dst + (size << 3); + // auto meta_addr = (uint64_t*) ((SIDE_METADATA_BASE_ADDRESS + (dst >> 6)) >> 3 << 3); + // auto val = *meta_addr; + // while (dst < limit) { + // if ((dst & 0b111111111) == 0) { + // auto val = *meta_addr; + // for (auto j = 0; j < kBitsInUInt; j++) { + // if (((val >> j) & 1) != 0) { + // auto e = dst + (j << 3); + // if (e < limit) { + // auto offset = e - origin_dst; + // record_array_copy_mmtk_slow((void*) (src + offset), (void*) e, size - (offset >> 3)); + // return; + // } + // } + // } + // dst += 1 << (kLogBitsInUInt + 3); + // meta_addr += kBytesInUInt; + // } else { + // auto shift = (dst >> 3) & (kBitsInUInt - 1); + // if ((val >> shift) & 1 != 0) { + // auto offset = dst - origin_dst; + // record_array_copy_mmtk_slow((void*) (src + offset), (void*) dst, size - (offset >> 3)); + // return; + // } + // dst += kBytesInUInt; + // } + // } // const auto origin_dst = dst; // size_t i = 0; From be47337757b159da719ae70242064ec2a03785e6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 29 Oct 2021 10:55:37 +1100 Subject: [PATCH 079/494] Don't emit nursery barriers --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 40 +++++++++++++++++++- openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 4 ++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index 1d145316..4ba03347 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -232,15 +232,51 @@ void MMTkFieldLoggingBarrierSetC1::record_modified_node(LIRAccess& access, LIR_O #define __ ideal. +bool MMTkFieldLoggingBarrierSetC2::can_remove_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr) const { + intptr_t offset = 0; + Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset); + AllocateNode* alloc = AllocateNode::Ideal_allocation(base, phase); + + if (offset == Type::OffsetBot) { + return false; // cannot unalias unless there are precise offsets + } + + if (alloc == NULL) { + return false; // No allocation found + } + + // Start search from Store node + Node* mem = kit->control(); + if (mem->is_Proj() && mem->in(0)->is_Initialize()) { + + InitializeNode* st_init = mem->in(0)->as_Initialize(); + AllocateNode* st_alloc = st_init->allocation(); + + // Make sure we are looking at the same allocation + if (alloc == st_alloc) { + return true; + } + } + + return false; +} + void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src, Node* slot, Node* val) const { + if (src == kit->just_allocated_object(kit->control())) { + return; + } + if (can_remove_barrier(kit, &kit->gvn(), slot)) { + return; + } + MMTkIdealKit ideal(kit, true); #if MMTK_ENABLE_BARRIER_FASTPATH Node* no_base = __ top(); float unlikely = PROB_UNLIKELY(0.999); - Node* unlogged_value = __ ConI(kUnloggedValue); + Node* zero = __ ConI(0); Node* addr = __ CastPX(__ ctrl(), slot); Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); @@ -248,7 +284,7 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); - __ if_then(result, BoolTest::eq, unlogged_value, unlikely); { + __ if_then(result, BoolTest::ne, zero, unlikely); { const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src, slot, val); } __ end_if(); diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index 2dbbf3a2..09366846 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -267,6 +267,7 @@ class MMTkFieldLoggingBarrierSetC1: public MMTkBarrierSetC1 { class MMTkFieldLoggingBarrierSetC2: public MMTkBarrierSetC2 { void record_modified_node(GraphKit* kit, Node* node, Node* slot, Node* val) const; void record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const; + bool can_remove_barrier(GraphKit* kit, PhaseTransform* phase, Node* adr) const; public: virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const { if (access.is_oop()) record_modified_node(access.kit(), access.base(), access.addr().node(), val.node()); @@ -297,6 +298,9 @@ class MMTkFieldLoggingBarrierSetC2: public MMTkBarrierSetC2 { CallLeafNode *call = node->as_CallLeaf(); return call->_name != NULL && (strcmp(call->_name, "record_modified_node") == 0 || strcmp(call->_name, "record_clone") == 0); } + virtual bool array_copy_requires_gc_barriers(BasicType type) const override { + return false; + } }; #undef __ From c579d507ae18e682621cf4207f07218cda7e553d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 29 Oct 2021 23:56:22 +1100 Subject: [PATCH 080/494] Get object start address in arraycopy barrier --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 23 +++++--------------- openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 17 ++++++++------- repos/openjdk | 2 +- 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index 4ba03347..e6cd19cd 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -31,7 +31,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t si void MMTkFieldLoggingBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) { // ::mmtk_object_reference_arraycopy((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src_obj, src_offset_in_bytes, (void*) dst_obj, dst_offset_in_bytes, length); - record_array_copy_inline((void*) (((intptr_t) (void*) src_obj) + src_offset_in_bytes), (void*) (((intptr_t) (void*) dst_obj) + dst_offset_in_bytes), length); + record_array_copy_inline((void*) (((intptr_t) (void*) src_obj) + src_offset_in_bytes), (void*) (((intptr_t) (void*) dst_obj) + dst_offset_in_bytes), length, (void*) dst_obj); } #define __ masm-> @@ -51,21 +51,10 @@ void MMTkFieldLoggingBarrierSetAssembler::oop_store_at(MacroAssembler* masm, Dec BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); } -void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { +void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) {} + +void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue2(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count, Register dst_obj) { if (type == T_OBJECT || type == T_ARRAY) { - Label slow, done; - // Bailout if count is zero - // __ cmpptr(count, 0); - // __ jcc(Assembler::equal, done); - // Fast path if count is one - __ cmpptr(count, 1); - __ jcc(Assembler::notEqual, slow); - __ push(rax); - record_modified_node(masm, Address(dst, 0), src, rax, rax); - __ pop(rax); - __ jmp(done); - // Slow path - __ bind(slow); __ pusha(); assert_different_registers(c_rarg0, dst); assert_different_registers(c_rarg0, count); @@ -89,9 +78,9 @@ void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue(MacroAssembler* mas __ movptr(c_rarg1, dst); __ movptr(c_rarg2, count); - __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_array_copy_slow), 3); + __ movptr(c_rarg3, dst_obj); + __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_array_copy_slow), 4); __ popa(); - __ bind(done); } } diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index 09366846..6014ec00 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -20,22 +20,22 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { public: static void record_modified_node_slow(void* src, void* slot, void* val); static void record_clone_slow(void* src, void* dst, size_t size); - static void record_array_copy_slow(void* src, void* dst, const size_t size) { - record_array_copy_inline(src, dst, size); + static void record_array_copy_slow(void* src, void* dst, const size_t size, void* dst_obj) { + record_array_copy_inline(src, dst, size, dst_obj); } - inline static void record_array_copy_mmtk_slow(const void* src, const void* dst, const size_t size) { + inline static void record_array_copy_mmtk_slow(const void* src, const void* dst, const size_t size, void* dst_obj) { ::mmtk_object_reference_arraycopy( (MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) ((intptr_t) src), 0, - (void*) ((intptr_t) dst), - 0, + (void*) ((intptr_t) dst_obj), + ((intptr_t) dst) - ((intptr_t) dst_obj), size ); } - inline static void record_array_copy_inline(const void* src_ptr, void* dst_ptr, const size_t size) { + inline static void record_array_copy_inline(const void* src_ptr, void* dst_ptr, const size_t size, void* dst_obj) { if (size == 0) return; if (size == 1) { intptr_t addr = (intptr_t) dst_ptr; @@ -43,12 +43,12 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { intptr_t shift = (addr >> 3) & 0b111; uint8_t byte_val = *meta_addr; if (((byte_val >> shift) & 1) != 0) { - record_modified_node_slow((void*) NULL, dst_ptr, (void*) NULL); + record_modified_node_slow(dst_obj, dst_ptr, (void*) NULL); } return; } - record_array_copy_mmtk_slow(src_ptr, dst_ptr, size); + record_array_copy_mmtk_slow(src_ptr, dst_ptr, size, dst_obj); // constexpr intptr_t kLogBitsInUInt = 6; // constexpr intptr_t kBitsInUInt = 1 << kLogBitsInUInt; @@ -158,6 +158,7 @@ class MMTkFieldLoggingBarrierSetAssembler: public MMTkBarrierSetAssembler { } } virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; + virtual void arraycopy_prologue2(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count, Register dst_obj) override; inline void gen_write_barrier_stub(LIR_Assembler* ce, MMTkFieldLoggingBarrierStub* stub); #define __ sasm-> void generate_c1_write_barrier_runtime_stub(StubAssembler* sasm) { diff --git a/repos/openjdk b/repos/openjdk index 6dc618e2..9f641740 160000 --- a/repos/openjdk +++ b/repos/openjdk @@ -1 +1 @@ -Subproject commit 6dc618e281128b6a38d40c7d8d2e345d610f0160 +Subproject commit 9f6417407870b5c28aa45ba2ee74de5cace90a4f From c9bfa2dfc85db8b066bfca1843903696cba3ed59 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 30 Oct 2021 14:36:05 +1100 Subject: [PATCH 081/494] Fix GC_FEATURES --- openjdk/CompileThirdPartyHeap.gmk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openjdk/CompileThirdPartyHeap.gmk b/openjdk/CompileThirdPartyHeap.gmk index f9b04d4a..67be60f2 100644 --- a/openjdk/CompileThirdPartyHeap.gmk +++ b/openjdk/CompileThirdPartyHeap.gmk @@ -40,8 +40,8 @@ else endif $(LIB_MMTK): FORCE - echo "cd $(MMTK_RUST_ROOT) && cargo build $(CARGO_PROFILE_FLAG) $(GC_FEATURES)" - cd $(MMTK_RUST_ROOT) && cargo build $(CARGO_PROFILE_FLAG) $(GC_FEATURES) + echo "cd $(MMTK_RUST_ROOT) && cargo build $(CARGO_PROFILE_FLAG) --features $(GC_FEATURES)" + cd $(MMTK_RUST_ROOT) && cargo build $(CARGO_PROFILE_FLAG) --features $(GC_FEATURES) cp $(MMTK_RUST_ROOT)/target/$(CARGO_PROFILE)/libmmtk_openjdk.so $(LIB_MMTK) JVM_LIBS += -L$(JVM_LIB_OUTPUTDIR) -lmmtk_openjdk From 687a633a9f27be1e8cf0d666c9c615a5d6b031c2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 30 Oct 2021 14:41:02 +1100 Subject: [PATCH 082/494] Update makefile --- openjdk/CompileThirdPartyHeap.gmk | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/openjdk/CompileThirdPartyHeap.gmk b/openjdk/CompileThirdPartyHeap.gmk index 67be60f2..ff45a7e3 100644 --- a/openjdk/CompileThirdPartyHeap.gmk +++ b/openjdk/CompileThirdPartyHeap.gmk @@ -39,9 +39,15 @@ else CARGO_PROFILE = debug endif +ifeq ($(strip $(GC_FEATURES)),) + CARGO_BUILD_FEATURES = +else + CARGO_BUILD_FEATURES = --features $(GC_FEATURES) +endif + $(LIB_MMTK): FORCE - echo "cd $(MMTK_RUST_ROOT) && cargo build $(CARGO_PROFILE_FLAG) --features $(GC_FEATURES)" - cd $(MMTK_RUST_ROOT) && cargo build $(CARGO_PROFILE_FLAG) --features $(GC_FEATURES) + echo "cd $(MMTK_RUST_ROOT) && cargo build $(CARGO_PROFILE_FLAG) $(CARGO_BUILD_FEATURES)" + cd $(MMTK_RUST_ROOT) && cargo build $(CARGO_PROFILE_FLAG) $(CARGO_BUILD_FEATURES) cp $(MMTK_RUST_ROOT)/target/$(CARGO_PROFILE)/libmmtk_openjdk.so $(LIB_MMTK) JVM_LIBS += -L$(JVM_LIB_OUTPUTDIR) -lmmtk_openjdk From da2bf1775deb7817e031f7d0b4afb29ea92be71b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 30 Oct 2021 14:54:25 +1100 Subject: [PATCH 083/494] Update cargo.toml --- mmtk/Cargo.toml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 878553f3..afb3dddc 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -27,7 +27,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "08cecbc080cc853383fa96f54436b8ca8026b130" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "6d46bc90ccbd159b6c303c2dc157c2fbdeeec202" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } @@ -54,3 +54,20 @@ gencopy = [] marksweep = [] pageprotect = [] immix = [] + +barrier_measurement = ["mmtk/barrier_measurement"] + +# Non-moving, Non-concurrent LXR +lxr_basic = ["mmtk/lxr_basic"] +# LXR (Concurrent marking) +lxr_cm = ["mmtk/lxr_cm"] +# LXR (Lazy RC processing) +lxr_lazy = ["mmtk/lxr_lazy"] +# LXR (Concurrent) +lxr_conc = ["mmtk/lxr_conc"] +# LXR (Evacuation) +lxr_evac = ["mmtk/lxr_evac"] +# LXR (Block only) +lxr_block = ["mmtk/lxr_block"] +# LXR (Default config with full features) +lxr = ["mmtk/lxr"] \ No newline at end of file From ef11f6752613ffca877c4bb6ddfd18a7fff1d15d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 30 Oct 2021 15:03:28 +1100 Subject: [PATCH 084/494] cleanup --- openjdk/mmtkBarrierSet.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index 2bdadb99..c781aca5 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -41,7 +41,6 @@ MMTkBarrierBase* get_selected_barrier() { static MMTkBarrierBase* selected_barrier = NULL; if (selected_barrier) return selected_barrier; const char* barrier = mmtk_active_barrier(); - printf("%s\n", barrier); if (strcmp(barrier, "NoBarrier") == 0) selected_barrier = new MMTkNoBarrier(); else if (strcmp(barrier, "ObjectBarrier") == 0) selected_barrier = new MMTkObjectBarrier(); else if (strcmp(barrier, "FieldLoggingBarrier") == 0) selected_barrier = new MMTkFieldLoggingBarrier(); From 7597eab2e5435ab1580b8d8257ecd7fafeaad457 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 30 Oct 2021 15:22:14 +1100 Subject: [PATCH 085/494] Update cargo.toml --- mmtk/Cargo.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index afb3dddc..60630b3c 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -27,7 +27,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "6d46bc90ccbd159b6c303c2dc157c2fbdeeec202" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "9ca4871cc34198d30b19948bca723c2a05049a2e" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } @@ -56,7 +56,9 @@ pageprotect = [] immix = [] barrier_measurement = ["mmtk/barrier_measurement"] - +log_gc = ["mmtk/log_gc"] +log_stages = ["mmtk/log_stages"] +log_work_packets = ["mmtk/log_work_packets"] # Non-moving, Non-concurrent LXR lxr_basic = ["mmtk/lxr_basic"] # LXR (Concurrent marking) From 87c52ed8e1ca28a71b6944f6347e019d7baaae72 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 30 Oct 2021 16:09:01 +1100 Subject: [PATCH 086/494] Update features --- mmtk/Cargo.toml | 1 + mmtk/src/api.rs | 6 ------ 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 60630b3c..4ba6bbca 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -55,6 +55,7 @@ marksweep = [] pageprotect = [] immix = [] +sanity = ["mmtk/sanity"] barrier_measurement = ["mmtk/barrier_measurement"] log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 24452855..c1f43ab7 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -148,12 +148,6 @@ pub extern "C" fn total_bytes() -> usize { memory_manager::total_bytes(&SINGLETON) } -#[no_mangle] -#[cfg(feature = "sanity")] -pub extern "C" fn scan_region() { - memory_manager::scan_region(&SINGLETON) -} - #[no_mangle] pub extern "C" fn handle_user_collection_request(tls: VMMutatorThread) { memory_manager::handle_user_collection_request::(&SINGLETON, tls); From 3a1f3b7dc0bf2414e5c4fbf9c833f11fca6e5de2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 30 Oct 2021 19:16:41 +1100 Subject: [PATCH 087/494] Update Cargo.toml --- mmtk/Cargo.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 4ba6bbca..a476ab76 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -14,9 +14,10 @@ codegen-units = 24 [profile.release] lto = true -# codegen-units = 24 codegen-units = 1 +# codegen-units = 24 # debug-assertions = true +# debug = true [dependencies] libc = "0.2" @@ -60,6 +61,9 @@ barrier_measurement = ["mmtk/barrier_measurement"] log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] log_work_packets = ["mmtk/log_work_packets"] + +ix_block_only = ["mmtk/ix_block_only"] + # Non-moving, Non-concurrent LXR lxr_basic = ["mmtk/lxr_basic"] # LXR (Concurrent marking) From 67f671c33e199399b77c89595b149741fbda0874 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 1 Nov 2021 09:16:58 +1100 Subject: [PATCH 088/494] ix_defrag feature --- mmtk/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index a476ab76..7bc0d865 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -28,7 +28,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "9ca4871cc34198d30b19948bca723c2a05049a2e" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "89b6d642f19dcc76e967977a40b5a9950070197b" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } @@ -63,6 +63,7 @@ log_stages = ["mmtk/log_stages"] log_work_packets = ["mmtk/log_work_packets"] ix_block_only = ["mmtk/ix_block_only"] +ix_defrag = ["mmtk/ix_defrag"] # Non-moving, Non-concurrent LXR lxr_basic = ["mmtk/lxr_basic"] From 4ea38bfd23b007ad4283066f735a75001cbcba18 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 2 Nov 2021 17:05:14 +1100 Subject: [PATCH 089/494] Force enable avx and sse --- mmtk/.cargo/config.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 mmtk/.cargo/config.toml diff --git a/mmtk/.cargo/config.toml b/mmtk/.cargo/config.toml new file mode 100644 index 00000000..d038a028 --- /dev/null +++ b/mmtk/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.x86_64-unknown-linux-gnu] +rustflags = ["-C", "target-feature=+avx,+avx2,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+sse4a,+ssse3"] \ No newline at end of file From be7d0fe314bd375ae5b58fc36b36c19000c47d35 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 4 Nov 2021 20:20:07 +1100 Subject: [PATCH 090/494] Update mmtk-core --- mmtk/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 7bc0d865..87df1ae2 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -28,7 +28,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "89b6d642f19dcc76e967977a40b5a9950070197b" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7f78e5895b5033176f0598b740af048c59e991fa" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } @@ -58,6 +58,7 @@ immix = [] sanity = ["mmtk/sanity"] barrier_measurement = ["mmtk/barrier_measurement"] +instrumentation = ["mmtk/instrumentation"] log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] log_work_packets = ["mmtk/log_work_packets"] From 7f98279c3edaf39c2106150ae37829031fe58f76 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 4 Nov 2021 20:40:45 +1100 Subject: [PATCH 091/494] Expose a MMTk API to disable allocation fast-path --- mmtk/Cargo.toml | 3 ++- mmtk/src/lib.rs | 3 +++ openjdk/mmtk.h | 5 +++++ openjdk/mmtkBarrierSetAssembler_x86.cpp | 4 ++-- openjdk/mmtkBarrierSetC2.cpp | 8 ++++---- openjdk/mmtkHeap.hpp | 2 +- 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 87df1ae2..63314ca3 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,6 +41,7 @@ nogc_no_zeroing = ["mmtk/nogc_no_zeroing"] # Use the env var GLOBAL_ALLOC_BIT=1 when building OpenJDK so the fastpath for setting the alloc bit will be compiled in. # See README. global_alloc_bit = ["mmtk/global_alloc_bit"] +no_fast_alloc = [] # This compile time constant places the mark bit in the header of the object instead of on the side. mark_bit_in_header = [] @@ -58,7 +59,7 @@ immix = [] sanity = ["mmtk/sanity"] barrier_measurement = ["mmtk/barrier_measurement"] -instrumentation = ["mmtk/instrumentation"] +instrumentation = ["mmtk/instrumentation", "no_fast_alloc"] log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] log_work_packets = ["mmtk/log_work_packets"] diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index ba4c2b48..21173c9f 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -109,6 +109,9 @@ pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS: uintptr_t = pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t = crate::mmtk::util::metadata::side_metadata::ALLOC_SIDE_METADATA_ADDR.as_usize(); +#[no_mangle] +pub static DISABLE_ALLOCATION_FAST_PATH: i32 = cfg!(feature = "no_fast_alloc") as _; + #[derive(Default)] pub struct OpenJDK; diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 80be501c..8c25408f 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -15,6 +15,11 @@ typedef void* MMTk_TraceLocal; extern const uintptr_t GLOBAL_SIDE_METADATA_BASE_ADDRESS; extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; extern const uintptr_t GLOBAL_ALLOC_BIT_ADDRESS; +extern const int DISABLE_ALLOCATION_FAST_PATH; + +inline bool disable_fast_alloc() { + return DISABLE_ALLOCATION_FAST_PATH != 0; +} /** * Allocation diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index e4792449..f6c20980 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -37,7 +37,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa assert(obj == rax, "obj must be in rax, for cmpxchg"); assert_different_registers(obj, var_size_in_bytes, t1); - if (!MMTK_ENABLE_ALLOCATION_FASTPATH) { + if (!MMTK_ENABLE_ALLOCATION_FASTPATH || disable_fast_alloc()) { __ jmp(slow_case); } else { // MMTk size check. If the alloc size is larger than the allowed max size for non los, @@ -136,7 +136,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa __ movptr(tmp3, obj); __ shrptr(tmp3, 6); __ movptr(rcx, ALLOC_BIT_BASE_ADDRESS); - __ movb(Address(rcx, tmp3), tmp2); + __ movb(Address(rcx, tmp3), tmp2); #endif diff --git a/openjdk/mmtkBarrierSetC2.cpp b/openjdk/mmtkBarrierSetC2.cpp index 4b86d91f..96f1b202 100644 --- a/openjdk/mmtkBarrierSetC2.cpp +++ b/openjdk/mmtkBarrierSetC2.cpp @@ -125,7 +125,7 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, // But we need to figure out which allocator we are using by querying MMTk. AllocatorSelector selector = get_allocator_mapping(AllocatorDefault); - if (x->C->env()->dtrace_alloc_probes() || !MMTK_ENABLE_ALLOCATION_FASTPATH + if (x->C->env()->dtrace_alloc_probes() || !MMTK_ENABLE_ALLOCATION_FASTPATH || disable_fast_alloc() // Malloc allocator has no fastpath || (selector.tag == TAG_MALLOC || selector.tag == TAG_LARGE_OBJECT)) { // Force slow-path allocation @@ -278,13 +278,13 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, fast_oop_rawmem = store_eden_top; #ifdef MMTK_ENABLE_GLOBAL_ALLOC_BIT - // set the alloc bit: + // set the alloc bit: // intptr_t addr = (intptr_t) (void*) fast_oop; // uint8_t* meta_addr = (uint8_t*) (ALLOC_BIT_BASE_ADDRESS + (addr >> 6)); // intptr_t shift = (addr >> 3) & 0b111; // uint8_t byte_val = *meta_addr; // uint8_t new_byte_val = byte_val | (1 << shift); - // *meta_addr = new_byte_val; + // *meta_addr = new_byte_val; Node *obj_addr = new CastP2XNode(fast_oop_ctrl, fast_oop); x->transform_later(obj_addr); @@ -321,7 +321,7 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, Node *const_one = ConINode::make(1); x->transform_later(const_one); - Node *shifted_masked_addr_i = new ConvL2INode(shifted_masked_addr); + Node *shifted_masked_addr_i = new ConvL2INode(shifted_masked_addr); x->transform_later(shifted_masked_addr_i); Node *set_bit = new LShiftINode(const_one, shifted_masked_addr_i); diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index 677638fc..e2426e83 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -100,7 +100,7 @@ class MMTkHeap : public CollectedHeap { bool supports_tlab_allocation() const; bool supports_inline_contig_alloc() const { - return MMTK_ENABLE_ALLOCATION_FASTPATH; + return MMTK_ENABLE_ALLOCATION_FASTPATH && !disable_fast_alloc(); } // The amount of space available for thread-local allocation buffers. From 7b9631f414b0d2f3208505ea73959bfcb5afe0f5 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 5 Nov 2021 16:00:07 +1100 Subject: [PATCH 092/494] inline methods --- mmtk/src/abi.rs | 20 ++++++++++++++++---- mmtk/src/object_scanning.rs | 16 ++++++++-------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 6fd33524..7ccf3a3a 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -147,22 +147,22 @@ pub struct InstanceKlass { impl InstanceKlass { const HEADER_SIZE: usize = mem::size_of::() / BYTES_IN_WORD; const VTABLE_START_OFFSET: usize = Self::HEADER_SIZE * BYTES_IN_WORD; - + #[inline(always)] fn start_of_vtable(&self) -> *const usize { unsafe { (self as *const _ as *const u8).add(Self::VTABLE_START_OFFSET) as _ } } - + #[inline(always)] fn start_of_itable(&self) -> *const usize { unsafe { self.start_of_vtable().add(self.klass.vtable_len as _) } } - + #[inline(always)] fn nonstatic_oop_map_count(&self) -> usize { let oop_map_block_size = mem::size_of::(); let oop_map_block_size_up = mmtk::util::conversions::raw_align_up(oop_map_block_size, BYTES_IN_WORD); self.nonstatic_oop_map_size as usize / (oop_map_block_size_up >> LOG_BYTES_IN_WORD) } - + #[inline(always)] pub fn nonstatic_oop_maps(&self) -> &'static [OopMapBlock] { let start_of_itable = self.start_of_itable(); let start = unsafe { start_of_itable.add(self.itable_len as _) as *const OopMapBlock }; @@ -177,6 +177,7 @@ pub struct InstanceMirrorKlass { } impl InstanceMirrorKlass { + #[inline(always)] fn offset_of_static_fields() -> usize { lazy_static! { pub static ref OFFSET_OF_STATIC_FIELDS: usize = @@ -184,6 +185,7 @@ impl InstanceMirrorKlass { } *OFFSET_OF_STATIC_FIELDS } + #[inline(always)] fn static_oop_field_count_offset() -> i32 { lazy_static! { pub static ref STATIC_OOP_FIELD_COUNT_OFFSET: i32 = @@ -191,9 +193,11 @@ impl InstanceMirrorKlass { } *STATIC_OOP_FIELD_COUNT_OFFSET } + #[inline(always)] pub fn start_of_static_fields(oop: Oop) -> Address { Address::from_ref(oop) + Self::offset_of_static_fields() } + #[inline(always)] pub fn static_oop_field_count(oop: Oop) -> usize { let offset = Self::static_oop_field_count_offset(); unsafe { oop.get_field_address(offset).load::() as _ } @@ -232,21 +236,25 @@ pub struct InstanceRefKlass { } impl InstanceRefKlass { + #[inline(always)] fn referent_offset() -> i32 { lazy_static! { pub static ref REFERENT_OFFSET: i32 = unsafe { ((*UPCALLS).referent_offset)() }; } *REFERENT_OFFSET } + #[inline(always)] fn discovered_offset() -> i32 { lazy_static! { pub static ref DISCOVERED_OFFSET: i32 = unsafe { ((*UPCALLS).discovered_offset)() }; } *DISCOVERED_OFFSET } + #[inline(always)] pub fn referent_address(oop: Oop) -> Address { oop.get_field_address(Self::referent_offset()) } + #[inline(always)] pub fn discovered_address(oop: Oop) -> Address { oop.get_field_address(Self::discovered_offset()) } @@ -336,6 +344,7 @@ pub type ArrayOop = &'static ArrayOopDesc; impl ArrayOopDesc { const ELEMENT_TYPE_SHOULD_BE_ALIGNED: bool = type_equal::() || type_equal::(); const LENGTH_OFFSET: usize = mem::size_of::(); + #[inline(always)] fn header_size() -> usize { let typesize_in_bytes = conversions::raw_align_up(Self::LENGTH_OFFSET + BYTES_IN_INT, BYTES_IN_LONG); @@ -345,13 +354,16 @@ impl ArrayOopDesc { typesize_in_bytes / BYTES_IN_WORD } } + #[inline(always)] fn length(&self) -> i32 { unsafe { *((self as *const _ as *const u8).add(Self::LENGTH_OFFSET) as *const i32) } } + #[inline(always)] fn base(&self) -> *const T { let base_offset_in_bytes = Self::header_size() * BYTES_IN_WORD; unsafe { (self as *const _ as *const u8).add(base_offset_in_bytes) as _ } } + #[inline(always)] pub fn data(&self) -> &[T] { unsafe { slice::from_raw_parts(self.base(), self.length() as _) } } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 7b2dfc2b..7d8c16ba 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -15,7 +15,7 @@ trait OopIterate: Sized { } impl OopIterate for OopMapBlock { - #[inline] + #[inline(always)] fn oop_iterate(&self, oop: Oop, closure: &mut impl TransitiveClosure) { let start = oop.get_field_address(self.offset); for i in 0..self.count as usize { @@ -26,7 +26,7 @@ impl OopIterate for OopMapBlock { } impl OopIterate for InstanceKlass { - #[inline] + #[inline(always)] fn oop_iterate(&self, oop: Oop, closure: &mut impl TransitiveClosure) { let oop_maps = self.nonstatic_oop_maps(); for map in oop_maps { @@ -36,7 +36,7 @@ impl OopIterate for InstanceKlass { } impl OopIterate for InstanceMirrorKlass { - #[inline] + #[inline(always)] fn oop_iterate(&self, oop: Oop, closure: &mut impl TransitiveClosure) { self.instance_klass.oop_iterate(oop, closure); // if (Devirtualizer::do_metadata(closure)) { @@ -77,7 +77,7 @@ impl OopIterate for InstanceMirrorKlass { } impl OopIterate for InstanceClassLoaderKlass { - #[inline] + #[inline(always)] fn oop_iterate(&self, oop: Oop, closure: &mut impl TransitiveClosure) { self.instance_klass.oop_iterate(oop, closure); // if (Devirtualizer::do_metadata(closure)) { @@ -91,7 +91,7 @@ impl OopIterate for InstanceClassLoaderKlass { } impl OopIterate for ObjArrayKlass { - #[inline] + #[inline(always)] fn oop_iterate(&self, oop: Oop, closure: &mut impl TransitiveClosure) { let array = unsafe { oop.as_array_oop::() }; for oop in array.data() { @@ -101,7 +101,7 @@ impl OopIterate for ObjArrayKlass { } impl OopIterate for TypeArrayKlass { - #[inline] + #[inline(always)] fn oop_iterate(&self, _oop: Oop, _closure: &mut impl TransitiveClosure) { // Performance tweak: We skip processing the klass pointer since all // TypeArrayKlasses are guaranteed processed via the null class loader. @@ -109,7 +109,7 @@ impl OopIterate for TypeArrayKlass { } impl OopIterate for InstanceRefKlass { - #[inline] + #[inline(always)] fn oop_iterate(&self, oop: Oop, closure: &mut impl TransitiveClosure) { self.instance_klass.oop_iterate(oop, closure); let referent_addr = Self::referent_address(oop); @@ -126,7 +126,7 @@ fn oop_iterate_slow(oop: Oop, closure: &mut impl TransitiveClosure, tls: OpaqueP } } -#[inline] +#[inline(always)] fn oop_iterate(oop: Oop, closure: &mut impl TransitiveClosure) { let klass_id = oop.klass.id; debug_assert!( From 3dca4074528570b01aa1e933484fb421c2f3f1eb Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 6 Nov 2021 22:49:42 +1100 Subject: [PATCH 093/494] Optimize root scanning --- openjdk/mmtkHeap.cpp | 58 +++++------------------------------- openjdk/mmtkHeap.hpp | 2 +- openjdk/mmtkRootsClosure.hpp | 2 +- openjdk/mmtkUpcalls.cpp | 5 +++- 4 files changed, 13 insertions(+), 54 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 23ddaed8..0587aa32 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -330,106 +330,63 @@ void MMTkHeap::print_tracing_info() const { // An object is scavengable if its location may move during a scavenge. // (A scavenge is a GC which is not a full GC.) -bool MMTkHeap::is_scavengable(oop obj) {return false;} +bool MMTkHeap::is_scavengable(oop obj) {return true;} // Registering and unregistering an nmethod (compiled code) with the heap. // Override with specific mechanism for each specialized heap type. +void MMTkHeap::register_nmethod(nmethod* nm) { + CodeCache::register_scavenge_root_nmethod(nm); +} + // Heap verification void MMTkHeap::verify(VerifyOption option) {} -template -struct MMTkRootScanWorkScope { - int* _num_root_scan_tasks; - int _current_task_ordinal; - MMTkRootScanWorkScope(int* num_root_scan_tasks): _num_root_scan_tasks(num_root_scan_tasks), _current_task_ordinal(0) { - _current_task_ordinal = Atomic::add(1, _num_root_scan_tasks); - if (_current_task_ordinal == 1) { - nmethod::oops_do_marking_prologue(); - } - } - ~MMTkRootScanWorkScope() { - if (_current_task_ordinal == MAX_TASKS) { - _current_task_ordinal = 0; - nmethod::oops_do_marking_epilogue(); - } - } -}; - void MMTkHeap::scan_static_roots(OopClosure& cl) { } void MMTkHeap::scan_universe_roots(OopClosure& cl) { - ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); Universe::oops_do(&cl); } void MMTkHeap::scan_jni_handle_roots(OopClosure& cl) { - ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); JNIHandles::oops_do(&cl); } void MMTkHeap::scan_object_synchronizer_roots(OopClosure& cl) { - ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); ObjectSynchronizer::oops_do(&cl); } void MMTkHeap::scan_management_roots(OopClosure& cl) { - ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); Management::oops_do(&cl); } void MMTkHeap::scan_jvmti_export_roots(OopClosure& cl) { - ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); JvmtiExport::oops_do(&cl); } void MMTkHeap::scan_aot_loader_roots(OopClosure& cl) { - ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); AOTLoader::oops_do(&cl); } void MMTkHeap::scan_system_dictionary_roots(OopClosure& cl) { - ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); SystemDictionary::oops_do(&cl); } void MMTkHeap::scan_code_cache_roots(OopClosure& cl) { - ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); - CodeCache::gc_prologue(); - CodeBlobToOopClosure cb_cl(&cl, true); - { - MutexLockerEx lock(CodeCache_lock, Mutex::_no_safepoint_check_flag); - CodeCache::scavenge_root_nmethods_do(&cb_cl); - CodeCache::blobs_do(&cb_cl); - } + MarkingCodeBlobClosure cb_cl(&cl, false); + CodeCache::scavenge_root_nmethods_do(&cb_cl); } void MMTkHeap::scan_string_table_roots(OopClosure& cl) { - ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); StringTable::oops_do(&cl); } void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl) { - ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); CLDToOopClosure cld_cl(&cl, false); ClassLoaderDataGraph::cld_do(&cld_cl); } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { - ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); WeakProcessor::oops_do(&cl); // (really needed???) } void MMTkHeap::scan_vm_thread_roots(OopClosure& cl) { ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); VMThread::vm_thread()->oops_do(&cl, NULL); } void MMTkHeap::scan_global_roots(OopClosure& cl) { ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); CodeBlobToOopClosure cb_cl(&cl, true); CLDToOopClosure cld_cl(&cl, false); @@ -457,7 +414,6 @@ void MMTkHeap::scan_global_roots(OopClosure& cl) { void MMTkHeap::scan_thread_roots(OopClosure& cl) { ResourceMark rm; - MMTkRootScanWorkScope<> root_scan_work(&_num_root_scan_tasks); Threads::possibly_parallel_oops_do(false, &cl, NULL); } diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index e2426e83..b12f352b 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -184,7 +184,7 @@ class MMTkHeap : public CollectedHeap { bool is_scavengable(oop obj); // Registering and unregistering an nmethod (compiled code) with the heap. // Override with specific mechanism for each specialized heap type. - + void register_nmethod(nmethod* nm); // Heap verification void verify(VerifyOption option); diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 7870b036..92ee4329 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -54,7 +54,7 @@ class MMTkRootsClosure2 : public OopClosure { size_t _cursor; template - void do_oop_work(T* p) { + inline void do_oop_work(T* p) { // T heap_oop = RawAccess<>::oop_load(p); // if (!CompressedOops::is_null(heap_oop)) { // oop obj = CompressedOops::decode_not_null(heap_oop); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index a0539a42..3b522233 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -55,6 +55,7 @@ static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(voi MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_suspended, true); log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); + nmethod::oops_do_marking_prologue(); { JavaThreadIteratorWithHandle jtiwh; while (JavaThread *cur = jtiwh.next()) { @@ -68,6 +69,7 @@ static void mmtk_resume_mutators(void *tls) { ClassLoaderDataGraph::purge(); CodeCache::gc_epilogue(); JvmtiExport::gc_epilogue(); + nmethod::oops_do_marking_epilogue(); #if COMPILER2_OR_JVMCI DerivedPointerTable::update_pointers(); #endif @@ -187,7 +189,8 @@ static void mmtk_scan_thread_root(ProcessEdgesFn process_edges, void* tls) { ResourceMark rm; JavaThread* thread = (JavaThread*) tls; MMTkRootsClosure2 cl(process_edges); - thread->oops_do(&cl, NULL); + MarkingCodeBlobClosure cb_cl(&cl, false); + thread->oops_do(&cl, &cb_cl); } static void mmtk_scan_object(void* trace, void* object, void* tls) { From 172f7b7af55b15576af8fbd589022c067819dfee Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 6 Nov 2021 23:00:20 +1100 Subject: [PATCH 094/494] safepoint workers --- openjdk/mmtkHeap.cpp | 7 ++++++- openjdk/mmtkHeap.hpp | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 0587aa32..4e5b6297 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -66,6 +66,11 @@ MMTkHeap::MMTkHeap(MMTkCollectorPolicy* policy) : CollectedHeap(), _last_gc_time // , _par_state_string(StringTable::weak_storage()) { _heap = this; + unsigned int ncpus = (unsigned int) os::initial_active_processor_count(); + _safepoint_workers = new WorkGang("GC Thread", ncpus, + /* are_GC_task_threads */true, + /* are_ConcurrentGC_threads */false); + _safepoint_workers->initialize_workers(); } jint MMTkHeap::initialize() { @@ -330,7 +335,7 @@ void MMTkHeap::print_tracing_info() const { // An object is scavengable if its location may move during a scavenge. // (A scavenge is a GC which is not a full GC.) -bool MMTkHeap::is_scavengable(oop obj) {return true;} +// bool MMTkHeap::is_scavengable(oop obj) {return true;} // Registering and unregistering an nmethod (compiled code) with the heap. // Override with specific mechanism for each specialized heap type. diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index b12f352b..c44856f5 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -59,6 +59,7 @@ class MMTkHeap : public CollectedHeap { ContiguousSpace* _space; int _num_root_scan_tasks; MMTkVMCompanionThread* _companion_thread; + WorkGang* _safepoint_workers; public: MMTkHeap(MMTkCollectorPolicy* policy); @@ -178,10 +179,11 @@ class MMTkHeap : public CollectedHeap { // Default implementation does nothing. void print_tracing_info() const ; + inline virtual WorkGang* get_safepoint_workers() { return _safepoint_workers; } // An object is scavengable if its location may move during a scavenge. // (A scavenge is a GC which is not a full GC.) - bool is_scavengable(oop obj); + inline bool is_scavengable(oop obj) { return true; } // Registering and unregistering an nmethod (compiled code) with the heap. // Override with specific mechanism for each specialized heap type. void register_nmethod(nmethod* nm); From 8f48d224670a5634e30ec159347903e5fd6e4602 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 7 Nov 2021 11:00:25 +1100 Subject: [PATCH 095/494] clear OopMapCache --- openjdk/mmtkVMOperation.cpp | 6 ++++++ openjdk/mmtkVMOperation.hpp | 1 + 2 files changed, 7 insertions(+) diff --git a/openjdk/mmtkVMOperation.cpp b/openjdk/mmtkVMOperation.cpp index ee56976d..ef3ce209 100644 --- a/openjdk/mmtkVMOperation.cpp +++ b/openjdk/mmtkVMOperation.cpp @@ -26,6 +26,7 @@ #include "mmtk.h" #include "mmtkVMCompanionThread.hpp" #include "mmtkVMOperation.hpp" +#include "interpreter/oopMapCache.hpp" VM_MMTkSTWOperation::VM_MMTkSTWOperation(MMTkVMCompanionThread *companion_thread): _companion_thread(companion_thread) { @@ -36,3 +37,8 @@ void VM_MMTkSTWOperation::doit() { _companion_thread->reach_suspended_and_wait_for_resume(); log_trace(vmthread)("Leaving VM_MMTkSTWOperation::doit()"); } + + +void VM_MMTkSTWOperation::doit_epilogue() { + OopMapCache::cleanup_old_entries(); +} \ No newline at end of file diff --git a/openjdk/mmtkVMOperation.hpp b/openjdk/mmtkVMOperation.hpp index 98d37ea4..402dedbe 100644 --- a/openjdk/mmtkVMOperation.hpp +++ b/openjdk/mmtkVMOperation.hpp @@ -40,6 +40,7 @@ class VM_MMTkSTWOperation : public VM_MMTkOperation { public: VM_MMTkSTWOperation(MMTkVMCompanionThread *companion_thread); virtual void doit() override; + virtual void doit_epilogue() override; // Note: Not called if mode is: _concurrent }; #endif // MMTK_OPENJDK_MMTK_VM_OPERATION_HPP From 66f522d6aaee36689aa2b17cdedf2e057a47f047 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 7 Nov 2021 11:45:07 +1100 Subject: [PATCH 096/494] Update --- mmtk/Cargo.toml | 3 ++- mmtk/src/collection.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 63314ca3..63763e21 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -28,7 +28,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7f78e5895b5033176f0598b740af048c59e991fa" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f8edc26271f95b389acfe021732d72dab38604b8" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } @@ -59,6 +59,7 @@ immix = [] sanity = ["mmtk/sanity"] barrier_measurement = ["mmtk/barrier_measurement"] +work_packet_timer = ["mmtk/work_packet_timer"] instrumentation = ["mmtk/instrumentation", "no_fast_alloc"] log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 7a4faff7..e6734b7d 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -13,7 +13,7 @@ extern "C" fn create_mutator_scan_work>( mutator: &'static mut Mutator, ) { mutator.flush(); - mmtk::memory_manager::add_work_packet( + mmtk::memory_manager::add_work_packet_no_notify( &SINGLETON, // FIXME: Should be Unconstrained! WorkBucketStage::ScanGlobalRoots, From 36c80b8fb518f6f3da787647ed4d6df437dc288d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 7 Nov 2021 13:10:54 +1100 Subject: [PATCH 097/494] Add feature --- mmtk/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 63763e21..3200b5f3 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -28,7 +28,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f8edc26271f95b389acfe021732d72dab38604b8" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f6bb233aa233944c1abb2b6170d28a8a24b32105" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } @@ -60,6 +60,7 @@ immix = [] sanity = ["mmtk/sanity"] barrier_measurement = ["mmtk/barrier_measurement"] work_packet_timer = ["mmtk/work_packet_timer"] +yield_and_roots_timer = ["mmtk/yield_and_roots_timer"] instrumentation = ["mmtk/instrumentation", "no_fast_alloc"] log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] From 1d4b102b91bdc079bf21d637250c9d5df8864e8d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 7 Nov 2021 13:51:16 +1100 Subject: [PATCH 098/494] satb_timer --- mmtk/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 3200b5f3..9de9163c 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -28,7 +28,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f6bb233aa233944c1abb2b6170d28a8a24b32105" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f6a8108fad8e7cb717edbd8f17977aef963ba1cb" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } @@ -61,6 +61,7 @@ sanity = ["mmtk/sanity"] barrier_measurement = ["mmtk/barrier_measurement"] work_packet_timer = ["mmtk/work_packet_timer"] yield_and_roots_timer = ["mmtk/yield_and_roots_timer"] +satb_timer = ["mmtk/satb_timer"] instrumentation = ["mmtk/instrumentation", "no_fast_alloc"] log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] From daff1f1cbba1f99ab602549f42341679f16faab6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 8 Nov 2021 03:56:48 +1100 Subject: [PATCH 099/494] Reduce buckets --- mmtk/src/collection.rs | 2 +- mmtk/src/scanning.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index e6734b7d..b5a99bdd 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -16,7 +16,7 @@ extern "C" fn create_mutator_scan_work>( mmtk::memory_manager::add_work_packet_no_notify( &SINGLETON, // FIXME: Should be Unconstrained! - WorkBucketStage::ScanGlobalRoots, + WorkBucketStage::Unconstrained, ScanStackRoot::(mutator), ); } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 90100f01..5a00474b 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -25,7 +25,7 @@ pub(crate) extern "C" fn create_process_edges_work for VMScanning { fn scan_vm_specific_roots>() { memory_manager::add_work_packets( &SINGLETON, - WorkBucketStage::ScanGlobalRoots, + WorkBucketStage::Unconstrained, vec![ box ScanUniverseRoots::::new(), box ScanJNIHandlesRoots::::new(), @@ -98,7 +98,7 @@ impl Scanning for VMScanning { if !(Self::SCAN_MUTATORS_IN_SAFEPOINT && Self::SINGLE_THREAD_MUTATOR_SCANNING) { memory_manager::add_work_packet( &SINGLETON, - WorkBucketStage::ScanGlobalRoots, + WorkBucketStage::Unconstrained, ScanVMThreadRoots::::new(), ); } From a12a553634cb93cbe7ca1154f2733f4f585b3fd0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 8 Nov 2021 10:01:27 +1100 Subject: [PATCH 100/494] cleanup --- mmtk/src/collection.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index b5a99bdd..9bb422cb 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -13,7 +13,7 @@ extern "C" fn create_mutator_scan_work>( mutator: &'static mut Mutator, ) { mutator.flush(); - mmtk::memory_manager::add_work_packet_no_notify( + mmtk::memory_manager::add_work_packet( &SINGLETON, // FIXME: Should be Unconstrained! WorkBucketStage::Unconstrained, From b6e9eab5fbca6b727680f360f5ec99eb1956d920 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 8 Nov 2021 10:17:39 +1100 Subject: [PATCH 101/494] report gc time without yield time --- mmtk/src/api.rs | 5 +++++ openjdk/mmtk.h | 2 ++ openjdk/mmtkUpcalls.cpp | 1 + 3 files changed, 8 insertions(+) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index c1f43ab7..b5e3fa3d 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -33,6 +33,11 @@ pub extern "C" fn mmtk_active_barrier() -> *const c_char { } } +#[no_mangle] +pub extern "C" fn mmtk_report_gc_start() { + mmtk::memory_manager::report_gc_start::(&SINGLETON) +} + /// # Safety /// Caller needs to make sure the ptr is a valid vector pointer. #[no_mangle] diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 8c25408f..d8b968ae 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -28,6 +28,8 @@ extern MMTk_Mutator bind_mutator(void *tls); extern void destroy_mutator(MMTk_Mutator mutator); extern void flush_mutator(MMTk_Mutator mutator); +extern void mmtk_report_gc_start(); + extern void* alloc(MMTk_Mutator mutator, size_t size, size_t align, size_t offset, int allocator); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 3b522233..c5a38dcf 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -54,6 +54,7 @@ static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(voi log_debug(gc)("Requesting the VM to suspend all mutators..."); MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_suspended, true); log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); + mmtk_report_gc_start(); nmethod::oops_do_marking_prologue(); { From 7055fb9b3735c39f46180bf4f661835c15fbc1eb Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 8 Nov 2021 11:11:49 +1100 Subject: [PATCH 102/494] Reduce yield time --- mmtk/src/collection.rs | 3 +-- mmtk/src/scanning.rs | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 9bb422cb..036ad4e2 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -12,11 +12,10 @@ pub struct VMCollection {} extern "C" fn create_mutator_scan_work>( mutator: &'static mut Mutator, ) { - mutator.flush(); mmtk::memory_manager::add_work_packet( &SINGLETON, // FIXME: Should be Unconstrained! - WorkBucketStage::Unconstrained, + WorkBucketStage::Prepare, ScanStackRoot::(mutator), ); } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 5a00474b..5ef6e5a0 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -80,7 +80,7 @@ impl Scanning for VMScanning { fn scan_vm_specific_roots>() { memory_manager::add_work_packets( &SINGLETON, - WorkBucketStage::Unconstrained, + WorkBucketStage::Prepare, vec![ box ScanUniverseRoots::::new(), box ScanJNIHandlesRoots::::new(), @@ -98,7 +98,7 @@ impl Scanning for VMScanning { if !(Self::SCAN_MUTATORS_IN_SAFEPOINT && Self::SINGLE_THREAD_MUTATOR_SCANNING) { memory_manager::add_work_packet( &SINGLETON, - WorkBucketStage::Unconstrained, + WorkBucketStage::Prepare, ScanVMThreadRoots::::new(), ); } From 21d9591e46e7593d0a78c8dc1f53e92518ac4fd9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 8 Nov 2021 19:18:46 +1100 Subject: [PATCH 103/494] Update features --- mmtk/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 9de9163c..937a29d9 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -28,7 +28,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f6a8108fad8e7cb717edbd8f17977aef963ba1cb" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "d1aa270ba527d8e7e1de30cd7c88c8b840f42669" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } @@ -79,6 +79,7 @@ lxr_lazy = ["mmtk/lxr_lazy"] # LXR (Concurrent) lxr_conc = ["mmtk/lxr_conc"] # LXR (Evacuation) +lxr_nevac = ["mmtk/lxr_nevac"] lxr_evac = ["mmtk/lxr_evac"] # LXR (Block only) lxr_block = ["mmtk/lxr_block"] From 53453d07550a814c522d734063860bbc82b9f963 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 9 Nov 2021 23:11:22 +1100 Subject: [PATCH 104/494] Fix root scan bucket --- mmtk/src/collection.rs | 2 +- mmtk/src/scanning.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 036ad4e2..4d2ddbbc 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -15,7 +15,7 @@ extern "C" fn create_mutator_scan_work>( mmtk::memory_manager::add_work_packet( &SINGLETON, // FIXME: Should be Unconstrained! - WorkBucketStage::Prepare, + WorkBucketStage::Unconstrained, ScanStackRoot::(mutator), ); } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 5ef6e5a0..5a00474b 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -80,7 +80,7 @@ impl Scanning for VMScanning { fn scan_vm_specific_roots>() { memory_manager::add_work_packets( &SINGLETON, - WorkBucketStage::Prepare, + WorkBucketStage::Unconstrained, vec![ box ScanUniverseRoots::::new(), box ScanJNIHandlesRoots::::new(), @@ -98,7 +98,7 @@ impl Scanning for VMScanning { if !(Self::SCAN_MUTATORS_IN_SAFEPOINT && Self::SINGLE_THREAD_MUTATOR_SCANNING) { memory_manager::add_work_packet( &SINGLETON, - WorkBucketStage::Prepare, + WorkBucketStage::Unconstrained, ScanVMThreadRoots::::new(), ); } From 7e1dcb0e3da1a30c57cfc0873646bccceee44453 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 10 Nov 2021 17:23:29 +1100 Subject: [PATCH 105/494] Minor fix --- mmtk/src/collection.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 4d2ddbbc..156bb817 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -15,7 +15,7 @@ extern "C" fn create_mutator_scan_work>( mmtk::memory_manager::add_work_packet( &SINGLETON, // FIXME: Should be Unconstrained! - WorkBucketStage::Unconstrained, + WorkBucketStage::Initial, ScanStackRoot::(mutator), ); } From 4334688d4b492b57972ad1f1e7d5a33ae9d5d2b0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 13 Nov 2021 02:18:11 +1100 Subject: [PATCH 106/494] clone barrier --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index e6cd19cd..34bec8aa 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -8,7 +8,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow(void* src, voi } void MMTkFieldLoggingBarrierSetRuntime::record_clone_slow(void* src, void* dst, size_t size) { - // ::mmtk_object_reference_clone((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, size); + ::mmtk_object_reference_clone((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, size); } void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t offset, oop val) { @@ -26,7 +26,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t } void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t size) { - // record_clone_slow((void*) src, (void*) dst, size); + record_clone_slow((void*) src, (void*) dst, size); } void MMTkFieldLoggingBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) { @@ -288,14 +288,14 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src } void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const { - // MMTkIdealKit ideal(kit, true); - // const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); - // Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); + MMTkIdealKit ideal(kit, true); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); + Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); - // kit->sync_kit(ideal); - // kit->insert_mem_bar(Op_MemBarCPUOrder); + kit->sync_kit(ideal); + kit->insert_mem_bar(Op_MemBarCPUOrder); - // kit->final_sync(ideal); // Final sync IdealKit and GraphKit. + kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } #undef __ \ No newline at end of file From 0feefad4f917870f7a439c79063bb5ce53f1ad46 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 13 Nov 2021 21:41:30 +1100 Subject: [PATCH 107/494] Minor update barrier code --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 42 ++++++++++++++++---- openjdk/mmtkBarrierSet.hpp | 1 + 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index 34bec8aa..8994fb50 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -26,7 +26,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_modified_node(oop src, ptrdiff_t } void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t size) { - record_clone_slow((void*) src, (void*) dst, size); + // record_clone_slow((void*) src, (void*) dst, size); } void MMTkFieldLoggingBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) { @@ -282,20 +282,46 @@ void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_modified_node_slow), "record_modified_node", src, slot, val); #endif kit->sync_kit(ideal); - kit->insert_mem_bar(Op_MemBarCPUOrder); + kit->insert_mem_bar(Op_MemBarVolatile); kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } +// static bool clone_needs_barrier(Node* src, PhaseGVN& gvn) { +// const TypeOopPtr* src_type = gvn.type(src)->is_oopptr(); +// if (src_type->isa_instptr() != NULL) { +// ciInstanceKlass* ik = src_type->klass()->as_instance_klass(); +// if ((src_type->klass_is_exact() || (!ik->is_interface() && !ik->has_subklass())) && !ik->has_injected_fields()) { +// return ik->has_object_fields(); +// } else { +// return true; +// } +// } else if (src_type->isa_aryptr()) { +// BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type(); +// if (src_elem == T_OBJECT || src_elem == T_ARRAY) { +// return true; +// } else { +// return false; +// } +// } else { +// return true; +// } +// } + void MMTkFieldLoggingBarrierSetC2::record_clone(GraphKit* kit, Node* src, Node* dst, Node* size) const { - MMTkIdealKit ideal(kit, true); - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); - Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); + // if (!clone_needs_barrier(src, kit->gvn())) return; + // if (src == kit->just_allocated_object(kit->control())) { + // return; + // } - kit->sync_kit(ideal); - kit->insert_mem_bar(Op_MemBarCPUOrder); + // MMTkIdealKit ideal(kit, true); + // const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeInt::INT); + // Node* x = __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_clone_slow), "record_clone", src, dst, size); - kit->final_sync(ideal); // Final sync IdealKit and GraphKit. + // kit->sync_kit(ideal); + // kit->insert_mem_bar(Op_MemBarVolatile); + + // kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } #undef __ \ No newline at end of file diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index b9af7691..065c661a 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -202,6 +202,7 @@ class MMTkBarrierSet : public BarrierSet { static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) { + runtime()->record_arraycopy(src_obj, src_offset_in_bytes, (oop*) src_raw, dst_obj, dst_offset_in_bytes, (oop*) dst_raw, length); return Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); } From ab145b18a76d21ff1f825ffac5f5df774926ed4d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 16 Nov 2021 23:35:14 +1100 Subject: [PATCH 108/494] Update immix-allocator layout --- mmtk/Cargo.toml | 2 +- mmtk/src/lib.rs | 3 +++ openjdk/mmtk.h | 1 + openjdk/mmtkMutator.cpp | 4 ++++ openjdk/mmtkMutator.hpp | 1 + 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 937a29d9..a61ff94c 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -28,7 +28,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "d1aa270ba527d8e7e1de30cd7c88c8b840f42669" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "577695a23476ca2277a15bb5da8ac2e878259357" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 21173c9f..12e95367 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -112,6 +112,9 @@ pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t = #[no_mangle] pub static DISABLE_ALLOCATION_FAST_PATH: i32 = cfg!(feature = "no_fast_alloc") as _; +#[no_mangle] +pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = std::mem::size_of::>(); + #[derive(Default)] pub struct OpenJDK; diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index d8b968ae..6ef96b04 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -16,6 +16,7 @@ extern const uintptr_t GLOBAL_SIDE_METADATA_BASE_ADDRESS; extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; extern const uintptr_t GLOBAL_ALLOC_BIT_ADDRESS; extern const int DISABLE_ALLOCATION_FAST_PATH; +extern const uintptr_t IMMIX_ALLOCATOR_SIZE; inline bool disable_fast_alloc() { return DISABLE_ALLOCATION_FAST_PATH != 0; diff --git a/openjdk/mmtkMutator.cpp b/openjdk/mmtkMutator.cpp index 57ee9760..bdb20986 100644 --- a/openjdk/mmtkMutator.cpp +++ b/openjdk/mmtkMutator.cpp @@ -6,6 +6,10 @@ size_t MMTkMutatorContext::max_non_los_default_alloc_bytes = 0; MMTkMutatorContext MMTkMutatorContext::bind(::Thread* current) { + if (IMMIX_ALLOCATOR_SIZE != sizeof(ImmixAllocator)) { + printf("ERROR: Unmatched immix allocator size: rs=%zu cpp=%zu\n", IMMIX_ALLOCATOR_SIZE, sizeof(ImmixAllocator)); + guarantee(false, "ERROR"); + } return *((MMTkMutatorContext*) ::bind_mutator((void*) current)); } diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp index d534227d..469d65d4 100644 --- a/openjdk/mmtkMutator.hpp +++ b/openjdk/mmtkMutator.hpp @@ -54,6 +54,7 @@ struct ImmixAllocator { uint8_t _align[7]; uint8_t line_opt_tag; uintptr_t line_opt; + uintptr_t mutator_recycled_blocks; }; struct MallocAllocator { From 5c27fe1d8b2e165f4a564747d851aae283e8cd5b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 17 Nov 2021 15:54:01 +1100 Subject: [PATCH 109/494] Fast oop field check --- mmtk/src/abi.rs | 16 +++++++ mmtk/src/object_scanning.rs | 93 +++++++++++++++++++++++++++++++++++++ mmtk/src/scanning.rs | 5 ++ 3 files changed, 114 insertions(+) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 7ccf3a3a..0b5dfb07 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -6,6 +6,7 @@ use mmtk::util::{Address, OpaquePointer}; use std::ffi::CStr; use std::fmt; use std::marker::PhantomData; +use std::ops::Range; use std::{mem, slice}; trait EqualTo { @@ -266,6 +267,15 @@ pub struct OopDesc { pub klass: &'static Klass, } +impl OopDesc { + #[inline(always)] + pub fn start(&self) -> Address { + unsafe { + mem::transmute(self) + } + } +} + impl fmt::Debug for OopDesc { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let c_string = unsafe { ((*UPCALLS).dump_object_string)(mem::transmute(self)) }; @@ -367,6 +377,12 @@ impl ArrayOopDesc { pub fn data(&self) -> &[T] { unsafe { slice::from_raw_parts(self.base(), self.length() as _) } } + #[inline(always)] + pub fn data_range(&self) -> Range
{ + let base = Address::from_ptr(self.base()); + let len = self.length() as usize; + base .. (base + (len << BYTES_IN_ADDRESS)) + } } #[repr(C)] diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 7d8c16ba..3b1bc073 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -12,6 +12,7 @@ use std::{mem, slice}; trait OopIterate: Sized { fn oop_iterate(&self, oop: Oop, closure: &mut impl TransitiveClosure); + fn is_oop_field(&self, oop: Oop, edge: Address) -> bool; } impl OopIterate for OopMapBlock { @@ -23,6 +24,12 @@ impl OopIterate for OopMapBlock { closure.process_edge(edge); } } + #[inline(always)] + fn is_oop_field(&self, oop: Oop, edge: Address) -> bool { + let start = oop.get_field_address(self.offset); + let end = start + ((self.count as usize) << LOG_BYTES_IN_ADDRESS); + edge >= start && edge < end + } } impl OopIterate for InstanceKlass { @@ -33,6 +40,16 @@ impl OopIterate for InstanceKlass { map.oop_iterate(oop, closure) } } + #[inline(always)] + fn is_oop_field(&self, oop: Oop, edge: Address) -> bool { + let oop_maps = self.nonstatic_oop_maps(); + for map in oop_maps { + if map.is_oop_field(oop, edge) { + return true; + } + } + false + } } impl OopIterate for InstanceMirrorKlass { @@ -74,6 +91,16 @@ impl OopIterate for InstanceMirrorKlass { closure.process_edge(Address::from_ref(oop as &Oop)); } } + #[inline(always)] + fn is_oop_field(&self, oop: Oop, edge: Address) -> bool { + if self.instance_klass.is_oop_field(oop, edge) { + return true; + } + let start = Self::start_of_static_fields(oop); + let len = Self::static_oop_field_count(oop); + let end = start + (len << LOG_BYTES_IN_ADDRESS); + edge >= start && edge < end + } } impl OopIterate for InstanceClassLoaderKlass { @@ -88,6 +115,10 @@ impl OopIterate for InstanceClassLoaderKlass { // } // } } + #[inline(always)] + fn is_oop_field(&self, oop: Oop, edge: Address) -> bool { + self.instance_klass.is_oop_field(oop, edge) + } } impl OopIterate for ObjArrayKlass { @@ -98,6 +129,12 @@ impl OopIterate for ObjArrayKlass { closure.process_edge(Address::from_ref(oop as &Oop)); } } + #[inline(always)] + fn is_oop_field(&self, oop: Oop, edge: Address) -> bool { + let array = unsafe { oop.as_array_oop::() }; + let data = array.data_range(); + edge >= data.start && edge < data.end + } } impl OopIterate for TypeArrayKlass { @@ -106,6 +143,10 @@ impl OopIterate for TypeArrayKlass { // Performance tweak: We skip processing the klass pointer since all // TypeArrayKlasses are guaranteed processed via the null class loader. } + #[inline(always)] + fn is_oop_field(&self, _oop: Oop, _edge: Address) -> bool { + false + } } impl OopIterate for InstanceRefKlass { @@ -117,6 +158,21 @@ impl OopIterate for InstanceRefKlass { let discovered_addr = Self::discovered_address(oop); closure.process_edge(discovered_addr); } + #[inline(always)] + fn is_oop_field(&self, oop: Oop, edge: Address) -> bool { + if self.instance_klass.is_oop_field(oop, edge) { + return true; + } + let referent_addr = Self::referent_address(oop); + if edge == referent_addr { + return true; + } + let discovered_addr = Self::discovered_address(oop); + if edge == discovered_addr { + return true; + } + false + } } #[allow(unused)] @@ -163,6 +219,43 @@ fn oop_iterate(oop: Oop, closure: &mut impl TransitiveClosure) { } } +#[inline(always)] +pub fn is_oop_field(oop: Oop, edge: Address) -> bool { + let klass_id = oop.klass.id; + debug_assert!( + klass_id as i32 >= 0 && (klass_id as i32) < 6, + "Invalid klass-id: {:x} for oop: {:x}", + klass_id as i32, + unsafe { mem::transmute::(oop) } + ); + match klass_id { + KlassID::Instance => { + let instance_klass = unsafe { oop.klass.cast::() }; + instance_klass.is_oop_field(oop, edge) + } + KlassID::InstanceClassLoader => { + let instance_klass = unsafe { oop.klass.cast::() }; + instance_klass.is_oop_field(oop, edge) + } + KlassID::InstanceMirror => { + let instance_klass = unsafe { oop.klass.cast::() }; + instance_klass.is_oop_field(oop, edge) + } + KlassID::ObjArray => { + let array_klass = unsafe { oop.klass.cast::() }; + array_klass.is_oop_field(oop, edge) + } + KlassID::TypeArray => { + let array_klass = unsafe { oop.klass.cast::() }; + array_klass.is_oop_field(oop, edge) + } + KlassID::InstanceRef => { + let instance_klass = unsafe { oop.klass.cast::() }; + instance_klass.is_oop_field(oop, edge) + } // _ => oop_iterate_slow(oop, closure, tls), + } +} + #[inline] pub fn scan_object( object: ObjectReference, diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 5a00474b..a1c32318 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -47,6 +47,11 @@ impl Scanning for VMScanning { crate::object_scanning::scan_object(object, trace, tls) } + #[inline(always)] + fn is_oop_field(o: ObjectReference, e: Address) -> bool { + crate::object_scanning::is_oop_field(unsafe { std::mem::transmute(o) }, e) + } + fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { // unimplemented!() // TODO From 76aa0c683c2c3157cf20756ed08dee80c69d0d54 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 17 Nov 2021 17:06:02 +1100 Subject: [PATCH 110/494] Switch back to arraycopy_prologue --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 52 ++++++++++++++++++-- openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 6 +-- repos/openjdk | 2 +- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index 8994fb50..86ed180b 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -51,10 +51,21 @@ void MMTkFieldLoggingBarrierSetAssembler::oop_store_at(MacroAssembler* masm, Dec BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2); } -void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) {} - -void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue2(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count, Register dst_obj) { +void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { if (type == T_OBJECT || type == T_ARRAY) { + Label slow, done; + // Bailout if count is zero + __ cmpptr(count, 0); + __ jcc(Assembler::equal, done); + // Fast path if count is one + __ cmpptr(count, 1); + __ jcc(Assembler::notEqual, slow); + __ push(rax); + record_modified_node(masm, Address(dst, 0), src, rax, rax); + __ pop(rax); + __ jmp(done); + // Slow path + __ bind(slow); __ pusha(); assert_different_registers(c_rarg0, dst); assert_different_registers(c_rarg0, count); @@ -78,12 +89,43 @@ void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue2(MacroAssembler* ma __ movptr(c_rarg1, dst); __ movptr(c_rarg2, count); - __ movptr(c_rarg3, dst_obj); - __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_array_copy_slow), 4); + __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_array_copy_slow), 3); __ popa(); + __ bind(done); } } +// void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue2(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count, Register dst_obj) { +// if (type == T_OBJECT || type == T_ARRAY) { +// __ pusha(); +// assert_different_registers(c_rarg0, dst); +// assert_different_registers(c_rarg0, count); +// // guarantee(c_rarg0 != dst, "X"); +// guarantee(rscratch1 != src, "X"); +// guarantee(rscratch1 != dst, "X"); +// guarantee(rscratch1 != count, "X"); + +// guarantee(c_rarg0 != count, "X"); + +// if (c_rarg0 == dst) { +// __ movptr(rscratch1, dst); +// } +// __ movptr(c_rarg0, src); +// if (c_rarg0 == dst) { +// __ movptr(dst, rscratch1); +// } + +// assert_different_registers(c_rarg1, count); +// guarantee(c_rarg1 != count, "X"); + +// __ movptr(c_rarg1, dst); +// __ movptr(c_rarg2, count); +// __ movptr(c_rarg3, dst_obj); +// __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_array_copy_slow), 4); +// __ popa(); +// } +// } + void MMTkFieldLoggingBarrierSetAssembler::record_modified_node(MacroAssembler* masm, Address dst, Register val, Register tmp1, Register tmp2) { #if MMTK_ENABLE_BARRIER_FASTPATH Label done; diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index 6014ec00..7babc8e0 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -20,8 +20,8 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { public: static void record_modified_node_slow(void* src, void* slot, void* val); static void record_clone_slow(void* src, void* dst, size_t size); - static void record_array_copy_slow(void* src, void* dst, const size_t size, void* dst_obj) { - record_array_copy_inline(src, dst, size, dst_obj); + static void record_array_copy_slow(void* src, void* dst, const size_t size) { + record_array_copy_inline(src, dst, size, NULL); } inline static void record_array_copy_mmtk_slow(const void* src, const void* dst, const size_t size, void* dst_obj) { @@ -158,7 +158,7 @@ class MMTkFieldLoggingBarrierSetAssembler: public MMTkBarrierSetAssembler { } } virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; - virtual void arraycopy_prologue2(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count, Register dst_obj) override; + // virtual void arraycopy_prologue2(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count, Register dst_obj) override; inline void gen_write_barrier_stub(LIR_Assembler* ce, MMTkFieldLoggingBarrierStub* stub); #define __ sasm-> void generate_c1_write_barrier_runtime_stub(StubAssembler* sasm) { diff --git a/repos/openjdk b/repos/openjdk index 9f641740..6dc618e2 160000 --- a/repos/openjdk +++ b/repos/openjdk @@ -1 +1 @@ -Subproject commit 9f6417407870b5c28aa45ba2ee74de5cace90a4f +Subproject commit 6dc618e281128b6a38d40c7d8d2e345d610f0160 From fc93ec826dcbab4930c1a7ddd35266ce23736464 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 18 Nov 2021 16:58:57 +1100 Subject: [PATCH 111/494] Update features --- mmtk/Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index a61ff94c..43e1430e 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -28,7 +28,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "577695a23476ca2277a15bb5da8ac2e878259357" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "a2365b2a4be1dfbdd5b2afb73125c73280e8319c" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } @@ -70,6 +70,8 @@ log_work_packets = ["mmtk/log_work_packets"] ix_block_only = ["mmtk/ix_block_only"] ix_defrag = ["mmtk/ix_defrag"] +lxr_heap_health_guided_gc = ["mmtk/lxr_heap_health_guided_gc"] + # Non-moving, Non-concurrent LXR lxr_basic = ["mmtk/lxr_basic"] # LXR (Concurrent marking) From 3d3b850251ae0893dbaa1196f3faaf549b207d5e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 19 Nov 2021 03:37:26 +1100 Subject: [PATCH 112/494] mutator performance --- openjdk/mmtkUpcalls.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index c5a38dcf..e0cdaafd 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -45,8 +45,8 @@ static size_t mmtk_start_the_world_count = 0; static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(void* mutator)) { MMTkHeap::_create_stack_scan_work = create_stack_scan_work; - ClassLoaderDataGraph::clear_claimed_marks(); - CodeCache::gc_prologue(); + // ClassLoaderDataGraph::clear_claimed_marks(); + // CodeCache::gc_prologue(); #if COMPILER2_OR_JVMCI DerivedPointerTable::clear(); #endif @@ -67,9 +67,9 @@ static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(voi } static void mmtk_resume_mutators(void *tls) { - ClassLoaderDataGraph::purge(); - CodeCache::gc_epilogue(); - JvmtiExport::gc_epilogue(); + // ClassLoaderDataGraph::purge(); + // CodeCache::gc_epilogue(); + // JvmtiExport::gc_epilogue(); nmethod::oops_do_marking_epilogue(); #if COMPILER2_OR_JVMCI DerivedPointerTable::update_pointers(); From 8c34bd436a2a6e754a5d2a696f47a22b0e5ac39d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 19 Nov 2021 14:51:16 +1100 Subject: [PATCH 113/494] don't force `no_fast_alloc` when doing instrumentation --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 43e1430e..65b4049c 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -62,7 +62,7 @@ barrier_measurement = ["mmtk/barrier_measurement"] work_packet_timer = ["mmtk/work_packet_timer"] yield_and_roots_timer = ["mmtk/yield_and_roots_timer"] satb_timer = ["mmtk/satb_timer"] -instrumentation = ["mmtk/instrumentation", "no_fast_alloc"] +instrumentation = ["mmtk/instrumentation"] log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] log_work_packets = ["mmtk/log_work_packets"] From 4adf7f835058c77cf882cfc1c6474a66224ebe0d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 3 Mar 2022 13:10:41 +1100 Subject: [PATCH 114/494] wip --- mmtk/src/api.rs | 19 ++++++++++++ mmtk/src/collection.rs | 16 ++++++++++ mmtk/src/gc_work.rs | 20 +++++++++++++ mmtk/src/lib.rs | 2 ++ mmtk/src/scanning.rs | 2 +- openjdk/mmtk.h | 6 ++++ openjdk/mmtkHeap.cpp | 3 +- openjdk/mmtkUpcalls.cpp | 65 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 131 insertions(+), 2 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index d7c79f84..923fb55d 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -286,3 +286,22 @@ pub extern "C" fn get_finalized_object() -> ObjectReference { None => unsafe { Address::ZERO.to_object_reference() }, } } + +/// Test if an object is live at the end of a GC. +/// Note: only call this method after the liveness tracing and before gc release. +#[no_mangle] +pub extern "C" fn mmtk_is_live(object: ObjectReference) -> usize { + if object.is_null() { + return 0; + } + object.is_live() as _ +} + +/// If the object is non-null and forwarded, return the forwarded pointer. Otherwise, return the original pointer. +#[no_mangle] +pub extern "C" fn mmtk_get_forwarded_ref(object: ObjectReference) -> ObjectReference { + if object.is_null() { + return object; + } + object.get_forwarded_object().unwrap_or(object) +} \ No newline at end of file diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 156bb817..a6f5048f 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -6,6 +6,7 @@ use mmtk::{Mutator, MutatorContext}; use crate::OpenJDK; use crate::{SINGLETON, UPCALLS}; +use crate::gc_work::*; pub struct VMCollection {} @@ -74,4 +75,19 @@ impl Collection for VMCollection { ((*UPCALLS).schedule_finalizer)(); } } + + fn process_weak_refs>(_worker: &mut GCWorker) { + println!("process_weak_refs"); + mmtk::memory_manager::add_work_packets( + &SINGLETON, + WorkBucketStage::Prepare, + vec![ + box ProcessWeakRef(0), + box ProcessWeakRef(1), + box ProcessWeakRef(2), + box ProcessWeakRef(3), + box ProcessNMethods, + ], + ); + } } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index ff75e4f2..19ac57ef 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -195,3 +195,23 @@ impl> GCWork for ScanVMThreadRoots } } } + +pub struct ProcessWeakRef(pub i32); + +impl GCWork for ProcessWeakRef { + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { + unsafe { + ((*UPCALLS).process_weak_refs)(self.0); + } + } +} + +pub struct ProcessNMethods; + +impl GCWork for ProcessNMethods { + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { + unsafe { + ((*UPCALLS).process_nmethods)(); + } + } +} \ No newline at end of file diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index ff607e2e..949eba3f 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -86,6 +86,8 @@ pub struct OpenJDK_Upcalls { pub schedule_finalizer: extern "C" fn(), pub prepare_for_roots_re_scanning: extern "C" fn(), pub object_alignment: extern "C" fn() -> i32, + pub process_weak_refs: extern "C" fn(id: i32), + pub process_nmethods: extern "C" fn(), } #[thread_local] diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 183a6500..b473c0ba 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -97,7 +97,7 @@ impl Scanning for VMScanning { box ScanCodeCacheRoots::::new(), box ScanStringTableRoots::::new(), box ScanClassLoaderDataGraphRoots::::new(), - box ScanWeakProcessorRoots::::new(), + // box ScanWeakProcessorRoots::::new(), ], ); if !(Self::SCAN_MUTATORS_IN_SAFEPOINT && Self::SINGLE_THREAD_MUTATOR_SCANNING) { diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index d52dd219..733ca11e 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -90,6 +90,9 @@ extern void handle_user_collection_request(void *tls); extern void start_control_collector(void *tls); extern void start_worker(void *tls, void* worker); +extern size_t mmtk_is_live(void* object); +extern void* mmtk_get_forwarded_ref(void* object); + /** * VM Accounting */ @@ -146,6 +149,9 @@ typedef struct { size_t (*number_of_mutators)(); void (*schedule_finalizer)(); void (*prepare_for_roots_re_scanning)(); + int32_t (*object_alignment)(); + void (*process_weak_ref)(int id); + void (*process_nmethods)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls, size_t heap_size); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index c136254b..a926befb 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -383,7 +383,8 @@ void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl) { ClassLoaderDataGraph::cld_do(&cld_cl); } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { - WeakProcessor::oops_do(&cl); // (really needed???) + // ShoudNotReach + // WeakProcessor::oops_do(&cl); // (really needed???) } void MMTkHeap::scan_vm_thread_roots(OopClosure& cl) { ResourceMark rm; diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index dc8f473a..a2b615da 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -39,6 +39,10 @@ #include "runtime/thread.hpp" #include "runtime/threadSMR.hpp" #include "runtime/vmThread.hpp" +#include "gc/shared/weakProcessor.hpp" +#include "prims/resolvedMethodTable.hpp" +#include "jfr/jfr.hpp" +#include "gc/shared/oopStorage.inline.hpp" static size_t mmtk_start_the_world_count = 0; @@ -71,6 +75,10 @@ static void mmtk_resume_mutators(void *tls) { // CodeCache::gc_epilogue(); // JvmtiExport::gc_epilogue(); nmethod::oops_do_marking_epilogue(); + // ClassLoaderDataGraph::purge(); + // BiasedLocking::restore_marks(); + // CodeCache::gc_epilogue(); + // JvmtiExport::gc_epilogue(); #if COMPILER2_OR_JVMCI DerivedPointerTable::update_pointers(); #endif @@ -291,6 +299,60 @@ static void mmtk_prepare_for_roots_re_scanning() { #endif } +static int32_t mmtk_object_alignment() { + ShouldNotReachHere(); + return 0; +} + +class MMTkIsAliveClosure : public BoolObjectClosure { +public: + virtual bool do_object_b(oop p) { + if (p == NULL) return false; + return mmtk_is_live((void*) p) != 0; + } +}; + +class MMTkForwardClosure : public BasicOopIterateClosure { + public: + virtual void do_oop(oop* slot) { + // *slot = (oop) mmtk_get_forwarded_ref((void*) *slot); + auto o = *slot; + if (o == NULL) return; + auto status = *((size_t*) (void*) o); + if ((status & (3ull << 56)) != 0) { + auto ptr = (oop) (void*) (status << 8 >> 8); + *slot = ptr; + } + } + virtual void do_oop(narrowOop* o) {} + virtual ReferenceIterationMode reference_iteration_mode() { return DO_FIELDS; } +}; + +/// Clean up the weak-ref storage and update pointers. +static void mmtk_process_weak_ref(int id) { + HandleMark hm; + + MMTkIsAliveClosure is_alive; + MMTkForwardClosure forward; + + JNIHandles::weak_oops_do(&is_alive, &forward); + JvmtiExport::weak_oops_do(&is_alive, &forward); + SystemDictionary::vm_weak_oop_storage()->weak_oops_do(&is_alive, &forward); + JFR_ONLY(Jfr::weak_oops_do(&is_alive, &forward);); + // if (id == 0) JNIHandles::weak_oops_do(&is_alive, &forward); + // else if (id == 1) JvmtiExport::weak_oops_do(&is_alive, &forward); + // else if (id == 2) SystemDictionary::vm_weak_oop_storage()->weak_oops_do(&is_alive, &forward); + // else { + // JFR_ONLY(Jfr::weak_oops_do(&is_alive, &forward);); + // } +printf("mmtk_process_weak_ref end\n"); +} + +static void mmtk_process_nmethods() { + // HandleMark hm; + // nmethod::oops_do_marking_epilogue(); +} + OpenJDK_Upcalls mmtk_upcalls = { mmtk_stop_all_mutators, mmtk_resume_mutators, @@ -331,4 +393,7 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_number_of_mutators, mmtk_schedule_finalizer, mmtk_prepare_for_roots_re_scanning, + mmtk_object_alignment, + mmtk_process_weak_ref, + mmtk_process_nmethods, }; From 0916064bbff3bc815352d257fe14906adf05d810 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 7 Mar 2022 21:30:46 +1100 Subject: [PATCH 115/494] WIP --- openjdk/mmtkUpcalls.cpp | 136 ++++++++++++++++++++++++++++------------ repos/openjdk | 2 +- 2 files changed, 98 insertions(+), 40 deletions(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index a2b615da..9f812d2f 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -46,6 +46,58 @@ static size_t mmtk_start_the_world_count = 0; +class MMTkSSIsAliveClosure : public BoolObjectClosure { + static constexpr size_t SS0_START = 0x20000000000ULL; + static constexpr size_t SS1_START = 0x40000000000ULL; + static constexpr size_t LOS_START = 0x80000000000ULL; + void* from_start = NULL; + void* from_limit = NULL; + void* to_start = NULL; + void* to_limit = NULL; +public: + MMTkSSIsAliveClosure() { + from_start = (void*) (high ? SS0_START : SS1_START); + from_limit = (void*) (((size_t) from_start) + 0x20000000000ULL); + to_start = (void*) (high ? SS1_START : SS0_START); + to_limit = (void*) (((size_t) to_start) + 0x20000000000ULL); + } + inline virtual bool do_object_b(oop p) { + if (p == NULL) return false; +#if INLINE_IS_ALIVE + auto x = (void*) p; + if (x >= to_start && x < to_limit) { + return true; + } + if (x >= from_start && x < from_limit) { + auto status = *((size_t*) (void*) p); + return (status & (3ull << 56)) != 0; + } + if (x >= (void*) LOS_START && x < ((void*) (LOS_START + 0x20000000000ULL))) { + return mmtk_is_live((void*) p) != 0;; + } + return false; +#else + return mmtk_is_live((void*) p) != 0; +#endif + } +}; + +class MMTkForwardClosure : public BasicOopIterateClosure { + public: + virtual void do_oop(oop* slot) { + // *slot = (oop) mmtk_get_forwarded_ref((void*) *slot); + auto o = *slot; + if (o == NULL) return; + auto status = *((size_t*) (void*) o); + if ((status & (3ull << 56)) != 0) { + auto ptr = (oop) (void*) (status << 8 >> 8); + *slot = ptr; + } + } + virtual void do_oop(narrowOop* o) {} + virtual ReferenceIterationMode reference_iteration_mode() { return DO_FIELDS; } +}; + static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(void* mutator)) { MMTkHeap::_create_stack_scan_work = create_stack_scan_work; @@ -71,6 +123,12 @@ static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(voi } static void mmtk_resume_mutators(void *tls) { + { + HandleMark hm; + MMTkSSIsAliveClosure is_alive; + MMTkForwardClosure forward; + WeakProcessor::weak_oops_do(&is_alive, &forward); + } // ClassLoaderDataGraph::purge(); // CodeCache::gc_epilogue(); // JvmtiExport::gc_epilogue(); @@ -304,48 +362,48 @@ static int32_t mmtk_object_alignment() { return 0; } -class MMTkIsAliveClosure : public BoolObjectClosure { -public: - virtual bool do_object_b(oop p) { - if (p == NULL) return false; - return mmtk_is_live((void*) p) != 0; - } -}; - -class MMTkForwardClosure : public BasicOopIterateClosure { - public: - virtual void do_oop(oop* slot) { - // *slot = (oop) mmtk_get_forwarded_ref((void*) *slot); - auto o = *slot; - if (o == NULL) return; - auto status = *((size_t*) (void*) o); - if ((status & (3ull << 56)) != 0) { - auto ptr = (oop) (void*) (status << 8 >> 8); - *slot = ptr; - } - } - virtual void do_oop(narrowOop* o) {} - virtual ReferenceIterationMode reference_iteration_mode() { return DO_FIELDS; } -}; +// class MMTkIsAliveClosure : public BoolObjectClosure { +// public: +// virtual bool do_object_b(oop p) { +// if (p == NULL) return false; +// return mmtk_is_live((void*) p) != 0; +// } +// }; + +// class MMTkForwardClosure : public BasicOopIterateClosure { +// public: +// virtual void do_oop(oop* slot) { +// // *slot = (oop) mmtk_get_forwarded_ref((void*) *slot); +// auto o = *slot; +// if (o == NULL) return; +// auto status = *((size_t*) (void*) o); +// if ((status & (3ull << 56)) != 0) { +// auto ptr = (oop) (void*) (status << 8 >> 8); +// *slot = ptr; +// } +// } +// virtual void do_oop(narrowOop* o) {} +// virtual ReferenceIterationMode reference_iteration_mode() { return DO_FIELDS; } +// }; /// Clean up the weak-ref storage and update pointers. static void mmtk_process_weak_ref(int id) { - HandleMark hm; - - MMTkIsAliveClosure is_alive; - MMTkForwardClosure forward; - - JNIHandles::weak_oops_do(&is_alive, &forward); - JvmtiExport::weak_oops_do(&is_alive, &forward); - SystemDictionary::vm_weak_oop_storage()->weak_oops_do(&is_alive, &forward); - JFR_ONLY(Jfr::weak_oops_do(&is_alive, &forward);); - // if (id == 0) JNIHandles::weak_oops_do(&is_alive, &forward); - // else if (id == 1) JvmtiExport::weak_oops_do(&is_alive, &forward); - // else if (id == 2) SystemDictionary::vm_weak_oop_storage()->weak_oops_do(&is_alive, &forward); - // else { - // JFR_ONLY(Jfr::weak_oops_do(&is_alive, &forward);); - // } -printf("mmtk_process_weak_ref end\n"); +// HandleMark hm; + +// MMTkIsAliveClosure is_alive; +// MMTkForwardClosure forward; + +// JNIHandles::weak_oops_do(&is_alive, &forward); +// JvmtiExport::weak_oops_do(&is_alive, &forward); +// SystemDictionary::vm_weak_oop_storage()->weak_oops_do(&is_alive, &forward); +// JFR_ONLY(Jfr::weak_oops_do(&is_alive, &forward);); +// // if (id == 0) JNIHandles::weak_oops_do(&is_alive, &forward); +// // else if (id == 1) JvmtiExport::weak_oops_do(&is_alive, &forward); +// // else if (id == 2) SystemDictionary::vm_weak_oop_storage()->weak_oops_do(&is_alive, &forward); +// // else { +// // JFR_ONLY(Jfr::weak_oops_do(&is_alive, &forward);); +// // } +// printf("mmtk_process_weak_ref end\n"); } static void mmtk_process_nmethods() { diff --git a/repos/openjdk b/repos/openjdk index 6dc618e2..f817e9d0 160000 --- a/repos/openjdk +++ b/repos/openjdk @@ -1 +1 @@ -Subproject commit 6dc618e281128b6a38d40c7d8d2e345d610f0160 +Subproject commit f817e9d00b2850221bb9443443a123e38e81a129 From d265ce5df35be083ebdf2c789914bd2ebc9acc28 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 7 Mar 2022 22:10:07 +1100 Subject: [PATCH 116/494] Update openjdk --- repos/openjdk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/openjdk b/repos/openjdk index 6dc618e2..f817e9d0 160000 --- a/repos/openjdk +++ b/repos/openjdk @@ -1 +1 @@ -Subproject commit 6dc618e281128b6a38d40c7d8d2e345d610f0160 +Subproject commit f817e9d00b2850221bb9443443a123e38e81a129 From ff059e28881cb55e112b3bff9267de1b718a7a78 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 10 Mar 2022 15:15:20 +1100 Subject: [PATCH 117/494] Check in Cargo.lock --- .gitignore | 2 +- mmtk/Cargo.lock | 634 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 635 insertions(+), 1 deletion(-) create mode 100644 mmtk/Cargo.lock diff --git a/.gitignore b/.gitignore index 5909989f..7a5ac51f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ target/ # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html -Cargo.lock +# Cargo.lock # These are backup files generated by rustfmt **/*.rs.bk diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock new file mode 100644 index 00000000..aba2f8c6 --- /dev/null +++ b/mmtk/Cargo.lock @@ -0,0 +1,634 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "array-macro" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06e97b4e522f9e55523001238ac59d13a8603af57f69980de5d8de4bbbe8ada6" + +[[package]] +name = "atomic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b88d82667eca772c4aa12f0f1348b3ae643424c8876448f3f7bd5787032e234c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "atomic-traits" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e609bfb99c65d95ebabff6df4641de32b4492ae10a6d77b628a03936ac6ad8d2" +dependencies = [ + "cfg-if 0.1.10", + "rustc_version", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "bindgen" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f8523b410d7187a43085e7e064416ea32ded16bd0a4e6fc025e21616d01258f" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "clap", + "env_logger", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "which", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cexpr" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clang-sys" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cc00842eed744b858222c4c9faf7243aafc6d33f92f96935263ef4d8a41ce21" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "lazy_static", + "maybe-uninit", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dd435b205a4842da59efd07628f921c096bc1cc0a156835b4fa0bcb9a19bcce" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.7", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" +dependencies = [ + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "custom_derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef8ae57c4978a2acd8b869ce6b9ca1dfe817bff704c220209fdef2c0b75a01b9" + +[[package]] +name = "downcast-rs" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" + +[[package]] +name = "enum-map" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4187999839f4ae8be35cf185d1381aa8dc32d2f5951349cc37ae49ebc4781855" +dependencies = [ + "array-macro", + "enum-map-derive", +] + +[[package]] +name = "enum-map-derive" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c450cf304c9e18d45db562025a14fb1ca0f5c769b6f609309f81d4c31de455" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "enum_derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "406ac2a8c9eedf8af9ee1489bee9e50029278a6456c740f7454cf8a158abc816" + +[[package]] +name = "env_logger" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" + +[[package]] +name = "libloading" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" +dependencies = [ + "cfg-if 1.0.0", + "winapi", +] + +[[package]] +name = "lock_api" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mmtk" +version = "0.7.0" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a2365b2a4be1dfbdd5b2afb73125c73280e8319c#a2365b2a4be1dfbdd5b2afb73125c73280e8319c" +dependencies = [ + "atomic", + "atomic-traits", + "crossbeam-deque", + "crossbeam-queue", + "custom_derive", + "downcast-rs", + "enum-map", + "enum_derive", + "env_logger", + "lazy_static", + "libc", + "log", + "num_cpus", + "pfm", + "spin", + "thread-priority", +] + +[[package]] +name = "mmtk_openjdk" +version = "0.8.0" +dependencies = [ + "lazy_static", + "libc", + "mmtk", +] + +[[package]] +name = "nom" +version = "5.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" +dependencies = [ + "memchr", + "version_check", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ad167a2f54e832b82dbe003a046280dceffe5227b5f79e08e363a29638cfddd" + +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + +[[package]] +name = "perf-event-open-sys" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce9bedf5da2c234fdf2391ede2b90fabf585355f33100689bc364a3ea558561a" +dependencies = [ + "libc", +] + +[[package]] +name = "pfm" +version = "0.1.0-beta.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f70ac95ca20c32999af1029c044e06414765fd6d65f8787d39dd7052f425b2" +dependencies = [ + "libc", + "perf-event-open-sys", + "pfm-sys", +] + +[[package]] +name = "pfm-sys" +version = "0.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e641198c90a32ac324bab66c5f95c11ee03ca3d9ce6dbc444534478c237e23f3" +dependencies = [ + "bindgen", +] + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "shlex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" + +[[package]] +name = "spin" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5" +dependencies = [ + "lock_api", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "syn" +version = "1.0.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thread-priority" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbb8681124ddad2b07e677d7a99f564c9edbd289cc53cdc0010fd110cb4818ec" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "thread_local" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +dependencies = [ + "once_cell", +] + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "which" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" +dependencies = [ + "libc", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" From 088fb7461e76cbdcc36cb5d621d2657ed14f7d28 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 23 Mar 2022 17:46:58 +1100 Subject: [PATCH 118/494] Fix leak --- mmtk/src/collection.rs | 16 -------- mmtk/src/gc_work.rs | 20 ---------- mmtk/src/lib.rs | 2 - mmtk/src/scanning.rs | 1 - openjdk/mmtk.h | 2 - openjdk/mmtkHeap.cpp | 8 ++-- openjdk/mmtkUpcalls.cpp | 85 ++++------------------------------------- 7 files changed, 12 insertions(+), 122 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index a6f5048f..156bb817 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -6,7 +6,6 @@ use mmtk::{Mutator, MutatorContext}; use crate::OpenJDK; use crate::{SINGLETON, UPCALLS}; -use crate::gc_work::*; pub struct VMCollection {} @@ -75,19 +74,4 @@ impl Collection for VMCollection { ((*UPCALLS).schedule_finalizer)(); } } - - fn process_weak_refs>(_worker: &mut GCWorker) { - println!("process_weak_refs"); - mmtk::memory_manager::add_work_packets( - &SINGLETON, - WorkBucketStage::Prepare, - vec![ - box ProcessWeakRef(0), - box ProcessWeakRef(1), - box ProcessWeakRef(2), - box ProcessWeakRef(3), - box ProcessNMethods, - ], - ); - } } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 19ac57ef..ff75e4f2 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -195,23 +195,3 @@ impl> GCWork for ScanVMThreadRoots } } } - -pub struct ProcessWeakRef(pub i32); - -impl GCWork for ProcessWeakRef { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { - unsafe { - ((*UPCALLS).process_weak_refs)(self.0); - } - } -} - -pub struct ProcessNMethods; - -impl GCWork for ProcessNMethods { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { - unsafe { - ((*UPCALLS).process_nmethods)(); - } - } -} \ No newline at end of file diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 949eba3f..ff607e2e 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -86,8 +86,6 @@ pub struct OpenJDK_Upcalls { pub schedule_finalizer: extern "C" fn(), pub prepare_for_roots_re_scanning: extern "C" fn(), pub object_alignment: extern "C" fn() -> i32, - pub process_weak_refs: extern "C" fn(id: i32), - pub process_nmethods: extern "C" fn(), } #[thread_local] diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index b473c0ba..77a2b42b 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -97,7 +97,6 @@ impl Scanning for VMScanning { box ScanCodeCacheRoots::::new(), box ScanStringTableRoots::::new(), box ScanClassLoaderDataGraphRoots::::new(), - // box ScanWeakProcessorRoots::::new(), ], ); if !(Self::SCAN_MUTATORS_IN_SAFEPOINT && Self::SINGLE_THREAD_MUTATOR_SCANNING) { diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 733ca11e..30bb8f61 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -150,8 +150,6 @@ typedef struct { void (*schedule_finalizer)(); void (*prepare_for_roots_re_scanning)(); int32_t (*object_alignment)(); - void (*process_weak_ref)(int id); - void (*process_nmethods)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls, size_t heap_size); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index a926befb..bb4bec07 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -379,12 +379,12 @@ void MMTkHeap::scan_string_table_roots(OopClosure& cl) { StringTable::oops_do(&cl); } void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl) { - CLDToOopClosure cld_cl(&cl, false); - ClassLoaderDataGraph::cld_do(&cld_cl); + ResourceMark rm; + CLDToOopClosure cld_cl(&cl); + ClassLoaderDataGraph::roots_cld_do(&cld_cl, &cld_cl); } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { - // ShoudNotReach - // WeakProcessor::oops_do(&cl); // (really needed???) + ShouldNotReachHere(); } void MMTkHeap::scan_vm_thread_roots(OopClosure& cl) { ResourceMark rm; diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 9f812d2f..77ba9da3 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -46,7 +46,7 @@ static size_t mmtk_start_the_world_count = 0; -class MMTkSSIsAliveClosure : public BoolObjectClosure { +class MMTkIsAliveClosure : public BoolObjectClosure { static constexpr size_t SS0_START = 0x20000000000ULL; static constexpr size_t SS1_START = 0x40000000000ULL; static constexpr size_t LOS_START = 0x80000000000ULL; @@ -55,7 +55,7 @@ class MMTkSSIsAliveClosure : public BoolObjectClosure { void* to_start = NULL; void* to_limit = NULL; public: - MMTkSSIsAliveClosure() { + MMTkIsAliveClosure() { from_start = (void*) (high ? SS0_START : SS1_START); from_limit = (void*) (((size_t) from_start) + 0x20000000000ULL); to_start = (void*) (high ? SS1_START : SS0_START); @@ -63,22 +63,7 @@ class MMTkSSIsAliveClosure : public BoolObjectClosure { } inline virtual bool do_object_b(oop p) { if (p == NULL) return false; -#if INLINE_IS_ALIVE - auto x = (void*) p; - if (x >= to_start && x < to_limit) { - return true; - } - if (x >= from_start && x < from_limit) { - auto status = *((size_t*) (void*) p); - return (status & (3ull << 56)) != 0; - } - if (x >= (void*) LOS_START && x < ((void*) (LOS_START + 0x20000000000ULL))) { - return mmtk_is_live((void*) p) != 0;; - } - return false; -#else return mmtk_is_live((void*) p) != 0; -#endif } }; @@ -101,8 +86,8 @@ class MMTkForwardClosure : public BasicOopIterateClosure { static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(void* mutator)) { MMTkHeap::_create_stack_scan_work = create_stack_scan_work; - // ClassLoaderDataGraph::clear_claimed_marks(); - // CodeCache::gc_prologue(); + ClassLoaderDataGraph::clear_claimed_marks(); + CodeCache::gc_prologue(); #if COMPILER2_OR_JVMCI DerivedPointerTable::clear(); #endif @@ -112,7 +97,6 @@ static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(voi log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); mmtk_report_gc_start(); - nmethod::oops_do_marking_prologue(); { JavaThreadIteratorWithHandle jtiwh; while (JavaThread *cur = jtiwh.next()) { @@ -120,23 +104,21 @@ static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(voi } } log_debug(gc)("Finished enumerating threads."); + nmethod::oops_do_marking_prologue(); } static void mmtk_resume_mutators(void *tls) { { HandleMark hm; - MMTkSSIsAliveClosure is_alive; + MMTkIsAliveClosure is_alive; MMTkForwardClosure forward; WeakProcessor::weak_oops_do(&is_alive, &forward); } - // ClassLoaderDataGraph::purge(); - // CodeCache::gc_epilogue(); - // JvmtiExport::gc_epilogue(); nmethod::oops_do_marking_epilogue(); // ClassLoaderDataGraph::purge(); // BiasedLocking::restore_marks(); - // CodeCache::gc_epilogue(); - // JvmtiExport::gc_epilogue(); + CodeCache::gc_epilogue(); + JvmtiExport::gc_epilogue(); #if COMPILER2_OR_JVMCI DerivedPointerTable::update_pointers(); #endif @@ -362,55 +344,6 @@ static int32_t mmtk_object_alignment() { return 0; } -// class MMTkIsAliveClosure : public BoolObjectClosure { -// public: -// virtual bool do_object_b(oop p) { -// if (p == NULL) return false; -// return mmtk_is_live((void*) p) != 0; -// } -// }; - -// class MMTkForwardClosure : public BasicOopIterateClosure { -// public: -// virtual void do_oop(oop* slot) { -// // *slot = (oop) mmtk_get_forwarded_ref((void*) *slot); -// auto o = *slot; -// if (o == NULL) return; -// auto status = *((size_t*) (void*) o); -// if ((status & (3ull << 56)) != 0) { -// auto ptr = (oop) (void*) (status << 8 >> 8); -// *slot = ptr; -// } -// } -// virtual void do_oop(narrowOop* o) {} -// virtual ReferenceIterationMode reference_iteration_mode() { return DO_FIELDS; } -// }; - -/// Clean up the weak-ref storage and update pointers. -static void mmtk_process_weak_ref(int id) { -// HandleMark hm; - -// MMTkIsAliveClosure is_alive; -// MMTkForwardClosure forward; - -// JNIHandles::weak_oops_do(&is_alive, &forward); -// JvmtiExport::weak_oops_do(&is_alive, &forward); -// SystemDictionary::vm_weak_oop_storage()->weak_oops_do(&is_alive, &forward); -// JFR_ONLY(Jfr::weak_oops_do(&is_alive, &forward);); -// // if (id == 0) JNIHandles::weak_oops_do(&is_alive, &forward); -// // else if (id == 1) JvmtiExport::weak_oops_do(&is_alive, &forward); -// // else if (id == 2) SystemDictionary::vm_weak_oop_storage()->weak_oops_do(&is_alive, &forward); -// // else { -// // JFR_ONLY(Jfr::weak_oops_do(&is_alive, &forward);); -// // } -// printf("mmtk_process_weak_ref end\n"); -} - -static void mmtk_process_nmethods() { - // HandleMark hm; - // nmethod::oops_do_marking_epilogue(); -} - OpenJDK_Upcalls mmtk_upcalls = { mmtk_stop_all_mutators, mmtk_resume_mutators, @@ -452,6 +385,4 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_schedule_finalizer, mmtk_prepare_for_roots_re_scanning, mmtk_object_alignment, - mmtk_process_weak_ref, - mmtk_process_nmethods, }; From d8970e78eba3c61b046a6734c0df63b1a7d5ddff Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 23 Mar 2022 18:00:51 +1100 Subject: [PATCH 119/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index aba2f8c6..d2a4660d 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -337,8 +337,8 @@ dependencies = [ [[package]] name = "mmtk" -version = "0.7.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a2365b2a4be1dfbdd5b2afb73125c73280e8319c#a2365b2a4be1dfbdd5b2afb73125c73280e8319c" +version = "0.8.0" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=21251948b856836b82ba295d726bf98cc480ae2d#21251948b856836b82ba295d726bf98cc480ae2d" dependencies = [ "atomic", "atomic-traits", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 0d703d8e..2728c047 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -28,7 +28,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "a2365b2a4be1dfbdd5b2afb73125c73280e8319c" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "21251948b856836b82ba295d726bf98cc480ae2d" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From 2ef06861325502f65f8a57e6fce98d9200f472c8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 23 Mar 2022 18:08:20 +1100 Subject: [PATCH 120/494] cleanup --- mmtk/Cargo.lock | 2 +- mmtk/Cargo.toml | 2 +- mmtk/src/gc_work.rs | 16 ---------------- 3 files changed, 2 insertions(+), 18 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index d2a4660d..0a71a161 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -338,7 +338,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.8.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=21251948b856836b82ba295d726bf98cc480ae2d#21251948b856836b82ba295d726bf98cc480ae2d" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=67212aaaa2a46fa35d1c7ab808f6fc32ca9b380b#67212aaaa2a46fa35d1c7ab808f6fc32ca9b380b" dependencies = [ "atomic", "atomic-traits", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 2728c047..b41260b8 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -28,7 +28,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "21251948b856836b82ba295d726bf98cc480ae2d" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "67212aaaa2a46fa35d1c7ab808f6fc32ca9b380b" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index ff75e4f2..c6d56a0f 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -164,22 +164,6 @@ impl> GCWork for ScanClassLoaderDataG } } -pub struct ScanWeakProcessorRoots>(PhantomData); - -impl> ScanWeakProcessorRoots { - pub fn new() -> Self { - Self(PhantomData) - } -} - -impl> GCWork for ScanWeakProcessorRoots { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { - unsafe { - ((*UPCALLS).scan_weak_processor_roots)(create_process_edges_work:: as _); - } - } -} - pub struct ScanVMThreadRoots>(PhantomData); impl> ScanVMThreadRoots { From 49b1541bb1db65e2b05405c26f7465c1f7d14fad Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 23 Mar 2022 21:11:32 +1100 Subject: [PATCH 121/494] Add lxr_rc_only variant --- mmtk/Cargo.lock | 2 +- mmtk/Cargo.toml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 0a71a161..7d11cb01 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -338,7 +338,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.8.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=67212aaaa2a46fa35d1c7ab808f6fc32ca9b380b#67212aaaa2a46fa35d1c7ab808f6fc32ca9b380b" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3ade3a10eaf07dabeaab74a5c452ec544e1d0419#3ade3a10eaf07dabeaab74a5c452ec544e1d0419" dependencies = [ "atomic", "atomic-traits", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index b41260b8..4d6fb6be 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -28,7 +28,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "67212aaaa2a46fa35d1c7ab808f6fc32ca9b380b" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "3ade3a10eaf07dabeaab74a5c452ec544e1d0419" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } @@ -85,5 +85,6 @@ lxr_nevac = ["mmtk/lxr_nevac"] lxr_evac = ["mmtk/lxr_evac"] # LXR (Block only) lxr_block = ["mmtk/lxr_block"] +lxr_rc_only = ["mmtk/lxr_rc_only", "lxr"] # LXR (Default config with full features) lxr = ["mmtk/lxr"] \ No newline at end of file From 59dfc1e2369f68a6bb374449afc08ce906569a97 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 27 Mar 2022 00:08:17 +1100 Subject: [PATCH 122/494] Fix pmd --- mmtk/Cargo.lock | 2 +- mmtk/Cargo.toml | 2 +- mmtk/src/api.rs | 2 +- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 33 +++++--------------- openjdk/barriers/mmtkFieldLoggingBarrier.hpp | 14 ++++----- 5 files changed, 18 insertions(+), 35 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 7d11cb01..477cf578 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -338,7 +338,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.8.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3ade3a10eaf07dabeaab74a5c452ec544e1d0419#3ade3a10eaf07dabeaab74a5c452ec544e1d0419" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7977dd4a0ca9a06417ea63d9b4f5e91a2f5a1012#7977dd4a0ca9a06417ea63d9b4f5e91a2f5a1012" dependencies = [ "atomic", "atomic-traits", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 4d6fb6be..3e655474 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -28,7 +28,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "3ade3a10eaf07dabeaab74a5c452ec544e1d0419" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7977dd4a0ca9a06417ea63d9b4f5e91a2f5a1012" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 923fb55d..8d20bb6a 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -291,7 +291,7 @@ pub extern "C" fn get_finalized_object() -> ObjectReference { /// Note: only call this method after the liveness tracing and before gc release. #[no_mangle] pub extern "C" fn mmtk_is_live(object: ObjectReference) -> usize { - if object.is_null() { + if object.is_null() || !object.to_address().is_mapped() { return 0; } object.is_live() as _ diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index 86ed180b..bc12a30c 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -31,7 +31,7 @@ void MMTkFieldLoggingBarrierSetRuntime::record_clone(oop src, oop dst, size_t si void MMTkFieldLoggingBarrierSetRuntime::record_arraycopy(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw, size_t length) { // ::mmtk_object_reference_arraycopy((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src_obj, src_offset_in_bytes, (void*) dst_obj, dst_offset_in_bytes, length); - record_array_copy_inline((void*) (((intptr_t) (void*) src_obj) + src_offset_in_bytes), (void*) (((intptr_t) (void*) dst_obj) + dst_offset_in_bytes), length, (void*) dst_obj); + record_array_copy_inline((void*) (((intptr_t) (void*) src_obj) + src_offset_in_bytes), (void*) (((intptr_t) (void*) dst_obj) + dst_offset_in_bytes), length); } #define __ masm-> @@ -52,7 +52,8 @@ void MMTkFieldLoggingBarrierSetAssembler::oop_store_at(MacroAssembler* masm, Dec } void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { - if (type == T_OBJECT || type == T_ARRAY) { + const bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; + if ((type == T_OBJECT || type == T_ARRAY) && !dest_uninitialized) { Label slow, done; // Bailout if count is zero __ cmpptr(count, 0); @@ -67,29 +68,11 @@ void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue(MacroAssembler* mas // Slow path __ bind(slow); __ pusha(); - assert_different_registers(c_rarg0, dst); - assert_different_registers(c_rarg0, count); - // guarantee(c_rarg0 != dst, "X"); - guarantee(rscratch1 != src, "X"); - guarantee(rscratch1 != dst, "X"); - guarantee(rscratch1 != count, "X"); - - guarantee(c_rarg0 != count, "X"); - - if (c_rarg0 == dst) { - __ movptr(rscratch1, dst); - } - __ movptr(c_rarg0, src); - if (c_rarg0 == dst) { - __ movptr(dst, rscratch1); - } - - assert_different_registers(c_rarg1, count); - guarantee(c_rarg1 != count, "X"); - - __ movptr(c_rarg1, dst); - __ movptr(c_rarg2, count); - __ call_VM_leaf_base(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_array_copy_slow), 3); + assert_different_registers(src, dst, count); + assert(src == rdi, "expected"); + assert(dst == rsi, "expected"); + assert(count == rdx, "expected"); + __ call_VM_leaf(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_array_copy_slow), src, dst, count); __ popa(); __ bind(done); } diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp index 7babc8e0..d5c85a6f 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.hpp @@ -21,21 +21,21 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { static void record_modified_node_slow(void* src, void* slot, void* val); static void record_clone_slow(void* src, void* dst, size_t size); static void record_array_copy_slow(void* src, void* dst, const size_t size) { - record_array_copy_inline(src, dst, size, NULL); + record_array_copy_inline(src, dst, size); } - inline static void record_array_copy_mmtk_slow(const void* src, const void* dst, const size_t size, void* dst_obj) { + inline static void record_array_copy_mmtk_slow(const void* src, const void* dst, const size_t size) { ::mmtk_object_reference_arraycopy( (MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) ((intptr_t) src), 0, - (void*) ((intptr_t) dst_obj), - ((intptr_t) dst) - ((intptr_t) dst_obj), + (void*) ((intptr_t) dst), + 0, size ); } - inline static void record_array_copy_inline(const void* src_ptr, void* dst_ptr, const size_t size, void* dst_obj) { + inline static void record_array_copy_inline(const void* src_ptr, void* dst_ptr, const size_t size) { if (size == 0) return; if (size == 1) { intptr_t addr = (intptr_t) dst_ptr; @@ -43,12 +43,12 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { intptr_t shift = (addr >> 3) & 0b111; uint8_t byte_val = *meta_addr; if (((byte_val >> shift) & 1) != 0) { - record_modified_node_slow(dst_obj, dst_ptr, (void*) NULL); + record_modified_node_slow(dst_ptr, dst_ptr, (void*) NULL); } return; } - record_array_copy_mmtk_slow(src_ptr, dst_ptr, size, dst_obj); + record_array_copy_mmtk_slow(src_ptr, dst_ptr, size); // constexpr intptr_t kLogBitsInUInt = 6; // constexpr intptr_t kBitsInUInt = 1 << kLogBitsInUInt; From 6d7c532d12801c492bd34ed23fa89c5cb17bc6cc Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 28 Mar 2022 22:30:46 +1100 Subject: [PATCH 123/494] minor --- mmtk/Cargo.lock | 2 +- mmtk/Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 477cf578..d2f297df 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -338,7 +338,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.8.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7977dd4a0ca9a06417ea63d9b4f5e91a2f5a1012#7977dd4a0ca9a06417ea63d9b4f5e91a2f5a1012" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ca834c0d2cb92aa57c266c9032904531d6db58d6#ca834c0d2cb92aa57c266c9032904531d6db58d6" dependencies = [ "atomic", "atomic-traits", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 3e655474..b75134e2 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -28,7 +28,7 @@ lazy_static = "1.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7977dd4a0ca9a06417ea63d9b4f5e91a2f5a1012" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "ca834c0d2cb92aa57c266c9032904531d6db58d6" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } @@ -85,6 +85,6 @@ lxr_nevac = ["mmtk/lxr_nevac"] lxr_evac = ["mmtk/lxr_evac"] # LXR (Block only) lxr_block = ["mmtk/lxr_block"] -lxr_rc_only = ["mmtk/lxr_rc_only", "lxr"] +lxr_rc_only = ["mmtk/lxr_rc_only"] # LXR (Default config with full features) lxr = ["mmtk/lxr"] \ No newline at end of file From 1a658065cce32600b4891e87d081e9e64456154a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 29 Mar 2022 20:34:45 +1100 Subject: [PATCH 124/494] fix get_forwarded_object --- openjdk/mmtkUpcalls.cpp | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 77ba9da3..3feebc60 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -47,20 +47,7 @@ static size_t mmtk_start_the_world_count = 0; class MMTkIsAliveClosure : public BoolObjectClosure { - static constexpr size_t SS0_START = 0x20000000000ULL; - static constexpr size_t SS1_START = 0x40000000000ULL; - static constexpr size_t LOS_START = 0x80000000000ULL; - void* from_start = NULL; - void* from_limit = NULL; - void* to_start = NULL; - void* to_limit = NULL; public: - MMTkIsAliveClosure() { - from_start = (void*) (high ? SS0_START : SS1_START); - from_limit = (void*) (((size_t) from_start) + 0x20000000000ULL); - to_start = (void*) (high ? SS1_START : SS0_START); - to_limit = (void*) (((size_t) to_start) + 0x20000000000ULL); - } inline virtual bool do_object_b(oop p) { if (p == NULL) return false; return mmtk_is_live((void*) p) != 0; @@ -69,14 +56,27 @@ class MMTkIsAliveClosure : public BoolObjectClosure { class MMTkForwardClosure : public BasicOopIterateClosure { public: + inline size_t read_forwarding_word(oop o) const { + return *((size_t*) (void*) o); + } + inline oop extract_forwarding_pointer(size_t status) const { + return (oop) (void*) (status << 8 >> 8); + } + inline bool is_forwarded(size_t status) const { + return (status & (3ull << 56)) != 0; + } virtual void do_oop(oop* slot) { // *slot = (oop) mmtk_get_forwarded_ref((void*) *slot); auto o = *slot; if (o == NULL) return; - auto status = *((size_t*) (void*) o); - if ((status & (3ull << 56)) != 0) { - auto ptr = (oop) (void*) (status << 8 >> 8); - *slot = ptr; + auto status = read_forwarding_word(o); + if (is_forwarded(status)) { + o = extract_forwarding_pointer(status); + auto status = read_forwarding_word(o); + if (is_forwarded(status)) { + o = extract_forwarding_pointer(status); + } + *slot = o; } } virtual void do_oop(narrowOop* o) {} From 7e1b638993799df6273841b3a6431d9c110a6db9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 31 Mar 2022 19:01:54 +1100 Subject: [PATCH 125/494] fix --- openjdk/mmtkHeap.cpp | 5 ++--- openjdk/mmtkUpcalls.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index bb4bec07..0544f6b1 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -379,9 +379,8 @@ void MMTkHeap::scan_string_table_roots(OopClosure& cl) { StringTable::oops_do(&cl); } void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl) { - ResourceMark rm; - CLDToOopClosure cld_cl(&cl); - ClassLoaderDataGraph::roots_cld_do(&cld_cl, &cld_cl); + CLDToOopClosure cld_cl(&cl, false); + ClassLoaderDataGraph::cld_do(&cld_cl); } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { ShouldNotReachHere(); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 3feebc60..e43ded75 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -86,8 +86,8 @@ class MMTkForwardClosure : public BasicOopIterateClosure { static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(void* mutator)) { MMTkHeap::_create_stack_scan_work = create_stack_scan_work; - ClassLoaderDataGraph::clear_claimed_marks(); - CodeCache::gc_prologue(); + // ClassLoaderDataGraph::clear_claimed_marks(); + // CodeCache::gc_prologue(); #if COMPILER2_OR_JVMCI DerivedPointerTable::clear(); #endif @@ -117,8 +117,8 @@ static void mmtk_resume_mutators(void *tls) { nmethod::oops_do_marking_epilogue(); // ClassLoaderDataGraph::purge(); // BiasedLocking::restore_marks(); - CodeCache::gc_epilogue(); - JvmtiExport::gc_epilogue(); + // CodeCache::gc_epilogue(); + // JvmtiExport::gc_epilogue(); #if COMPILER2_OR_JVMCI DerivedPointerTable::update_pointers(); #endif From fa88e926e4a952b7229b39f40ae0f26127f837ec Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 2 Apr 2022 23:03:51 +1100 Subject: [PATCH 126/494] minor --- mmtk/src/api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 8d20bb6a..1711581a 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -291,7 +291,7 @@ pub extern "C" fn get_finalized_object() -> ObjectReference { /// Note: only call this method after the liveness tracing and before gc release. #[no_mangle] pub extern "C" fn mmtk_is_live(object: ObjectReference) -> usize { - if object.is_null() || !object.to_address().is_mapped() { + if object.is_null() || !object.to_address().is_mapped() || !object.class_is_valid() { return 0; } object.is_live() as _ From 408b1b1b8fdb264b4451e1b1d7eb722327af28c0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 4 Apr 2022 18:44:11 +1000 Subject: [PATCH 127/494] fix barrier elimination --- openjdk/barriers/mmtkFieldLoggingBarrier.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp index bc12a30c..ed051112 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldLoggingBarrier.cpp @@ -277,10 +277,10 @@ bool MMTkFieldLoggingBarrierSetC2::can_remove_barrier(GraphKit* kit, PhaseTransf void MMTkFieldLoggingBarrierSetC2::record_modified_node(GraphKit* kit, Node* src, Node* slot, Node* val) const { - if (src == kit->just_allocated_object(kit->control())) { + if (!disable_fast_alloc() && src == kit->just_allocated_object(kit->control())) { return; } - if (can_remove_barrier(kit, &kit->gvn(), slot)) { + if (!disable_fast_alloc() && can_remove_barrier(kit, &kit->gvn(), slot)) { return; } From 971de128667ceb7814f7cb0687336feb43869815 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 6 Apr 2022 10:42:28 +1000 Subject: [PATCH 128/494] add array related functions --- mmtk/src/object_scanning.rs | 27 +++++++++++++++++++++++---- mmtk/src/scanning.rs | 15 +++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 3b1bc073..1c0e2bc7 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -208,17 +208,36 @@ fn oop_iterate(oop: Oop, closure: &mut impl TransitiveClosure) { let array_klass = unsafe { oop.klass.cast::() }; array_klass.oop_iterate(oop, closure); } - KlassID::TypeArray => { - let array_klass = unsafe { oop.klass.cast::() }; - array_klass.oop_iterate(oop, closure); - } + // KlassID::TypeArray => { + // // let array_klass = unsafe { oop.klass.cast::() }; + // // array_klass.oop_iterate(oop, closure); + // } KlassID::InstanceRef => { let instance_klass = unsafe { oop.klass.cast::() }; instance_klass.oop_iterate(oop, closure); } // _ => oop_iterate_slow(oop, closure, tls), + _ => {} } } +#[inline(always)] +pub fn is_obj_array(oop: Oop ) -> bool { + let klass_id = oop.klass.id; + klass_id == KlassID::ObjArray +} + +#[inline(always)] +pub fn obj_array_data(oop: Oop) -> &'static [ObjectReference] { + let array = unsafe { oop.as_array_oop::() }; + array.data() +} + +#[inline(always)] +pub fn is_type_array(oop: Oop ) -> bool { + let klass_id = oop.klass.id; + klass_id == KlassID::TypeArray +} + #[inline(always)] pub fn is_oop_field(oop: Oop, edge: Address) -> bool { let klass_id = oop.klass.id; diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 77a2b42b..d05bddc1 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -47,6 +47,21 @@ impl Scanning for VMScanning { crate::object_scanning::scan_object(object, trace, tls) } + #[inline(always)] + fn obj_array_data(o: ObjectReference) -> &'static [ObjectReference] { + crate::object_scanning::obj_array_data(unsafe { std::mem::transmute(o) }) + } + + #[inline(always)] + fn is_obj_array(o: ObjectReference) -> bool { + crate::object_scanning::is_obj_array(unsafe { std::mem::transmute(o) }) + } + + #[inline(always)] + fn is_type_array(o: ObjectReference) -> bool { + crate::object_scanning::is_type_array(unsafe { std::mem::transmute(o) }) + } + #[inline(always)] fn is_oop_field(o: ObjectReference, e: Address) -> bool { crate::object_scanning::is_oop_field(unsafe { std::mem::transmute(o) }, e) From 6cf9ae9efb965a2816fa2f78e9ae7b39dd72d106 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 7 Apr 2022 22:02:39 +1000 Subject: [PATCH 129/494] performance tweaks on code cache roots --- mmtk/Cargo.lock | 1 + mmtk/Cargo.toml | 1 + mmtk/src/abi.rs | 14 ++++++++------ mmtk/src/api.rs | 23 +++++++++++++++++++++++ mmtk/src/collection.rs | 2 +- mmtk/src/gc_work.rs | 11 ++++++++--- mmtk/src/lib.rs | 7 +++++++ mmtk/src/scanning.rs | 21 +++++++++++++++++++-- openjdk/mmtk.h | 5 +++++ openjdk/mmtkHeap.cpp | 18 +++++++++++++++++- openjdk/mmtkHeap.hpp | 3 ++- 11 files changed, 92 insertions(+), 14 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index d2f297df..1b1ed983 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -365,6 +365,7 @@ dependencies = [ "lazy_static", "libc", "mmtk", + "spin", ] [[package]] diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index b75134e2..ae5fce6d 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -22,6 +22,7 @@ codegen-units = 1 [dependencies] libc = "0.2" lazy_static = "1.1" +spin = "0.9.2" # Be very careful to commit any changes to the following mmtk dependency, as our CI scripts (including mmtk-core CI) # rely on matching these lines to modify them: e.g. comment out the git dependency and use the local path. # These changes are safe: diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 0b5dfb07..bdd1f64b 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -236,19 +236,21 @@ pub struct InstanceRefKlass { pub instance_klass: InstanceKlass, } +static DISCOVERED_OFFSET: spin::Lazy = spin::Lazy::new(|| { + unsafe { ((*UPCALLS).discovered_offset)() } +}); + +static REFERENT_OFFSET: spin::Lazy = spin::Lazy::new(|| { + unsafe { ((*UPCALLS).referent_offset)() } +}); + impl InstanceRefKlass { #[inline(always)] fn referent_offset() -> i32 { - lazy_static! { - pub static ref REFERENT_OFFSET: i32 = unsafe { ((*UPCALLS).referent_offset)() }; - } *REFERENT_OFFSET } #[inline(always)] fn discovered_offset() -> i32 { - lazy_static! { - pub static ref DISCOVERED_OFFSET: i32 = unsafe { ((*UPCALLS).discovered_offset)() }; - } *DISCOVERED_OFFSET } #[inline(always)] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 1711581a..c878f075 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -15,6 +15,7 @@ use mmtk::MutatorContext; use mmtk::MMTK; use std::ffi::{CStr, CString}; use std::lazy::SyncLazy; +use std::sync::atomic::Ordering; // Supported barriers: static NO_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("NoBarrier").unwrap()); @@ -304,4 +305,26 @@ pub extern "C" fn mmtk_get_forwarded_ref(object: ObjectReference) -> ObjectRefer return object; } object.get_forwarded_object().unwrap_or(object) +} + +#[thread_local] +static mut NMETHOD_SLOTS: Vec
= vec![]; + +#[no_mangle] +pub unsafe extern "C" fn mmtk_add_nmethod_oop(addr: Address) { + NMETHOD_SLOTS.push(addr) +} +#[no_mangle] +pub unsafe extern "C" fn mmtk_register_nmethod(nm: Address) { + if NMETHOD_SLOTS.len() == 0 { return; } + let mut slots = vec![]; + std::mem::swap(&mut slots, &mut NMETHOD_SLOTS); + crate::TOTAL_SIZE.fetch_add(slots.len(), Ordering::Relaxed); + crate::CODE_CACHE_ROOTS.lock().insert(nm, slots); +} +#[no_mangle] +pub unsafe extern "C" fn mmtk_unregister_nmethod(nm: Address) { + if let Some(slots) = crate::CODE_CACHE_ROOTS.lock().remove(&nm) { + crate::TOTAL_SIZE.fetch_sub(slots.len(), Ordering::Relaxed); + } } \ No newline at end of file diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 156bb817..ebfaa8b7 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -15,7 +15,7 @@ extern "C" fn create_mutator_scan_work>( mmtk::memory_manager::add_work_packet( &SINGLETON, // FIXME: Should be Unconstrained! - WorkBucketStage::Initial, + WorkBucketStage::RCProcessIncs, ScanStackRoot::(mutator), ); } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index c6d56a0f..621d9a7d 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -1,8 +1,9 @@ use super::{OpenJDK, UPCALLS}; -use crate::scanning::create_process_edges_work; +use crate::scanning::{create_process_edges_work, create_process_edges_work_vec}; use mmtk::scheduler::*; use mmtk::MMTK; use std::marker::PhantomData; +use std::sync::atomic::Ordering; pub struct ScanUniverseRoots>(PhantomData); @@ -126,9 +127,13 @@ impl> ScanCodeCacheRoots { impl> GCWork for ScanCodeCacheRoots { fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { - unsafe { - ((*UPCALLS).scan_code_cache_roots)(create_process_edges_work:: as _); + let mut vec = Vec::with_capacity(crate::TOTAL_SIZE.load(Ordering::Relaxed)); + for (_, roots) in &*crate::CODE_CACHE_ROOTS.lock() { + for r in roots { + vec.push(*r) + } } + create_process_edges_work_vec::(vec) } } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index ff607e2e..d5175f3d 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -12,8 +12,11 @@ extern crate libc; extern crate mmtk; #[macro_use] extern crate lazy_static; +extern crate spin; +use std::collections::HashMap; use std::ptr::null_mut; +use std::sync::atomic::AtomicUsize; use libc::{c_char, c_void, uintptr_t}; use mmtk::scheduler::GCWorker; @@ -22,6 +25,7 @@ use mmtk::util::{Address, ObjectReference}; use mmtk::vm::VMBinding; use mmtk::Mutator; use mmtk::MMTK; +use spin::Mutex; mod abi; pub mod active_plan; pub mod api; @@ -150,3 +154,6 @@ lazy_static! { #[no_mangle] pub static MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES: usize = mmtk::util::alloc::MarkCompactAllocator::::HEADER_RESERVED_IN_BYTES; + +static CODE_CACHE_ROOTS: spin::Lazy>>> = spin::Lazy::new(|| Mutex::new(HashMap::new())); +static TOTAL_SIZE: AtomicUsize = AtomicUsize::new(0); \ No newline at end of file diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index d05bddc1..d2530108 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -12,6 +12,23 @@ use mmtk::{Mutator, TransitiveClosure}; pub struct VMScanning {} +pub(crate) fn create_process_edges_work_vec>( + buf: Vec
, +) { + if !buf.is_empty() { + let w = W::new(buf, true, &SINGLETON); + if W::RC_ROOTS { + crate::current_worker().add_work(WorkBucketStage::RCProcessIncs, w); + } else { + memory_manager::add_work_packet( + &SINGLETON, + WorkBucketStage::Closure, + w, + ); + } + } +} + pub(crate) extern "C" fn create_process_edges_work>( ptr: *mut Address, length: usize, @@ -21,7 +38,7 @@ pub(crate) extern "C" fn create_process_edges_work::from_raw_parts(ptr, length, capacity) }; let w = W::new(buf, true, &SINGLETON); if W::RC_ROOTS { - crate::current_worker().add_work(WorkBucketStage::Unconstrained, w); + crate::current_worker().add_work(WorkBucketStage::RCProcessIncs, w); } else { memory_manager::add_work_packet( &SINGLETON, @@ -100,7 +117,7 @@ impl Scanning for VMScanning { fn scan_vm_specific_roots>() { memory_manager::add_work_packets( &SINGLETON, - WorkBucketStage::Unconstrained, + WorkBucketStage::RCProcessIncs, vec![ box ScanUniverseRoots::::new(), box ScanJNIHandlesRoots::::new(), diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 30bb8f61..df65ca55 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -93,6 +93,11 @@ extern void start_worker(void *tls, void* worker); extern size_t mmtk_is_live(void* object); extern void* mmtk_get_forwarded_ref(void* object); + +extern size_t mmtk_add_nmethod_oop(void* object); +extern size_t mmtk_register_nmethod(void* nm); +extern size_t mmtk_unregister_nmethod(void* nm); + /** * VM Accounting */ diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 0544f6b1..a78faf33 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -339,8 +339,24 @@ void MMTkHeap::print_tracing_info() const { // Registering and unregistering an nmethod (compiled code) with the heap. // Override with specific mechanism for each specialized heap type. +class MMTkRegisterNMethodOopClosure: public OopClosure { + template void do_oop_work(T* p) { + mmtk_add_nmethod_oop((void*) p); + } +public: + void do_oop(oop* p) { do_oop_work(p); } + void do_oop(narrowOop* p) { do_oop_work(p); } +}; + + void MMTkHeap::register_nmethod(nmethod* nm) { - CodeCache::register_scavenge_root_nmethod(nm); + MMTkRegisterNMethodOopClosure reg_cl; + nm->oops_do(®_cl); + mmtk_register_nmethod((void*) nm); +} + +void MMTkHeap::unregister_nmethod(nmethod* nm) { + mmtk_unregister_nmethod((void*) nm); } // Heap verification diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index c44856f5..8f6a2b3e 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -186,7 +186,8 @@ class MMTkHeap : public CollectedHeap { inline bool is_scavengable(oop obj) { return true; } // Registering and unregistering an nmethod (compiled code) with the heap. // Override with specific mechanism for each specialized heap type. - void register_nmethod(nmethod* nm); + virtual void register_nmethod(nmethod* nm); + virtual void unregister_nmethod(nmethod* nm); // Heap verification void verify(VerifyOption option); From c4506217d7f1f5ebef5640f0ac35db88fe25afa9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 8 Apr 2022 13:11:45 +1000 Subject: [PATCH 130/494] fix forwarding checks --- openjdk/mmtkUpcalls.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index e43ded75..55b44ce5 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -63,7 +63,7 @@ class MMTkForwardClosure : public BasicOopIterateClosure { return (oop) (void*) (status << 8 >> 8); } inline bool is_forwarded(size_t status) const { - return (status & (3ull << 56)) != 0; + return (status & (0xffull << 56)) != 0; } virtual void do_oop(oop* slot) { // *slot = (oop) mmtk_get_forwarded_ref((void*) *slot); From abbdd1dbbe7277e4b0dfd1d37ba613c53f5ca50d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 12 Apr 2022 17:35:10 +1000 Subject: [PATCH 131/494] update mmtk-core --- mmtk/Cargo.lock | 2 +- mmtk/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 1b1ed983..0bd1091b 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -338,7 +338,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.8.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ca834c0d2cb92aa57c266c9032904531d6db58d6#ca834c0d2cb92aa57c266c9032904531d6db58d6" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=df8d30a39237a5bf5a8e27ca5a6f46acc6080c94#df8d30a39237a5bf5a8e27ca5a6f46acc6080c94" dependencies = [ "atomic", "atomic-traits", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index ae5fce6d..f5d038e9 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -29,7 +29,7 @@ spin = "0.9.2" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "ca834c0d2cb92aa57c266c9032904531d6db58d6" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "df8d30a39237a5bf5a8e27ca5a6f46acc6080c94" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From fe4efef9a7b1e204e4647d3d15aeb7fac05911cf Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 27 Apr 2022 11:51:19 +1000 Subject: [PATCH 132/494] Update Cargo.toml --- mmtk/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index f5d038e9..04c96150 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -29,7 +29,7 @@ spin = "0.9.2" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "df8d30a39237a5bf5a8e27ca5a6f46acc6080c94" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "47232c1a0a789edd578d3239a813cdd42b02dd3c" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } @@ -88,4 +88,4 @@ lxr_evac = ["mmtk/lxr_evac"] lxr_block = ["mmtk/lxr_block"] lxr_rc_only = ["mmtk/lxr_rc_only"] # LXR (Default config with full features) -lxr = ["mmtk/lxr"] \ No newline at end of file +lxr = ["mmtk/lxr"] From 36c49d2449c6cff1de9cc0aa9da38d87f2deed89 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 28 Apr 2022 02:43:12 +1000 Subject: [PATCH 133/494] fix weak processor --- mmtk/src/collection.rs | 6 ++++++ mmtk/src/lib.rs | 1 + openjdk/mmtk.h | 1 + openjdk/mmtkUpcalls.cpp | 8 ++++++++ 4 files changed, 16 insertions(+) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index ebfaa8b7..7f4558bd 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -74,4 +74,10 @@ impl Collection for VMCollection { ((*UPCALLS).schedule_finalizer)(); } } + + fn update_weak_processor() { + unsafe { + ((*UPCALLS).update_weak_processor)(); + } + } } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index d5175f3d..fbbacf65 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -90,6 +90,7 @@ pub struct OpenJDK_Upcalls { pub schedule_finalizer: extern "C" fn(), pub prepare_for_roots_re_scanning: extern "C" fn(), pub object_alignment: extern "C" fn() -> i32, + pub update_weak_processor: extern "C" fn(), } #[thread_local] diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index df65ca55..b92139a4 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -155,6 +155,7 @@ typedef struct { void (*schedule_finalizer)(); void (*prepare_for_roots_re_scanning)(); int32_t (*object_alignment)(); + void (*mmtk_update_weak_processor)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls, size_t heap_size); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 55b44ce5..5d4bf518 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -107,6 +107,13 @@ static void mmtk_stop_all_mutators(void *tls, void (*create_stack_scan_work)(voi nmethod::oops_do_marking_prologue(); } +static void mmtk_update_weak_processor() { + HandleMark hm; + MMTkIsAliveClosure is_alive; + MMTkForwardClosure forward; + WeakProcessor::weak_oops_do(&is_alive, &forward); +} + static void mmtk_resume_mutators(void *tls) { { HandleMark hm; @@ -385,4 +392,5 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_schedule_finalizer, mmtk_prepare_for_roots_re_scanning, mmtk_object_alignment, + mmtk_update_weak_processor, }; From 729dd2973361d0e322901fb1a6ce9da0bd1dc23c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 4 May 2022 13:57:01 +1000 Subject: [PATCH 134/494] Update Cargo.lock --- mmtk/Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 0bd1091b..c2737154 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -338,7 +338,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.8.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=df8d30a39237a5bf5a8e27ca5a6f46acc6080c94#df8d30a39237a5bf5a8e27ca5a6f46acc6080c94" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=47232c1a0a789edd578d3239a813cdd42b02dd3c#47232c1a0a789edd578d3239a813cdd42b02dd3c" dependencies = [ "atomic", "atomic-traits", From 4236d3699336e8770d99082a5322cc3592614c03 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 6 May 2022 20:17:05 +1000 Subject: [PATCH 135/494] fix --- mmtk/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 262fd7ee..01c74284 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -20,6 +20,7 @@ use std::sync::atomic::AtomicUsize; use libc::{c_char, c_void, uintptr_t}; use mmtk::util::opaque_pointer::*; +use mmtk::scheduler::GCWorker; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::VMBinding; use mmtk::Mutator; From f4c1e3af072e627fb8141f8b09b3ea5acf327715 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 8 May 2022 16:38:33 +1000 Subject: [PATCH 136/494] Update .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7a5ac51f..7aebf836 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,5 @@ target/ /*.dylib /*.so -/repos/mmtk-core \ No newline at end of file +/repos/mmtk-core +/repos/openjdk \ No newline at end of file From 66395ef9f614b1d623348d087c84c2a71cf009d2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 11 May 2022 11:40:37 +1000 Subject: [PATCH 137/494] stable --- mmtk/Cargo.lock | 5 ++- mmtk/Cargo.toml | 1 + mmtk/rust-toolchain | 2 +- mmtk/src/abi.rs | 83 ++++++++++++++++++++++--------------- mmtk/src/api.rs | 29 +++++++------ mmtk/src/lib.rs | 23 ++++------ mmtk/src/object_scanning.rs | 12 +++--- mmtk/src/scanning.rs | 28 ++++++++----- 8 files changed, 103 insertions(+), 80 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 173e883a..9fd6ffd4 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -365,6 +365,7 @@ dependencies = [ "lazy_static", "libc", "mmtk", + "once_cell", "spin", ] @@ -390,9 +391,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.6.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad167a2f54e832b82dbe003a046280dceffe5227b5f79e08e363a29638cfddd" +checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" [[package]] name = "peeking_take_while" diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 2b0de36c..27c333ee 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -29,6 +29,7 @@ openjdk_version = "67d5d2b16aacb2ea948552fab2323ebd0abbe924" libc = "0.2" lazy_static = "1.1" spin = "0.9.2" +once_cell = "1.10.0" # Be very careful to commit any changes to the following mmtk dependency, as our CI scripts (including mmtk-core CI) # rely on matching these lines to modify them: e.g. comment out the git dependency and use the local path. # These changes are safe: diff --git a/mmtk/rust-toolchain b/mmtk/rust-toolchain index 8445feb8..2bf5ad04 100644 --- a/mmtk/rust-toolchain +++ b/mmtk/rust-toolchain @@ -1 +1 @@ -nightly-2022-02-11 +stable diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index c6db8e95..8a03b81a 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -5,26 +5,9 @@ use mmtk::util::ObjectReference; use mmtk::util::{Address, OpaquePointer}; use std::ffi::CStr; use std::fmt; -use std::marker::PhantomData; use std::ops::Range; use std::{mem, slice}; -trait EqualTo { - const VALUE: bool; -} - -impl EqualTo for T { - default const VALUE: bool = false; -} - -impl EqualTo for T { - const VALUE: bool = true; -} - -pub const fn type_equal() -> bool { - >::VALUE -} - #[repr(i32)] #[derive(Copy, Clone, PartialEq, Eq, Debug)] #[allow(dead_code)] @@ -37,6 +20,30 @@ pub enum KlassID { ObjArray, } +#[repr(i32)] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[allow(dead_code)] +#[allow(non_camel_case_types)] +pub enum BasicType { + T_BOOLEAN = 4, + T_CHAR = 5, + T_FLOAT = 6, + T_DOUBLE = 7, + T_BYTE = 8, + T_SHORT = 9, + T_INT = 10, + T_LONG = 11, + T_OBJECT = 12, + T_ARRAY = 13, + T_VOID = 14, + T_ADDRESS = 15, + T_NARROWOOP = 16, + T_METADATA = 17, + T_NARROWKLASS = 18, + T_CONFLICT = 19, // for stack value type with conflicting contents + T_ILLEGAL = 99, +} + #[repr(C)] pub struct Klass { vptr: OpaquePointer, @@ -289,14 +296,14 @@ impl fmt::Debug for OopDesc { pub type Oop = &'static OopDesc; -impl const From for Oop { +impl From for Oop { #[inline(always)] fn from(o: ObjectReference) -> Self { unsafe { mem::transmute(o) } } } -impl const Into for &OopDesc { +impl Into for &OopDesc { #[inline(always)] fn into(self) -> ObjectReference { unsafe { mem::transmute::<&OopDesc, _>(self) } @@ -305,8 +312,8 @@ impl const Into for &OopDesc { impl OopDesc { #[inline(always)] - pub const unsafe fn as_array_oop(&self) -> ArrayOop { - &*(self as *const OopDesc as *const ArrayOopDesc) + pub unsafe fn as_array_oop(&self) -> ArrayOop { + &*(self as *const OopDesc as *const ArrayOopDesc) } #[inline(always)] @@ -335,7 +342,7 @@ impl OopDesc { } else if lh <= Klass::LH_NEUTRAL_VALUE { if lh < Klass::LH_NEUTRAL_VALUE { // Calculate array size - let array_length = self.as_array_oop::<()>().length(); + let array_length = self.as_array_oop().length(); let mut size_in_bytes: usize = (array_length as usize) << Klass::layout_helper_log2_element_size(lh); size_in_bytes += Klass::layout_helper_header_size(lh) as usize; @@ -350,18 +357,22 @@ impl OopDesc { } #[repr(C)] -pub struct ArrayOopDesc(OopDesc, PhantomData); +pub struct ArrayOopDesc(OopDesc); -pub type ArrayOop = &'static ArrayOopDesc; +pub type ArrayOop = &'static ArrayOopDesc; -impl ArrayOopDesc { - const ELEMENT_TYPE_SHOULD_BE_ALIGNED: bool = type_equal::() || type_equal::(); +impl ArrayOopDesc { const LENGTH_OFFSET: usize = mem::size_of::(); + + fn element_type_should_be_aligned(ty: BasicType) -> bool { + ty == BasicType::T_DOUBLE || ty == BasicType::T_LONG + } + #[inline(always)] - fn header_size() -> usize { + fn header_size(ty: BasicType) -> usize { let typesize_in_bytes = conversions::raw_align_up(Self::LENGTH_OFFSET + BYTES_IN_INT, BYTES_IN_LONG); - if Self::ELEMENT_TYPE_SHOULD_BE_ALIGNED { + if Self::element_type_should_be_aligned(ty) { conversions::raw_align_up(typesize_in_bytes / BYTES_IN_WORD, BYTES_IN_LONG) } else { typesize_in_bytes / BYTES_IN_WORD @@ -372,17 +383,21 @@ impl ArrayOopDesc { unsafe { *((self as *const _ as *const u8).add(Self::LENGTH_OFFSET) as *const i32) } } #[inline(always)] - fn base(&self) -> *const T { - let base_offset_in_bytes = Self::header_size() * BYTES_IN_WORD; - unsafe { (self as *const _ as *const u8).add(base_offset_in_bytes) as _ } + fn base(&self, ty: BasicType) -> Address { + let base_offset_in_bytes = Self::header_size(ty) * BYTES_IN_WORD; + Address::from_ptr(unsafe { (self as *const _ as *const u8).add(base_offset_in_bytes) }) } + // This provides an easy way to access the array data in Rust. However, the array data + // is Java types, so we have to map Java types to Rust types. The caller needs to guarantee: + // 1. matches the actual Java type + // 2. matches the argument, BasicType `ty` #[inline(always)] - pub fn data(&self) -> &[T] { - unsafe { slice::from_raw_parts(self.base(), self.length() as _) } + pub unsafe fn data(&self, ty: BasicType) -> &[T] { + slice::from_raw_parts(self.base(ty).to_ptr(), self.length() as _) } #[inline(always)] pub fn data_range(&self) -> Range
{ - let base = Address::from_ptr(self.base()); + let base = self.base(BasicType::T_OBJECT); let len = self.length() as usize; base..(base + (len << BYTES_IN_ADDRESS)) } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 0d7b98c0..c60fe261 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -14,14 +14,15 @@ use mmtk::AllocationSemantics; use mmtk::Mutator; use mmtk::MutatorContext; use mmtk::MMTK; +use once_cell::sync; +use std::cell::RefCell; use std::ffi::{CStr, CString}; -use std::lazy::SyncLazy; use std::sync::atomic::Ordering; // Supported barriers: -static NO_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("NoBarrier").unwrap()); -static OBJECT_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("ObjectBarrier").unwrap()); -static FIELD_LOGGING_BARRIER: SyncLazy = SyncLazy::new(|| CString::new("FieldLoggingBarrier").unwrap()); +static NO_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("NoBarrier").unwrap()); +static OBJECT_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("ObjectBarrier").unwrap()); +static FIELD_LOGGING_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("FieldLoggingBarrier").unwrap()); #[no_mangle] pub extern "C" fn mmtk_active_barrier() -> *const c_char { @@ -135,9 +136,7 @@ pub extern "C" fn start_control_collector( // We trust the worker pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] pub extern "C" fn start_worker(tls: VMWorkerThread, worker: *mut GCWorker) { - unsafe { - crate::CURRENT_WORKER = Some(&mut *worker); - } + crate::CURRENT_WORKER.with(|x| x.replace(Some(unsafe { &mut *worker }))); let mut worker = unsafe { Box::from_raw(worker) }; memory_manager::start_worker::(&SINGLETON, tls, &mut worker) } @@ -323,18 +322,22 @@ pub extern "C" fn mmtk_get_forwarded_ref(object: ObjectReference) -> ObjectRefer object.get_forwarded_object().unwrap_or(object) } -#[thread_local] -static mut NMETHOD_SLOTS: Vec
= vec![]; +thread_local! { + static NMETHOD_SLOTS: RefCell> = RefCell::new(vec![]); +} #[no_mangle] pub unsafe extern "C" fn mmtk_add_nmethod_oop(addr: Address) { - NMETHOD_SLOTS.push(addr) + NMETHOD_SLOTS.with(|x| x.borrow_mut().push(addr)) } #[no_mangle] pub unsafe extern "C" fn mmtk_register_nmethod(nm: Address) { - if NMETHOD_SLOTS.len() == 0 { return; } - let mut slots = vec![]; - std::mem::swap(&mut slots, &mut NMETHOD_SLOTS); + let slots = NMETHOD_SLOTS.with(|x| { + if x.borrow().len() == 0 { return None; } + Some(x.replace(vec![])) + }); + if slots.is_none() { return; } + let slots = slots.unwrap(); crate::TOTAL_SIZE.fetch_add(slots.len(), Ordering::Relaxed); crate::CODE_CACHE_ROOTS.lock().insert(nm, slots); } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index e5315fa9..8d1c1a01 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -1,19 +1,11 @@ -// specialization is considered as an incomplete feature. -#![allow(incomplete_features)] -#![feature(specialization)] -#![feature(box_syntax)] -#![feature(vec_into_raw_parts)] -#![feature(once_cell)] -#![feature(const_raw_ptr_deref)] -#![feature(const_trait_impl)] -#![feature(thread_local)] - extern crate libc; extern crate mmtk; #[macro_use] extern crate lazy_static; extern crate spin; +extern crate once_cell; +use std::cell::RefCell; use std::collections::HashMap; use std::ptr::null_mut; use std::sync::atomic::AtomicUsize; @@ -93,13 +85,16 @@ pub struct OpenJDK_Upcalls { pub prepare_for_roots_re_scanning: extern "C" fn(), pub update_weak_processor: extern "C" fn(), } - -#[thread_local] -pub static mut CURRENT_WORKER: Option<&'static mut GCWorker> = None; +thread_local! { + pub static CURRENT_WORKER: RefCell>> = RefCell::new(None); +} #[inline(always)] pub fn current_worker() -> &'static mut GCWorker { - unsafe { CURRENT_WORKER.as_mut().unwrap() } + CURRENT_WORKER.with(|x| { + let ptr = x.borrow().unwrap() ; + unsafe {&mut *ptr} + }) } pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 1c0e2bc7..a6737b8f 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -124,14 +124,14 @@ impl OopIterate for InstanceClassLoaderKlass { impl OopIterate for ObjArrayKlass { #[inline(always)] fn oop_iterate(&self, oop: Oop, closure: &mut impl TransitiveClosure) { - let array = unsafe { oop.as_array_oop::() }; - for oop in array.data() { + let array = unsafe { oop.as_array_oop() }; + for oop in unsafe { array.data::(BasicType::T_OBJECT) } { closure.process_edge(Address::from_ref(oop as &Oop)); } } #[inline(always)] fn is_oop_field(&self, oop: Oop, edge: Address) -> bool { - let array = unsafe { oop.as_array_oop::() }; + let array = unsafe { oop.as_array_oop() }; let data = array.data_range(); edge >= data.start && edge < data.end } @@ -228,8 +228,10 @@ pub fn is_obj_array(oop: Oop ) -> bool { #[inline(always)] pub fn obj_array_data(oop: Oop) -> &'static [ObjectReference] { - let array = unsafe { oop.as_array_oop::() }; - array.data() + unsafe { + let array = oop.as_array_oop(); + array.data::(BasicType::T_OBJECT) + } } #[inline(always)] diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index d2530108..53dcfd3b 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -47,7 +47,13 @@ pub(crate) extern "C" fn create_process_edges_work for VMScanning { &SINGLETON, WorkBucketStage::RCProcessIncs, vec![ - box ScanUniverseRoots::::new(), - box ScanJNIHandlesRoots::::new(), - box ScanObjectSynchronizerRoots::::new(), - box ScanManagementRoots::::new(), - box ScanJvmtiExportRoots::::new(), - box ScanAOTLoaderRoots::::new(), - box ScanSystemDictionaryRoots::::new(), - box ScanCodeCacheRoots::::new(), - box ScanStringTableRoots::::new(), - box ScanClassLoaderDataGraphRoots::::new(), + Box::new(ScanUniverseRoots::::new()), + Box::new(ScanJNIHandlesRoots::::new()), + Box::new(ScanObjectSynchronizerRoots::::new()), + Box::new(ScanManagementRoots::::new()), + Box::new(ScanJvmtiExportRoots::::new()), + Box::new(ScanAOTLoaderRoots::::new()), + Box::new(ScanSystemDictionaryRoots::::new()), + Box::new(ScanCodeCacheRoots::::new()), + Box::new(ScanStringTableRoots::::new()), + Box::new(ScanClassLoaderDataGraphRoots::::new()), ], ); if !(Self::SCAN_MUTATORS_IN_SAFEPOINT && Self::SINGLE_THREAD_MUTATOR_SCANNING) { From b3af28b7113d7c85f413812da43cb57e71ae3d80 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 12 May 2022 10:18:57 +1000 Subject: [PATCH 138/494] cargo fmt --- mmtk/src/api.rs | 18 ++++++++++++------ mmtk/src/lib.rs | 16 +++++++++------- mmtk/src/object_scanning.rs | 4 ++-- mmtk/src/scanning.rs | 16 +++------------- 4 files changed, 26 insertions(+), 28 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 9c3ec76b..51577dae 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -21,8 +21,10 @@ use std::sync::atomic::Ordering; // Supported barriers: static NO_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("NoBarrier").unwrap()); -static OBJECT_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("ObjectBarrier").unwrap()); -static FIELD_LOGGING_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("FieldLoggingBarrier").unwrap()); +static OBJECT_BARRIER: sync::Lazy = + sync::Lazy::new(|| CString::new("ObjectBarrier").unwrap()); +static FIELD_LOGGING_BARRIER: sync::Lazy = + sync::Lazy::new(|| CString::new("FieldLoggingBarrier").unwrap()); #[no_mangle] pub extern "C" fn mmtk_active_barrier() -> *const c_char { @@ -327,16 +329,20 @@ thread_local! { } #[no_mangle] -pub unsafe extern "C" fn mmtk_add_nmethod_oop(addr: Address) { +pub unsafe extern "C" fn mmtk_add_nmethod_oop(addr: Address) { NMETHOD_SLOTS.with(|x| x.borrow_mut().push(addr)) } #[no_mangle] pub unsafe extern "C" fn mmtk_register_nmethod(nm: Address) { let slots = NMETHOD_SLOTS.with(|x| { - if x.borrow().len() == 0 { return None; } + if x.borrow().len() == 0 { + return None; + } Some(x.replace(vec![])) }); - if slots.is_none() { return; } + if slots.is_none() { + return; + } let slots = slots.unwrap(); crate::TOTAL_SIZE.fetch_add(slots.len(), Ordering::Relaxed); crate::CODE_CACHE_ROOTS.lock().insert(nm, slots); @@ -346,4 +352,4 @@ pub unsafe extern "C" fn mmtk_unregister_nmethod(nm: Address) { if let Some(slots) = crate::CODE_CACHE_ROOTS.lock().remove(&nm) { crate::TOTAL_SIZE.fetch_sub(slots.len(), Ordering::Relaxed); } -} \ No newline at end of file +} diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 4bef7eef..b6257df0 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -2,8 +2,8 @@ extern crate libc; extern crate mmtk; #[macro_use] extern crate lazy_static; -extern crate spin; extern crate once_cell; +extern crate spin; use std::cell::RefCell; use std::collections::HashMap; @@ -11,9 +11,9 @@ use std::ptr::null_mut; use std::sync::atomic::AtomicUsize; use libc::{c_char, c_void, uintptr_t}; +use mmtk::scheduler::GCWorker; use mmtk::util::alloc::AllocationError; use mmtk::util::opaque_pointer::*; -use mmtk::scheduler::GCWorker; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::VMBinding; use mmtk::Mutator; @@ -93,8 +93,8 @@ thread_local! { #[inline(always)] pub fn current_worker() -> &'static mut GCWorker { CURRENT_WORKER.with(|x| { - let ptr = x.borrow().unwrap() ; - unsafe {&mut *ptr} + let ptr = x.borrow().unwrap(); + unsafe { &mut *ptr } }) } @@ -116,7 +116,8 @@ pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t = pub static DISABLE_ALLOCATION_FAST_PATH: i32 = cfg!(feature = "no_fast_alloc") as _; #[no_mangle] -pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = std::mem::size_of::>(); +pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = + std::mem::size_of::>(); #[derive(Default)] pub struct OpenJDK; @@ -153,5 +154,6 @@ lazy_static! { pub static MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES: usize = mmtk::util::alloc::MarkCompactAllocator::::HEADER_RESERVED_IN_BYTES; -static CODE_CACHE_ROOTS: spin::Lazy>>> = spin::Lazy::new(|| Mutex::new(HashMap::new())); -static TOTAL_SIZE: AtomicUsize = AtomicUsize::new(0); \ No newline at end of file +static CODE_CACHE_ROOTS: spin::Lazy>>> = + spin::Lazy::new(|| Mutex::new(HashMap::new())); +static TOTAL_SIZE: AtomicUsize = AtomicUsize::new(0); diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 104e2eef..b04dbb66 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -249,7 +249,7 @@ fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { } #[inline(always)] -pub fn is_obj_array(oop: Oop ) -> bool { +pub fn is_obj_array(oop: Oop) -> bool { let klass_id = oop.klass.id; klass_id == KlassID::ObjArray } @@ -263,7 +263,7 @@ pub fn obj_array_data(oop: Oop) -> &'static [ObjectReference] { } #[inline(always)] -pub fn is_type_array(oop: Oop ) -> bool { +pub fn is_type_array(oop: Oop) -> bool { let klass_id = oop.klass.id; klass_id == KlassID::TypeArray } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 9fb3a5e9..ce322d05 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -12,19 +12,13 @@ use mmtk::MutatorContext; pub struct VMScanning {} -pub(crate) fn create_process_edges_work_vec>( - buf: Vec
, -) { +pub(crate) fn create_process_edges_work_vec>(buf: Vec
) { if !buf.is_empty() { let w = W::new(buf, true, &SINGLETON); if W::RC_ROOTS { crate::current_worker().add_work(WorkBucketStage::RCProcessIncs, w); } else { - memory_manager::add_work_packet( - &SINGLETON, - WorkBucketStage::Closure, - w, - ); + memory_manager::add_work_packet(&SINGLETON, WorkBucketStage::Closure, w); } } } @@ -40,11 +34,7 @@ pub(crate) extern "C" fn create_process_edges_work Date: Thu, 12 May 2022 10:35:10 +1000 Subject: [PATCH 139/494] cleanup --- mmtk/Cargo.lock | 11 ++++- mmtk/Cargo.toml | 2 +- mmtk/src/object_model.rs | 8 --- mmtk/src/object_scanning.rs | 99 ------------------------------------- mmtk/src/scanning.rs | 10 ---- 5 files changed, 10 insertions(+), 120 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 3a6fc718..edf7d1cd 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -45,6 +45,12 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "atomic_refcell" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b5e5f48b927f04e952dedc932f31995a65a0bf65ec971c74436e51bf6e970d" + [[package]] name = "atty" version = "0.2.14" @@ -337,11 +343,12 @@ dependencies = [ [[package]] name = "mmtk" -version = "0.8.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=47232c1a0a789edd578d3239a813cdd42b02dd3c#47232c1a0a789edd578d3239a813cdd42b02dd3c" +version = "0.11.0" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=cabaaf9cf22c13d3aab7ec60a1a4a27aad75374d#cabaaf9cf22c13d3aab7ec60a1a4a27aad75374d" dependencies = [ "atomic", "atomic-traits", + "atomic_refcell", "crossbeam-deque", "crossbeam-queue", "custom_derive", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 87f0ca22..4ed7cec2 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -37,7 +37,7 @@ once_cell = "1.10.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "47232c1a0a789edd578d3239a813cdd42b02dd3c" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "cabaaf9cf22c13d3aab7ec60a1a4a27aad75374d" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 3006c77e..d4c9f10d 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -1,4 +1,3 @@ -use std::ffi::CStr; use std::sync::atomic::Ordering; use super::UPCALLS; @@ -158,11 +157,4 @@ impl ObjectModel for VMObjectModel { ((*UPCALLS).dump_object)(object); } } - - fn dump_object_s(object: ObjectReference) -> String { - unsafe { - let c_str = ((*UPCALLS).dump_object_string)(object); - CStr::from_ptr(c_str).to_str().unwrap().to_owned() - } - } } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index b04dbb66..f3561020 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -8,7 +8,6 @@ use std::{mem, slice}; trait OopIterate: Sized { fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor); - fn is_oop_field(&self, oop: Oop, edge: Address) -> bool; } impl OopIterate for OopMapBlock { @@ -20,12 +19,6 @@ impl OopIterate for OopMapBlock { closure.visit_edge(edge); } } - #[inline(always)] - fn is_oop_field(&self, oop: Oop, edge: Address) -> bool { - let start = oop.get_field_address(self.offset); - let end = start + ((self.count as usize) << LOG_BYTES_IN_ADDRESS); - edge >= start && edge < end - } } impl OopIterate for InstanceKlass { @@ -36,16 +29,6 @@ impl OopIterate for InstanceKlass { map.oop_iterate(oop, closure) } } - #[inline(always)] - fn is_oop_field(&self, oop: Oop, edge: Address) -> bool { - let oop_maps = self.nonstatic_oop_maps(); - for map in oop_maps { - if map.is_oop_field(oop, edge) { - return true; - } - } - false - } } impl OopIterate for InstanceMirrorKlass { @@ -87,16 +70,6 @@ impl OopIterate for InstanceMirrorKlass { closure.visit_edge(Address::from_ref(oop as &Oop)); } } - #[inline(always)] - fn is_oop_field(&self, oop: Oop, edge: Address) -> bool { - if self.instance_klass.is_oop_field(oop, edge) { - return true; - } - let start = Self::start_of_static_fields(oop); - let len = Self::static_oop_field_count(oop); - let end = start + (len << LOG_BYTES_IN_ADDRESS); - edge >= start && edge < end - } } impl OopIterate for InstanceClassLoaderKlass { @@ -111,10 +84,6 @@ impl OopIterate for InstanceClassLoaderKlass { // } // } } - #[inline(always)] - fn is_oop_field(&self, oop: Oop, edge: Address) -> bool { - self.instance_klass.is_oop_field(oop, edge) - } } impl OopIterate for ObjArrayKlass { @@ -125,12 +94,6 @@ impl OopIterate for ObjArrayKlass { closure.visit_edge(Address::from_ref(oop as &Oop)); } } - #[inline(always)] - fn is_oop_field(&self, oop: Oop, edge: Address) -> bool { - let array = unsafe { oop.as_array_oop() }; - let data = array.data_range(); - edge >= data.start && edge < data.end - } } impl OopIterate for TypeArrayKlass { @@ -139,10 +102,6 @@ impl OopIterate for TypeArrayKlass { // Performance tweak: We skip processing the klass pointer since all // TypeArrayKlasses are guaranteed processed via the null class loader. } - #[inline(always)] - fn is_oop_field(&self, _oop: Oop, _edge: Address) -> bool { - false - } } impl OopIterate for InstanceRefKlass { @@ -171,21 +130,6 @@ impl OopIterate for InstanceRefKlass { Self::process_ref_as_strong(oop, closure); } } - #[inline(always)] - fn is_oop_field(&self, oop: Oop, edge: Address) -> bool { - if self.instance_klass.is_oop_field(oop, edge) { - return true; - } - let referent_addr = Self::referent_address(oop); - if edge == referent_addr { - return true; - } - let discovered_addr = Self::discovered_address(oop); - if edge == discovered_addr { - return true; - } - false - } } impl InstanceRefKlass { @@ -262,49 +206,6 @@ pub fn obj_array_data(oop: Oop) -> &'static [ObjectReference] { } } -#[inline(always)] -pub fn is_type_array(oop: Oop) -> bool { - let klass_id = oop.klass.id; - klass_id == KlassID::TypeArray -} - -#[inline(always)] -pub fn is_oop_field(oop: Oop, edge: Address) -> bool { - let klass_id = oop.klass.id; - debug_assert!( - klass_id as i32 >= 0 && (klass_id as i32) < 6, - "Invalid klass-id: {:x} for oop: {:x}", - klass_id as i32, - unsafe { mem::transmute::(oop) } - ); - match klass_id { - KlassID::Instance => { - let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.is_oop_field(oop, edge) - } - KlassID::InstanceClassLoader => { - let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.is_oop_field(oop, edge) - } - KlassID::InstanceMirror => { - let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.is_oop_field(oop, edge) - } - KlassID::ObjArray => { - let array_klass = unsafe { oop.klass.cast::() }; - array_klass.is_oop_field(oop, edge) - } - KlassID::TypeArray => { - let array_klass = unsafe { oop.klass.cast::() }; - array_klass.is_oop_field(oop, edge) - } - KlassID::InstanceRef => { - let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.is_oop_field(oop, edge) - } // _ => oop_iterate_slow(oop, closure, tls), - } -} - #[inline] pub fn scan_object(object: ObjectReference, closure: &mut impl EdgeVisitor, _tls: VMWorkerThread) { // println!("*****scan_object(0x{:x}) -> \n 0x{:x}, 0x{:x} \n", diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index ce322d05..c92ae841 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -70,16 +70,6 @@ impl Scanning for VMScanning { crate::object_scanning::is_obj_array(unsafe { std::mem::transmute(o) }) } - #[inline(always)] - fn is_type_array(o: ObjectReference) -> bool { - crate::object_scanning::is_type_array(unsafe { std::mem::transmute(o) }) - } - - #[inline(always)] - fn is_oop_field(o: ObjectReference, e: Address) -> bool { - crate::object_scanning::is_oop_field(unsafe { std::mem::transmute(o) }, e) - } - fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { // unimplemented!() // TODO From c9e22ab52714074079bdad65a9901ca8f706d93b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 14 Jul 2022 13:55:25 +1000 Subject: [PATCH 140/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index b52cffea..fccae57f 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -553,9 +553,9 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "spin" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5" +checksum = "c530c2b0d0bf8b69304b39fe2001993e267461948b890cd037d8ad4293fa1a0d" dependencies = [ "lock_api", ] diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index aecfd835..1656de55 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -29,7 +29,7 @@ openjdk_version = "ca90b43f0f51d9ddf754e6ab134c5030cf54118b" [dependencies] libc = "0.2" lazy_static = "1.1" -spin = "0.9.2" +spin = {version = "0.9.3", features = ["std"]} once_cell = "1.10.0" # Be very careful to commit any changes to the following mmtk dependency, as our CI scripts (including mmtk-core CI) # rely on matching these lines to modify them: e.g. comment out the git dependency and use the local path. @@ -37,7 +37,7 @@ once_cell = "1.10.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "1e4361c86a76e04cc1aac999cb344f77b4c137ed" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8430420d4d36e022f4b6b9a2f3a7ccbf79a7997c" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From 11fc4fbe0df1da216179cbf267210890873db90c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 21 Jul 2022 14:51:34 +1000 Subject: [PATCH 141/494] minor --- mmtk/Cargo.lock | 139 +++++++++++++++++++++++++++--------------------- mmtk/Cargo.toml | 22 -------- 2 files changed, 78 insertions(+), 83 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index fccae57f..d631f693 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -20,12 +20,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "array-macro" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06e97b4e522f9e55523001238ac59d13a8603af57f69980de5d8de4bbbe8ada6" - [[package]] name = "atomic" version = "0.5.1" @@ -144,29 +138,52 @@ dependencies = [ "vec_map", ] +[[package]] +name = "crossbeam" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae5588f6b3c3cb05239e90bd110f257254aecd01e4635400391aeae07497845" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c02a4d71819009c192cf4872265391563fd6a84c81ff2c0f2a7026ca4c1d85c" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" -version = "0.7.4" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" dependencies = [ + "cfg-if 1.0.0", "crossbeam-epoch", - "crossbeam-utils 0.7.2", - "maybe-uninit", + "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.8.2" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +checksum = "07db9d94cbd326813772c968ccd25999e5f8ae22f4f8d1b11effa37ef6ce281d" dependencies = [ "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", + "cfg-if 1.0.0", + "crossbeam-utils", "memoffset", + "once_cell", "scopeguard", ] @@ -177,18 +194,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dd435b205a4842da59efd07628f921c096bc1cc0a156835b4fa0bcb9a19bcce" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.7", -] - -[[package]] -name = "crossbeam-utils" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "lazy_static", + "crossbeam-utils", ] [[package]] @@ -201,12 +207,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "custom_derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8ae57c4978a2acd8b869ce6b9ca1dfe817bff704c220209fdef2c0b75a01b9" - [[package]] name = "downcast-rs" version = "1.2.0" @@ -215,31 +215,24 @@ checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" [[package]] name = "enum-map" -version = "0.6.4" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4187999839f4ae8be35cf185d1381aa8dc32d2f5951349cc37ae49ebc4781855" +checksum = "0348b2a57c82f98b9dbd8098b1abb2416f221823d3e50cbe24eaebdd16896826" dependencies = [ - "array-macro", "enum-map-derive", ] [[package]] name = "enum-map-derive" -version = "0.4.6" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5c450cf304c9e18d45db562025a14fb1ca0f5c769b6f609309f81d4c31de455" +checksum = "a63b7a0ddec6f38dcec5e36257750b7a8fcaf4227e12ceb306e341d63634da05" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "enum_derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406ac2a8c9eedf8af9ee1489bee9e50029278a6456c740f7454cf8a158abc816" - [[package]] name = "env_logger" version = "0.8.3" @@ -259,6 +252,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" + [[package]] name = "hermit-abi" version = "0.1.18" @@ -320,12 +319,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "memchr" version = "2.3.4" @@ -334,27 +327,24 @@ checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" [[package]] name = "memoffset" -version = "0.5.6" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" dependencies = [ "autocfg", ] [[package]] name = "mmtk" -version = "0.12.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=1e4361c86a76e04cc1aac999cb344f77b4c137ed#1e4361c86a76e04cc1aac999cb344f77b4c137ed" +version = "0.13.0" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8430420d4d36e022f4b6b9a2f3a7ccbf79a7997c#8430420d4d36e022f4b6b9a2f3a7ccbf79a7997c" dependencies = [ "atomic", "atomic-traits", "atomic_refcell", - "crossbeam-deque", - "crossbeam-queue", - "custom_derive", + "crossbeam", "downcast-rs", "enum-map", - "enum_derive", "env_logger", "lazy_static", "libc", @@ -363,13 +353,15 @@ dependencies = [ "num_cpus", "pfm", "spin", + "strum", + "strum_macros", "thread-priority", ] [[package]] name = "mmtk-macros" -version = "0.12.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=1e4361c86a76e04cc1aac999cb344f77b4c137ed#1e4361c86a76e04cc1aac999cb344f77b4c137ed" +version = "0.13.0" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8430420d4d36e022f4b6b9a2f3a7ccbf79a7997c#8430420d4d36e022f4b6b9a2f3a7ccbf79a7997c" dependencies = [ "proc-macro-error", "proc-macro2", @@ -524,6 +516,12 @@ dependencies = [ "semver", ] +[[package]] +name = "rustversion" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24c8ad4f0c00e1eb5bc7614d236a7f1300e3dbd76b68cac8e06fb00b015ad8d8" + [[package]] name = "scopeguard" version = "1.1.0" @@ -566,6 +564,25 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" + +[[package]] +name = "strum_macros" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4faebde00e8ff94316c01800f9054fd2ba77d30d9e922541913051d1d978918b" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "syn" version = "1.0.95" diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 1656de55..50eb1900 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -75,25 +75,3 @@ instrumentation = ["mmtk/instrumentation"] log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] log_work_packets = ["mmtk/log_work_packets"] - -ix_block_only = ["mmtk/ix_block_only"] -ix_defrag = ["mmtk/ix_defrag"] - -lxr_heap_health_guided_gc = ["mmtk/lxr_heap_health_guided_gc"] - -# Non-moving, Non-concurrent LXR -lxr_basic = ["mmtk/lxr_basic"] -# LXR (Concurrent marking) -lxr_cm = ["mmtk/lxr_cm"] -# LXR (Lazy RC processing) -lxr_lazy = ["mmtk/lxr_lazy"] -# LXR (Concurrent) -lxr_conc = ["mmtk/lxr_conc"] -# LXR (Evacuation) -lxr_nevac = ["mmtk/lxr_nevac"] -lxr_evac = ["mmtk/lxr_evac"] -# LXR (Block only) -lxr_block = ["mmtk/lxr_block"] -lxr_rc_only = ["mmtk/lxr_rc_only"] -# LXR (Default config with full features) -lxr = ["mmtk/lxr"] From 1ee42eaa28a35bece2ced5aa7444cc5461c2488d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 9 Aug 2022 12:12:32 +1000 Subject: [PATCH 142/494] wip --- mmtk/src/abi.rs | 88 ++++++++++++++++++++--------- mmtk/src/lib.rs | 2 + mmtk/src/object_model.rs | 1 + mmtk/src/object_scanning.rs | 30 +++++----- openjdk/mmtk.h | 2 + openjdk/mmtkFinalizerThread.cpp | 2 + openjdk/mmtkHeap.cpp | 21 +++---- openjdk/mmtkRootsClosure.hpp | 53 ++--------------- openjdk/mmtkUpcalls.cpp | 22 +++++++- openjdk/mmtkVMCompanionThread.cpp | 4 +- openjdk/thirdPartyHeapArguments.cpp | 4 +- 11 files changed, 125 insertions(+), 104 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 5a2620cd..324bea37 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -7,6 +7,11 @@ use std::ffi::CStr; use std::fmt; use std::{mem, slice}; +lazy_static! { + static ref COMPRESSED_KLASS_BASE: Address = unsafe { ((*UPCALLS).compressed_klass_base)() }; + static ref COMPRESSED_KLASS_SHIFT: usize = unsafe { ((*UPCALLS).compressed_klass_shift)() }; +} + #[repr(i32)] #[derive(Copy, Clone, PartialEq, Eq, Debug)] #[allow(dead_code)] @@ -277,14 +282,31 @@ impl InstanceRefKlass { #[repr(C)] pub struct OopDesc { pub mark: usize, - pub klass: &'static Klass, } impl OopDesc { + pub fn header_size() -> usize { + 12 // 8 byte mark word + 4 byte compressed klass + } #[inline(always)] pub fn start(&self) -> Address { unsafe { mem::transmute(self) } } + #[inline(always)] + pub fn klass(&self) -> &'static Klass { + let compressed = unsafe { (self.start() + std::mem::size_of::()).load::() }; + let addr = + unsafe { *COMPRESSED_KLASS_BASE + ((compressed as usize) << *COMPRESSED_KLASS_SHIFT) }; + // println!( + // "oop({:?}) klass: c={:x} base={:?} shift={:?} addr={:?}", + // self.start(), + // compressed, + // *COMPRESSED_KLASS_BASE, + // *COMPRESSED_KLASS_SHIFT, + // addr, + // ); + unsafe { &*addr.to_ptr::() } + } } impl fmt::Debug for OopDesc { @@ -298,6 +320,15 @@ impl fmt::Debug for OopDesc { pub type Oop = &'static OopDesc; +#[repr(transparent)] +pub struct NarrowOop(u32); + +impl NarrowOop { + pub fn slot(&self) -> Address { + Address::from_ref(self) + } +} + /// Convert ObjectReference to Oop impl From for &OopDesc { #[inline(always)] @@ -332,29 +363,30 @@ impl OopDesc { /// Calculate object instance size #[inline(always)] pub unsafe fn size(&self) -> usize { - let klass = self.klass; - let lh = klass.layout_helper; - // The (scalar) instance size is pre-recorded in the TIB? - if lh > Klass::LH_NEUTRAL_VALUE { - if !Klass::layout_helper_needs_slow_path(lh) { - lh as _ - } else { - self.size_slow() - } - } else if lh <= Klass::LH_NEUTRAL_VALUE { - if lh < Klass::LH_NEUTRAL_VALUE { - // Calculate array size - let array_length = self.as_array_oop().length(); - let mut size_in_bytes: usize = - (array_length as usize) << Klass::layout_helper_log2_element_size(lh); - size_in_bytes += Klass::layout_helper_header_size(lh) as usize; - (size_in_bytes + 0b111) & !0b111 - } else { - self.size_slow() - } - } else { - unreachable!() - } + self.size_slow() + // let klass = self.klass(); + // let lh = klass.layout_helper; + // // The (scalar) instance size is pre-recorded in the TIB? + // if lh > Klass::LH_NEUTRAL_VALUE { + // if !Klass::layout_helper_needs_slow_path(lh) { + // lh as _ + // } else { + // self.size_slow() + // } + // } else if lh <= Klass::LH_NEUTRAL_VALUE { + // if lh < Klass::LH_NEUTRAL_VALUE { + // // Calculate array size + // let array_length = self.as_array_oop().length(); + // let mut size_in_bytes: usize = + // (array_length as usize) << Klass::layout_helper_log2_element_size(lh); + // size_in_bytes += Klass::layout_helper_header_size(lh) as usize; + // (size_in_bytes + 0b111) & !0b111 + // } else { + // self.size_slow() + // } + // } else { + // unreachable!() + // } } } @@ -364,7 +396,9 @@ pub struct ArrayOopDesc(OopDesc); pub type ArrayOop = &'static ArrayOopDesc; impl ArrayOopDesc { - const LENGTH_OFFSET: usize = mem::size_of::(); + fn length_offset() -> usize { + OopDesc::header_size() + } fn element_type_should_be_aligned(ty: BasicType) -> bool { ty == BasicType::T_DOUBLE || ty == BasicType::T_LONG @@ -372,7 +406,7 @@ impl ArrayOopDesc { fn header_size(ty: BasicType) -> usize { let typesize_in_bytes = - conversions::raw_align_up(Self::LENGTH_OFFSET + BYTES_IN_INT, BYTES_IN_LONG); + conversions::raw_align_up(Self::length_offset() + BYTES_IN_INT, BYTES_IN_LONG); if Self::element_type_should_be_aligned(ty) { conversions::raw_align_up(typesize_in_bytes / BYTES_IN_WORD, BYTES_IN_LONG) } else { @@ -380,7 +414,7 @@ impl ArrayOopDesc { } } fn length(&self) -> i32 { - unsafe { *((self as *const _ as *const u8).add(Self::LENGTH_OFFSET) as *const i32) } + unsafe { *((self as *const _ as *const u8).add(Self::length_offset()) as *const i32) } } fn base(&self, ty: BasicType) -> Address { let base_offset_in_bytes = Self::header_size(ty) * BYTES_IN_WORD; diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 7226163f..56150681 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -94,6 +94,8 @@ pub struct OpenJDK_Upcalls { pub schedule_finalizer: extern "C" fn(), pub prepare_for_roots_re_scanning: extern "C" fn(), pub enqueue_references: extern "C" fn(objects: *const ObjectReference, len: usize), + pub compressed_klass_base: extern "C" fn() -> Address, + pub compressed_klass_shift: extern "C" fn() -> usize, } pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index d4c9f10d..67538863 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -94,6 +94,7 @@ impl ObjectModel for VMObjectModel { let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::(), 0, copy); // Copy let src = from.to_address(); + // println!("cp {:?} -> {:?}", src..src + bytes, dst..dst + bytes); unsafe { std::ptr::copy_nonoverlapping::(src.to_ptr(), dst.to_mut_ptr(), bytes) } let to_obj = unsafe { dst.to_object_reference() }; copy_context.post_copy(to_obj, bytes, copy); diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index ae9277dc..1716d0ab 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -15,7 +15,7 @@ impl OopIterate for OopMapBlock { fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { let start = oop.get_field_address(self.offset); for i in 0..self.count as usize { - let edge = start + (i << LOG_BYTES_IN_ADDRESS); + let edge = start + (i << 2); closure.visit_edge(edge); } } @@ -63,11 +63,11 @@ impl OopIterate for InstanceMirrorKlass { // } // static fields - let start: *const Oop = Self::start_of_static_fields(oop).to_ptr::(); + let start: *const NarrowOop = Self::start_of_static_fields(oop).to_ptr::(); let len = Self::static_oop_field_count(oop); let slice = unsafe { slice::from_raw_parts(start, len as _) }; - for oop in slice { - closure.visit_edge(Address::from_ref(oop as &Oop)); + for narrow_oop in slice { + closure.visit_edge(narrow_oop.slot()); } } } @@ -90,8 +90,8 @@ impl OopIterate for ObjArrayKlass { #[inline] fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { let array = unsafe { oop.as_array_oop() }; - for oop in unsafe { array.data::(BasicType::T_OBJECT) } { - closure.visit_edge(Address::from_ref(oop as &Oop)); + for narrow_oop in unsafe { array.data::(BasicType::T_OBJECT) } { + closure.visit_edge(narrow_oop.slot()); } } } @@ -112,6 +112,7 @@ impl OopIterate for InstanceRefKlass { self.instance_klass.oop_iterate(oop, closure); if Self::should_scan_weak_refs() { + unreachable!(); let reference = ObjectReference::from(oop); match self.instance_klass.reference_type { ReferenceType::None => { @@ -156,36 +157,38 @@ fn oop_iterate_slow(oop: Oop, closure: &mut impl EdgeVisitor, tls: OpaquePointer #[inline] fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { - let klass_id = oop.klass.id; + let klass = oop.klass(); + let klass_id = oop.klass().id; debug_assert!( klass_id as i32 >= 0 && (klass_id as i32) < 6, "Invalid klass-id: {:x} for oop: {:x}", klass_id as i32, unsafe { mem::transmute::(oop) } ); + // println!("type {:?}", klass_id); match klass_id { KlassID::Instance => { - let instance_klass = unsafe { oop.klass.cast::() }; + let instance_klass = unsafe { klass.cast::() }; instance_klass.oop_iterate(oop, closure); } KlassID::InstanceClassLoader => { - let instance_klass = unsafe { oop.klass.cast::() }; + let instance_klass = unsafe { klass.cast::() }; instance_klass.oop_iterate(oop, closure); } KlassID::InstanceMirror => { - let instance_klass = unsafe { oop.klass.cast::() }; + let instance_klass = unsafe { klass.cast::() }; instance_klass.oop_iterate(oop, closure); } KlassID::ObjArray => { - let array_klass = unsafe { oop.klass.cast::() }; + let array_klass = unsafe { klass.cast::() }; array_klass.oop_iterate(oop, closure); } KlassID::TypeArray => { - let array_klass = unsafe { oop.klass.cast::() }; + let array_klass = unsafe { klass.cast::() }; array_klass.oop_iterate(oop, closure); } KlassID::InstanceRef => { - let instance_klass = unsafe { oop.klass.cast::() }; + let instance_klass = unsafe { klass.cast::() }; instance_klass.oop_iterate(oop, closure); } // _ => oop_iterate_slow(oop, closure, tls), } @@ -193,6 +196,7 @@ fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { #[inline] pub fn scan_object(object: ObjectReference, closure: &mut impl EdgeVisitor, _tls: VMWorkerThread) { + // println!("Scan {:?}", object); // println!("*****scan_object(0x{:x}) -> \n 0x{:x}, 0x{:x} \n", // object, // unsafe { *(object.value() as *const usize) }, diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 181bc005..a7e29e60 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -162,6 +162,8 @@ typedef struct { void (*schedule_finalizer)(); void (*prepare_for_roots_re_scanning)(); void (*enqueue_references)(void** objects, size_t len); + void* (*compressed_klass_base)(); + size_t (*compressed_klass_shift)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); diff --git a/openjdk/mmtkFinalizerThread.cpp b/openjdk/mmtkFinalizerThread.cpp index f18f0e24..2d9b6452 100644 --- a/openjdk/mmtkFinalizerThread.cpp +++ b/openjdk/mmtkFinalizerThread.cpp @@ -100,6 +100,7 @@ void MMTkFinalizerThread::finalizer_thread_entry(JavaThread* thread, TRAPS) { void* obj_ref = get_finalized_object(); if (obj_ref != NULL) { instanceOop obj = (instanceOop) obj_ref; + // printf("finalize %p\n", obj); // Invoke finalize() { @@ -111,6 +112,7 @@ void MMTkFinalizerThread::finalizer_thread_entry(JavaThread* thread, TRAPS) { JavaCalls::call_virtual(&ret, handle_obj, obj->klass(), finalize_method, sig, this_thread); } + // printf("finalize done %p\n", obj); } else { break; } diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 2c00897c..55a5b0ba 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -71,8 +71,6 @@ MMTkHeap::MMTkHeap(MMTkCollectorPolicy* policy) : CollectedHeap(), _last_gc_time jint MMTkHeap::initialize() { assert(!UseTLAB , "should disable UseTLAB"); - assert(!UseCompressedOops , "should disable CompressedOops"); - assert(!UseCompressedClassPointers , "should disable UseCompressedClassPointers"); const size_t heap_size = collector_policy()->max_heap_byte_size(); // printf("policy max heap size %zu, min heap size %zu\n", heap_size, collector_policy()->min_heap_byte_size()); @@ -105,10 +103,15 @@ jint MMTkHeap::initialize() { _start = (HeapWord*) starting_heap_address(); _end = (HeapWord*) last_heap_address(); - // printf("start: %p, end: %p\n", _start, _end); + printf("start: %p, end: %p, base: %p\n", _start, _end, (void*) (intptr_t(_start) - 4096)); - initialize_reserved_region(_start, _end); + Universe::set_narrow_oop_base((address) (intptr_t(_start))); + Universe::set_narrow_oop_shift(LogMinObjAlignmentInBytes); + initialize_reserved_region((HeapWord*) (intptr_t(_start) + (1ull << 30)), _end); + + + printf("compressed oops: base=%p, shift=%zu\n", Universe::narrow_oop_base(), Universe::narrow_oop_shift()); MMTkBarrierSet* const barrier_set = new MMTkBarrierSet(reserved_region()); //barrier_set->initialize(); @@ -277,18 +280,16 @@ void MMTkHeap::safe_object_iterate(ObjectClosure* cl) { //not sure..many depende } HeapWord* MMTkHeap::block_start(const void* addr) const {//OK - guarantee(false, "block start not supported"); - return NULL; + return (HeapWord*) starting_heap_address(); } size_t MMTkHeap::block_size(const HeapWord* addr) const { //OK - guarantee(false, "block size not supported"); - return 0; + return size_t(last_heap_address()) - size_t(starting_heap_address()); } bool MMTkHeap::block_is_obj(const HeapWord* addr) const { //OK - guarantee(false, "block is obj not supported"); - return false; + // guarantee(false, "block is obj not supported"); + return size_t(addr) >= size_t(starting_heap_address()) && size_t(addr) < size_t(last_heap_address()); } jlong MMTkHeap::millis_since_last_gc() {//later when gc is implemented in rust diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 990f6d5e..30bbb107 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -9,44 +9,6 @@ #define ROOTS_BUFFER_SIZE 4096 -class MMTkRootsClosure : public OopClosure { - void* _trace; - void* _buffer[ROOTS_BUFFER_SIZE]; - size_t _cursor; - - template - void do_oop_work(T* p) { - // T heap_oop = RawAccess<>::oop_load(p); - // if (!CompressedOops::is_null(heap_oop)) { - // oop obj = CompressedOops::decode_not_null(heap_oop); - // oop fwd = (oop) trace_root_object(_trace, obj); - // RawAccess<>::oop_store(p, fwd); - // } - _buffer[_cursor++] = (void*) p; - if (_cursor >= ROOTS_BUFFER_SIZE) { - flush(); - } - } - - NOINLINE void flush() { - // bulk_report_delayed_root_edge(_trace, _buffer, _cursor); - _cursor = 0; - } - -public: - MMTkRootsClosure(void* trace): _trace(trace), _cursor(0) {} - - ~MMTkRootsClosure() { - if (_cursor > 0) flush(); - } - - virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { - // printf("narrowoop root %p -> %d %p %p\n", (void*) p, *p, *((void**) p), (void*) oopDesc::load_decode_heap_oop(p)); - do_oop_work(p); - } -}; - class MMTkRootsClosure2 : public OopClosure { EdgesClosure _edges_closure; void** _buffer; @@ -55,15 +17,12 @@ class MMTkRootsClosure2 : public OopClosure { template void do_oop_work(T* p) { - // T heap_oop = RawAccess<>::oop_load(p); - // if (!CompressedOops::is_null(heap_oop)) { - // oop obj = CompressedOops::decode_not_null(heap_oop); - // oop fwd = (oop) trace_root_object(_trace, obj); - // RawAccess<>::oop_store(p, fwd); - // } - _buffer[_cursor++] = (void*) p; - if (_cursor >= _cap) { - flush(); + T heap_oop = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(heap_oop)) { + _buffer[_cursor++] = (void*) p; + if (_cursor >= _cap) { + flush(); + } } } diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 0eee0aa8..18b4d55d 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -46,6 +46,8 @@ static volatile size_t mmtk_start_the_world_count = 0; static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure) { + + printf("mmtk_stop_all_mutators\n"); ClassLoaderDataGraph::clear_claimed_marks(); CodeCache::gc_prologue(); #if COMPILER2_OR_JVMCI @@ -62,11 +64,12 @@ static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, M closure.invoke((void*)&cur->third_party_heap_mutator); } } - log_debug(gc)("Finished enumerating threads."); + printf("Finished enumerating threads.\n"); nmethod::oops_do_marking_prologue(); } static void mmtk_resume_mutators(void *tls) { + printf("mmtk_resume_mutators\n"); nmethod::oops_do_marking_epilogue(); // ClassLoaderDataGraph::purge(); CodeCache::gc_epilogue(); @@ -80,7 +83,7 @@ static void mmtk_resume_mutators(void *tls) { // otherwise, mutators might see a stale value Atomic::inc(&mmtk_start_the_world_count); - log_debug(gc)("Requesting the VM to resume all mutators..."); + printf("Requesting the VM to resume all mutators...\n"); MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_resumed, true); log_debug(gc)("Mutators resumed. Now notify any mutators waiting for GC to finish..."); @@ -212,6 +215,7 @@ static void mmtk_reset_mutator_iterator() { static void mmtk_scan_all_thread_roots(EdgesClosure closure) { + guarantee(false, "unreachable"); MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_thread_roots(cl); } @@ -224,6 +228,7 @@ static void mmtk_scan_thread_roots(EdgesClosure closure, void* tls) { } static void mmtk_scan_object(void* trace, void* object, void* tls) { + guarantee(false, "unreachable"); MMTkScanObjectClosure cl(trace); ((oop) object)->oop_iterate(&cl); } @@ -321,6 +326,7 @@ static void mmtk_prepare_for_roots_re_scanning() { } static void mmtk_enqueue_references(void** objects, size_t len) { + guarantee(false, "unreachable"); if (len == 0) { return; } @@ -341,6 +347,14 @@ static void mmtk_enqueue_references(void** objects, size_t len) { assert(Universe::has_reference_pending_list(), "Reference pending list is empty after swap"); } +static void* mmtk_compressed_klass_base() { + return (void*) Universe::narrow_klass_base(); +} + +static size_t mmtk_compressed_klass_shift() { + return (size_t) Universe::narrow_klass_shift(); +} + OpenJDK_Upcalls mmtk_upcalls = { mmtk_stop_all_mutators, mmtk_resume_mutators, @@ -379,5 +393,7 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_number_of_mutators, mmtk_schedule_finalizer, mmtk_prepare_for_roots_re_scanning, - mmtk_enqueue_references + mmtk_enqueue_references, + mmtk_compressed_klass_base, + mmtk_compressed_klass_shift, }; diff --git a/openjdk/mmtkVMCompanionThread.cpp b/openjdk/mmtkVMCompanionThread.cpp index 17f2d2c1..d5e16d8a 100644 --- a/openjdk/mmtkVMCompanionThread.cpp +++ b/openjdk/mmtkVMCompanionThread.cpp @@ -59,7 +59,7 @@ void MMTkVMCompanionThread::run() { } // Let the VM thread stop the world. - log_trace(gc)("MMTkVMCompanionThread: Letting VMThread execute VM op..."); + printf("MMTkVMCompanionThread: Letting VMThread execute VM op...\n"); VM_MMTkSTWOperation op(this); // VMThread::execute() is blocking. The companion thread will be blocked // here waiting for the VM thread to execute op, and the VM thread will @@ -68,7 +68,7 @@ void MMTkVMCompanionThread::run() { VMThread::execute(&op); // Tell the waiter thread that the world has resumed. - log_trace(gc)("MMTkVMCompanionThread: Notifying threads resumption..."); + printf("MMTkVMCompanionThread: Notifying threads resumption...\n"); { MutexLockerEx locker(_lock, Mutex::_no_safepoint_check_flag); assert(_desired_state == _threads_resumed, "start-the-world should be requested."); diff --git a/openjdk/thirdPartyHeapArguments.cpp b/openjdk/thirdPartyHeapArguments.cpp index 745250df..1f14fa63 100644 --- a/openjdk/thirdPartyHeapArguments.cpp +++ b/openjdk/thirdPartyHeapArguments.cpp @@ -43,8 +43,8 @@ void ThirdPartyHeapArguments::initialize() { GCArguments::initialize(); assert(UseThirdPartyHeap , "Error, should UseThirdPartyHeap"); FLAG_SET_DEFAULT(UseTLAB, false); - FLAG_SET_DEFAULT(UseCompressedOops, false); - FLAG_SET_DEFAULT(UseCompressedClassPointers, false); + FLAG_SET_DEFAULT(UseCompressedOops, true); + FLAG_SET_DEFAULT(UseCompressedClassPointers, true); } CollectedHeap* ThirdPartyHeapArguments::create_heap() { From f655e17d6cf94e1779428ba7c68f74da1329eafa Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 16 Aug 2022 21:17:35 +1000 Subject: [PATCH 143/494] wip --- mmtk/src/abi.rs | 57 ++++++++++------ mmtk/src/lib.rs | 2 + mmtk/src/object_model.rs | 6 ++ mmtk/src/object_scanning.rs | 102 ++++++++++++++++++---------- openjdk/mmtk.h | 2 + openjdk/mmtkBarrierSet.hpp | 2 +- openjdk/mmtkHeap.cpp | 2 +- openjdk/mmtkRootsClosure.hpp | 34 ++++++---- openjdk/mmtkUpcalls.cpp | 20 ++++-- openjdk/thirdPartyHeapArguments.cpp | 2 +- 10 files changed, 149 insertions(+), 80 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 324bea37..37538378 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -261,13 +261,21 @@ pub struct InstanceRefKlass { impl InstanceRefKlass { fn referent_offset() -> i32 { lazy_static! { - pub static ref REFERENT_OFFSET: i32 = unsafe { ((*UPCALLS).referent_offset)() }; + pub static ref REFERENT_OFFSET: i32 = unsafe { + let referent_offset = ((*UPCALLS).referent_offset)(); + println!("referent_offset 0x{:x}", referent_offset); + referent_offset + }; } *REFERENT_OFFSET } fn discovered_offset() -> i32 { lazy_static! { - pub static ref DISCOVERED_OFFSET: i32 = unsafe { ((*UPCALLS).discovered_offset)() }; + pub static ref DISCOVERED_OFFSET: i32 = unsafe { + let discovered_offset = ((*UPCALLS).discovered_offset)(); + println!("discovered_offset 0x{:x}", discovered_offset); + discovered_offset + }; } *DISCOVERED_OFFSET } @@ -282,30 +290,29 @@ impl InstanceRefKlass { #[repr(C)] pub struct OopDesc { pub mark: usize, + klass: &'static Klass, } impl OopDesc { - pub fn header_size() -> usize { - 12 // 8 byte mark word + 4 byte compressed klass - } #[inline(always)] pub fn start(&self) -> Address { unsafe { mem::transmute(self) } } #[inline(always)] pub fn klass(&self) -> &'static Klass { - let compressed = unsafe { (self.start() + std::mem::size_of::()).load::() }; - let addr = - unsafe { *COMPRESSED_KLASS_BASE + ((compressed as usize) << *COMPRESSED_KLASS_SHIFT) }; - // println!( - // "oop({:?}) klass: c={:x} base={:?} shift={:?} addr={:?}", - // self.start(), - // compressed, - // *COMPRESSED_KLASS_BASE, - // *COMPRESSED_KLASS_SHIFT, - // addr, - // ); - unsafe { &*addr.to_ptr::() } + self.klass + // let compressed = unsafe { (self.start() + std::mem::size_of::()).load::() }; + // let addr = + // unsafe { *COMPRESSED_KLASS_BASE + ((compressed as usize) << *COMPRESSED_KLASS_SHIFT) }; + // // println!( + // // "oop({:?}) klass: c={:x} base={:?} shift={:?} addr={:?}", + // // self.start(), + // // compressed, + // // *COMPRESSED_KLASS_BASE, + // // *COMPRESSED_KLASS_SHIFT, + // // addr, + // // ); + // unsafe { &*addr.to_ptr::() } } } @@ -397,7 +404,11 @@ pub type ArrayOop = &'static ArrayOopDesc; impl ArrayOopDesc { fn length_offset() -> usize { - OopDesc::header_size() + lazy_static! { + static ref OOP_ARRAY_LENGTH_OFFSET_IN_BYTES: usize = + unsafe { ((*UPCALLS).oop_array_length_offset_in_bytes)() }; + } + *OOP_ARRAY_LENGTH_OFFSET_IN_BYTES } fn element_type_should_be_aligned(ty: BasicType) -> bool { @@ -417,8 +428,14 @@ impl ArrayOopDesc { unsafe { *((self as *const _ as *const u8).add(Self::length_offset()) as *const i32) } } fn base(&self, ty: BasicType) -> Address { - let base_offset_in_bytes = Self::header_size(ty) * BYTES_IN_WORD; - Address::from_ptr(unsafe { (self as *const _ as *const u8).add(base_offset_in_bytes) }) + debug_assert_eq!(ty, BasicType::T_OBJECT); + lazy_static! { + static ref OOP_ARRAY_BASE_OFFSET_IN_BYTES: usize = + unsafe { ((*UPCALLS).oop_array_base_offset_in_bytes)() }; + } + Address::from_ptr(unsafe { + (self as *const _ as *const u8).add(*OOP_ARRAY_BASE_OFFSET_IN_BYTES) + }) } // This provides an easy way to access the array data in Rust. However, the array data // is Java types, so we have to map Java types to Rust types. The caller needs to guarantee: diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 56150681..1fd90910 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -96,6 +96,8 @@ pub struct OpenJDK_Upcalls { pub enqueue_references: extern "C" fn(objects: *const ObjectReference, len: usize), pub compressed_klass_base: extern "C" fn() -> Address, pub compressed_klass_shift: extern "C" fn() -> usize, + pub oop_array_base_offset_in_bytes: extern "C" fn() -> usize, + pub oop_array_length_offset_in_bytes: extern "C" fn() -> usize, } pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 67538863..025df37c 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -98,10 +98,16 @@ impl ObjectModel for VMObjectModel { unsafe { std::ptr::copy_nonoverlapping::(src.to_ptr(), dst.to_mut_ptr(), bytes) } let to_obj = unsafe { dst.to_object_reference() }; copy_context.post_copy(to_obj, bytes, copy); + unsafe { + let a = dst; + debug_assert_ne!(a, Address::from_usize(0x20000000000)); + debug_assert!(a < Address::from_usize(0x200c0000000)); + } to_obj } fn copy_to(from: ObjectReference, to: ObjectReference, region: Address) -> Address { + unreachable!(); let need_copy = from != to; let bytes = unsafe { ((*UPCALLS).get_object_size)(from) }; if need_copy { diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 1716d0ab..c5a0077d 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -4,6 +4,7 @@ use mmtk::util::constants::*; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::EdgeVisitor; +use std::ffi::CStr; use std::{mem, slice}; trait OopIterate: Sized { @@ -149,58 +150,85 @@ impl InstanceRefKlass { } #[allow(unused)] -fn oop_iterate_slow(oop: Oop, closure: &mut impl EdgeVisitor, tls: OpaquePointer) { +fn oop_iterate_slow(oop: Oop, process_edge: extern "C" fn(Address), tls: OpaquePointer) { unsafe { - ((*UPCALLS).scan_object)(closure as *mut _ as _, mem::transmute(oop), tls); + ((*UPCALLS).scan_object)(process_edge as _, mem::transmute(oop), tls); } } +static mut FIELDS: Vec
= Vec::new(); + #[inline] fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { - let klass = oop.klass(); - let klass_id = oop.klass().id; - debug_assert!( - klass_id as i32 >= 0 && (klass_id as i32) < 6, - "Invalid klass-id: {:x} for oop: {:x}", - klass_id as i32, - unsafe { mem::transmute::(oop) } - ); - // println!("type {:?}", klass_id); - match klass_id { - KlassID::Instance => { - let instance_klass = unsafe { klass.cast::() }; - instance_klass.oop_iterate(oop, closure); - } - KlassID::InstanceClassLoader => { - let instance_klass = unsafe { klass.cast::() }; - instance_klass.oop_iterate(oop, closure); - } - KlassID::InstanceMirror => { - let instance_klass = unsafe { klass.cast::() }; - instance_klass.oop_iterate(oop, closure); - } - KlassID::ObjArray => { - let array_klass = unsafe { klass.cast::() }; - array_klass.oop_iterate(oop, closure); - } - KlassID::TypeArray => { - let array_klass = unsafe { klass.cast::() }; - array_klass.oop_iterate(oop, closure); + unsafe { + FIELDS = Vec::new(); + } + extern "C" fn report_edge(a: Address) { + unsafe { FIELDS.push(a) } + } + oop_iterate_slow(oop, report_edge, OpaquePointer::UNINITIALIZED); + unsafe { + for e in &FIELDS { + closure.visit_edge(*e) } - KlassID::InstanceRef => { - let instance_klass = unsafe { klass.cast::() }; - instance_klass.oop_iterate(oop, closure); - } // _ => oop_iterate_slow(oop, closure, tls), } + // let klass = oop.klass(); + // let klass_id = oop.klass().id; + // debug_assert!( + // klass_id as i32 >= 0 && (klass_id as i32) < 6, + // "Invalid klass-id: {:x} for oop: {:x}", + // klass_id as i32, + // unsafe { mem::transmute::(oop) } + // ); + // unsafe { + // ((*UPCALLS).dump_object)(mem::transmute(oop)); + // } + // // println!("oop {:?}", unsafe { + // // let c_string = ((*UPCALLS).dump_object)(mem::transmute(oop)); + // // let c_str: &CStr = unsafe { CStr::from_ptr(c_string) }; + // // c_str + // // }); + // match klass_id { + // KlassID::Instance => { + // let instance_klass = unsafe { klass.cast::() }; + // instance_klass.oop_iterate(oop, closure); + // } + // KlassID::InstanceClassLoader => { + // let instance_klass = unsafe { klass.cast::() }; + // instance_klass.oop_iterate(oop, closure); + // } + // KlassID::InstanceMirror => { + // let instance_klass = unsafe { klass.cast::() }; + // instance_klass.oop_iterate(oop, closure); + // } + // KlassID::ObjArray => { + // let array_klass = unsafe { klass.cast::() }; + // array_klass.oop_iterate(oop, closure); + // } + // KlassID::TypeArray => { + // let array_klass = unsafe { klass.cast::() }; + // array_klass.oop_iterate(oop, closure); + // } + // KlassID::InstanceRef => { + // let instance_klass = unsafe { klass.cast::() }; + // instance_klass.oop_iterate(oop, closure); + // } // _ => oop_iterate_slow(oop, closure, tls), + // } } #[inline] -pub fn scan_object(object: ObjectReference, closure: &mut impl EdgeVisitor, _tls: VMWorkerThread) { - // println!("Scan {:?}", object); +pub fn scan_object(object: ObjectReference, closure: &mut T, _tls: VMWorkerThread) { + // println!( + // "Scan {:?} (klass={:?} id={:?})", + // object, + // unsafe { (object.to_address() + 8usize).load::
() }, + // unsafe { mem::transmute::<_, Oop>(object).klass().id } + // ); // println!("*****scan_object(0x{:x}) -> \n 0x{:x}, 0x{:x} \n", // object, // unsafe { *(object.value() as *const usize) }, // unsafe { *((object.value() + 8) as *const usize) } // ); + let closure = unsafe { &mut *(closure as *mut T) }; unsafe { oop_iterate(mem::transmute(object), closure) } } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index a7e29e60..2b5ea196 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -164,6 +164,8 @@ typedef struct { void (*enqueue_references)(void** objects, size_t len); void* (*compressed_klass_base)(); size_t (*compressed_klass_shift)(); + size_t (*oop_array_base_offset_in_bytes)(); + size_t (*oop_array_length_offset_in_bytes)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 5ca0ee73..51dbb3c9 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -36,7 +36,7 @@ #include "oops/oopsHierarchy.hpp" #include "utilities/fakeRttiSupport.hpp" -#define MMTK_ENABLE_ALLOCATION_FASTPATH true +#define MMTK_ENABLE_ALLOCATION_FASTPATH false const intptr_t ALLOC_BIT_BASE_ADDRESS = GLOBAL_ALLOC_BIT_ADDRESS; diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 55a5b0ba..6faae50a 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -345,7 +345,7 @@ class MMTkRegisterNMethodOopClosure: public OopClosure { } public: void do_oop(oop* p) { do_oop_work(p); } - void do_oop(narrowOop* p) { do_oop_work(p); } + void do_oop(narrowOop* p) { guarantee(false, "unreachable"); } }; diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 30bbb107..8308a83f 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -16,14 +16,18 @@ class MMTkRootsClosure2 : public OopClosure { size_t _cursor; template - void do_oop_work(T* p) { + void do_oop_work(T* p, bool narrow) { T heap_oop = RawAccess<>::oop_load(p); - if (!CompressedOops::is_null(heap_oop)) { + // if (!CompressedOops::is_null(heap_oop)) { + if (narrow) { + guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); + p = (T*) (uintptr_t(p) | (1ull << 63)); + } _buffer[_cursor++] = (void*) p; if (_cursor >= _cap) { flush(); } - } + // } } void flush() { @@ -49,28 +53,28 @@ class MMTkRootsClosure2 : public OopClosure { } } - virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p, false); } + virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } }; class MMTkScanObjectClosure : public BasicOopIterateClosure { - void* _trace; + void (*_trace)(void* edge); CLDToOopClosure follow_cld_closure; template - void do_oop_work(T* p) { - // oop ref = (void*) oopDesc::decode_heap_oop(oopDesc::load_heap_oop(p)); - // process_edge(_trace, (void*) p); + void do_oop_work(T* p, bool narrow) { + if (narrow) { + guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); + p = (T*) (uintptr_t(p) | (1ull << 63)); + } + _trace((void*) p); } public: - MMTkScanObjectClosure(void* trace): _trace(trace), follow_cld_closure(this, false) {} + MMTkScanObjectClosure(void* trace): _trace((void (*)(void*)) trace), follow_cld_closure(this, false) {} - virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { - // printf("narrowoop edge %p -> %d %p %p\n", (void*) p, *p, *((void**) p), (void*) oopDesc::load_decode_heap_oop(p)); - do_oop_work(p); - } + virtual void do_oop(oop* p) { do_oop_work(p, false); } + virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } virtual bool do_metadata() { return true; diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 18b4d55d..72b9340b 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -228,7 +228,6 @@ static void mmtk_scan_thread_roots(EdgesClosure closure, void* tls) { } static void mmtk_scan_object(void* trace, void* object, void* tls) { - guarantee(false, "unreachable"); MMTkScanObjectClosure cl(trace); ((oop) object)->oop_iterate(&cl); } @@ -238,7 +237,7 @@ static void mmtk_dump_object(void* object) { // o->print(); o->print_value(); - printf("\n"); + // printf("\n"); // o->print_address(); } @@ -246,8 +245,8 @@ static void mmtk_dump_object(void* object) { static size_t mmtk_get_object_size(void* object) { oop o = (oop) object; // Slow-dispatch only. The fast-path code is moved to rust. - auto klass = o->klass(); - return klass->oop_size(o) << LogHeapWordSize; + // auto klass = o->klass(); + return o->size() << LogHeapWordSize; } static void mmtk_harness_begin() { @@ -293,8 +292,9 @@ static int discovered_offset() { } static char* dump_object_string(void* object) { + ResourceMark rm; oop o = (oop) object; - return o->print_value_string(); + return o->print_string(); } static void mmtk_schedule_finalizer() { @@ -355,6 +355,14 @@ static size_t mmtk_compressed_klass_shift() { return (size_t) Universe::narrow_klass_shift(); } +static size_t mmtk_oop_array_base_offset_in_bytes() { + return size_t(arrayOopDesc::base_offset_in_bytes(T_OBJECT)); +} + +static size_t mmtk_oop_array_length_offset_in_bytes() { + return size_t(arrayOopDesc::length_offset_in_bytes()); +} + OpenJDK_Upcalls mmtk_upcalls = { mmtk_stop_all_mutators, mmtk_resume_mutators, @@ -396,4 +404,6 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_enqueue_references, mmtk_compressed_klass_base, mmtk_compressed_klass_shift, + mmtk_oop_array_base_offset_in_bytes, + mmtk_oop_array_length_offset_in_bytes, }; diff --git a/openjdk/thirdPartyHeapArguments.cpp b/openjdk/thirdPartyHeapArguments.cpp index 1f14fa63..94fa2941 100644 --- a/openjdk/thirdPartyHeapArguments.cpp +++ b/openjdk/thirdPartyHeapArguments.cpp @@ -44,7 +44,7 @@ void ThirdPartyHeapArguments::initialize() { assert(UseThirdPartyHeap , "Error, should UseThirdPartyHeap"); FLAG_SET_DEFAULT(UseTLAB, false); FLAG_SET_DEFAULT(UseCompressedOops, true); - FLAG_SET_DEFAULT(UseCompressedClassPointers, true); + FLAG_SET_DEFAULT(UseCompressedClassPointers, false); } CollectedHeap* ThirdPartyHeapArguments::create_heap() { From a8d9d1caef46e51ef139a693ed778d6e8c2217f9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 17 Aug 2022 10:18:25 +1000 Subject: [PATCH 144/494] Single gc thread working --- openjdk/mmtkBarrierSet.hpp | 2 +- openjdk/mmtkRootsClosure.hpp | 10 ++++++++++ openjdk/mmtkUpcalls.cpp | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 51dbb3c9..5ca0ee73 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -36,7 +36,7 @@ #include "oops/oopsHierarchy.hpp" #include "utilities/fakeRttiSupport.hpp" -#define MMTK_ENABLE_ALLOCATION_FASTPATH false +#define MMTK_ENABLE_ALLOCATION_FASTPATH true const intptr_t ALLOC_BIT_BASE_ADDRESS = GLOBAL_ALLOC_BIT_ADDRESS; diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 8308a83f..4d564f3c 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -103,4 +103,14 @@ class MMTkScanObjectClosure : public BasicOopIterateClosure { // } // }; +class CodeBlobFixRelocationClosure: public CodeBlobClosure { + public: + inline virtual void do_code_blob(CodeBlob* cb) { + nmethod* nm = cb->as_nmethod_or_null(); + if (nm != NULL) { + nm->fix_oop_relocations(); + } + } +}; + #endif // MMTK_OPENJDK_MMTK_ROOTS_CLOSURE_HPP diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 72b9340b..8e9e562e 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -77,6 +77,10 @@ static void mmtk_resume_mutators(void *tls) { #if COMPILER2_OR_JVMCI DerivedPointerTable::update_pointers(); #endif + { + CodeBlobFixRelocationClosure cb_cl; + CodeCache::blobs_do(&cb_cl); + } // Note: we don't have to hold gc_lock to increment the counter. // The increment has to be done before mutators can be resumed From 1ea39e882cadedf49e603e6d8fea6350376c204d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 17 Aug 2022 10:32:18 +1000 Subject: [PATCH 145/494] cleanup --- mmtk/src/object_scanning.rs | 94 +++++++++++-------------------- openjdk/mmtkRootsClosure.hpp | 4 +- openjdk/mmtkUpcalls.cpp | 14 ++--- openjdk/mmtkVMCompanionThread.cpp | 4 +- 4 files changed, 40 insertions(+), 76 deletions(-) diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index c5a0077d..e5cb507f 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -156,79 +156,49 @@ fn oop_iterate_slow(oop: Oop, process_edge: extern "C" fn(Address), tls: OpaqueP } } -static mut FIELDS: Vec
= Vec::new(); - #[inline] fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { - unsafe { - FIELDS = Vec::new(); - } - extern "C" fn report_edge(a: Address) { - unsafe { FIELDS.push(a) } - } - oop_iterate_slow(oop, report_edge, OpaquePointer::UNINITIALIZED); - unsafe { - for e in &FIELDS { - closure.visit_edge(*e) + let klass_id = oop.klass().id; + debug_assert!( + klass_id as i32 >= 0 && (klass_id as i32) < 6, + "Invalid klass-id: {:x} for oop: {:x}", + klass_id as i32, + unsafe { mem::transmute::(oop) } + ); + match klass_id { + KlassID::Instance => { + let instance_klass = unsafe { oop.klass().cast::() }; + instance_klass.oop_iterate(oop, closure); + } + KlassID::InstanceClassLoader => { + let instance_klass = unsafe { oop.klass().cast::() }; + instance_klass.oop_iterate(oop, closure); } + KlassID::InstanceMirror => { + let instance_klass = unsafe { oop.klass().cast::() }; + instance_klass.oop_iterate(oop, closure); + } + KlassID::ObjArray => { + let array_klass = unsafe { oop.klass().cast::() }; + array_klass.oop_iterate(oop, closure); + } + KlassID::TypeArray => { + let array_klass = unsafe { oop.klass().cast::() }; + array_klass.oop_iterate(oop, closure); + } + KlassID::InstanceRef => { + let instance_klass = unsafe { oop.klass().cast::() }; + instance_klass.oop_iterate(oop, closure); + } // _ => oop_iterate_slow(oop, closure, tls), } - // let klass = oop.klass(); - // let klass_id = oop.klass().id; - // debug_assert!( - // klass_id as i32 >= 0 && (klass_id as i32) < 6, - // "Invalid klass-id: {:x} for oop: {:x}", - // klass_id as i32, - // unsafe { mem::transmute::(oop) } - // ); - // unsafe { - // ((*UPCALLS).dump_object)(mem::transmute(oop)); - // } - // // println!("oop {:?}", unsafe { - // // let c_string = ((*UPCALLS).dump_object)(mem::transmute(oop)); - // // let c_str: &CStr = unsafe { CStr::from_ptr(c_string) }; - // // c_str - // // }); - // match klass_id { - // KlassID::Instance => { - // let instance_klass = unsafe { klass.cast::() }; - // instance_klass.oop_iterate(oop, closure); - // } - // KlassID::InstanceClassLoader => { - // let instance_klass = unsafe { klass.cast::() }; - // instance_klass.oop_iterate(oop, closure); - // } - // KlassID::InstanceMirror => { - // let instance_klass = unsafe { klass.cast::() }; - // instance_klass.oop_iterate(oop, closure); - // } - // KlassID::ObjArray => { - // let array_klass = unsafe { klass.cast::() }; - // array_klass.oop_iterate(oop, closure); - // } - // KlassID::TypeArray => { - // let array_klass = unsafe { klass.cast::() }; - // array_klass.oop_iterate(oop, closure); - // } - // KlassID::InstanceRef => { - // let instance_klass = unsafe { klass.cast::() }; - // instance_klass.oop_iterate(oop, closure); - // } // _ => oop_iterate_slow(oop, closure, tls), - // } } #[inline] -pub fn scan_object(object: ObjectReference, closure: &mut T, _tls: VMWorkerThread) { - // println!( - // "Scan {:?} (klass={:?} id={:?})", - // object, - // unsafe { (object.to_address() + 8usize).load::
() }, - // unsafe { mem::transmute::<_, Oop>(object).klass().id } - // ); +pub fn scan_object(object: ObjectReference, closure: &mut impl EdgeVisitor, _tls: VMWorkerThread) { // println!("*****scan_object(0x{:x}) -> \n 0x{:x}, 0x{:x} \n", // object, // unsafe { *(object.value() as *const usize) }, // unsafe { *((object.value() + 8) as *const usize) } // ); - let closure = unsafe { &mut *(closure as *mut T) }; unsafe { oop_iterate(mem::transmute(object), closure) } } diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 4d564f3c..970e39a6 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -18,7 +18,7 @@ class MMTkRootsClosure2 : public OopClosure { template void do_oop_work(T* p, bool narrow) { T heap_oop = RawAccess<>::oop_load(p); - // if (!CompressedOops::is_null(heap_oop)) { + if (!CompressedOops::is_null(heap_oop)) { if (narrow) { guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); p = (T*) (uintptr_t(p) | (1ull << 63)); @@ -27,7 +27,7 @@ class MMTkRootsClosure2 : public OopClosure { if (_cursor >= _cap) { flush(); } - // } + } } void flush() { diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 8e9e562e..2a818826 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -46,8 +46,6 @@ static volatile size_t mmtk_start_the_world_count = 0; static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure) { - - printf("mmtk_stop_all_mutators\n"); ClassLoaderDataGraph::clear_claimed_marks(); CodeCache::gc_prologue(); #if COMPILER2_OR_JVMCI @@ -64,12 +62,11 @@ static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, M closure.invoke((void*)&cur->third_party_heap_mutator); } } - printf("Finished enumerating threads.\n"); + log_debug(gc)("Finished enumerating threads."); nmethod::oops_do_marking_prologue(); } static void mmtk_resume_mutators(void *tls) { - printf("mmtk_resume_mutators\n"); nmethod::oops_do_marking_epilogue(); // ClassLoaderDataGraph::purge(); CodeCache::gc_epilogue(); @@ -87,7 +84,7 @@ static void mmtk_resume_mutators(void *tls) { // otherwise, mutators might see a stale value Atomic::inc(&mmtk_start_the_world_count); - printf("Requesting the VM to resume all mutators...\n"); + log_debug(gc)("Requesting the VM to resume all mutators..."); MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_resumed, true); log_debug(gc)("Mutators resumed. Now notify any mutators waiting for GC to finish..."); @@ -219,7 +216,6 @@ static void mmtk_reset_mutator_iterator() { static void mmtk_scan_all_thread_roots(EdgesClosure closure) { - guarantee(false, "unreachable"); MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_thread_roots(cl); } @@ -241,7 +237,7 @@ static void mmtk_dump_object(void* object) { // o->print(); o->print_value(); - // printf("\n"); + printf("\n"); // o->print_address(); } @@ -296,9 +292,8 @@ static int discovered_offset() { } static char* dump_object_string(void* object) { - ResourceMark rm; oop o = (oop) object; - return o->print_string(); + return o->print_value_string(); } static void mmtk_schedule_finalizer() { @@ -330,7 +325,6 @@ static void mmtk_prepare_for_roots_re_scanning() { } static void mmtk_enqueue_references(void** objects, size_t len) { - guarantee(false, "unreachable"); if (len == 0) { return; } diff --git a/openjdk/mmtkVMCompanionThread.cpp b/openjdk/mmtkVMCompanionThread.cpp index d5e16d8a..17f2d2c1 100644 --- a/openjdk/mmtkVMCompanionThread.cpp +++ b/openjdk/mmtkVMCompanionThread.cpp @@ -59,7 +59,7 @@ void MMTkVMCompanionThread::run() { } // Let the VM thread stop the world. - printf("MMTkVMCompanionThread: Letting VMThread execute VM op...\n"); + log_trace(gc)("MMTkVMCompanionThread: Letting VMThread execute VM op..."); VM_MMTkSTWOperation op(this); // VMThread::execute() is blocking. The companion thread will be blocked // here waiting for the VM thread to execute op, and the VM thread will @@ -68,7 +68,7 @@ void MMTkVMCompanionThread::run() { VMThread::execute(&op); // Tell the waiter thread that the world has resumed. - printf("MMTkVMCompanionThread: Notifying threads resumption...\n"); + log_trace(gc)("MMTkVMCompanionThread: Notifying threads resumption..."); { MutexLockerEx locker(_lock, Mutex::_no_safepoint_check_flag); assert(_desired_state == _threads_resumed, "start-the-world should be requested."); From 547de4eddf4cafa02cd475196b24230f6d6d9c6b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 17 Aug 2022 10:55:26 +1000 Subject: [PATCH 146/494] cleanup --- mmtk/src/abi.rs | 76 +++++++++++------------------ mmtk/src/object_model.rs | 7 --- mmtk/src/object_scanning.rs | 3 -- openjdk/mmtkFinalizerThread.cpp | 2 - openjdk/mmtkUpcalls.cpp | 4 +- openjdk/thirdPartyHeapArguments.cpp | 2 +- 6 files changed, 32 insertions(+), 62 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 37538378..c05cadbe 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -261,21 +261,13 @@ pub struct InstanceRefKlass { impl InstanceRefKlass { fn referent_offset() -> i32 { lazy_static! { - pub static ref REFERENT_OFFSET: i32 = unsafe { - let referent_offset = ((*UPCALLS).referent_offset)(); - println!("referent_offset 0x{:x}", referent_offset); - referent_offset - }; + pub static ref REFERENT_OFFSET: i32 = unsafe { ((*UPCALLS).referent_offset)() }; } *REFERENT_OFFSET } fn discovered_offset() -> i32 { lazy_static! { - pub static ref DISCOVERED_OFFSET: i32 = unsafe { - let discovered_offset = ((*UPCALLS).discovered_offset)(); - println!("discovered_offset 0x{:x}", discovered_offset); - discovered_offset - }; + pub static ref DISCOVERED_OFFSET: i32 = unsafe { ((*UPCALLS).discovered_offset)() }; } *DISCOVERED_OFFSET } @@ -300,19 +292,10 @@ impl OopDesc { } #[inline(always)] pub fn klass(&self) -> &'static Klass { - self.klass - // let compressed = unsafe { (self.start() + std::mem::size_of::()).load::() }; - // let addr = - // unsafe { *COMPRESSED_KLASS_BASE + ((compressed as usize) << *COMPRESSED_KLASS_SHIFT) }; - // // println!( - // // "oop({:?}) klass: c={:x} base={:?} shift={:?} addr={:?}", - // // self.start(), - // // compressed, - // // *COMPRESSED_KLASS_BASE, - // // *COMPRESSED_KLASS_SHIFT, - // // addr, - // // ); - // unsafe { &*addr.to_ptr::() } + // self.klass + let compressed = unsafe { (self.start() + std::mem::size_of::()).load::() }; + let addr = *COMPRESSED_KLASS_BASE + ((compressed as usize) << *COMPRESSED_KLASS_SHIFT); + unsafe { &*addr.to_ptr::() } } } @@ -370,30 +353,29 @@ impl OopDesc { /// Calculate object instance size #[inline(always)] pub unsafe fn size(&self) -> usize { - self.size_slow() - // let klass = self.klass(); - // let lh = klass.layout_helper; - // // The (scalar) instance size is pre-recorded in the TIB? - // if lh > Klass::LH_NEUTRAL_VALUE { - // if !Klass::layout_helper_needs_slow_path(lh) { - // lh as _ - // } else { - // self.size_slow() - // } - // } else if lh <= Klass::LH_NEUTRAL_VALUE { - // if lh < Klass::LH_NEUTRAL_VALUE { - // // Calculate array size - // let array_length = self.as_array_oop().length(); - // let mut size_in_bytes: usize = - // (array_length as usize) << Klass::layout_helper_log2_element_size(lh); - // size_in_bytes += Klass::layout_helper_header_size(lh) as usize; - // (size_in_bytes + 0b111) & !0b111 - // } else { - // self.size_slow() - // } - // } else { - // unreachable!() - // } + let klass = self.klass(); + let lh = klass.layout_helper; + // The (scalar) instance size is pre-recorded in the TIB? + if lh > Klass::LH_NEUTRAL_VALUE { + if !Klass::layout_helper_needs_slow_path(lh) { + lh as _ + } else { + self.size_slow() + } + } else if lh <= Klass::LH_NEUTRAL_VALUE { + if lh < Klass::LH_NEUTRAL_VALUE { + // Calculate array size + let array_length = self.as_array_oop().length(); + let mut size_in_bytes: usize = + (array_length as usize) << Klass::layout_helper_log2_element_size(lh); + size_in_bytes += Klass::layout_helper_header_size(lh) as usize; + (size_in_bytes + 0b111) & !0b111 + } else { + self.size_slow() + } + } else { + unreachable!() + } } } diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 025df37c..d4c9f10d 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -94,20 +94,13 @@ impl ObjectModel for VMObjectModel { let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::(), 0, copy); // Copy let src = from.to_address(); - // println!("cp {:?} -> {:?}", src..src + bytes, dst..dst + bytes); unsafe { std::ptr::copy_nonoverlapping::(src.to_ptr(), dst.to_mut_ptr(), bytes) } let to_obj = unsafe { dst.to_object_reference() }; copy_context.post_copy(to_obj, bytes, copy); - unsafe { - let a = dst; - debug_assert_ne!(a, Address::from_usize(0x20000000000)); - debug_assert!(a < Address::from_usize(0x200c0000000)); - } to_obj } fn copy_to(from: ObjectReference, to: ObjectReference, region: Address) -> Address { - unreachable!(); let need_copy = from != to; let bytes = unsafe { ((*UPCALLS).get_object_size)(from) }; if need_copy { diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index e5cb507f..fb806ea9 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -1,10 +1,8 @@ use super::abi::*; use super::UPCALLS; -use mmtk::util::constants::*; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::EdgeVisitor; -use std::ffi::CStr; use std::{mem, slice}; trait OopIterate: Sized { @@ -113,7 +111,6 @@ impl OopIterate for InstanceRefKlass { self.instance_klass.oop_iterate(oop, closure); if Self::should_scan_weak_refs() { - unreachable!(); let reference = ObjectReference::from(oop); match self.instance_klass.reference_type { ReferenceType::None => { diff --git a/openjdk/mmtkFinalizerThread.cpp b/openjdk/mmtkFinalizerThread.cpp index 2d9b6452..f18f0e24 100644 --- a/openjdk/mmtkFinalizerThread.cpp +++ b/openjdk/mmtkFinalizerThread.cpp @@ -100,7 +100,6 @@ void MMTkFinalizerThread::finalizer_thread_entry(JavaThread* thread, TRAPS) { void* obj_ref = get_finalized_object(); if (obj_ref != NULL) { instanceOop obj = (instanceOop) obj_ref; - // printf("finalize %p\n", obj); // Invoke finalize() { @@ -112,7 +111,6 @@ void MMTkFinalizerThread::finalizer_thread_entry(JavaThread* thread, TRAPS) { JavaCalls::call_virtual(&ret, handle_obj, obj->klass(), finalize_method, sig, this_thread); } - // printf("finalize done %p\n", obj); } else { break; } diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 2a818826..52f9ae29 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -245,8 +245,8 @@ static void mmtk_dump_object(void* object) { static size_t mmtk_get_object_size(void* object) { oop o = (oop) object; // Slow-dispatch only. The fast-path code is moved to rust. - // auto klass = o->klass(); - return o->size() << LogHeapWordSize; + auto klass = o->klass(); + return klass->oop_size(o) << LogHeapWordSize; } static void mmtk_harness_begin() { diff --git a/openjdk/thirdPartyHeapArguments.cpp b/openjdk/thirdPartyHeapArguments.cpp index 94fa2941..1f14fa63 100644 --- a/openjdk/thirdPartyHeapArguments.cpp +++ b/openjdk/thirdPartyHeapArguments.cpp @@ -44,7 +44,7 @@ void ThirdPartyHeapArguments::initialize() { assert(UseThirdPartyHeap , "Error, should UseThirdPartyHeap"); FLAG_SET_DEFAULT(UseTLAB, false); FLAG_SET_DEFAULT(UseCompressedOops, true); - FLAG_SET_DEFAULT(UseCompressedClassPointers, false); + FLAG_SET_DEFAULT(UseCompressedClassPointers, true); } CollectedHeap* ThirdPartyHeapArguments::create_heap() { From e9a88e25d757385c87ea8a031dcc2eb9cb084a18 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 17 Aug 2022 11:33:19 +1000 Subject: [PATCH 147/494] Remove placeholder space --- openjdk/mmtkHeap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 6faae50a..169459d3 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -105,7 +105,7 @@ jint MMTkHeap::initialize() { _end = (HeapWord*) last_heap_address(); printf("start: %p, end: %p, base: %p\n", _start, _end, (void*) (intptr_t(_start) - 4096)); - Universe::set_narrow_oop_base((address) (intptr_t(_start))); + Universe::set_narrow_oop_base((address) (intptr_t(_start) - 4096)); Universe::set_narrow_oop_shift(LogMinObjAlignmentInBytes); initialize_reserved_region((HeapWord*) (intptr_t(_start) + (1ull << 30)), _end); From 99c5ba33df8464852623f11d00bda6d255cc73d2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 18 Aug 2022 12:39:17 +1000 Subject: [PATCH 148/494] cleanup --- mmtk/src/abi.rs | 47 ++++++++++++++++------------- mmtk/src/api.rs | 5 +++ mmtk/src/lib.rs | 21 +++++++++++-- mmtk/src/object_model.rs | 42 ++++++++++++++++++++++++++ mmtk/src/object_scanning.rs | 33 ++++++++++++++------ openjdk/mmtk.h | 4 +-- openjdk/mmtkHeap.cpp | 21 ++++++++----- openjdk/mmtkUpcalls.cpp | 10 ------ openjdk/thirdPartyHeapArguments.cpp | 6 ++-- 9 files changed, 136 insertions(+), 53 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index c05cadbe..f00c7607 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -7,11 +7,6 @@ use std::ffi::CStr; use std::fmt; use std::{mem, slice}; -lazy_static! { - static ref COMPRESSED_KLASS_BASE: Address = unsafe { ((*UPCALLS).compressed_klass_base)() }; - static ref COMPRESSED_KLASS_SHIFT: usize = unsafe { ((*UPCALLS).compressed_klass_shift)() }; -} - #[repr(i32)] #[derive(Copy, Clone, PartialEq, Eq, Debug)] #[allow(dead_code)] @@ -279,10 +274,16 @@ impl InstanceRefKlass { } } +#[repr(C)] +union KlassField { + klass: &'static Klass, + narrow_klass: u32, +} + #[repr(C)] pub struct OopDesc { pub mark: usize, - klass: &'static Klass, + klass: KlassField, } impl OopDesc { @@ -293,9 +294,19 @@ impl OopDesc { #[inline(always)] pub fn klass(&self) -> &'static Klass { // self.klass - let compressed = unsafe { (self.start() + std::mem::size_of::()).load::() }; - let addr = *COMPRESSED_KLASS_BASE + ((compressed as usize) << *COMPRESSED_KLASS_SHIFT); - unsafe { &*addr.to_ptr::() } + if *crate::USE_COMPRESSED_OOPS { + lazy_static! { + static ref COMPRESSED_KLASS_BASE: Address = + unsafe { ((*UPCALLS).compressed_klass_base)() }; + static ref COMPRESSED_KLASS_SHIFT: usize = + unsafe { ((*UPCALLS).compressed_klass_shift)() }; + } + let compressed = unsafe { self.klass.narrow_klass }; + let addr = *COMPRESSED_KLASS_BASE + ((compressed as usize) << *COMPRESSED_KLASS_SHIFT); + unsafe { &*addr.to_ptr::() } + } else { + unsafe { self.klass.klass } + } } } @@ -386,11 +397,11 @@ pub type ArrayOop = &'static ArrayOopDesc; impl ArrayOopDesc { fn length_offset() -> usize { - lazy_static! { - static ref OOP_ARRAY_LENGTH_OFFSET_IN_BYTES: usize = - unsafe { ((*UPCALLS).oop_array_length_offset_in_bytes)() }; + if *crate::USE_COMPRESSED_OOPS { + mem::size_of::() + mem::size_of::() + } else { + mem::size_of::() } - *OOP_ARRAY_LENGTH_OFFSET_IN_BYTES } fn element_type_should_be_aligned(ty: BasicType) -> bool { @@ -410,14 +421,8 @@ impl ArrayOopDesc { unsafe { *((self as *const _ as *const u8).add(Self::length_offset()) as *const i32) } } fn base(&self, ty: BasicType) -> Address { - debug_assert_eq!(ty, BasicType::T_OBJECT); - lazy_static! { - static ref OOP_ARRAY_BASE_OFFSET_IN_BYTES: usize = - unsafe { ((*UPCALLS).oop_array_base_offset_in_bytes)() }; - } - Address::from_ptr(unsafe { - (self as *const _ as *const u8).add(*OOP_ARRAY_BASE_OFFSET_IN_BYTES) - }) + let base_offset_in_bytes = Self::header_size(ty) * BYTES_IN_WORD; + Address::from_ptr(unsafe { (self as *const _ as *const u8).add(base_offset_in_bytes) }) } // This provides an easy way to access the array data in Rust. However, the array data // is Java types, so we have to map Java types to Rust types. The caller needs to guarantee: diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 66e21f9a..f454b32c 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -203,6 +203,11 @@ pub extern "C" fn handle_user_collection_request(tls: VMMutatorThread) { memory_manager::handle_user_collection_request::(&SINGLETON, tls); } +#[no_mangle] +pub extern "C" fn mmtk_use_compressed_ptrs() -> bool { + *crate::USE_COMPRESSED_OOPS +} + #[no_mangle] pub extern "C" fn is_in_mmtk_spaces(object: ObjectReference) -> bool { memory_manager::is_in_mmtk_spaces(object) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 1fd90910..93bd07b3 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -96,8 +96,6 @@ pub struct OpenJDK_Upcalls { pub enqueue_references: extern "C" fn(objects: *const ObjectReference, len: usize), pub compressed_klass_base: extern "C" fn() -> Address, pub compressed_klass_shift: extern "C" fn() -> usize, - pub oop_array_base_offset_in_bytes: extern "C" fn() -> usize, - pub oop_array_length_offset_in_bytes: extern "C" fn() -> usize, } pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); @@ -114,6 +112,25 @@ pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS: uintptr_t = pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t = crate::mmtk::util::metadata::side_metadata::ALLOC_SIDE_METADATA_ADDR.as_usize(); +static mut LOG_BYTES_IN_OOP: usize = 3; + +#[inline(always)] +fn log_bytes_in_oop() -> usize { + unsafe { LOG_BYTES_IN_OOP } +} + +lazy_static! { + static ref USE_COMPRESSED_OOPS: bool = { + let enable = std::env::var("MMTK_COMPRESSED_PTRS") + .map(|s| s.trim() != "0" || s.trim() != "false") + .unwrap_or(false); + if enable { + unsafe { LOG_BYTES_IN_OOP = 2 } + } + enable + }; +} + #[derive(Default)] pub struct OpenJDK; diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index d4c9f10d..f9fb9b6c 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -5,6 +5,7 @@ use crate::abi::Oop; use crate::{vm_metadata, OpenJDK}; use mmtk::util::alloc::fill_alignment_gap; use mmtk::util::copy::*; +use mmtk::util::heap::layout::vm_layout_constants::HEAP_START; use mmtk::util::metadata::header_metadata::HeaderMetadataSpec; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::*; @@ -22,6 +23,47 @@ impl ObjectModel for VMObjectModel { const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = vm_metadata::MARKING_METADATA_SPEC; const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = vm_metadata::LOS_METADATA_SPEC; + #[inline(always)] + fn load_reference_field(slot: Address, root: bool) -> ObjectReference { + if *crate::USE_COMPRESSED_OOPS { + let narrow_root = slot.as_usize() & (1usize << 63) != 0; + let slot = unsafe { Address::from_usize(slot.as_usize() << 1 >> 1) }; + if root && !narrow_root { + unsafe { slot.load::() } + } else { + debug_assert!(root || !narrow_root); + let v = unsafe { slot.load::() }; + if v == 0 { + ObjectReference::NULL + } else { + unsafe { (HEAP_START + ((v as usize) << 3) - 4096).to_object_reference() } + } + } + } else { + unsafe { slot.load() } + } + } + + #[inline(always)] + fn store_reference_field(slot: Address, object: ObjectReference, root: bool) { + if *crate::USE_COMPRESSED_OOPS { + let narrow_root = slot.as_usize() & (1usize << 63) != 0; + let slot = unsafe { Address::from_usize(slot.as_usize() << 1 >> 1) }; + if root && !narrow_root { + unsafe { slot.store(object) } + } else { + debug_assert!(root || !narrow_root); + if object.is_null() { + unsafe { slot.store(0u32) }; + } else { + unsafe { slot.store(((object.to_address() - HEAP_START + 4096) >> 3) as u32) } + } + } + } else { + unsafe { slot.store(object) } + } + } + #[inline(always)] fn load_metadata( metadata_spec: &HeaderMetadataSpec, diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index fb806ea9..c96d78b5 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -14,7 +14,7 @@ impl OopIterate for OopMapBlock { fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { let start = oop.get_field_address(self.offset); for i in 0..self.count as usize { - let edge = start + (i << 2); + let edge = start + (i << crate::log_bytes_in_oop()); closure.visit_edge(edge); } } @@ -62,11 +62,20 @@ impl OopIterate for InstanceMirrorKlass { // } // static fields - let start: *const NarrowOop = Self::start_of_static_fields(oop).to_ptr::(); + let start = Self::start_of_static_fields(oop); let len = Self::static_oop_field_count(oop); - let slice = unsafe { slice::from_raw_parts(start, len as _) }; - for narrow_oop in slice { - closure.visit_edge(narrow_oop.slot()); + if *crate::USE_COMPRESSED_OOPS { + let start: *const NarrowOop = start.to_ptr::(); + let slice = unsafe { slice::from_raw_parts(start, len as _) }; + for narrow_oop in slice { + closure.visit_edge(narrow_oop.slot()); + } + } else { + let start: *const Oop = start.to_ptr::(); + let slice = unsafe { slice::from_raw_parts(start, len as _) }; + for oop in slice { + closure.visit_edge(Address::from_ref(oop as &Oop)); + } } } } @@ -89,8 +98,14 @@ impl OopIterate for ObjArrayKlass { #[inline] fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { let array = unsafe { oop.as_array_oop() }; - for narrow_oop in unsafe { array.data::(BasicType::T_OBJECT) } { - closure.visit_edge(narrow_oop.slot()); + if *crate::USE_COMPRESSED_OOPS { + for narrow_oop in unsafe { array.data::(BasicType::T_OBJECT) } { + closure.visit_edge(narrow_oop.slot()); + } + } else { + for oop in unsafe { array.data::(BasicType::T_OBJECT) } { + closure.visit_edge(Address::from_ref(oop as &Oop)); + } } } } @@ -147,9 +162,9 @@ impl InstanceRefKlass { } #[allow(unused)] -fn oop_iterate_slow(oop: Oop, process_edge: extern "C" fn(Address), tls: OpaquePointer) { +fn oop_iterate_slow(oop: Oop, closure: &mut impl EdgeVisitor, tls: OpaquePointer) { unsafe { - ((*UPCALLS).scan_object)(process_edge as _, mem::transmute(oop), tls); + ((*UPCALLS).scan_object)(closure as *mut _ as _, mem::transmute(oop), tls); } } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 2b5ea196..d6cf8c28 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -164,8 +164,6 @@ typedef struct { void (*enqueue_references)(void** objects, size_t len); void* (*compressed_klass_base)(); size_t (*compressed_klass_shift)(); - size_t (*oop_array_base_offset_in_bytes)(); - size_t (*oop_array_length_offset_in_bytes)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); @@ -173,6 +171,8 @@ extern bool openjdk_is_gc_initialized(); extern bool mmtk_set_heap_size(size_t size); +extern bool mmtk_use_compressed_ptrs(); + extern size_t used_bytes(); extern void* starting_heap_address(); extern void* last_heap_address(); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 169459d3..f1476c4e 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -71,6 +71,14 @@ MMTkHeap::MMTkHeap(MMTkCollectorPolicy* policy) : CollectedHeap(), _last_gc_time jint MMTkHeap::initialize() { assert(!UseTLAB , "should disable UseTLAB"); + auto compressed_oops = mmtk_use_compressed_ptrs(); + if (compressed_oops) { + assert(UseCompressedOops , "should enable CompressedOops"); + assert(UseCompressedClassPointers , "should enable UseCompressedClassPointers"); + } else { + assert(!UseCompressedOops , "should disable CompressedOops"); + assert(!UseCompressedClassPointers , "should disable UseCompressedClassPointers"); + } const size_t heap_size = collector_policy()->max_heap_byte_size(); // printf("policy max heap size %zu, min heap size %zu\n", heap_size, collector_policy()->min_heap_byte_size()); @@ -103,15 +111,14 @@ jint MMTkHeap::initialize() { _start = (HeapWord*) starting_heap_address(); _end = (HeapWord*) last_heap_address(); - printf("start: %p, end: %p, base: %p\n", _start, _end, (void*) (intptr_t(_start) - 4096)); - - Universe::set_narrow_oop_base((address) (intptr_t(_start) - 4096)); - Universe::set_narrow_oop_shift(LogMinObjAlignmentInBytes); - - initialize_reserved_region((HeapWord*) (intptr_t(_start) + (1ull << 30)), _end); + // printf("start: %p, end: %p\n", _start, _end); + if (compressed_oops) { + Universe::set_narrow_oop_base((address) (intptr_t(_start) - 4096)); + Universe::set_narrow_oop_shift(LogMinObjAlignmentInBytes); + } + initialize_reserved_region(_start, _end); - printf("compressed oops: base=%p, shift=%zu\n", Universe::narrow_oop_base(), Universe::narrow_oop_shift()); MMTkBarrierSet* const barrier_set = new MMTkBarrierSet(reserved_region()); //barrier_set->initialize(); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 52f9ae29..0457cec1 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -353,14 +353,6 @@ static size_t mmtk_compressed_klass_shift() { return (size_t) Universe::narrow_klass_shift(); } -static size_t mmtk_oop_array_base_offset_in_bytes() { - return size_t(arrayOopDesc::base_offset_in_bytes(T_OBJECT)); -} - -static size_t mmtk_oop_array_length_offset_in_bytes() { - return size_t(arrayOopDesc::length_offset_in_bytes()); -} - OpenJDK_Upcalls mmtk_upcalls = { mmtk_stop_all_mutators, mmtk_resume_mutators, @@ -402,6 +394,4 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_enqueue_references, mmtk_compressed_klass_base, mmtk_compressed_klass_shift, - mmtk_oop_array_base_offset_in_bytes, - mmtk_oop_array_length_offset_in_bytes, }; diff --git a/openjdk/thirdPartyHeapArguments.cpp b/openjdk/thirdPartyHeapArguments.cpp index 1f14fa63..cd019347 100644 --- a/openjdk/thirdPartyHeapArguments.cpp +++ b/openjdk/thirdPartyHeapArguments.cpp @@ -43,8 +43,10 @@ void ThirdPartyHeapArguments::initialize() { GCArguments::initialize(); assert(UseThirdPartyHeap , "Error, should UseThirdPartyHeap"); FLAG_SET_DEFAULT(UseTLAB, false); - FLAG_SET_DEFAULT(UseCompressedOops, true); - FLAG_SET_DEFAULT(UseCompressedClassPointers, true); + auto compressed_oops = mmtk_use_compressed_ptrs(); + printf("MMTk Compressed Pointers: %s\n", compressed_oops ? "enabled" : "disabled"); + FLAG_SET_DEFAULT(UseCompressedOops, compressed_oops); + FLAG_SET_DEFAULT(UseCompressedClassPointers, compressed_oops); } CollectedHeap* ThirdPartyHeapArguments::create_heap() { From d8932872c9d62ac49c1fc6f9e5a00287739817ec Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 18 Aug 2022 12:59:04 +1000 Subject: [PATCH 149/494] cleanup --- openjdk/mmtkHeap.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 72341dd7..6c27dd09 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -299,7 +299,6 @@ size_t MMTkHeap::block_size(const HeapWord* addr) const { //OK } bool MMTkHeap::block_is_obj(const HeapWord* addr) const { //OK - // guarantee(false, "block is obj not supported"); return size_t(addr) >= size_t(starting_heap_address()) && size_t(addr) < size_t(last_heap_address()); } From 2e95d7f0e2bc61cd0f0ef2dfdb4bc867b54c2c3d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 22 Aug 2022 10:18:01 +1000 Subject: [PATCH 150/494] relocate nmethod in work packets --- mmtk/src/gc_work.rs | 19 ++++++++++--------- mmtk/src/lib.rs | 1 + openjdk/mmtk.h | 1 + openjdk/mmtkUpcalls.cpp | 9 +++++---- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 098092e5..f65fa334 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -1,5 +1,3 @@ -use std::sync::atomic::Ordering; - use super::{OpenJDK, UPCALLS}; use mmtk::scheduler::*; use mmtk::vm::RootsWorkFactory; @@ -56,14 +54,17 @@ impl ScanCodeCacheRoots { impl GCWork for ScanCodeCacheRoots { fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { // Collect all the cached roots - let mut edges = Vec::with_capacity(crate::CODE_CACHE_ROOTS_SIZE.load(Ordering::Relaxed)); - for roots in (*crate::CODE_CACHE_ROOTS.lock().unwrap()).values() { - for r in roots { - edges.push(*r) - } + for (nmethod, roots) in (*crate::CODE_CACHE_ROOTS.lock().unwrap()).iter() { + let edges = roots.clone(); + let nmethod = *nmethod; + // Create work packet + self.factory.create_process_edge_roots_work_with_finalizer( + edges, + Box::new(move || unsafe { + ((*UPCALLS).nmethod_fix_relocation)(nmethod); + }), + ); } - // Create work packet - self.factory.create_process_edge_roots_work(edges); // Use the following code to scan CodeCache directly, instead of scanning the "remembered set". // unsafe { // ((*UPCALLS).scan_code_cache_roots)(create_process_edges_work:: as _); diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 72a8a7c3..654a8ca9 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -97,6 +97,7 @@ pub struct OpenJDK_Upcalls { pub enqueue_references: extern "C" fn(objects: *const ObjectReference, len: usize), pub compressed_klass_base: extern "C" fn() -> Address, pub compressed_klass_shift: extern "C" fn() -> usize, + pub nmethod_fix_relocation: extern "C" fn(Address), } pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 9a0e29e7..dd73436f 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -166,6 +166,7 @@ typedef struct { void (*enqueue_references)(void** objects, size_t len); void* (*compressed_klass_base)(); size_t (*compressed_klass_shift)(); + void (*nmethod_fix_relocation)(void* nmethod); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 0457cec1..4110546e 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -74,10 +74,6 @@ static void mmtk_resume_mutators(void *tls) { #if COMPILER2_OR_JVMCI DerivedPointerTable::update_pointers(); #endif - { - CodeBlobFixRelocationClosure cb_cl; - CodeCache::blobs_do(&cb_cl); - } // Note: we don't have to hold gc_lock to increment the counter. // The increment has to be done before mutators can be resumed @@ -353,6 +349,10 @@ static size_t mmtk_compressed_klass_shift() { return (size_t) Universe::narrow_klass_shift(); } +static void nmethod_fix_relocation(void* nm) { + ((nmethod*) nm)->fix_oop_relocations(); +} + OpenJDK_Upcalls mmtk_upcalls = { mmtk_stop_all_mutators, mmtk_resume_mutators, @@ -394,4 +394,5 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_enqueue_references, mmtk_compressed_klass_base, mmtk_compressed_klass_shift, + nmethod_fix_relocation, }; From 84dccde0fcc0926bb2a51fd470398f0b93c7d01f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 22 Aug 2022 10:35:58 +1000 Subject: [PATCH 151/494] minor --- mmtk/src/gc_work.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index f65fa334..ea2e958f 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -53,17 +53,22 @@ impl ScanCodeCacheRoots { impl GCWork for ScanCodeCacheRoots { fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { + let use_compressed_ptrs = *crate::USE_COMPRESSED_OOPS; // Collect all the cached roots for (nmethod, roots) in (*crate::CODE_CACHE_ROOTS.lock().unwrap()).iter() { let edges = roots.clone(); let nmethod = *nmethod; // Create work packet - self.factory.create_process_edge_roots_work_with_finalizer( - edges, - Box::new(move || unsafe { - ((*UPCALLS).nmethod_fix_relocation)(nmethod); - }), - ); + if use_compressed_ptrs { + self.factory.create_process_edge_roots_work_with_finalizer( + edges, + Box::new(move || unsafe { + ((*UPCALLS).nmethod_fix_relocation)(nmethod); + }), + ); + } else { + self.factory.create_process_edge_roots_work(edges); + } } // Use the following code to scan CodeCache directly, instead of scanning the "remembered set". // unsafe { From 886efcec84aea8a042aabc494362f3c1a218e3ac Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 1 Sep 2022 15:27:36 +1000 Subject: [PATCH 152/494] performance fix --- mmtk/src/gc_work.rs | 22 ++++++++-------------- openjdk/mmtkHeap.cpp | 10 +++++++--- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index ea2e958f..098092e5 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -1,3 +1,5 @@ +use std::sync::atomic::Ordering; + use super::{OpenJDK, UPCALLS}; use mmtk::scheduler::*; use mmtk::vm::RootsWorkFactory; @@ -53,23 +55,15 @@ impl ScanCodeCacheRoots { impl GCWork for ScanCodeCacheRoots { fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { - let use_compressed_ptrs = *crate::USE_COMPRESSED_OOPS; // Collect all the cached roots - for (nmethod, roots) in (*crate::CODE_CACHE_ROOTS.lock().unwrap()).iter() { - let edges = roots.clone(); - let nmethod = *nmethod; - // Create work packet - if use_compressed_ptrs { - self.factory.create_process_edge_roots_work_with_finalizer( - edges, - Box::new(move || unsafe { - ((*UPCALLS).nmethod_fix_relocation)(nmethod); - }), - ); - } else { - self.factory.create_process_edge_roots_work(edges); + let mut edges = Vec::with_capacity(crate::CODE_CACHE_ROOTS_SIZE.load(Ordering::Relaxed)); + for roots in (*crate::CODE_CACHE_ROOTS.lock().unwrap()).values() { + for r in roots { + edges.push(*r) } } + // Create work packet + self.factory.create_process_edge_roots_work(edges); // Use the following code to scan CodeCache directly, instead of scanning the "remembered set". // unsafe { // ((*UPCALLS).scan_code_cache_roots)(create_process_edges_work:: as _); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 6c27dd09..37e0a750 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -350,12 +350,16 @@ void MMTkHeap::print_tracing_info() const { // Registering and unregistering an nmethod (compiled code) with the heap. // Override with specific mechanism for each specialized heap type. class MMTkRegisterNMethodOopClosure: public OopClosure { - template void do_oop_work(T* p) { + template void do_oop_work(T* p, bool narrow) { + if (narrow) { + guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); + p = (T*) (uintptr_t(p) | (1ull << 63)); + } mmtk_add_nmethod_oop((void*) p); } public: - void do_oop(oop* p) { do_oop_work(p); } - void do_oop(narrowOop* p) { guarantee(false, "unreachable"); } + void do_oop(oop* p) { do_oop_work(p, false); } + void do_oop(narrowOop* p) { do_oop_work(p, true); } }; From 2c3fa6dff5e9c55c060b1c535c62214805addf12 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 2 Sep 2022 16:58:24 +1000 Subject: [PATCH 153/494] cleanup --- mmtk/src/object_model.rs | 42 ------------------------------------- mmtk/src/object_scanning.rs | 1 - 2 files changed, 43 deletions(-) diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index f9fb9b6c..d4c9f10d 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -5,7 +5,6 @@ use crate::abi::Oop; use crate::{vm_metadata, OpenJDK}; use mmtk::util::alloc::fill_alignment_gap; use mmtk::util::copy::*; -use mmtk::util::heap::layout::vm_layout_constants::HEAP_START; use mmtk::util::metadata::header_metadata::HeaderMetadataSpec; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::*; @@ -23,47 +22,6 @@ impl ObjectModel for VMObjectModel { const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = vm_metadata::MARKING_METADATA_SPEC; const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = vm_metadata::LOS_METADATA_SPEC; - #[inline(always)] - fn load_reference_field(slot: Address, root: bool) -> ObjectReference { - if *crate::USE_COMPRESSED_OOPS { - let narrow_root = slot.as_usize() & (1usize << 63) != 0; - let slot = unsafe { Address::from_usize(slot.as_usize() << 1 >> 1) }; - if root && !narrow_root { - unsafe { slot.load::() } - } else { - debug_assert!(root || !narrow_root); - let v = unsafe { slot.load::() }; - if v == 0 { - ObjectReference::NULL - } else { - unsafe { (HEAP_START + ((v as usize) << 3) - 4096).to_object_reference() } - } - } - } else { - unsafe { slot.load() } - } - } - - #[inline(always)] - fn store_reference_field(slot: Address, object: ObjectReference, root: bool) { - if *crate::USE_COMPRESSED_OOPS { - let narrow_root = slot.as_usize() & (1usize << 63) != 0; - let slot = unsafe { Address::from_usize(slot.as_usize() << 1 >> 1) }; - if root && !narrow_root { - unsafe { slot.store(object) } - } else { - debug_assert!(root || !narrow_root); - if object.is_null() { - unsafe { slot.store(0u32) }; - } else { - unsafe { slot.store(((object.to_address() - HEAP_START + 4096) >> 3) as u32) } - } - } - } else { - unsafe { slot.store(object) } - } - } - #[inline(always)] fn load_metadata( metadata_spec: &HeaderMetadataSpec, diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index b1de3dbd..bf46ecfc 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -1,6 +1,5 @@ use super::abi::*; use super::{OpenJDKEdge, UPCALLS}; -use mmtk::util::constants::*; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::EdgeVisitor; From 0263434c92f29cf03784d60ea51a80cd6a227f86 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 10 Sep 2022 08:34:01 +0000 Subject: [PATCH 154/494] wip --- openjdk/mmtkHeap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 37e0a750..edfff75f 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -111,7 +111,7 @@ jint MMTkHeap::initialize() { _start = (HeapWord*) starting_heap_address(); _end = (HeapWord*) last_heap_address(); - // printf("start: %p, end: %p\n", _start, _end); + printf("start: %p, end: %p\n", _start, _end); if (compressed_oops) { Universe::set_narrow_oop_base((address) (intptr_t(_start) - 4096)); Universe::set_narrow_oop_shift(LogMinObjAlignmentInBytes); From fd1ca919e3413c1db5ab486c88cb7611338d24bf Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 10 Sep 2022 11:25:13 +0000 Subject: [PATCH 155/494] dynamic switch address space --- mmtk/src/lib.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index daca8190..cfb293b6 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -11,7 +11,7 @@ use std::sync::Mutex; use libc::{c_char, c_void, uintptr_t}; use mmtk::util::alloc::AllocationError; -use mmtk::util::heap::layout::vm_layout_constants::HEAP_START; +use mmtk::util::heap::layout::vm_layout_constants::VM_LAYOUT_CONSTANTS; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::edge_shape::Edge; @@ -168,7 +168,7 @@ impl Edge for OpenJDKEdge { if v == 0 { ObjectReference::NULL } else { - unsafe { (HEAP_START - 4096 + (v << 3)).to_object_reference() } + unsafe { (VM_LAYOUT_CONSTANTS.heap_start - 4096 + (v << 3)).to_object_reference() } } } else { unsafe { slot.load::() } @@ -182,7 +182,11 @@ impl Edge for OpenJDKEdge { if object.is_null() { unsafe { slot.store(0u32) }; } else { - unsafe { slot.store(((object.to_address() - HEAP_START + 4096) >> 3) as u32) } + unsafe { + slot.store( + ((object.to_address() - VM_LAYOUT_CONSTANTS.heap_start + 4096) >> 3) as u32, + ) + } } } else { unsafe { slot.store(object) } @@ -208,7 +212,10 @@ pub static MMTK_INITIALIZED: AtomicBool = AtomicBool::new(false); lazy_static! { pub static ref BUILDER: Mutex = Mutex::new(MMTKBuilder::new()); pub static ref SINGLETON: MMTK = { - let builder = BUILDER.lock().unwrap(); + let mut builder = BUILDER.lock().unwrap(); + if *USE_COMPRESSED_OOPS { + builder.set_option("use_35bit_address_space", "true"); + } assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); From d76258c7f52ea20095f787486f17a36967efa767 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 19 Sep 2022 05:00:22 +0000 Subject: [PATCH 156/494] wip --- mmtk/src/lib.rs | 86 +++++++++++++++++++++----------- mmtk/src/object_model.rs | 4 ++ mmtk/src/object_scanning.rs | 97 ++++++++++++++++++++++-------------- mmtk/src/scanning.rs | 14 ++++-- openjdk/mmtkHeap.cpp | 2 +- openjdk/mmtkRootsClosure.hpp | 4 +- 6 files changed, 135 insertions(+), 72 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index cfb293b6..b5cbfd0b 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -116,22 +116,11 @@ pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS: uintptr_t = pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t = crate::mmtk::util::metadata::side_metadata::ALLOC_SIDE_METADATA_ADDR.as_usize(); -static mut LOG_BYTES_IN_OOP: usize = 3; - -#[inline(always)] -fn log_bytes_in_oop() -> usize { - unsafe { LOG_BYTES_IN_OOP } -} - lazy_static! { static ref USE_COMPRESSED_OOPS: bool = { - let enable = std::env::var("MMTK_COMPRESSED_PTRS") + std::env::var("MMTK_COMPRESSED_PTRS") .map(|s| s.trim() != "0" || s.trim() != "false") - .unwrap_or(false); - if enable { - unsafe { LOG_BYTES_IN_OOP = 2 } - } - enable + .unwrap_or(false) }; } @@ -147,49 +136,88 @@ impl OpenJDKEdge { const MASK: usize = 1usize << 63; const fn is_compressed(&self) -> bool { - self.0.as_usize() & Self::MASK != 0 + self.0.as_usize() & Self::MASK == 0 } const fn untagged_address(&self) -> Address { unsafe { Address::from_usize(self.0.as_usize() << 1 >> 1) } } - - pub const fn as_compressed(&self) -> Self { - unsafe { Self(Address::from_usize(self.0.as_usize() | Self::MASK)) } - } } impl Edge for OpenJDKEdge { /// Load object reference from the edge. - fn load(&self) -> ObjectReference { - let slot = self.untagged_address(); - if self.is_compressed() { - let v = unsafe { slot.load::() as usize }; + #[inline(always)] + fn load(&self) -> ObjectReference { + if cfg!(debug_assertions) { + if !ROOT { + if *USE_COMPRESSED_OOPS { + assert!(self.is_compressed()) + } else { + assert!(!self.is_compressed()) + } + } else { + if !*USE_COMPRESSED_OOPS { + assert!(!self.is_compressed()) + } + } + } + if ROOT && COMPRESSED { + let slot = self.untagged_address(); + if self.is_compressed() { + let v = unsafe { slot.load::() as usize }; + if v == 0 { + ObjectReference::NULL + } else { + unsafe { + (VM_LAYOUT_CONSTANTS.heap_start - 4096 + (v << 3)).to_object_reference() + } + } + } else { + unsafe { slot.load::() } + } + } else if COMPRESSED { + let v = unsafe { self.0.load::() as usize }; if v == 0 { ObjectReference::NULL } else { unsafe { (VM_LAYOUT_CONSTANTS.heap_start - 4096 + (v << 3)).to_object_reference() } } } else { - unsafe { slot.load::() } + unsafe { self.0.load::() } } } /// Store the object reference `object` into the edge. - fn store(&self, object: ObjectReference) { - let slot = self.untagged_address(); - if self.is_compressed() { + #[inline(always)] + fn store(&self, object: ObjectReference) { + if ROOT && COMPRESSED { + let slot = self.untagged_address(); + if self.is_compressed() { + if object.is_null() { + unsafe { slot.store(0u32) }; + } else { + unsafe { + slot.store( + ((object.to_address() - VM_LAYOUT_CONSTANTS.heap_start + 4096) >> 3) + as u32, + ) + } + } + } else { + unsafe { slot.store(object) } + } + } else if COMPRESSED { if object.is_null() { - unsafe { slot.store(0u32) }; + unsafe { self.0.store(0u32) }; } else { unsafe { - slot.store( + self.0.store( ((object.to_address() - VM_LAYOUT_CONSTANTS.heap_start + 4096) >> 3) as u32, ) } } } else { - unsafe { slot.store(object) } + unsafe { self.0.store(object) } } } } diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index d4c9f10d..8019d06a 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -157,4 +157,8 @@ impl ObjectModel for VMObjectModel { ((*UPCALLS).dump_object)(object); } } + + fn compressed_pointers_enabled() -> bool { + *crate::USE_COMPRESSED_OOPS + } } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index bf46ecfc..012fc652 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -6,15 +6,24 @@ use mmtk::vm::EdgeVisitor; use std::{mem, slice}; trait OopIterate: Sized { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor); + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut C, + ); } impl OopIterate for OopMapBlock { #[inline] - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut C, + ) { + let log_bytes_in_oop = if COMPRESSED { 2 } else { 3 }; let start = oop.get_field_address(self.offset); for i in 0..self.count as usize { - let edge = OpenJDKEdge(start + (i << crate::log_bytes_in_oop())); + let edge = OpenJDKEdge(start + (i << log_bytes_in_oop)); closure.visit_edge(edge); } } @@ -22,18 +31,27 @@ impl OopIterate for OopMapBlock { impl OopIterate for InstanceKlass { #[inline] - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut C, + ) { let oop_maps = self.nonstatic_oop_maps(); for map in oop_maps { - map.oop_iterate(oop, closure) + map.oop_iterate::(oop, closure) } } } impl OopIterate for InstanceMirrorKlass { #[inline] - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { - self.instance_klass.oop_iterate(oop, closure); + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut C, + ) { + self.instance_klass + .oop_iterate::(oop, closure); // if (Devirtualizer::do_metadata(closure)) { // Klass* klass = java_lang_Class::as_Klass(obj); // // We'll get NULL for primitive mirrors. @@ -82,8 +100,13 @@ impl OopIterate for InstanceMirrorKlass { impl OopIterate for InstanceClassLoaderKlass { #[inline] - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { - self.instance_klass.oop_iterate(oop, closure); + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut C, + ) { + self.instance_klass + .oop_iterate::(oop, closure); // if (Devirtualizer::do_metadata(closure)) { // ClassLoaderData* cld = java_lang_ClassLoader::loader_data(obj); // // cld can be null if we have a non-registered class loader. @@ -96,9 +119,13 @@ impl OopIterate for InstanceClassLoaderKlass { impl OopIterate for ObjArrayKlass { #[inline] - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut C, + ) { let array = unsafe { oop.as_array_oop() }; - if *crate::USE_COMPRESSED_OOPS { + if COMPRESSED { for narrow_oop in unsafe { array.data::(BasicType::T_OBJECT) } { closure.visit_edge(OpenJDKEdge(narrow_oop.slot())); } @@ -112,7 +139,11 @@ impl OopIterate for ObjArrayKlass { impl OopIterate for TypeArrayKlass { #[inline] - fn oop_iterate(&self, _oop: Oop, _closure: &mut impl EdgeVisitor) { + fn oop_iterate, const COMPRESSED: bool>( + &self, + _oop: Oop, + _closure: &mut C, + ) { // Performance tweak: We skip processing the klass pointer since all // TypeArrayKlasses are guaranteed processed via the null class loader. } @@ -120,10 +151,15 @@ impl OopIterate for TypeArrayKlass { impl OopIterate for InstanceRefKlass { #[inline] - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut C, + ) { use crate::abi::*; use crate::api::{add_phantom_candidate, add_soft_candidate, add_weak_candidate}; - self.instance_klass.oop_iterate(oop, closure); + self.instance_klass + .oop_iterate::(oop, closure); if Self::should_scan_weak_refs() { let reference = ObjectReference::from(oop); @@ -169,7 +205,7 @@ fn oop_iterate_slow(oop: Oop, closure: &mut impl EdgeVisitor, tls: } #[inline] -fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { +fn oop_iterate, const COMPRESSED: bool>(oop: Oop, closure: &mut C) { let klass_id = oop.klass().id; debug_assert!( klass_id as i32 >= 0 && (klass_id as i32) < 6, @@ -180,49 +216,36 @@ fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { match klass_id { KlassID::Instance => { let instance_klass = unsafe { oop.klass().cast::() }; - instance_klass.oop_iterate(oop, closure); + instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceClassLoader => { let instance_klass = unsafe { oop.klass().cast::() }; - instance_klass.oop_iterate(oop, closure); + instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceMirror => { let instance_klass = unsafe { oop.klass().cast::() }; - instance_klass.oop_iterate(oop, closure); + instance_klass.oop_iterate::(oop, closure); } KlassID::ObjArray => { let array_klass = unsafe { oop.klass().cast::() }; - array_klass.oop_iterate(oop, closure); + array_klass.oop_iterate::(oop, closure); } KlassID::TypeArray => { let array_klass = unsafe { oop.klass().cast::() }; - array_klass.oop_iterate(oop, closure); + array_klass.oop_iterate::(oop, closure); } KlassID::InstanceRef => { let instance_klass = unsafe { oop.klass().cast::() }; - instance_klass.oop_iterate(oop, closure); + instance_klass.oop_iterate::(oop, closure); } // _ => oop_iterate_slow(oop, closure, tls), } } #[inline] -pub fn scan_object( +pub fn scan_object, const COMPRESSED: bool>( object: ObjectReference, - closure: &mut impl EdgeVisitor, + closure: &mut C, _tls: VMWorkerThread, ) { - // println!("*****scan_object(0x{:x}) -> \n 0x{:x}, 0x{:x} \n", - // object, - // unsafe { *(object.value() as *const usize) }, - // unsafe { *((object.value() + 8) as *const usize) } - // ); - if *crate::USE_COMPRESSED_OOPS { - unsafe { - oop_iterate(mem::transmute(object), &mut |edge: OpenJDKEdge| { - closure.visit_edge(edge.as_compressed()) - }) - } - } else { - unsafe { oop_iterate(mem::transmute(object), closure) } - } + unsafe { oop_iterate::(mem::transmute(object), closure) } } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index f46dd69d..9b1d6a6d 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -46,12 +46,20 @@ impl Scanning for VMScanning { const SCAN_MUTATORS_IN_SAFEPOINT: bool = false; const SINGLE_THREAD_MUTATOR_SCANNING: bool = false; - fn scan_object>( + fn scan_objects>( tls: VMWorkerThread, - object: ObjectReference, + objects: &[ObjectReference], edge_visitor: &mut EV, ) { - crate::object_scanning::scan_object(object, edge_visitor, tls) + if *crate::USE_COMPRESSED_OOPS { + for o in objects { + crate::object_scanning::scan_object::(*o, edge_visitor, tls) + } + } else { + for o in objects { + crate::object_scanning::scan_object::(*o, edge_visitor, tls) + } + } } fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index edfff75f..d4fc02de 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -351,7 +351,7 @@ void MMTkHeap::print_tracing_info() const { // Override with specific mechanism for each specialized heap type. class MMTkRegisterNMethodOopClosure: public OopClosure { template void do_oop_work(T* p, bool narrow) { - if (narrow) { + if (UseCompressedOops && !narrow) { guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); p = (T*) (uintptr_t(p) | (1ull << 63)); } diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 970e39a6..8b6d2b4f 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -19,7 +19,7 @@ class MMTkRootsClosure2 : public OopClosure { void do_oop_work(T* p, bool narrow) { T heap_oop = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(heap_oop)) { - if (narrow) { + if (UseCompressedOops && !narrow) { guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); p = (T*) (uintptr_t(p) | (1ull << 63)); } @@ -63,7 +63,7 @@ class MMTkScanObjectClosure : public BasicOopIterateClosure { template void do_oop_work(T* p, bool narrow) { - if (narrow) { + if (UseCompressedOops && !narrow) { guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); p = (T*) (uintptr_t(p) | (1ull << 63)); } From 5a9114780294c1e2cf64509b7acf100fc7ca293c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 20 Sep 2022 06:33:10 +0000 Subject: [PATCH 157/494] Compressed ptr constant for object scanning --- mmtk/src/scanning.rs | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 9b1d6a6d..0b582618 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -46,20 +46,12 @@ impl Scanning for VMScanning { const SCAN_MUTATORS_IN_SAFEPOINT: bool = false; const SINGLE_THREAD_MUTATOR_SCANNING: bool = false; - fn scan_objects>( + fn scan_object, const COMPRESSED: bool>( tls: VMWorkerThread, - objects: &[ObjectReference], + object: ObjectReference, edge_visitor: &mut EV, ) { - if *crate::USE_COMPRESSED_OOPS { - for o in objects { - crate::object_scanning::scan_object::(*o, edge_visitor, tls) - } - } else { - for o in objects { - crate::object_scanning::scan_object::(*o, edge_visitor, tls) - } - } + crate::object_scanning::scan_object::(object, edge_visitor, tls); } fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { From e1cbbd4e4dc805caa3f71eb4cf129f069ade7cdc Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 20 Sep 2022 08:39:03 +0000 Subject: [PATCH 158/494] Fix --- mmtk/src/api.rs | 34 ++++++++++++++++------ mmtk/src/lib.rs | 77 +++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 97 insertions(+), 14 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 893bc4a1..d2d2014a 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -1,4 +1,6 @@ use crate::OpenJDK; +use crate::OpenJDKEdge; +use crate::OpenJDKEdgeRange; use crate::OpenJDK_Upcalls; use crate::BUILDER; use crate::SINGLETON; @@ -324,7 +326,7 @@ pub extern "C" fn mmtk_object_reference_write_pre( ) { mutator .barrier() - .object_reference_write_pre(src, slot, target); + .object_reference_write_pre(src, OpenJDKEdge(slot), target); } /// Full post barrier @@ -337,7 +339,7 @@ pub extern "C" fn mmtk_object_reference_write_post( ) { mutator .barrier() - .object_reference_write_post(src, slot, target); + .object_reference_write_post(src, OpenJDKEdge(slot), target); } /// Barrier slow-path call @@ -350,7 +352,7 @@ pub extern "C" fn mmtk_object_reference_write_slow( ) { mutator .barrier() - .object_reference_write_slow(src, slot, target); + .object_reference_write_slow(src, OpenJDKEdge(slot), target); } /// Array-copy pre-barrier @@ -362,9 +364,16 @@ pub extern "C" fn mmtk_array_copy_pre( count: usize, ) { let bytes = count << LOG_BYTES_IN_ADDRESS; - mutator - .barrier() - .memory_region_copy_pre(src..src + bytes, dst..dst + bytes); + mutator.barrier().memory_region_copy_pre( + OpenJDKEdgeRange { + start: OpenJDKEdge(src), + end: OpenJDKEdge(src + bytes), + }, + OpenJDKEdgeRange { + start: OpenJDKEdge(dst), + end: OpenJDKEdge(dst + bytes), + }, + ); } /// Array-copy post-barrier @@ -376,9 +385,16 @@ pub extern "C" fn mmtk_array_copy_post( count: usize, ) { let bytes = count << LOG_BYTES_IN_ADDRESS; - mutator - .barrier() - .memory_region_copy_post(src..src + bytes, dst..dst + bytes); + mutator.barrier().memory_region_copy_post( + OpenJDKEdgeRange { + start: OpenJDKEdge(src), + end: OpenJDKEdge(src + bytes), + }, + OpenJDKEdgeRange { + start: OpenJDKEdge(dst), + end: OpenJDKEdge(dst + bytes), + }, + ); } // finalization diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 4ea2e71e..94482b7d 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -5,17 +5,17 @@ extern crate lazy_static; extern crate once_cell; use std::collections::HashMap; -use std::ops::Range; use std::ptr::null_mut; use std::sync::atomic::AtomicUsize; use std::sync::Mutex; use libc::{c_char, c_void, uintptr_t}; use mmtk::util::alloc::AllocationError; +use mmtk::util::constants::{BYTES_IN_ADDRESS, LOG_BYTES_IN_ADDRESS}; use mmtk::util::heap::layout::vm_layout_constants::VM_LAYOUT_CONSTANTS; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; -use mmtk::vm::edge_shape::Edge; +use mmtk::vm::edge_shape::{Edge, MemorySlice}; use mmtk::vm::VMBinding; use mmtk::MMTKBuilder; use mmtk::Mutator; @@ -154,11 +154,11 @@ impl Edge for OpenJDKEdge { if *USE_COMPRESSED_OOPS { assert!(self.is_compressed()) } else { - assert!(!self.is_compressed()) + assert!(self.0.as_usize() & Self::MASK == 0) } } else { if !*USE_COMPRESSED_OOPS { - assert!(!self.is_compressed()) + assert!(self.0.as_usize() & Self::MASK == 0) } } } @@ -223,6 +223,73 @@ impl Edge for OpenJDKEdge { } } +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +pub struct OpenJDKEdgeRange { + pub start: OpenJDKEdge, + pub end: OpenJDKEdge, +} + +/// Iterate edges within `Range
`. +pub struct AddressRangeIterator { + cursor: Address, + limit: Address, +} + +impl Iterator for AddressRangeIterator { + type Item = OpenJDKEdge; + + #[inline] + fn next(&mut self) -> Option { + if self.cursor >= self.limit { + None + } else { + let edge = self.cursor; + self.cursor += BYTES_IN_ADDRESS; + Some(OpenJDKEdge(edge)) + } + } +} + +impl MemorySlice for OpenJDKEdgeRange { + type Edge = OpenJDKEdge; + type EdgeIterator = AddressRangeIterator; + + #[inline] + fn iter_edges(&self) -> Self::EdgeIterator { + AddressRangeIterator { + cursor: self.start.0, + limit: self.end.0, + } + } + + #[inline] + fn start(&self) -> Address { + self.start.0 + } + + #[inline] + fn bytes(&self) -> usize { + self.end.0 - self.start.0 + } + + #[inline] + fn copy(src: &Self, tgt: &Self) { + debug_assert_eq!(src.bytes(), tgt.bytes()); + debug_assert_eq!( + src.bytes() & ((1 << LOG_BYTES_IN_ADDRESS) - 1), + 0, + "bytes are not a multiple of words" + ); + // Raw memory copy + unsafe { + let words = tgt.bytes() >> LOG_BYTES_IN_ADDRESS; + let src = src.start().to_ptr::(); + let tgt = tgt.start().to_mut_ptr::(); + std::ptr::copy(src, tgt, words) + } + } +} + impl VMBinding for OpenJDK { type VMObjectModel = object_model::VMObjectModel; type VMScanning = scanning::VMScanning; @@ -231,7 +298,7 @@ impl VMBinding for OpenJDK { type VMReferenceGlue = reference_glue::VMReferenceGlue; type VMEdge = OpenJDKEdge; - type VMMemorySlice = Range
; + type VMMemorySlice = OpenJDKEdgeRange; } use std::sync::atomic::AtomicBool; From 66c5c4ec458a1f499dca0900fbe9da1f5661582e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 24 Sep 2022 04:23:06 +0000 Subject: [PATCH 159/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index d631f693..5871cb7c 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -337,7 +337,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.13.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8430420d4d36e022f4b6b9a2f3a7ccbf79a7997c#8430420d4d36e022f4b6b9a2f3a7ccbf79a7997c" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=24a11fd478fb45b5258fad49aa33ee2f95807e58#24a11fd478fb45b5258fad49aa33ee2f95807e58" dependencies = [ "atomic", "atomic-traits", @@ -361,7 +361,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.13.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8430420d4d36e022f4b6b9a2f3a7ccbf79a7997c#8430420d4d36e022f4b6b9a2f3a7ccbf79a7997c" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=24a11fd478fb45b5258fad49aa33ee2f95807e58#24a11fd478fb45b5258fad49aa33ee2f95807e58" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 50eb1900..fbc4484d 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -37,7 +37,7 @@ once_cell = "1.10.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8430420d4d36e022f4b6b9a2f3a7ccbf79a7997c" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "24a11fd478fb45b5258fad49aa33ee2f95807e58" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From 3ace1a17676fde88b5b01c4968527596767abeed Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 25 Sep 2022 05:07:36 +0000 Subject: [PATCH 160/494] Update openjdk revision --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index fbc4484d..6556ffdf 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -24,7 +24,7 @@ codegen-units = 1 [package.metadata.openjdk] # Our CI matches the following line and extract mmtk/openjdk. If this line is updated, please check ci yaml files and make sure it works. openjdk_repo = "https://github.com/mmtk/openjdk.git" -openjdk_version = "ca90b43f0f51d9ddf754e6ab134c5030cf54118b" +openjdk_version = "4fa9d18163e325d1b99b42140c87b7eeb499a129" [dependencies] libc = "0.2" From 0d9f9bb29457d596a1fa3e40f8a87ffaa0f3c6eb Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 6 Oct 2022 05:55:10 +0000 Subject: [PATCH 161/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 5871cb7c..807fcc67 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -337,7 +337,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.13.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=24a11fd478fb45b5258fad49aa33ee2f95807e58#24a11fd478fb45b5258fad49aa33ee2f95807e58" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b6f18a9f293d2c32b8acf0ba3b8710662b79b2c4#b6f18a9f293d2c32b8acf0ba3b8710662b79b2c4" dependencies = [ "atomic", "atomic-traits", @@ -361,7 +361,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.13.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=24a11fd478fb45b5258fad49aa33ee2f95807e58#24a11fd478fb45b5258fad49aa33ee2f95807e58" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b6f18a9f293d2c32b8acf0ba3b8710662b79b2c4#b6f18a9f293d2c32b8acf0ba3b8710662b79b2c4" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 6556ffdf..c516a4c4 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -37,7 +37,7 @@ once_cell = "1.10.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "24a11fd478fb45b5258fad49aa33ee2f95807e58" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "b6f18a9f293d2c32b8acf0ba3b8710662b79b2c4" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From c16350e1fd514bf7c848629df33dbe5d6afb3ef6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 7 Oct 2022 01:53:08 +0000 Subject: [PATCH 162/494] Choose pointer compression policy based on heap size --- mmtk/src/api.rs | 10 ++++++ mmtk/src/lib.rs | 80 ++++++++++++++++++++++++++------------------ openjdk/mmtk.h | 2 ++ openjdk/mmtkHeap.cpp | 7 ++-- 4 files changed, 63 insertions(+), 36 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index d2d2014a..082ebdc4 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -296,6 +296,16 @@ pub extern "C" fn process_bulk(options: *const c_char) -> bool { memory_manager::process_bulk(&mut builder, options_str.to_str().unwrap()) } +#[no_mangle] +pub extern "C" fn mmtk_narrow_oop_base() -> Address { + unsafe { crate::BASE } +} + +#[no_mangle] +pub extern "C" fn mmtk_narrow_oop_shift() -> usize { + unsafe { crate::SHIFT } +} + #[no_mangle] pub extern "C" fn starting_heap_address() -> Address { memory_manager::starting_heap_address() diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 94482b7d..2bcee86c 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -125,6 +125,45 @@ lazy_static! { }; } +static mut BASE: Address = Address::ZERO; +static mut SHIFT: usize = 0; + +fn compress(o: ObjectReference) -> u32 { + if o.is_null() { + 0u32 + } else { + unsafe { ((o.to_address() - BASE) >> SHIFT) as u32 } + } +} + +fn decompress(v: u32) -> ObjectReference { + if v == 0 { + ObjectReference::NULL + } else { + unsafe { (BASE + ((v as usize) << SHIFT)).to_object_reference() } + } +} + +fn initialize_compressed_oops() { + let heap_end = VM_LAYOUT_CONSTANTS.heap_end.as_usize(); + if heap_end <= (4usize << 30) { + unsafe { + BASE = Address::ZERO; + SHIFT = 0; + } + } else if heap_end <= (32usize << 30) { + unsafe { + BASE = Address::ZERO; + SHIFT = 3; + } + } else { + unsafe { + BASE = VM_LAYOUT_CONSTANTS.heap_start - 4096; + SHIFT = 3; + } + } +} + #[derive(Default)] pub struct OpenJDK; @@ -165,24 +204,12 @@ impl Edge for OpenJDKEdge { if ROOT && COMPRESSED { let slot = self.untagged_address(); if self.is_compressed() { - let v = unsafe { slot.load::() as usize }; - if v == 0 { - ObjectReference::NULL - } else { - unsafe { - (VM_LAYOUT_CONSTANTS.heap_start - 4096 + (v << 3)).to_object_reference() - } - } + decompress(unsafe { slot.load::() }) } else { unsafe { slot.load::() } } } else if COMPRESSED { - let v = unsafe { self.0.load::() as usize }; - if v == 0 { - ObjectReference::NULL - } else { - unsafe { (VM_LAYOUT_CONSTANTS.heap_start - 4096 + (v << 3)).to_object_reference() } - } + decompress(unsafe { self.0.load::() }) } else { unsafe { self.0.load::() } } @@ -194,29 +221,12 @@ impl Edge for OpenJDKEdge { if ROOT && COMPRESSED { let slot = self.untagged_address(); if self.is_compressed() { - if object.is_null() { - unsafe { slot.store(0u32) }; - } else { - unsafe { - slot.store( - ((object.to_address() - VM_LAYOUT_CONSTANTS.heap_start + 4096) >> 3) - as u32, - ) - } - } + unsafe { slot.store(compress(object)) } } else { unsafe { slot.store(object) } } } else if COMPRESSED { - if object.is_null() { - unsafe { self.0.store(0u32) }; - } else { - unsafe { - self.0.store( - ((object.to_address() - VM_LAYOUT_CONSTANTS.heap_start + 4096) >> 3) as u32, - ) - } - } + unsafe { self.0.store(compress(object)) } } else { unsafe { self.0.store(object) } } @@ -312,10 +322,14 @@ lazy_static! { let mut builder = BUILDER.lock().unwrap(); if *USE_COMPRESSED_OOPS { builder.set_option("use_35bit_address_space", "true"); + builder.set_option("use_35bit_address_space", "true"); } assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); + if *USE_COMPRESSED_OOPS { + initialize_compressed_oops(); + } *ret }; } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index fe0ba249..12f3f685 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -188,6 +188,8 @@ extern bool openjdk_is_gc_initialized(); extern bool mmtk_set_heap_size(size_t size); extern bool mmtk_use_compressed_ptrs(); +extern void* mmtk_narrow_oop_base(); +extern size_t mmtk_narrow_oop_shift(); extern size_t used_bytes(); extern void* starting_heap_address(); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index d4fc02de..83c2fba5 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -111,10 +111,11 @@ jint MMTkHeap::initialize() { _start = (HeapWord*) starting_heap_address(); _end = (HeapWord*) last_heap_address(); - printf("start: %p, end: %p\n", _start, _end); + printf("start: %p, end: %p\n", _start, _end); if (compressed_oops) { - Universe::set_narrow_oop_base((address) (intptr_t(_start) - 4096)); - Universe::set_narrow_oop_shift(LogMinObjAlignmentInBytes); + Universe::set_narrow_oop_base((address) mmtk_narrow_oop_base()); + Universe::set_narrow_oop_shift(mmtk_narrow_oop_shift()); + printf("narrow_oop_mode: %s\n", Universe::narrow_oop_mode_to_string(Universe::narrow_oop_mode())); } initialize_reserved_region(_start, _end); From 4990c3aacd09f030309d8f6304170312c118a8de Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 10 Oct 2022 23:46:18 +0000 Subject: [PATCH 163/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 4acf3ee9..7d446f04 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8572adc696e8c52eab7eb2cace53a8a9634a2ccf#8572adc696e8c52eab7eb2cace53a8a9634a2ccf" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6779dff4f1b10173b03b39e1c42455c2e79bd879#6779dff4f1b10173b03b39e1c42455c2e79bd879" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8572adc696e8c52eab7eb2cace53a8a9634a2ccf#8572adc696e8c52eab7eb2cace53a8a9634a2ccf" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6779dff4f1b10173b03b39e1c42455c2e79bd879#6779dff4f1b10173b03b39e1c42455c2e79bd879" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 1d538f0e..e8eaa3d0 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -38,7 +38,7 @@ once_cell = "1.10.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8572adc696e8c52eab7eb2cace53a8a9634a2ccf" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "6779dff4f1b10173b03b39e1c42455c2e79bd879" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From efbd3b954ff620b60a86d3c1954f65b95d806e35 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 11 Oct 2022 05:13:02 +0000 Subject: [PATCH 164/494] Support C1 --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 7d446f04..c6deb2c3 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6779dff4f1b10173b03b39e1c42455c2e79bd879#6779dff4f1b10173b03b39e1c42455c2e79bd879" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e6e3c1880caf297dcdc53ce7c47745a9ebce39f2#e6e3c1880caf297dcdc53ce7c47745a9ebce39f2" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6779dff4f1b10173b03b39e1c42455c2e79bd879#6779dff4f1b10173b03b39e1c42455c2e79bd879" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e6e3c1880caf297dcdc53ce7c47745a9ebce39f2#e6e3c1880caf297dcdc53ce7c47745a9ebce39f2" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index e8eaa3d0..8e6a7d15 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -25,7 +25,7 @@ codegen-units = 1 [package.metadata.openjdk] # Our CI matches the following line and extract mmtk/openjdk. If this line is updated, please check ci yaml files and make sure it works. openjdk_repo = "https://github.com/mmtk/openjdk.git" -openjdk_version = "c82e5c44adced4383162826c2c3933a83cfb139b" +openjdk_version = "f24111bb0f220aec18136eeb020aadb7276225ca" [dependencies] libc = "0.2" @@ -38,7 +38,7 @@ once_cell = "1.10.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "6779dff4f1b10173b03b39e1c42455c2e79bd879" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e6e3c1880caf297dcdc53ce7c47745a9ebce39f2" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From ed8ac66722340e7284e3382f4957feed3f7d0f81 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 13 Oct 2022 04:49:48 +0000 Subject: [PATCH 165/494] rename barrier --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/api.rs | 4 ++-- ...LoggingBarrier.cpp => mmtkFieldBarrier.cpp} | 14 +++++++------- ...LoggingBarrier.hpp => mmtkFieldBarrier.hpp} | 18 +++++++++--------- openjdk/mmtkBarrierSet.cpp | 4 ++-- 6 files changed, 23 insertions(+), 23 deletions(-) rename openjdk/barriers/{mmtkFieldLoggingBarrier.cpp => mmtkFieldBarrier.cpp} (88%) rename openjdk/barriers/{mmtkFieldLoggingBarrier.hpp => mmtkFieldBarrier.hpp} (78%) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index c6deb2c3..5acbb6d6 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e6e3c1880caf297dcdc53ce7c47745a9ebce39f2#e6e3c1880caf297dcdc53ce7c47745a9ebce39f2" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b754df6d916509aa45715d541b160babbbba43f8#b754df6d916509aa45715d541b160babbbba43f8" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e6e3c1880caf297dcdc53ce7c47745a9ebce39f2#e6e3c1880caf297dcdc53ce7c47745a9ebce39f2" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b754df6d916509aa45715d541b160babbbba43f8#b754df6d916509aa45715d541b160babbbba43f8" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 8e6a7d15..eb0a3e2b 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -38,7 +38,7 @@ once_cell = "1.10.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e6e3c1880caf297dcdc53ce7c47745a9ebce39f2" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "b754df6d916509aa45715d541b160babbbba43f8" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index c5fe8ff3..8ec1e322 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -25,7 +25,7 @@ static NO_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("NoBarr static OBJECT_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("ObjectBarrier").unwrap()); static FIELD_LOGGING_BARRIER: sync::Lazy = - sync::Lazy::new(|| CString::new("FieldLoggingBarrier").unwrap()); + sync::Lazy::new(|| CString::new("FieldBarrier").unwrap()); #[no_mangle] pub extern "C" fn get_mmtk_version() -> *const c_char { @@ -37,7 +37,7 @@ pub extern "C" fn mmtk_active_barrier() -> *const c_char { match SINGLETON.get_plan().constraints().barrier { BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), - BarrierSelector::FieldLoggingBarrier => FIELD_LOGGING_BARRIER.as_ptr(), + BarrierSelector::FieldBarrier => FIELD_LOGGING_BARRIER.as_ptr(), // In case we have more barriers in mmtk-core. #[allow(unreachable_patterns)] _ => unimplemented!(), diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp similarity index 88% rename from openjdk/barriers/mmtkFieldLoggingBarrier.cpp rename to openjdk/barriers/mmtkFieldBarrier.cpp index c4410c11..97b84969 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -1,9 +1,9 @@ -#include "mmtkFieldLoggingBarrier.hpp" +#include "mmtkFieldBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" constexpr int kUnloggedValue = 1; -void MMTkFieldLoggingBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, oop target) const { +void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, oop target) const { #if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = ((intptr_t) (void*) slot); uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); @@ -20,7 +20,7 @@ void MMTkFieldLoggingBarrierSetRuntime::object_reference_write_pre(oop src, oop* #define __ masm-> -void MMTkFieldLoggingBarrierSetAssembler::object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const { +void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const { if (can_remove_barrier(decorators, val, /* skip_const_null */ false)) return; #if MMTK_ENABLE_BARRIER_FASTPATH Label done; @@ -65,7 +65,7 @@ void MMTkFieldLoggingBarrierSetAssembler::object_reference_write_pre(MacroAssemb #endif } -void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { +void MMTkFieldBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { const bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; if ((type == T_OBJECT || type == T_ARRAY) && !dest_uninitialized) { // Label slow, done; @@ -86,7 +86,7 @@ void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue(MacroAssembler* mas // assert(src == rdi, "expected"); // assert(dst == rsi, "expected"); // assert(count == rdx, "expected"); - // __ call_VM_leaf(CAST_FROM_FN_PTR(address, MMTkFieldLoggingBarrierSetRuntime::record_array_copy_slow), src, dst, count); + // __ call_VM_leaf(CAST_FROM_FN_PTR(address, MMTkFieldBarrierSetRuntime::record_array_copy_slow), src, dst, count); // __ popa(); // __ bind(done); __ pusha(); @@ -106,7 +106,7 @@ void MMTkFieldLoggingBarrierSetAssembler::arraycopy_prologue(MacroAssembler* mas #define __ gen->lir()-> #endif -void MMTkFieldLoggingBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const { +void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const { LIRGenerator* gen = access.gen(); DecoratorSet decorators = access.decorators(); if ((decorators & IN_HEAP) == 0) return; @@ -179,7 +179,7 @@ void MMTkFieldLoggingBarrierSetC1::object_reference_write_pre(LIRAccess& access, #define __ ideal. -void MMTkFieldLoggingBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const { +void MMTkFieldBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const { if (can_remove_barrier(kit, &kit->gvn(), src, slot, val, /* skip_const_null */ false)) return; MMTkIdealKit ideal(kit, true); diff --git a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp similarity index 78% rename from openjdk/barriers/mmtkFieldLoggingBarrier.hpp rename to openjdk/barriers/mmtkFieldBarrier.hpp index b279cacb..c6afb103 100644 --- a/openjdk/barriers/mmtkFieldLoggingBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -16,7 +16,7 @@ #define LOG_BYTES_IN_CHUNK 22 #define CHUNK_MASK ((1L << LOG_BYTES_IN_CHUNK) - 1) -class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { +class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { public: // Interfaces called by `MMTkBarrierSet::AccessBarrier` virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; @@ -25,14 +25,14 @@ class MMTkFieldLoggingBarrierSetRuntime: public MMTkBarrierSetRuntime { } }; -class MMTkFieldLoggingBarrierSetAssembler: public MMTkBarrierSetAssembler { +class MMTkFieldBarrierSetAssembler: public MMTkBarrierSetAssembler { protected: virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const override; public: virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; }; -class MMTkFieldLoggingBarrierSetC1: public MMTkBarrierSetC1 { +class MMTkFieldBarrierSetC1: public MMTkBarrierSetC1 { protected: virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const override; @@ -41,16 +41,16 @@ class MMTkFieldLoggingBarrierSetC1: public MMTkBarrierSetC1 { } }; -class MMTkFieldLoggingBarrierSetC2: public MMTkBarrierSetC2 { +class MMTkFieldBarrierSetC2: public MMTkBarrierSetC2 { protected: virtual void object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const override; }; -struct MMTkFieldLoggingBarrier: MMTkBarrierImpl< - MMTkFieldLoggingBarrierSetRuntime, - MMTkFieldLoggingBarrierSetAssembler, - MMTkFieldLoggingBarrierSetC1, - MMTkFieldLoggingBarrierSetC2 +struct MMTkFieldBarrier: MMTkBarrierImpl< + MMTkFieldBarrierSetRuntime, + MMTkFieldBarrierSetAssembler, + MMTkFieldBarrierSetC1, + MMTkFieldBarrierSetC2 > {}; #endif diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index 98b39082..175c69a7 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -25,7 +25,7 @@ #include "precompiled.hpp" #include "barriers/mmtkNoBarrier.hpp" #include "barriers/mmtkObjectBarrier.hpp" -#include "barriers/mmtkFieldLoggingBarrier.hpp" +#include "barriers/mmtkFieldBarrier.hpp" #include "mmtkBarrierSet.hpp" #include "mmtkBarrierSetAssembler_x86.hpp" #include "runtime/interfaceSupport.inline.hpp" @@ -81,7 +81,7 @@ MMTkBarrierBase* get_selected_barrier() { const char* barrier = mmtk_active_barrier(); if (strcmp(barrier, "NoBarrier") == 0) selected_barrier = new MMTkNoBarrier(); else if (strcmp(barrier, "ObjectBarrier") == 0) selected_barrier = new MMTkObjectBarrier(); - else if (strcmp(barrier, "FieldLoggingBarrier") == 0) selected_barrier = new MMTkFieldLoggingBarrier(); + else if (strcmp(barrier, "FieldBarrier") == 0) selected_barrier = new MMTkFieldBarrier(); else guarantee(false, "Unimplemented"); return selected_barrier; } From 6a5911624a4b0bb67319035778a8f08eb57b754d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 18 Oct 2022 22:47:11 +0000 Subject: [PATCH 166/494] wip --- mmtk/src/object_scanning.rs | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 4ad2b43a..6bd1d4a1 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -117,11 +117,34 @@ impl OopIterate for InstanceRefKlass { ReferenceType::None => { panic!("oop_iterate on InstanceRefKlass with reference_type as None") } - ReferenceType::Weak => add_weak_candidate(reference), - ReferenceType::Soft => add_soft_candidate(reference), - ReferenceType::Phantom => add_phantom_candidate(reference), + ReferenceType::Weak => { + if !crate::SINGLETON.reference_processors.allow_new_candidate() { + Self::process_ref_as_strong(oop, closure) + } else { + add_weak_candidate(reference) + } + } + ReferenceType::Soft => { + // let oop = Oop::from(reff); + // unsafe { InstanceRefKlass::referent_address(oop).store(ObjectReference::NULL) }; + if !crate::SINGLETON.reference_processors.allow_new_candidate() { + Self::process_ref_as_strong(oop, closure) + } else { + add_soft_candidate(reference) + } + // Self::process_ref_as_strong(oop, closure) + } + ReferenceType::Phantom => { + if !crate::SINGLETON.reference_processors.allow_new_candidate() { + Self::process_ref_as_strong(oop, closure) + } else { + add_phantom_candidate(reference) + } + } // Process these two types normally (as if they are strong refs) // We will handle final reference later + // ReferenceType::Weak + // | ReferenceType::Phantom ReferenceType::Final | ReferenceType::Other => { Self::process_ref_as_strong(oop, closure) } From ff316be24566cc59a859cbdf7b3901e980f5b746 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 19 Oct 2022 00:52:15 +0000 Subject: [PATCH 167/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 5acbb6d6..377732a3 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b754df6d916509aa45715d541b160babbbba43f8#b754df6d916509aa45715d541b160babbbba43f8" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=92eadbcee4299682760ffc4b5d49d84c73bf0158#92eadbcee4299682760ffc4b5d49d84c73bf0158" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b754df6d916509aa45715d541b160babbbba43f8#b754df6d916509aa45715d541b160babbbba43f8" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=92eadbcee4299682760ffc4b5d49d84c73bf0158#92eadbcee4299682760ffc4b5d49d84c73bf0158" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index eb0a3e2b..6fc935fc 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -38,7 +38,7 @@ once_cell = "1.10.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "b754df6d916509aa45715d541b160babbbba43f8" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "92eadbcee4299682760ffc4b5d49d84c73bf0158" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 5c5b890ae62de86e1f40bb17dd7d2afdb3e08abd Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 19 Oct 2022 00:56:14 +0000 Subject: [PATCH 168/494] Fix C1 pre-barrier code patching --- openjdk/barriers/mmtkFieldBarrier.cpp | 60 ++++++++++++++----------- openjdk/mmtkBarrierSetAssembler_x86.cpp | 23 +++++++++- openjdk/mmtkBarrierSetC1.hpp | 14 +++++- 3 files changed, 69 insertions(+), 28 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 97b84969..155929ca 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -110,6 +110,7 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op LIRGenerator* gen = access.gen(); DecoratorSet decorators = access.decorators(); if ((decorators & IN_HEAP) == 0) return; + bool needs_patching = (decorators & C1_NEEDS_PATCHING) != 0; if (!src->is_register()) { LIR_Opr reg = gen->new_pointer_register(); if (src->is_constant()) { @@ -120,7 +121,7 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op src = reg; } assert(src->is_register(), "must be a register at this point"); - if (!slot->is_register()) { + if (!slot->is_register() && !needs_patching) { LIR_Address* address = slot->as_address_ptr(); LIR_Opr ptr = gen->new_pointer_register(); if (!address->index()->is_valid() && address->disp() == 0) { @@ -130,8 +131,11 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op __ leal(slot, ptr); } slot = ptr; + } else if (needs_patching && !slot->is_address()) { + assert(slot->is_register(), "must be"); + slot = LIR_OprFact::address(new LIR_Address(slot, T_OBJECT)); } - assert(slot->is_register(), "must be a register at this point"); + assert(needs_patching || slot->is_register(), "must be a register at this point unless needs_patching"); if (!new_val->is_register()) { LIR_Opr new_val_reg = gen->new_register(T_OBJECT); if (new_val->is_constant()) { @@ -142,31 +146,37 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op new_val = new_val_reg; } assert(new_val->is_register(), "must be a register at this point"); - CodeStub* slow = new MMTkC1BarrierStub(src, slot, new_val); + MMTkC1BarrierStub* slow = new MMTkC1BarrierStub(src, slot, new_val, access.patch_emit_info(), needs_patching ? lir_patch_normal : lir_patch_none); + if (needs_patching) slow->scratch = gen->new_register(T_OBJECT); #if MMTK_ENABLE_BARRIER_FASTPATH - LIR_Opr addr = slot; - // uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); - LIR_Opr offset = gen->new_pointer_register(); - __ move(addr, offset); - __ unsigned_shift_right(offset, 6, offset); - LIR_Opr base = gen->new_pointer_register(); - __ move(LIR_OprFact::longConst(SIDE_METADATA_BASE_ADDRESS), base); - LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); - // uint8_t byte_val = *meta_addr; - LIR_Opr byte_val = gen->new_register(T_INT); - __ move(meta_addr, byte_val); - // intptr_t shift = (addr >> 3) & 0b111; - LIR_Opr shift = gen->new_register(T_INT); - __ move(addr, shift); - __ unsigned_shift_right(shift, 3, shift); - __ logical_and(shift, LIR_OprFact::intConst(0b111), shift); - // if (((byte_val >> shift) & 1) == 1) slow; - LIR_Opr result = byte_val; - __ unsigned_shift_right(result, shift, result, LIR_OprFact::illegalOpr); - __ logical_and(result, LIR_OprFact::intConst(1), result); - __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(1)); - __ branch(lir_cond_equal, T_BYTE, slow); + if (needs_patching) { + // FIXME: Jump to a medium-path for code patching without entering slow-path + __ jump(slow); + } else { + LIR_Opr addr = slot; + // uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + LIR_Opr offset = gen->new_pointer_register(); + __ move(addr, offset); + __ unsigned_shift_right(offset, 6, offset); + LIR_Opr base = gen->new_pointer_register(); + __ move(LIR_OprFact::longConst(SIDE_METADATA_BASE_ADDRESS), base); + LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); + // uint8_t byte_val = *meta_addr; + LIR_Opr byte_val = gen->new_register(T_INT); + __ move(meta_addr, byte_val); + // intptr_t shift = (addr >> 3) & 0b111; + LIR_Opr shift = gen->new_register(T_INT); + __ move(addr, shift); + __ unsigned_shift_right(shift, 3, shift); + __ logical_and(shift, LIR_OprFact::intConst(0b111), shift); + // if (((byte_val >> shift) & 1) == 1) slow; + LIR_Opr result = byte_val; + __ unsigned_shift_right(result, shift, result, LIR_OprFact::illegalOpr); + __ logical_and(result, LIR_OprFact::intConst(1), result); + __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(1)); + __ branch(lir_cond_equal, T_BYTE, slow); + } #else __ jump(slow); #endif diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index 4d5a8e7f..7e037f58 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -22,6 +22,8 @@ * */ +#define private public // too lazy to change openjdk... + #include "precompiled.hpp" #include "asm/macroAssembler.inline.hpp" #include "interpreter/interp_masm.hpp" @@ -188,8 +190,27 @@ void MMTkBarrierSetAssembler::generate_c1_write_barrier_runtime_stub(StubAssembl void MMTkBarrierSetAssembler::generate_c1_write_barrier_stub_call(LIR_Assembler* ce, MMTkC1BarrierStub* stub) { MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1(); __ bind(*stub->entry()); + + // For pre-barriers, stub->slot may not be a resolved address. + // Manually patch the address and goes to the slow-path unconditionally. + if (stub->patch_code != lir_patch_none) { + // Patch + assert(stub->scratch->is_single_cpu(), "must be"); + assert(stub->scratch->is_register(), "Precondition."); + ce->mem2reg(stub->slot, stub->scratch, T_OBJECT, stub->patch_code, stub->info, false /*wide*/, false /*unaligned*/); + // Resolve address + auto masm = ce->masm(); + LIR_Address* addr = stub->slot->as_address_ptr(); + Address from_addr = ce->as_Address(addr); + __ movptr(stub->scratch->as_register(), from_addr); + // Store parameter + ce->store_parameter(stub->scratch->as_pointer_register(), 1); + } else { + // Store parameter + ce->store_parameter(stub->slot->as_pointer_register(), 1); + } + ce->store_parameter(stub->src->as_pointer_register(), 0); - ce->store_parameter(stub->slot->as_pointer_register(), 1); ce->store_parameter(stub->new_val->as_pointer_register(), 2); __ call(RuntimeAddress(bs->_write_barrier_c1_runtime_code_blob->code_begin())); __ jmp(*stub->continuation()); diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index dea0712e..99e81583 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -69,16 +69,26 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { /// Barrier implementations may inherit from this class, and override `emit_code` to perform a specialized slow-path call. struct MMTkC1BarrierStub: CodeStub { LIR_Opr src, slot, new_val; + CodeEmitInfo* info; // Code patching info + LIR_PatchCode patch_code; // Enable code patching? + LIR_Opr scratch = NULL; // Scratch register for the resolved field - MMTkC1BarrierStub(LIR_Opr src, LIR_Opr slot, LIR_Opr new_val): src(src), slot(slot), new_val(new_val) {} + MMTkC1BarrierStub(LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info = NULL, LIR_PatchCode patch_code = lir_patch_none): src(src), slot(slot), new_val(new_val), info(info), patch_code(patch_code) {} virtual void emit_code(LIR_Assembler* ce) override; virtual void visit(LIR_OpVisitState* visitor) override { - visitor->do_slow_case(); + if (info != NULL) + visitor->do_slow_case(info); + else + visitor->do_slow_case(); if (src != NULL) visitor->do_input(src); if (slot != NULL) visitor->do_input(slot); if (new_val != NULL) visitor->do_input(new_val); + if (scratch != NULL) { + assert(scratch->is_oop(), "must be"); + visitor->do_temp(scratch); + } } NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1BarrierStub"); }); From e2c6436611bb85d43baead1aa127bcd3849bb749 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 23 Oct 2022 22:55:16 +0000 Subject: [PATCH 169/494] wip --- mmtk/src/object_scanning.rs | 43 +++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 6bd1d4a1..a732f820 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -112,33 +112,15 @@ impl OopIterate for InstanceRefKlass { self.instance_klass.oop_iterate(oop, closure); if Self::should_scan_weak_refs() { + unreachable!(); let reference = ObjectReference::from(oop); match self.instance_klass.reference_type { ReferenceType::None => { panic!("oop_iterate on InstanceRefKlass with reference_type as None") } - ReferenceType::Weak => { - if !crate::SINGLETON.reference_processors.allow_new_candidate() { + rt @ (ReferenceType::Weak | ReferenceType::Soft | ReferenceType::Phantom) => { + if !Self::discover_reference(oop, rt) { Self::process_ref_as_strong(oop, closure) - } else { - add_weak_candidate(reference) - } - } - ReferenceType::Soft => { - // let oop = Oop::from(reff); - // unsafe { InstanceRefKlass::referent_address(oop).store(ObjectReference::NULL) }; - if !crate::SINGLETON.reference_processors.allow_new_candidate() { - Self::process_ref_as_strong(oop, closure) - } else { - add_soft_candidate(reference) - } - // Self::process_ref_as_strong(oop, closure) - } - ReferenceType::Phantom => { - if !crate::SINGLETON.reference_processors.allow_new_candidate() { - Self::process_ref_as_strong(oop, closure) - } else { - add_phantom_candidate(reference) } } // Process these two types normally (as if they are strong refs) @@ -168,6 +150,25 @@ impl InstanceRefKlass { let discovered_addr = Self::discovered_address(oop); closure.visit_edge(discovered_addr); } + #[inline] + fn discover_reference(oop: Oop, rt: ReferenceType) -> bool { + use crate::api::{add_phantom_candidate, add_soft_candidate, add_weak_candidate}; + if !crate::SINGLETON.reference_processors.allow_new_candidate() { + return false; + } + let referent: ObjectReference = unsafe { InstanceRefKlass::referent_address(oop).load() }; + if referent.is_live() { + return false; + } + let reference = ObjectReference::from(oop); + match rt { + ReferenceType::Weak => add_weak_candidate(reference), + ReferenceType::Soft => add_soft_candidate(reference), + ReferenceType::Phantom => add_phantom_candidate(reference), + _ => unreachable!(), + } + true + } } #[allow(unused)] From d732d7acccc705aa63faf0e2001264c3b9a9a03a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 23 Oct 2022 23:31:14 +0000 Subject: [PATCH 170/494] cleanup --- mmtk/src/object_scanning.rs | 3 --- mmtk/src/reference_glue.rs | 9 ++++----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index a732f820..94310ab4 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -108,12 +108,9 @@ impl OopIterate for InstanceRefKlass { #[inline] fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { use crate::abi::*; - use crate::api::{add_phantom_candidate, add_soft_candidate, add_weak_candidate}; self.instance_klass.oop_iterate(oop, closure); if Self::should_scan_weak_refs() { - unreachable!(); - let reference = ObjectReference::from(oop); match self.instance_klass.reference_type { ReferenceType::None => { panic!("oop_iterate on InstanceRefKlass with reference_type as None") diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 966f4e92..7817e8d2 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -1,6 +1,5 @@ use crate::abi::{InstanceRefKlass, Oop}; use crate::OpenJDK; -use crate::UPCALLS; use mmtk::util::opaque_pointer::VMWorkerThread; use mmtk::util::ObjectReference; use mmtk::vm::ReferenceGlue; @@ -18,9 +17,9 @@ impl ReferenceGlue for VMReferenceGlue { let oop = Oop::from(object); unsafe { InstanceRefKlass::referent_address(oop).load::() } } - fn enqueue_references(references: &[ObjectReference], _tls: VMWorkerThread) { - unsafe { - ((*UPCALLS).enqueue_references)(references.as_ptr(), references.len()); - } + fn enqueue_references(_references: &[ObjectReference], _tls: VMWorkerThread) { + // unsafe { + // ((*UPCALLS).enqueue_references)(references.as_ptr(), references.len()); + // } } } From 3493a1f36337984e9948b442840fe331badb1cc5 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 23 Oct 2022 23:51:51 +0000 Subject: [PATCH 171/494] Add comments --- mmtk/src/object_scanning.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 94310ab4..b9a060fc 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -150,13 +150,17 @@ impl InstanceRefKlass { #[inline] fn discover_reference(oop: Oop, rt: ReferenceType) -> bool { use crate::api::{add_phantom_candidate, add_soft_candidate, add_weak_candidate}; + // Do not discover new refs during reference processing. if !crate::SINGLETON.reference_processors.allow_new_candidate() { return false; } + // Do not discover if the referent is true. let referent: ObjectReference = unsafe { InstanceRefKlass::referent_address(oop).load() }; if referent.is_live() { return false; } + // TODO: Do not discover if the referent is a nursery object. + // TODO: Fast list insertion let reference = ObjectReference::from(oop); match rt { ReferenceType::Weak => add_weak_candidate(reference), From 74d31927ff8c98f1281108caf94def44ea4cee6a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 25 Oct 2022 10:44:47 +0000 Subject: [PATCH 172/494] fast ref proc --- mmtk/Cargo.lock | 12 +- mmtk/Cargo.toml | 1 + mmtk/src/abi.rs | 2 +- mmtk/src/collection.rs | 11 ++ mmtk/src/lib.rs | 3 + mmtk/src/object_scanning.rs | 29 +++-- mmtk/src/reference_glue.rs | 243 +++++++++++++++++++++++++++++++++++- 7 files changed, 289 insertions(+), 12 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 0eb4862a..477e33ec 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -17,6 +17,15 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64f46ca51dca4837f1520754d1c8c36636356b81553d928dc9c177025369a06e" +[[package]] +name = "atomic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b88d82667eca772c4aa12f0f1348b3ae643424c8876448f3f7bd5787032e234c" +dependencies = [ + "autocfg", +] + [[package]] name = "atomic-traits" version = "0.2.0" @@ -342,7 +351,7 @@ name = "mmtk" version = "0.15.0" source = "git+https://github.com/mmtk/mmtk-core.git?rev=f82848847a6392537fec6b914d7ad95fbf28b0a2#f82848847a6392537fec6b914d7ad95fbf28b0a2" dependencies = [ - "atomic", + "atomic 0.4.6", "atomic-traits", "atomic_refcell", "built", @@ -377,6 +386,7 @@ dependencies = [ name = "mmtk_openjdk" version = "0.15.0" dependencies = [ + "atomic 0.5.1", "built", "lazy_static", "libc", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index f17eea1c..7efa1ace 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -23,6 +23,7 @@ openjdk_version = "c82e5c44adced4383162826c2c3933a83cfb139b" libc = "0.2" lazy_static = "1.1" once_cell = "1.10.0" +atomic = "0.5.1" # Be very careful to commit any changes to the following mmtk dependency, as our CI scripts (including mmtk-core CI) # rely on matching these lines to modify them: e.g. comment out the git dependency and use the local path. # These changes are safe: diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 5a2620cd..8b0d325f 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -155,7 +155,7 @@ pub struct InstanceKlass { } #[repr(u8)] -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] #[allow(dead_code)] pub enum ReferenceType { None, // Regular class diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index ef1a7ceb..14fabe1e 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -1,8 +1,10 @@ +use mmtk::scheduler::ProcessEdgesWork; use mmtk::util::alloc::AllocationError; use mmtk::util::opaque_pointer::*; use mmtk::vm::{Collection, GCThreadContext, Scanning, VMBinding}; use mmtk::{Mutator, MutatorContext}; +use crate::reference_glue::DISCOVERED_LISTS; use crate::UPCALLS; use crate::{MutatorClosure, OpenJDK}; @@ -51,6 +53,7 @@ impl Collection for VMCollection { } fn resume_mutators(tls: VMWorkerThread) { + DISCOVERED_LISTS.enable_discover(); unsafe { ((*UPCALLS).resume_mutators)(tls); } @@ -96,4 +99,12 @@ impl Collection for VMCollection { ((*UPCALLS).schedule_finalizer)(); } } + + fn process_weak_refs>( + worker: &mut mmtk::scheduler::GCWorker, + ) { + if crate::VM_REF_PROCESSOR { + DISCOVERED_LISTS.process::(worker) + } + } } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 7231330f..1d8ead81 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -2,6 +2,7 @@ extern crate libc; extern crate mmtk; #[macro_use] extern crate lazy_static; +extern crate atomic; extern crate once_cell; use std::collections::HashMap; @@ -160,3 +161,5 @@ lazy_static! { /// A counter tracking the total size of the `CODE_CACHE_ROOTS`. static CODE_CACHE_ROOTS_SIZE: AtomicUsize = AtomicUsize::new(0); + +const VM_REF_PROCESSOR: bool = true; diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index b9a060fc..f8da92ac 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -1,3 +1,5 @@ +use crate::reference_glue::DISCOVERED_LISTS; + use super::abi::*; use super::{OpenJDKEdge, UPCALLS}; use mmtk::util::constants::*; @@ -151,10 +153,16 @@ impl InstanceRefKlass { fn discover_reference(oop: Oop, rt: ReferenceType) -> bool { use crate::api::{add_phantom_candidate, add_soft_candidate, add_weak_candidate}; // Do not discover new refs during reference processing. - if !crate::SINGLETON.reference_processors.allow_new_candidate() { - return false; + if crate::VM_REF_PROCESSOR { + if !DISCOVERED_LISTS.allow_discover() { + return false; + } + } else { + if !crate::SINGLETON.reference_processors.allow_new_candidate() { + return false; + } } - // Do not discover if the referent is true. + // Do not discover if the referent is live. let referent: ObjectReference = unsafe { InstanceRefKlass::referent_address(oop).load() }; if referent.is_live() { return false; @@ -162,11 +170,16 @@ impl InstanceRefKlass { // TODO: Do not discover if the referent is a nursery object. // TODO: Fast list insertion let reference = ObjectReference::from(oop); - match rt { - ReferenceType::Weak => add_weak_candidate(reference), - ReferenceType::Soft => add_soft_candidate(reference), - ReferenceType::Phantom => add_phantom_candidate(reference), - _ => unreachable!(), + if crate::VM_REF_PROCESSOR { + // crate::reference_glue::set_referent(reference, ObjectReference::NULL); + DISCOVERED_LISTS.get(rt).add(reference); + } else { + match rt { + ReferenceType::Weak => add_weak_candidate(reference), + ReferenceType::Soft => add_soft_candidate(reference), + ReferenceType::Phantom => add_phantom_candidate(reference), + _ => unreachable!(), + } } true } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 7817e8d2..4fc2fca8 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -1,8 +1,43 @@ -use crate::abi::{InstanceRefKlass, Oop}; +use std::marker::PhantomData; +use std::sync::atomic::{AtomicBool, AtomicUsize}; +use std::sync::Mutex; + +use crate::abi::{InstanceRefKlass, Oop, ReferenceType}; use crate::OpenJDK; +use atomic::{Atomic, Ordering}; +use mmtk::scheduler::{GCWork, GCWorker, ProcessEdgesWork, WorkBucketStage}; use mmtk::util::opaque_pointer::VMWorkerThread; -use mmtk::util::ObjectReference; +use mmtk::util::{Address, ObjectReference}; use mmtk::vm::ReferenceGlue; +use mmtk::MMTK; + +#[inline(always)] +pub fn set_referent(reff: ObjectReference, referent: ObjectReference) { + let oop = Oop::from(reff); + unsafe { InstanceRefKlass::referent_address(oop).store(referent) }; +} + +#[inline(always)] +fn get_referent(object: ObjectReference) -> ObjectReference { + let oop = Oop::from(object); + unsafe { InstanceRefKlass::referent_address(oop).load::() } +} + +#[inline(always)] +fn get_next_reference_slot(object: ObjectReference) -> Address { + let oop = Oop::from(object); + unsafe { InstanceRefKlass::discovered_address(oop) } +} + +#[inline(always)] +fn get_next_reference(object: ObjectReference) -> ObjectReference { + unsafe { get_next_reference_slot(object).load() } +} + +#[inline(always)] +fn set_next_reference(object: ObjectReference, next: ObjectReference) { + unsafe { get_next_reference_slot(object).store(next) } +} pub struct VMReferenceGlue {} @@ -23,3 +58,207 @@ impl ReferenceGlue for VMReferenceGlue { // } } } + +pub struct DiscoveredList { + head: Atomic, + rt: ReferenceType, +} + +impl DiscoveredList { + fn new(rt: ReferenceType) -> Self { + Self { + head: Atomic::new(ObjectReference::NULL), + rt, + } + } + + #[inline] + pub fn add(&self, obj: ObjectReference) { + // let _g = LOCK.lock().unwrap(); + let oop = Oop::from(obj); + let head = self.head.load(Ordering::SeqCst); + let discovered_addr = unsafe { + InstanceRefKlass::discovered_address(oop).as_ref::>() + }; + let next_discovered = if head.is_null() { obj } else { head }; + debug_assert!(!next_discovered.is_null()); + if discovered_addr + .compare_exchange( + ObjectReference::NULL, + next_discovered, + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + // println!("add {:?} {:?}", self.rt, obj); + self.head.store(obj, Ordering::SeqCst); + } + debug_assert!(!get_next_reference(obj).is_null()); + } +} + +pub struct DiscoveredLists { + pub soft: Vec, + pub weak: Vec, + pub phantom: Vec, + allow_discover: AtomicBool, +} + +impl DiscoveredLists { + pub fn new() -> Self { + let workers = crate::SINGLETON.scheduler.num_workers(); + let build_lists = |rt| (0..workers).map(|_| DiscoveredList::new(rt)).collect(); + Self { + soft: build_lists(ReferenceType::Soft), + weak: build_lists(ReferenceType::Weak), + phantom: build_lists(ReferenceType::Phantom), + allow_discover: AtomicBool::new(true), + } + } + + #[inline] + pub fn enable_discover(&self) { + self.allow_discover.store(true, Ordering::SeqCst) + } + + #[inline(always)] + pub fn allow_discover(&self) -> bool { + self.allow_discover.load(Ordering::SeqCst) + } + + #[inline] + pub fn get_by_rt_and_index(&self, rt: ReferenceType, index: usize) -> &DiscoveredList { + match rt { + ReferenceType::Soft => &self.soft[index], + ReferenceType::Weak => &self.weak[index], + ReferenceType::Phantom => &self.phantom[index], + _ => unimplemented!(), + } + } + + #[inline] + pub fn get(&self, rt: ReferenceType) -> &DiscoveredList { + let id = mmtk::scheduler::worker::current_worker_ordinal().unwrap(); + self.get_by_rt_and_index(rt, id) + } + + pub fn process_lists>( + &self, + worker: &mut GCWorker, + rt: ReferenceType, + lists: &[DiscoveredList], + ) { + let mut packets = vec![]; + for i in 0..lists.len() { + let head = lists[i].head.load(Ordering::SeqCst); + if head.is_null() { + continue; + } + let w = ProcessDiscoveredList { + list_index: i, + head, + rt, + _p: PhantomData::, + }; + packets.push(Box::new(w) as Box>); + } + worker.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets); + } + + pub fn process>(&self, worker: &mut GCWorker) { + println!( + "process refs {}", + worker.mmtk.plan.is_emergency_collection() + ); + self.allow_discover.store(false, Ordering::SeqCst); + self.process_lists::(worker, ReferenceType::Soft, &self.soft); + self.process_lists::(worker, ReferenceType::Weak, &self.weak); + self.process_lists::(worker, ReferenceType::Phantom, &self.phantom); + } +} + +lazy_static! { + pub static ref LOCK: Mutex<()> = Mutex::new(()); + pub static ref DISCOVERED_LISTS: DiscoveredLists = DiscoveredLists::new(); +} + +pub struct ProcessDiscoveredList> { + list_index: usize, + head: ObjectReference, + rt: ReferenceType, + _p: PhantomData, +} + +impl> GCWork for ProcessDiscoveredList { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { + let mut trace = E::new(vec![], false, mmtk); + trace.set_worker(worker); + // mmtk.reference_processors.scan_soft_refs(&mut w, mmtk); + let retain = true || self.rt == ReferenceType::Soft && !mmtk.plan.is_emergency_collection(); + // if rt == ReferenceType::Soft && !mmtk.plan.is_emergency_collection() { + // // Retain soft refs + // let mut obj = self.head; + // let mut prev_obj = ObjectReference::NULL; + // while obj != prev_obj { + // if !reference.is_live() {} + // prev_obj = obj; + // obj = get_next_reference(obj); + // } + // } + let mut cursor = &mut self.head; + let mut holder: Option = None; + loop { + debug_assert!(!cursor.is_null()); + let mut reference = *cursor; + // Remove reference from list if it is dead + if !reference.is_live() { + set_referent(reference, ObjectReference::NULL); + let next_ref = get_next_reference(reference); + if next_ref == reference { + if let Some(holder) = holder { + *cursor = holder; + } else { + *cursor = ObjectReference::NULL; + } + break; + } else { + debug_assert!(!next_ref.is_null()); + *cursor = next_ref; + debug_assert!(!cursor.is_null()); + continue; + } + } else { + let new_reference = reference.get_forwarded_object().unwrap_or(reference); + // println!("reference {:?} => {:?}", reference, new_reference); + reference = new_reference; + *cursor = reference; + } + let referent = get_referent(reference); + if !referent.is_null() && (referent.is_live() || retain) { + // Keep this ref + let forwarded = trace.trace_object(referent); + set_referent(reference, forwarded); + } else { + // Clear this ref + set_referent(reference, ObjectReference::NULL); + } + cursor = unsafe { &mut *get_next_reference_slot(reference).to_mut_ptr() }; + holder = Some(reference); + let next_ref = get_next_reference(reference); + if next_ref.is_live() { + let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); + if next_ref == reference { + set_next_reference(reference, next_ref); + break; + } + } + debug_assert!(!cursor.is_null(), "{:?}", reference); + } + trace.flush(); + DISCOVERED_LISTS + .get_by_rt_and_index(self.rt, self.list_index) + .head + .store(self.head, Ordering::SeqCst); + } +} From 4d91fcbf0be9499beac5e2015ad4523c35e28ae4 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 25 Oct 2022 12:01:29 +0000 Subject: [PATCH 173/494] Fast reference processing --- mmtk/Cargo.lock | 4 +- mmtk/Cargo.toml | 2 +- mmtk/src/object_scanning.rs | 3 +- mmtk/src/reference_glue.rs | 109 +++++++++++++++--------------------- 4 files changed, 51 insertions(+), 67 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 477e33ec..8955b0b2 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -349,7 +349,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=f82848847a6392537fec6b914d7ad95fbf28b0a2#f82848847a6392537fec6b914d7ad95fbf28b0a2" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7d56f0eefb2c47653b2f6a4a2c2a6afff37cafd2#7d56f0eefb2c47653b2f6a4a2c2a6afff37cafd2" dependencies = [ "atomic 0.4.6", "atomic-traits", @@ -374,7 +374,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=f82848847a6392537fec6b914d7ad95fbf28b0a2#f82848847a6392537fec6b914d7ad95fbf28b0a2" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7d56f0eefb2c47653b2f6a4a2c2a6afff37cafd2#7d56f0eefb2c47653b2f6a4a2c2a6afff37cafd2" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 7efa1ace..020a96e8 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -30,7 +30,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "f82848847a6392537fec6b914d7ad95fbf28b0a2" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7d56f0eefb2c47653b2f6a4a2c2a6afff37cafd2" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index f8da92ac..8b192a6d 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -168,7 +168,8 @@ impl InstanceRefKlass { return false; } // TODO: Do not discover if the referent is a nursery object. - // TODO: Fast list insertion + + // Add to reference list let reference = ObjectReference::from(oop); if crate::VM_REF_PROCESSOR { // crate::reference_glue::set_referent(reference, ObjectReference::NULL); diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 4fc2fca8..3da7d2ee 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -1,5 +1,5 @@ use std::marker::PhantomData; -use std::sync::atomic::{AtomicBool, AtomicUsize}; +use std::sync::atomic::AtomicBool; use std::sync::Mutex; use crate::abi::{InstanceRefKlass, Oop, ReferenceType}; @@ -26,7 +26,7 @@ fn get_referent(object: ObjectReference) -> ObjectReference { #[inline(always)] fn get_next_reference_slot(object: ObjectReference) -> Address { let oop = Oop::from(object); - unsafe { InstanceRefKlass::discovered_address(oop) } + InstanceRefKlass::discovered_address(oop) } #[inline(always)] @@ -61,22 +61,21 @@ impl ReferenceGlue for VMReferenceGlue { pub struct DiscoveredList { head: Atomic, - rt: ReferenceType, + _rt: ReferenceType, } impl DiscoveredList { fn new(rt: ReferenceType) -> Self { Self { head: Atomic::new(ObjectReference::NULL), - rt, + _rt: rt, } } #[inline] pub fn add(&self, obj: ObjectReference) { - // let _g = LOCK.lock().unwrap(); let oop = Oop::from(obj); - let head = self.head.load(Ordering::SeqCst); + let head = self.head.load(Ordering::Relaxed); let discovered_addr = unsafe { InstanceRefKlass::discovered_address(oop).as_ref::>() }; @@ -92,7 +91,7 @@ impl DiscoveredList { .is_ok() { // println!("add {:?} {:?}", self.rt, obj); - self.head.store(obj, Ordering::SeqCst); + self.head.store(obj, Ordering::Relaxed); } debug_assert!(!get_next_reference(obj).is_null()); } @@ -127,7 +126,7 @@ impl DiscoveredLists { self.allow_discover.load(Ordering::SeqCst) } - #[inline] + #[inline(always)] pub fn get_by_rt_and_index(&self, rt: ReferenceType, index: usize) -> &DiscoveredList { match rt { ReferenceType::Soft => &self.soft[index], @@ -137,7 +136,7 @@ impl DiscoveredLists { } } - #[inline] + #[inline(always)] pub fn get(&self, rt: ReferenceType) -> &DiscoveredList { let id = mmtk::scheduler::worker::current_worker_ordinal().unwrap(); self.get_by_rt_and_index(rt, id) @@ -152,6 +151,7 @@ impl DiscoveredLists { let mut packets = vec![]; for i in 0..lists.len() { let head = lists[i].head.load(Ordering::SeqCst); + lists[i].head.store(ObjectReference::NULL, Ordering::SeqCst); if head.is_null() { continue; } @@ -167,10 +167,10 @@ impl DiscoveredLists { } pub fn process>(&self, worker: &mut GCWorker) { - println!( - "process refs {}", - worker.mmtk.plan.is_emergency_collection() - ); + // println!( + // "process refs {}", + // worker.mmtk.plan.is_emergency_collection() + // ); self.allow_discover.store(false, Ordering::SeqCst); self.process_lists::(worker, ReferenceType::Soft, &self.soft); self.process_lists::(worker, ReferenceType::Weak, &self.weak); @@ -194,71 +194,54 @@ impl> GCWork for ProcessDiscoveredLis fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { let mut trace = E::new(vec![], false, mmtk); trace.set_worker(worker); - // mmtk.reference_processors.scan_soft_refs(&mut w, mmtk); - let retain = true || self.rt == ReferenceType::Soft && !mmtk.plan.is_emergency_collection(); - // if rt == ReferenceType::Soft && !mmtk.plan.is_emergency_collection() { - // // Retain soft refs - // let mut obj = self.head; - // let mut prev_obj = ObjectReference::NULL; - // while obj != prev_obj { - // if !reference.is_live() {} - // prev_obj = obj; - // obj = get_next_reference(obj); - // } - // } + let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); let mut cursor = &mut self.head; let mut holder: Option = None; loop { debug_assert!(!cursor.is_null()); - let mut reference = *cursor; - // Remove reference from list if it is dead - if !reference.is_live() { - set_referent(reference, ObjectReference::NULL); - let next_ref = get_next_reference(reference); - if next_ref == reference { - if let Some(holder) = holder { - *cursor = holder; - } else { - *cursor = ObjectReference::NULL; - } - break; - } else { - debug_assert!(!next_ref.is_null()); - *cursor = next_ref; - debug_assert!(!cursor.is_null()); - continue; - } - } else { - let new_reference = reference.get_forwarded_object().unwrap_or(reference); - // println!("reference {:?} => {:?}", reference, new_reference); - reference = new_reference; - *cursor = reference; - } + let reference = *cursor; + debug_assert!(reference.is_live()); + debug_assert!(reference.get_forwarded_object().is_none()); let referent = get_referent(reference); if !referent.is_null() && (referent.is_live() || retain) { // Keep this ref let forwarded = trace.trace_object(referent); set_referent(reference, forwarded); + // Remove from list + let next_ref = get_next_reference(reference); + set_next_reference(reference, ObjectReference::NULL); + // End of list. + if next_ref.is_live() { + let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); + if next_ref == reference { + // End of list + if let Some(holder) = holder { + set_next_reference(holder, holder); + } + break; + } + } + // Move to next + *cursor = next_ref; + // holder remains the same } else { // Clear this ref set_referent(reference, ObjectReference::NULL); - } - cursor = unsafe { &mut *get_next_reference_slot(reference).to_mut_ptr() }; - holder = Some(reference); - let next_ref = get_next_reference(reference); - if next_ref.is_live() { - let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); - if next_ref == reference { - set_next_reference(reference, next_ref); - break; + // Move to next + let next_ref = get_next_reference(reference); + cursor = unsafe { &mut *get_next_reference_slot(reference).to_mut_ptr() }; + holder = Some(reference); + if next_ref.is_live() { + let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); + if next_ref == reference { + // End of list + set_next_reference(reference, next_ref); + break; + } } } - debug_assert!(!cursor.is_null(), "{:?}", reference); } trace.flush(); - DISCOVERED_LISTS - .get_by_rt_and_index(self.rt, self.list_index) - .head - .store(self.head, Ordering::SeqCst); + // TODO: Flush the list to the Universe::pending_list } } From e3b58d52546ccd4b21ae41f0ed9f88f747af7411 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 31 Oct 2022 01:12:00 +0000 Subject: [PATCH 174/494] weak ref pending list --- mmtk/src/lib.rs | 1 + mmtk/src/reference_glue.rs | 22 ++++++++++++++++------ openjdk/mmtk.h | 1 + openjdk/mmtkUpcalls.cpp | 7 ++++++- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 1d8ead81..020fb87f 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -97,6 +97,7 @@ pub struct OpenJDK_Upcalls { pub schedule_finalizer: extern "C" fn(), pub prepare_for_roots_re_scanning: extern "C" fn(), pub enqueue_references: extern "C" fn(objects: *const ObjectReference, len: usize), + pub swap_reference_pending_list: extern "C" fn(objects: ObjectReference) -> ObjectReference, } pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 3da7d2ee..9967f501 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -167,10 +167,6 @@ impl DiscoveredLists { } pub fn process>(&self, worker: &mut GCWorker) { - // println!( - // "process refs {}", - // worker.mmtk.plan.is_emergency_collection() - // ); self.allow_discover.store(false, Ordering::SeqCst); self.process_lists::(worker, ReferenceType::Soft, &self.soft); self.process_lists::(worker, ReferenceType::Weak, &self.weak); @@ -197,6 +193,7 @@ impl> GCWork for ProcessDiscoveredLis let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); let mut cursor = &mut self.head; let mut holder: Option = None; + let mut tail: Option; loop { debug_assert!(!cursor.is_null()); let reference = *cursor; @@ -217,13 +214,16 @@ impl> GCWork for ProcessDiscoveredLis // End of list if let Some(holder) = holder { set_next_reference(holder, holder); + tail = Some(holder); + } else { + *cursor = ObjectReference::NULL; + tail = None; } break; } } // Move to next *cursor = next_ref; - // holder remains the same } else { // Clear this ref set_referent(reference, ObjectReference::NULL); @@ -236,12 +236,22 @@ impl> GCWork for ProcessDiscoveredLis if next_ref == reference { // End of list set_next_reference(reference, next_ref); + tail = Some(reference); break; } } } } + // Flush the list to the Universe::pending_list + let head = *cursor; + if let Some(tail) = tail { + assert!(!head.is_null()); + assert_eq!(tail, get_next_reference(tail)); + let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; + set_next_reference(tail, old_head); + } else { + assert!(head.is_null()); + } trace.flush(); - // TODO: Flush the list to the Universe::pending_list } } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index dd4560e9..09a22dc6 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -177,6 +177,7 @@ typedef struct { void (*schedule_finalizer)(); void (*prepare_for_roots_re_scanning)(); void (*enqueue_references)(void** objects, size_t len); + void* (*swap_reference_pending_list)(void* objects); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 3b983bcb..b43cbdd1 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -341,6 +341,10 @@ static void mmtk_enqueue_references(void** objects, size_t len) { assert(Universe::has_reference_pending_list(), "Reference pending list is empty after swap"); } +static void* mmtk_swap_reference_pending_list(void* object) { + return Universe::swap_reference_pending_list((oop) object); +} + OpenJDK_Upcalls mmtk_upcalls = { mmtk_stop_all_mutators, mmtk_resume_mutators, @@ -379,5 +383,6 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_number_of_mutators, mmtk_schedule_finalizer, mmtk_prepare_for_roots_re_scanning, - mmtk_enqueue_references + mmtk_enqueue_references, + mmtk_swap_reference_pending_list }; From 97bd4028c1a7214aeeca0abd8cffdb115758d0c6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 31 Oct 2022 02:22:04 +0000 Subject: [PATCH 175/494] refactor --- mmtk/src/reference_glue.rs | 98 +++++++++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 29 deletions(-) diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 9967f501..1e4ffe35 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -166,12 +166,24 @@ impl DiscoveredLists { worker.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets); } + pub fn reconsider_soft_refs>( + &self, + worker: &mut GCWorker, + ) { + } + pub fn process>(&self, worker: &mut GCWorker) { self.allow_discover.store(false, Ordering::SeqCst); self.process_lists::(worker, ReferenceType::Soft, &self.soft); self.process_lists::(worker, ReferenceType::Weak, &self.weak); self.process_lists::(worker, ReferenceType::Phantom, &self.phantom); } + + pub fn resurrect_final_refs>( + &self, + worker: &mut GCWorker, + ) { + } } lazy_static! { @@ -191,23 +203,58 @@ impl> GCWork for ProcessDiscoveredLis let mut trace = E::new(vec![], false, mmtk); trace.set_worker(worker); let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); - let mut cursor = &mut self.head; - let mut holder: Option = None; - let mut tail: Option; - loop { - debug_assert!(!cursor.is_null()); - let reference = *cursor; - debug_assert!(reference.is_live()); - debug_assert!(reference.get_forwarded_object().is_none()); + let new_list = iterate_list(self.head, |reference| { let referent = get_referent(reference); if !referent.is_null() && (referent.is_live() || retain) { - // Keep this ref + // Keep this referent let forwarded = trace.trace_object(referent); set_referent(reference, forwarded); - // Remove from list + // Remove the reference from the discovered list + return IterationResult::Removed; + } else { + // Clear this referent + set_referent(reference, ObjectReference::NULL); + // Keep the reference from the discovered list + return IterationResult::Kept; + } + }); + // Flush the list to the Universe::pending_list + if let Some((head, tail)) = new_list { + assert!(!head.is_null() && !tail.is_null()); + assert_eq!(tail, get_next_reference(tail)); + let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; + set_next_reference(tail, old_head); + } + trace.flush(); + } +} + +enum IterationResult { + Kept, + Removed, +} + +fn iterate_list( + mut head: ObjectReference, + mut visitor: impl FnMut(ObjectReference) -> IterationResult, +) -> Option<(ObjectReference, ObjectReference)> { + // let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); + let mut cursor = &mut head; + let mut holder: Option = None; + let mut tail: Option; + loop { + debug_assert!(!cursor.is_null()); + let reference = *cursor; + debug_assert!(reference.is_live()); + debug_assert!(reference.get_forwarded_object().is_none()); + let next_ref = get_next_reference(reference); + let result = visitor(reference); + match result { + IterationResult::Removed => { + // Remove `reference` from list let next_ref = get_next_reference(reference); set_next_reference(reference, ObjectReference::NULL); - // End of list. + // Reached the end of the list? if next_ref.is_live() { let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); if next_ref == reference { @@ -224,13 +271,9 @@ impl> GCWork for ProcessDiscoveredLis } // Move to next *cursor = next_ref; - } else { - // Clear this ref - set_referent(reference, ObjectReference::NULL); - // Move to next - let next_ref = get_next_reference(reference); - cursor = unsafe { &mut *get_next_reference_slot(reference).to_mut_ptr() }; - holder = Some(reference); + } + IterationResult::Kept => { + // Reached the end of the list? if next_ref.is_live() { let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); if next_ref == reference { @@ -240,18 +283,15 @@ impl> GCWork for ProcessDiscoveredLis break; } } + // Move to next + cursor = unsafe { &mut *get_next_reference_slot(reference).to_mut_ptr() }; + holder = Some(reference); } } - // Flush the list to the Universe::pending_list - let head = *cursor; - if let Some(tail) = tail { - assert!(!head.is_null()); - assert_eq!(tail, get_next_reference(tail)); - let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; - set_next_reference(tail, old_head); - } else { - assert!(head.is_null()); - } - trace.flush(); + } + if head.is_null() { + None + } else { + Some((head, tail.unwrap())) } } From 014f09347042c2833fcca2967083c4d15219429e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 31 Oct 2022 05:20:31 +0000 Subject: [PATCH 176/494] Fix finalizables --- mmtk/src/api.rs | 4 +- mmtk/src/collection.rs | 22 ++++-- mmtk/src/object_scanning.rs | 23 +++--- mmtk/src/reference_glue.rs | 112 ++++++++++++++++++++++++++---- openjdk/mmtkVMCompanionThread.cpp | 6 -- openjdk/mmtkVMOperation.cpp | 17 +++++ openjdk/mmtkVMOperation.hpp | 2 + 7 files changed, 149 insertions(+), 37 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 20d6736d..2b5f4e7d 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -378,8 +378,8 @@ pub extern "C" fn mmtk_array_copy_post( // finalization #[no_mangle] -pub extern "C" fn add_finalizer(object: ObjectReference) { - memory_manager::add_finalizer(&SINGLETON, object); +pub extern "C" fn add_finalizer(_object: ObjectReference) { + unreachable!() } #[no_mangle] diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 14fabe1e..4295394f 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -1,4 +1,4 @@ -use mmtk::scheduler::ProcessEdgesWork; +use mmtk::scheduler::{GCWorker, ProcessEdgesWork}; use mmtk::util::alloc::AllocationError; use mmtk::util::opaque_pointer::*; use mmtk::vm::{Collection, GCThreadContext, Scanning, VMBinding}; @@ -95,16 +95,24 @@ impl Collection for VMCollection { } fn schedule_finalization(_tls: VMWorkerThread) { - unsafe { - ((*UPCALLS).schedule_finalizer)(); + unreachable!() + } + + fn process_weak_refs>(worker: &mut GCWorker) { + if crate::VM_REF_PROCESSOR { + DISCOVERED_LISTS.process_soft_weak_final_refs::(worker) } } - fn process_weak_refs>( - worker: &mut mmtk::scheduler::GCWorker, - ) { + fn process_final_refs>(worker: &mut GCWorker) { + if crate::VM_REF_PROCESSOR { + DISCOVERED_LISTS.resurrect_final_refs::(worker) + } + } + + fn process_phantom_refs>(worker: &mut GCWorker) { if crate::VM_REF_PROCESSOR { - DISCOVERED_LISTS.process::(worker) + DISCOVERED_LISTS.process_phantom_refs::(worker) } } } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 8b192a6d..5ced3ae0 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -1,4 +1,5 @@ use crate::reference_glue::DISCOVERED_LISTS; +use crate::SINGLETON; use super::abi::*; use super::{OpenJDKEdge, UPCALLS}; @@ -112,12 +113,15 @@ impl OopIterate for InstanceRefKlass { use crate::abi::*; self.instance_klass.oop_iterate(oop, closure); - if Self::should_scan_weak_refs() { + if Self::should_discover_refs(self.instance_klass.reference_type) { match self.instance_klass.reference_type { ReferenceType::None => { panic!("oop_iterate on InstanceRefKlass with reference_type as None") } - rt @ (ReferenceType::Weak | ReferenceType::Soft | ReferenceType::Phantom) => { + rt @ (ReferenceType::Weak + | ReferenceType::Soft + | ReferenceType::Phantom + | ReferenceType::Final) => { if !Self::discover_reference(oop, rt) { Self::process_ref_as_strong(oop, closure) } @@ -126,9 +130,7 @@ impl OopIterate for InstanceRefKlass { // We will handle final reference later // ReferenceType::Weak // | ReferenceType::Phantom - ReferenceType::Final | ReferenceType::Other => { - Self::process_ref_as_strong(oop, closure) - } + ReferenceType::Other => Self::process_ref_as_strong(oop, closure), } } else { Self::process_ref_as_strong(oop, closure); @@ -138,9 +140,14 @@ impl OopIterate for InstanceRefKlass { impl InstanceRefKlass { #[inline] - fn should_scan_weak_refs() -> bool { - use SINGLETON; - !*SINGLETON.get_options().no_reference_types + fn should_discover_refs(rt: ReferenceType) -> bool { + if *SINGLETON.get_options().no_finalizer && rt == ReferenceType::Final { + return false; + } + if *SINGLETON.get_options().no_reference_types && rt != ReferenceType::Final { + return false; + } + true } #[inline] fn process_ref_as_strong(oop: Oop, closure: &mut impl EdgeVisitor) { diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 1e4ffe35..86feff9f 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -3,7 +3,7 @@ use std::sync::atomic::AtomicBool; use std::sync::Mutex; use crate::abi::{InstanceRefKlass, Oop, ReferenceType}; -use crate::OpenJDK; +use crate::{OpenJDK, SINGLETON}; use atomic::{Atomic, Ordering}; use mmtk::scheduler::{GCWork, GCWorker, ProcessEdgesWork, WorkBucketStage}; use mmtk::util::opaque_pointer::VMWorkerThread; @@ -100,6 +100,7 @@ impl DiscoveredList { pub struct DiscoveredLists { pub soft: Vec, pub weak: Vec, + pub r#final: Vec, pub phantom: Vec, allow_discover: AtomicBool, } @@ -111,6 +112,7 @@ impl DiscoveredLists { Self { soft: build_lists(ReferenceType::Soft), weak: build_lists(ReferenceType::Weak), + r#final: build_lists(ReferenceType::Final), phantom: build_lists(ReferenceType::Phantom), allow_discover: AtomicBool::new(true), } @@ -132,6 +134,7 @@ impl DiscoveredLists { ReferenceType::Soft => &self.soft[index], ReferenceType::Weak => &self.weak[index], ReferenceType::Phantom => &self.phantom[index], + ReferenceType::Final => &self.r#final[index], _ => unimplemented!(), } } @@ -147,11 +150,14 @@ impl DiscoveredLists { worker: &mut GCWorker, rt: ReferenceType, lists: &[DiscoveredList], + clear: bool, ) { let mut packets = vec![]; for i in 0..lists.len() { let head = lists[i].head.load(Ordering::SeqCst); - lists[i].head.store(ObjectReference::NULL, Ordering::SeqCst); + if clear { + lists[i].head.store(ObjectReference::NULL, Ordering::SeqCst); + } if head.is_null() { continue; } @@ -168,21 +174,53 @@ impl DiscoveredLists { pub fn reconsider_soft_refs>( &self, - worker: &mut GCWorker, + _worker: &mut GCWorker, ) { } - pub fn process>(&self, worker: &mut GCWorker) { + pub fn process_soft_weak_final_refs>( + &self, + worker: &mut GCWorker, + ) { self.allow_discover.store(false, Ordering::SeqCst); - self.process_lists::(worker, ReferenceType::Soft, &self.soft); - self.process_lists::(worker, ReferenceType::Weak, &self.weak); - self.process_lists::(worker, ReferenceType::Phantom, &self.phantom); + if !*SINGLETON.get_options().no_reference_types { + self.process_lists::(worker, ReferenceType::Soft, &self.soft, true); + self.process_lists::(worker, ReferenceType::Weak, &self.weak, true); + } + if !*SINGLETON.get_options().no_finalizer { + self.process_lists::(worker, ReferenceType::Final, &self.r#final, false); + } } pub fn resurrect_final_refs>( &self, worker: &mut GCWorker, ) { + assert!(!*SINGLETON.get_options().no_finalizer); + let lists = &self.r#final; + let mut packets = vec![]; + for i in 0..lists.len() { + let head = lists[i].head.load(Ordering::SeqCst); + lists[i].head.store(ObjectReference::NULL, Ordering::SeqCst); + if head.is_null() { + continue; + } + let w = ResurrectFinalizables { + list_index: i, + head, + _p: PhantomData::, + }; + packets.push(Box::new(w) as Box>); + } + worker.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets); + } + + pub fn process_phantom_refs>( + &self, + worker: &mut GCWorker, + ) { + assert!(!*SINGLETON.get_options().no_reference_types); + self.process_lists::(worker, ReferenceType::Phantom, &self.phantom, true); } } @@ -205,16 +243,21 @@ impl> GCWork for ProcessDiscoveredLis let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); let new_list = iterate_list(self.head, |reference| { let referent = get_referent(reference); - if !referent.is_null() && (referent.is_live() || retain) { + if referent.is_null() { + // Remove from the discovered list + return IterationResult::Removed; + } else if referent.is_live() || retain { // Keep this referent let forwarded = trace.trace_object(referent); set_referent(reference, forwarded); - // Remove the reference from the discovered list + // Remove from the discovered list return IterationResult::Removed; } else { - // Clear this referent - set_referent(reference, ObjectReference::NULL); - // Keep the reference from the discovered list + if self.rt != ReferenceType::Final { + // Clear this referent + set_referent(reference, ObjectReference::NULL); + } + // Keep the reference return IterationResult::Kept; } }); @@ -222,6 +265,48 @@ impl> GCWork for ProcessDiscoveredLis if let Some((head, tail)) = new_list { assert!(!head.is_null() && !tail.is_null()); assert_eq!(tail, get_next_reference(tail)); + if self.rt == ReferenceType::Final { + DISCOVERED_LISTS.r#final[self.list_index] + .head + .store(head, Ordering::SeqCst); + } else { + let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; + set_next_reference(tail, old_head); + } + } else { + if self.rt == ReferenceType::Final { + DISCOVERED_LISTS.r#final[self.list_index] + .head + .store(ObjectReference::NULL, Ordering::SeqCst); + } + } + trace.flush(); + } +} + +pub struct ResurrectFinalizables> { + list_index: usize, + head: ObjectReference, + _p: PhantomData, +} + +impl> GCWork for ResurrectFinalizables { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { + let mut trace = E::new(vec![], false, mmtk); + trace.set_worker(worker); + let new_list = iterate_list(self.head, |reference| { + let referent = get_referent(reference); + let forwarded = trace.trace_object(referent); + set_referent(reference, forwarded); + return IterationResult::Kept; + }); + + if let Some((head, tail)) = new_list { + assert!(!head.is_null() && !tail.is_null()); + assert_eq!(tail, get_next_reference(tail)); + DISCOVERED_LISTS.r#final[self.list_index] + .head + .store(ObjectReference::NULL, Ordering::SeqCst); let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; set_next_reference(tail, old_head); } @@ -238,10 +323,9 @@ fn iterate_list( mut head: ObjectReference, mut visitor: impl FnMut(ObjectReference) -> IterationResult, ) -> Option<(ObjectReference, ObjectReference)> { - // let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); let mut cursor = &mut head; let mut holder: Option = None; - let mut tail: Option; + let tail: Option; loop { debug_assert!(!cursor.is_null()); let reference = *cursor; diff --git a/openjdk/mmtkVMCompanionThread.cpp b/openjdk/mmtkVMCompanionThread.cpp index 17f2d2c1..4f99efba 100644 --- a/openjdk/mmtkVMCompanionThread.cpp +++ b/openjdk/mmtkVMCompanionThread.cpp @@ -76,12 +76,6 @@ void MMTkVMCompanionThread::run() { _reached_state = _threads_resumed; _lock->notify_all(); } - { - MutexLocker x(Heap_lock); - if (Universe::has_reference_pending_list()) { - Heap_lock->notify_all(); - } - } } } diff --git a/openjdk/mmtkVMOperation.cpp b/openjdk/mmtkVMOperation.cpp index 73676eec..fcbd14c0 100644 --- a/openjdk/mmtkVMOperation.cpp +++ b/openjdk/mmtkVMOperation.cpp @@ -27,13 +27,30 @@ #include "mmtkVMCompanionThread.hpp" #include "mmtkVMOperation.hpp" #include "logging/log.hpp" +#include "interpreter/oopMapCache.hpp" VM_MMTkSTWOperation::VM_MMTkSTWOperation(MMTkVMCompanionThread *companion_thread): _companion_thread(companion_thread) { } +bool VM_MMTkSTWOperation::doit_prologue() { + Heap_lock->lock(); + return true; +} + void VM_MMTkSTWOperation::doit() { log_trace(vmthread)("Entered VM_MMTkSTWOperation::doit()."); _companion_thread->reach_suspended_and_wait_for_resume(); log_trace(vmthread)("Leaving VM_MMTkSTWOperation::doit()"); } + +void VM_MMTkSTWOperation::doit_epilogue() { + // Clean up old interpreter OopMap entries that were replaced + // during the GC thread root traversal. + OopMapCache::cleanup_old_entries(); + // Notify the reference processing thread + if (Universe::has_reference_pending_list()) { + Heap_lock->notify_all(); + } + Heap_lock->unlock(); +} diff --git a/openjdk/mmtkVMOperation.hpp b/openjdk/mmtkVMOperation.hpp index 98d37ea4..49897ce7 100644 --- a/openjdk/mmtkVMOperation.hpp +++ b/openjdk/mmtkVMOperation.hpp @@ -39,7 +39,9 @@ class VM_MMTkSTWOperation : public VM_MMTkOperation { public: VM_MMTkSTWOperation(MMTkVMCompanionThread *companion_thread); + virtual bool doit_prologue() override; virtual void doit() override; + virtual void doit_epilogue() override; }; #endif // MMTK_OPENJDK_MMTK_VM_OPERATION_HPP From 67884f8a6d356c1f3ce4c58083310a319ad28269 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 1 Nov 2022 05:52:51 +0000 Subject: [PATCH 177/494] minor --- mmtk/src/abi.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index e6331253..511487fa 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -259,19 +259,19 @@ pub struct InstanceRefKlass { pub instance_klass: InstanceKlass, } -static DISCOVERED_OFFSET: spin::Lazy = - spin::Lazy::new(|| unsafe { ((*UPCALLS).discovered_offset)() }); - -static REFERENT_OFFSET: spin::Lazy = - spin::Lazy::new(|| unsafe { ((*UPCALLS).referent_offset)() }); - impl InstanceRefKlass { #[inline(always)] fn referent_offset() -> i32 { + lazy_static! { + pub static ref REFERENT_OFFSET: i32 = unsafe { ((*UPCALLS).referent_offset)() }; + } *REFERENT_OFFSET } #[inline(always)] fn discovered_offset() -> i32 { + lazy_static! { + pub static ref DISCOVERED_OFFSET: i32 = unsafe { ((*UPCALLS).discovered_offset)() }; + } *DISCOVERED_OFFSET } #[inline(always)] From b16d2fffb514d4e6174883f1e12d51c1dd18b690 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 1 Nov 2022 06:25:28 +0000 Subject: [PATCH 178/494] Update mmtk-core --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 1f738a25..4c1784b2 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "54b07fca9e121590226fec31e052da556e0cf7ae" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "eb8629b6e827684a443b1d1426ac012dcddeb213" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From eda11b365c46fd130ad3b084c9593e49570b0666 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 2 Nov 2022 04:37:22 +0000 Subject: [PATCH 179/494] Update Cargo.lock --- mmtk/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 00d4c06f..85b7e269 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=54b07fca9e121590226fec31e052da556e0cf7ae#54b07fca9e121590226fec31e052da556e0cf7ae" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eb8629b6e827684a443b1d1426ac012dcddeb213#eb8629b6e827684a443b1d1426ac012dcddeb213" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=54b07fca9e121590226fec31e052da556e0cf7ae#54b07fca9e121590226fec31e052da556e0cf7ae" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eb8629b6e827684a443b1d1426ac012dcddeb213#eb8629b6e827684a443b1d1426ac012dcddeb213" dependencies = [ "proc-macro-error", "proc-macro2", From 8a322c6811e89f5659f26eb7cbc44f43160371e0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 2 Nov 2022 05:01:52 +0000 Subject: [PATCH 180/494] cleanup --- mmtk/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 4c1784b2..4a0baf8c 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -76,7 +76,6 @@ immix = [] sanity = ["mmtk/sanity"] barrier_measurement = ["mmtk/barrier_measurement"] -work_packet_timer = ["mmtk/work_packet_timer"] yield_and_roots_timer = ["mmtk/yield_and_roots_timer"] satb_timer = ["mmtk/satb_timer"] instrumentation = ["mmtk/instrumentation"] From add25452dfc8512fbc91ce33a610456aca35ed00 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 3 Nov 2022 02:38:47 +0000 Subject: [PATCH 181/494] Fix C1 code patching --- openjdk/mmtkBarrierSetAssembler_x86.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index 7e037f58..f7998e6a 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -202,7 +202,7 @@ void MMTkBarrierSetAssembler::generate_c1_write_barrier_stub_call(LIR_Assembler* auto masm = ce->masm(); LIR_Address* addr = stub->slot->as_address_ptr(); Address from_addr = ce->as_Address(addr); - __ movptr(stub->scratch->as_register(), from_addr); + __ lea(stub->scratch->as_register(), from_addr); // Store parameter ce->store_parameter(stub->scratch->as_pointer_register(), 1); } else { From fb453977dd4691dea1ba11f3bb36ba524288693b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 4 Nov 2022 03:53:07 +0000 Subject: [PATCH 182/494] Fix arraycopy barrier --- openjdk/barriers/mmtkFieldBarrier.cpp | 2 +- openjdk/barriers/mmtkFieldBarrier.hpp | 4 ++++ openjdk/mmtkBarrierSet.hpp | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 155929ca..6aac0500 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -6,7 +6,7 @@ constexpr int kUnloggedValue = 1; void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, oop target) const { #if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = ((intptr_t) (void*) slot); - uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + const volatile uint8_t * meta_addr = (const volatile uint8_t *) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); intptr_t shift = (addr >> 3) & 0b111; uint8_t byte_val = *meta_addr; if (((byte_val >> shift) & 1) == kUnloggedValue) { diff --git a/openjdk/barriers/mmtkFieldBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp index c6afb103..b9986f87 100644 --- a/openjdk/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -44,6 +44,10 @@ class MMTkFieldBarrierSetC1: public MMTkBarrierSetC1 { class MMTkFieldBarrierSetC2: public MMTkBarrierSetC2 { protected: virtual void object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const override; +public: + virtual bool array_copy_requires_gc_barriers(BasicType type) const override { + return false; + } }; struct MMTkFieldBarrier: MMTkBarrierImpl< diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index c9149b5c..aa3ad76f 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -204,7 +204,7 @@ class MMTkBarrierSet : public BarrierSet { } template - static bool oop_arraycopy_in_heap_impl(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, + static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) { T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); From afe8150bdaf216093a85dda3dd161dfa86611f0d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 6 Nov 2022 02:58:14 +0000 Subject: [PATCH 183/494] WIP: Weak ref working --- mmtk/src/api.rs | 5 + mmtk/src/object_scanning.rs | 50 +++++++--- mmtk/src/reference_glue.rs | 138 +++++++++++++++----------- mmtk/src/scanning.rs | 3 +- openjdk/barriers/mmtkFieldBarrier.cpp | 85 ++++++++++++++++ openjdk/barriers/mmtkFieldBarrier.hpp | 17 ++++ openjdk/mmtk.h | 2 + openjdk/mmtkBarrierSet.hpp | 21 ++++ 8 files changed, 247 insertions(+), 74 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index f7dae302..b7fc68db 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -312,6 +312,11 @@ pub extern "C" fn executable() -> bool { true } +#[no_mangle] +pub extern "C" fn mmtk_load_reference(mutator: &'static mut Mutator, o: ObjectReference) { + mutator.barrier().load_reference(o); +} + /// Full pre barrier #[no_mangle] pub extern "C" fn mmtk_object_reference_write_pre( diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 46672936..aca96348 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -11,6 +11,14 @@ use std::{mem, slice}; trait OopIterate: Sized { fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor); + fn oop_iterate_with_discovery( + &self, + _oop: Oop, + _closure: &mut impl EdgeVisitor, + _discover: bool, + ) { + unimplemented!() + } } impl OopIterate for OopMapBlock { @@ -110,18 +118,24 @@ impl OopIterate for TypeArrayKlass { impl OopIterate for InstanceRefKlass { #[inline(always)] fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + unreachable!() + } + + fn oop_iterate_with_discovery( + &self, + oop: Oop, + closure: &mut impl EdgeVisitor, + disable_discovery: bool, + ) { use crate::abi::*; self.instance_klass.oop_iterate(oop, closure); - if Self::should_discover_refs(self.instance_klass.reference_type) { + if Self::should_discover_refs(self.instance_klass.reference_type, disable_discovery) { match self.instance_klass.reference_type { ReferenceType::None => { panic!("oop_iterate on InstanceRefKlass with reference_type as None") } - rt @ (ReferenceType::Weak - | ReferenceType::Soft - | ReferenceType::Phantom - | ReferenceType::Final) => { + rt @ (ReferenceType::Weak | ReferenceType::Phantom | ReferenceType::Soft) => { if !Self::discover_reference(oop, rt) { Self::process_ref_as_strong(oop, closure) } @@ -130,7 +144,9 @@ impl OopIterate for InstanceRefKlass { // We will handle final reference later // ReferenceType::Weak // | ReferenceType::Phantom - ReferenceType::Other => Self::process_ref_as_strong(oop, closure), + ReferenceType::Other | ReferenceType::Final => { + Self::process_ref_as_strong(oop, closure) + } } } else { Self::process_ref_as_strong(oop, closure); @@ -140,14 +156,14 @@ impl OopIterate for InstanceRefKlass { impl InstanceRefKlass { #[inline] - fn should_discover_refs(rt: ReferenceType) -> bool { - if *SINGLETON.get_options().no_finalizer && rt == ReferenceType::Final { + fn should_discover_refs(rt: ReferenceType, disable_discovery: bool) -> bool { + if disable_discovery { return false; } - if *SINGLETON.get_options().no_reference_types && rt != ReferenceType::Final { + if *SINGLETON.get_options().no_finalizer && rt == ReferenceType::Final { return false; } - if !SINGLETON.get_plan().should_process_refs_at_current_gc() { + if *SINGLETON.get_options().no_reference_types && rt != ReferenceType::Final { return false; } true @@ -174,7 +190,12 @@ impl InstanceRefKlass { } // Do not discover if the referent is live. let referent: ObjectReference = unsafe { InstanceRefKlass::referent_address(oop).load() }; - if referent.is_live() { + // Skip live or null referents + if referent.is_reachable() || referent.is_null() { + return false; + } + // Skip young referents + if mmtk::util::rc::count(referent) == 0 { return false; } // TODO: Do not discover if the referent is a nursery object. @@ -204,7 +225,7 @@ fn oop_iterate_slow(oop: Oop, closure: &mut impl EdgeVisitor, tls: } #[inline(always)] -fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { +fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor, disable_discovery: bool) { let klass_id = oop.klass.id; debug_assert!( klass_id as i32 >= 0 && (klass_id as i32) < 6, @@ -235,7 +256,7 @@ fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { // } KlassID::InstanceRef => { let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.oop_iterate(oop, closure); + instance_klass.oop_iterate_with_discovery(oop, closure, disable_discovery); } // _ => oop_iterate_slow(oop, closure, tls), _ => {} } @@ -260,11 +281,12 @@ pub fn scan_object( object: ObjectReference, closure: &mut impl EdgeVisitor, _tls: VMWorkerThread, + discover_references: bool, ) { // println!("*****scan_object(0x{:x}) -> \n 0x{:x}, 0x{:x} \n", // object, // unsafe { *(object.value() as *const usize) }, // unsafe { *((object.value() + 8) as *const usize) } // ); - unsafe { oop_iterate(mem::transmute(object), closure) } + unsafe { oop_iterate(mem::transmute(object), closure, discover_references) } } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 86feff9f..fb3c61ce 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -14,7 +14,12 @@ use mmtk::MMTK; #[inline(always)] pub fn set_referent(reff: ObjectReference, referent: ObjectReference) { let oop = Oop::from(reff); - unsafe { InstanceRefKlass::referent_address(oop).store(referent) }; + mmtk::memory_manager::vm_write_field( + &SINGLETON, + reff, + InstanceRefKlass::referent_address(oop), + referent, + ); } #[inline(always)] @@ -36,7 +41,7 @@ fn get_next_reference(object: ObjectReference) -> ObjectReference { #[inline(always)] fn set_next_reference(object: ObjectReference, next: ObjectReference) { - unsafe { get_next_reference_slot(object).store(next) } + mmtk::memory_manager::vm_write_field(&SINGLETON, object, get_next_reference_slot(object), next); } pub struct VMReferenceGlue {} @@ -45,8 +50,7 @@ impl ReferenceGlue for VMReferenceGlue { type FinalizableType = ObjectReference; fn set_referent(reff: ObjectReference, referent: ObjectReference) { - let oop = Oop::from(reff); - unsafe { InstanceRefKlass::referent_address(oop).store(referent) }; + unreachable!() } fn get_referent(object: ObjectReference) -> ObjectReference { let oop = Oop::from(object); @@ -74,6 +78,7 @@ impl DiscoveredList { #[inline] pub fn add(&self, obj: ObjectReference) { + mmtk::util::rc::inc(obj); let oop = Oop::from(obj); let head = self.head.load(Ordering::Relaxed); let discovered_addr = unsafe { @@ -182,6 +187,7 @@ impl DiscoveredLists { &self, worker: &mut GCWorker, ) { + println!("process_soft_weak_final_refs"); self.allow_discover.store(false, Ordering::SeqCst); if !*SINGLETON.get_options().no_reference_types { self.process_lists::(worker, ReferenceType::Soft, &self.soft, true); @@ -196,6 +202,7 @@ impl DiscoveredLists { &self, worker: &mut GCWorker, ) { + println!("resurrect_final_refs"); assert!(!*SINGLETON.get_options().no_finalizer); let lists = &self.r#final; let mut packets = vec![]; @@ -219,6 +226,7 @@ impl DiscoveredLists { &self, worker: &mut GCWorker, ) { + println!("process_phantom_refs"); assert!(!*SINGLETON.get_options().no_reference_types); self.process_lists::(worker, ReferenceType::Phantom, &self.phantom, true); } @@ -245,32 +253,45 @@ impl> GCWork for ProcessDiscoveredLis let referent = get_referent(reference); if referent.is_null() { // Remove from the discovered list - return IterationResult::Removed; - } else if referent.is_live() || retain { + return DiscoveredListIterationResult::Remove; + } else if referent.is_reachable() || retain { // Keep this referent let forwarded = trace.trace_object(referent); + assert!(forwarded.get_forwarded_object().is_none()); + assert!(reference.get_forwarded_object().is_none()); + assert!(forwarded.is_reachable()); set_referent(reference, forwarded); // Remove from the discovered list - return IterationResult::Removed; + return DiscoveredListIterationResult::Remove; } else { if self.rt != ReferenceType::Final { // Clear this referent set_referent(reference, ObjectReference::NULL); + println!( + " - [{:?}] {:?} {} => {:?} {} {} swpt", + self.rt, + reference, + mmtk::util::rc::count(reference), + referent, + mmtk::util::rc::count(referent), + referent.get_forwarded_object().is_none() + ); } // Keep the reference - return IterationResult::Kept; + return DiscoveredListIterationResult::Enqueue; } }); // Flush the list to the Universe::pending_list if let Some((head, tail)) = new_list { assert!(!head.is_null() && !tail.is_null()); - assert_eq!(tail, get_next_reference(tail)); + assert_eq!(ObjectReference::NULL, get_next_reference(tail)); if self.rt == ReferenceType::Final { DISCOVERED_LISTS.r#final[self.list_index] .head .store(head, Ordering::SeqCst); } else { let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; + println!("swap_reference_pending_list {:?} {:?}\n", head, old_head); set_next_reference(tail, old_head); } } else { @@ -298,7 +319,9 @@ impl> GCWork for ResurrectFinalizable let referent = get_referent(reference); let forwarded = trace.trace_object(referent); set_referent(reference, forwarded); - return IterationResult::Kept; + assert!(forwarded.get_forwarded_object().is_none()); + println!(" - [final] {:?} => {:?} forwarded", reference, forwarded); + return DiscoveredListIterationResult::Enqueue; }); if let Some((head, tail)) = new_list { @@ -307,6 +330,7 @@ impl> GCWork for ResurrectFinalizable DISCOVERED_LISTS.r#final[self.list_index] .head .store(ObjectReference::NULL, Ordering::SeqCst); + println!("swap_reference_pending_list\n"); let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; set_next_reference(tail, old_head); } @@ -314,68 +338,64 @@ impl> GCWork for ResurrectFinalizable } } -enum IterationResult { - Kept, - Removed, +enum DiscoveredListIterationResult { + Remove, + Enqueue, } fn iterate_list( - mut head: ObjectReference, - mut visitor: impl FnMut(ObjectReference) -> IterationResult, + head: ObjectReference, + mut visitor: impl FnMut(ObjectReference) -> DiscoveredListIterationResult, ) -> Option<(ObjectReference, ObjectReference)> { - let mut cursor = &mut head; - let mut holder: Option = None; - let tail: Option; + let mut new_head: Option = None; + let mut new_tail: Option = None; + let mut reference = head; loop { - debug_assert!(!cursor.is_null()); - let reference = *cursor; + debug_assert!(!reference.is_null()); debug_assert!(reference.is_live()); - debug_assert!(reference.get_forwarded_object().is_none()); - let next_ref = get_next_reference(reference); + let mut old_ref = reference; + if let Some(forwarded) = reference.get_forwarded_object() { + println!(" - {:?} => {:?}", reference, forwarded); + reference = forwarded; + } + assert!(reference.get_forwarded_object().is_none()); + assert_ne!(mmtk::util::rc::count(reference), 0); + assert!(reference.is_reachable()); + let old_next_ref = get_next_reference(reference); + + let next_ref = old_next_ref.get_forwarded_object().unwrap_or(old_next_ref); + let end_of_list = { + old_ref == next_ref + || old_ref == old_next_ref + || reference == next_ref + || reference == old_next_ref + }; + assert!(next_ref.get_forwarded_object().is_none()); let result = visitor(reference); + // Remove `reference` from current list + set_next_reference(reference, ObjectReference::NULL); match result { - IterationResult::Removed => { - // Remove `reference` from list - let next_ref = get_next_reference(reference); - set_next_reference(reference, ObjectReference::NULL); - // Reached the end of the list? - if next_ref.is_live() { - let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); - if next_ref == reference { - // End of list - if let Some(holder) = holder { - set_next_reference(holder, holder); - tail = Some(holder); - } else { - *cursor = ObjectReference::NULL; - tail = None; - } - break; - } - } - // Move to next - *cursor = next_ref; - } - IterationResult::Kept => { - // Reached the end of the list? - if next_ref.is_live() { - let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); - if next_ref == reference { - // End of list - set_next_reference(reference, next_ref); - tail = Some(reference); - break; - } + DiscoveredListIterationResult::Remove => {} + DiscoveredListIterationResult::Enqueue => { + // Add to new list + if let Some(new_head) = new_head { + set_next_reference(reference, new_head) + } else { + new_tail = Some(reference); } - // Move to next - cursor = unsafe { &mut *get_next_reference_slot(reference).to_mut_ptr() }; - holder = Some(reference); + new_head = Some(reference); } } + // Reached the end of the list? + if end_of_list { + break; + } + // Move to next + reference = next_ref; } - if head.is_null() { + if new_head.is_none() { None } else { - Some((head, tail.unwrap())) + Some((new_head.unwrap(), new_tail.unwrap())) } } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 1a534722..847d716c 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -50,8 +50,9 @@ impl Scanning for VMScanning { tls: VMWorkerThread, object: ObjectReference, edge_visitor: &mut EV, + disable_discovery: bool, ) { - crate::object_scanning::scan_object(object, edge_visitor, tls) + crate::object_scanning::scan_object(object, edge_visitor, tls, disable_discovery) } #[inline(always)] diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 6aac0500..cc1d171a 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -18,8 +18,26 @@ void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, #endif } +void MMTkFieldBarrierSetRuntime::load_reference_call(void* ref) { + ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); +} + #define __ masm-> +void MMTkFieldBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { + bool on_oop = type == T_OBJECT || type == T_ARRAY; + bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; + bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; + bool on_reference = on_weak || on_phantom; + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); + if (on_oop && on_reference) { + __ pusha(); + __ mov(c_rarg0, dst); + __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkFieldBarrierSetRuntime::load_reference_call), 1); + __ popa(); + } +} + void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const { if (can_remove_barrier(decorators, val, /* skip_const_null */ false)) return; #if MMTK_ENABLE_BARRIER_FASTPATH @@ -106,6 +124,31 @@ void MMTkFieldBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Deco #define __ gen->lir()-> #endif +void MMTkFieldBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { + DecoratorSet decorators = access.decorators(); + bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0; + bool is_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; + bool is_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; + LIRGenerator *gen = access.gen(); + + BarrierSetC1::load_at_resolved(access, result); + + if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) { + Unimplemented(); + // // Register the value in the referent field with the pre-barrier + // LabelObj *Lcont_anonymous; + // if (is_anonymous) { + // Lcont_anonymous = new LabelObj(); + // generate_referent_check(access, Lcont_anonymous); + // } + // pre_barrier(access, LIR_OprFact::illegalOpr /* addr_opr */, + // result /* pre_val */, access.patch_emit_info() /* info */); + // if (is_anonymous) { + // __ branch_destination(Lcont_anonymous->label()); + // } + } +} + void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const { LIRGenerator* gen = access.gen(); DecoratorSet decorators = access.decorators(); @@ -220,4 +263,46 @@ void MMTkFieldBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } +Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { + + DecoratorSet decorators = access.decorators(); + GraphKit* kit = access.kit(); + MMTkIdealKit ideal(kit, true); + + Node* adr = access.addr().node(); + Node* obj = access.base(); + + bool mismatched = (decorators & C2_MISMATCHED) != 0; + bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0; + bool in_heap = (decorators & IN_HEAP) != 0; + bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; + bool is_unordered = (decorators & MO_UNORDERED) != 0; + bool need_cpu_mem_bar = !is_unordered || mismatched || !in_heap; + + Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : kit->top(); + Node* load = BarrierSetC2::load_at_resolved(access, val_type); + + // If we are reading the value of the referent field of a Reference + // object (either by using Unsafe directly or through reflection) + // then, if G1 is enabled, we need to record the referent in an + // SATB log buffer using the pre-barrier mechanism. + // Also we need to add memory barrier to prevent commoning reads + // from this field across safepoint since GC can change its value. + bool need_read_barrier = in_heap && (on_weak || (unknown && offset != kit->top() && obj != kit->top())); + + if (!access.is_oop() || !need_read_barrier) { + return load; + } + + // TODO: Skip slow-call if concurrent marking is not in progress + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkFieldBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", load); + kit->sync_kit(ideal); + kit->insert_mem_bar(Op_MemBarVolatile); + kit->final_sync(ideal); // Final sync IdealKit and GraphKit. + + return load; +} + + #undef __ \ No newline at end of file diff --git a/openjdk/barriers/mmtkFieldBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp index b9986f87..f06c8d91 100644 --- a/openjdk/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -19,10 +19,22 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { public: // Interfaces called by `MMTkBarrierSet::AccessBarrier` + static void load_reference_call(void* ref); virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const override { object_reference_array_copy_pre_call((void*) src, (void*) dst, count); } + virtual bool is_slow_path_call(address call) const override { + return call == CAST_FROM_FN_PTR(address, object_reference_write_pre_call) + || call == CAST_FROM_FN_PTR(address, object_reference_write_post_call) + || call == CAST_FROM_FN_PTR(address, object_reference_write_slow_call) + || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call) + || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_post_call) + || call == CAST_FROM_FN_PTR(address, load_reference_call); + } + virtual void load_reference(DecoratorSet decorators, oop value) const override { + load_reference_call((void*) value); + }; }; class MMTkFieldBarrierSetAssembler: public MMTkBarrierSetAssembler { @@ -30,12 +42,16 @@ class MMTkFieldBarrierSetAssembler: public MMTkBarrierSetAssembler { virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const override; public: virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; + + virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) override; }; class MMTkFieldBarrierSetC1: public MMTkBarrierSetC1 { protected: virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const override; + virtual void load_at_resolved(LIRAccess& access, LIR_Opr result) override; + virtual LIR_Opr resolve_address(LIRAccess& access, bool resolve_in_register) override { return MMTkBarrierSetC1::resolve_address_in_register(access, resolve_in_register); } @@ -48,6 +64,7 @@ class MMTkFieldBarrierSetC2: public MMTkBarrierSetC2 { virtual bool array_copy_requires_gc_barriers(BasicType type) const override { return false; } + virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const override; }; struct MMTkFieldBarrier: MMTkBarrierImpl< diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 69721a13..ba1da95b 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -53,6 +53,8 @@ extern void* alloc_slow_largeobject(MMTk_Mutator mutator, size_t size, extern void post_alloc(MMTk_Mutator mutator, void* refer, int bytes, int allocator); +extern void mmtk_load_reference(MMTk_Mutator mutator, void* obj); + /// Full pre-barrier extern void mmtk_object_reference_write_pre(MMTk_Mutator mutator, void* src, void* slot, void* target); diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index aa3ad76f..4afcee7e 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -88,6 +88,7 @@ class MMTkBarrierSetRuntime: public CHeapObj { virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const {}; /// Full arraycopy post-barrier virtual void object_reference_array_copy_post(oop* src, oop* dst, size_t count) const {}; + virtual void load_reference(DecoratorSet decorators, oop value) const {}; }; class MMTkBarrierC1; @@ -166,6 +167,26 @@ class MMTkBarrierSet : public BarrierSet { private: typedef BarrierSet::AccessBarrier Raw; public: + + // Needed for weak references + static oop oop_load_in_heap_at(oop base, ptrdiff_t offset) { + oop value = Raw::oop_load_in_heap_at(base, offset); + const bool on_strong_oop_ref = (decorators & ON_STRONG_OOP_REF) != 0; + const bool peek = (decorators & AS_NO_KEEPALIVE) != 0; + const bool needs_enqueue = (!peek && !on_strong_oop_ref); + if (needs_enqueue && value != NULL) { + runtime()->load_reference(decorators, value); + } + return value; + } + + // Defensive: will catch weak oops at addresses in heap + template + static oop oop_load_in_heap(T* addr) { + UNREACHABLE(); + return NULL; + } + template static void oop_store_in_heap(T* addr, oop value) { UNREACHABLE(); From f2adf0ce406f99de14c45d698519cf052892ac9a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 6 Nov 2022 03:22:10 +0000 Subject: [PATCH 184/494] cleanup --- mmtk/src/object_scanning.rs | 26 +++++--------------------- mmtk/src/reference_glue.rs | 7 +++---- mmtk/src/scanning.rs | 3 +-- openjdk/mmtk.h | 1 + openjdk/mmtkBarrierSet.hpp | 2 +- 5 files changed, 11 insertions(+), 28 deletions(-) diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index aca96348..e283274c 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -11,14 +11,6 @@ use std::{mem, slice}; trait OopIterate: Sized { fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor); - fn oop_iterate_with_discovery( - &self, - _oop: Oop, - _closure: &mut impl EdgeVisitor, - _discover: bool, - ) { - unimplemented!() - } } impl OopIterate for OopMapBlock { @@ -118,18 +110,11 @@ impl OopIterate for TypeArrayKlass { impl OopIterate for InstanceRefKlass { #[inline(always)] fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { - unreachable!() - } - - fn oop_iterate_with_discovery( - &self, - oop: Oop, - closure: &mut impl EdgeVisitor, - disable_discovery: bool, - ) { use crate::abi::*; self.instance_klass.oop_iterate(oop, closure); + let disable_discovery = !closure.should_discover_references(); + if Self::should_discover_refs(self.instance_klass.reference_type, disable_discovery) { match self.instance_klass.reference_type { ReferenceType::None => { @@ -225,7 +210,7 @@ fn oop_iterate_slow(oop: Oop, closure: &mut impl EdgeVisitor, tls: } #[inline(always)] -fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor, disable_discovery: bool) { +fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { let klass_id = oop.klass.id; debug_assert!( klass_id as i32 >= 0 && (klass_id as i32) < 6, @@ -256,7 +241,7 @@ fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor, disable_di // } KlassID::InstanceRef => { let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.oop_iterate_with_discovery(oop, closure, disable_discovery); + instance_klass.oop_iterate(oop, closure); } // _ => oop_iterate_slow(oop, closure, tls), _ => {} } @@ -281,12 +266,11 @@ pub fn scan_object( object: ObjectReference, closure: &mut impl EdgeVisitor, _tls: VMWorkerThread, - discover_references: bool, ) { // println!("*****scan_object(0x{:x}) -> \n 0x{:x}, 0x{:x} \n", // object, // unsafe { *(object.value() as *const usize) }, // unsafe { *((object.value() + 8) as *const usize) } // ); - unsafe { oop_iterate(mem::transmute(object), closure, discover_references) } + unsafe { oop_iterate(mem::transmute(object), closure) } } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index fb3c61ce..a59ad639 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -49,7 +49,7 @@ pub struct VMReferenceGlue {} impl ReferenceGlue for VMReferenceGlue { type FinalizableType = ObjectReference; - fn set_referent(reff: ObjectReference, referent: ObjectReference) { + fn set_referent(_reff: ObjectReference, _referent: ObjectReference) { unreachable!() } fn get_referent(object: ObjectReference) -> ObjectReference { @@ -78,7 +78,7 @@ impl DiscoveredList { #[inline] pub fn add(&self, obj: ObjectReference) { - mmtk::util::rc::inc(obj); + let _ = mmtk::util::rc::inc(obj); let oop = Oop::from(obj); let head = self.head.load(Ordering::Relaxed); let discovered_addr = unsafe { @@ -353,9 +353,8 @@ fn iterate_list( loop { debug_assert!(!reference.is_null()); debug_assert!(reference.is_live()); - let mut old_ref = reference; + let old_ref = reference; if let Some(forwarded) = reference.get_forwarded_object() { - println!(" - {:?} => {:?}", reference, forwarded); reference = forwarded; } assert!(reference.get_forwarded_object().is_none()); diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 847d716c..1a534722 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -50,9 +50,8 @@ impl Scanning for VMScanning { tls: VMWorkerThread, object: ObjectReference, edge_visitor: &mut EV, - disable_discovery: bool, ) { - crate::object_scanning::scan_object(object, edge_visitor, tls, disable_discovery) + crate::object_scanning::scan_object(object, edge_visitor, tls) } #[inline(always)] diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index ba1da95b..619e4b98 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -53,6 +53,7 @@ extern void* alloc_slow_largeobject(MMTk_Mutator mutator, size_t size, extern void post_alloc(MMTk_Mutator mutator, void* refer, int bytes, int allocator); +/// java.lang.Reference load barrier extern void mmtk_load_reference(MMTk_Mutator mutator, void* obj); /// Full pre-barrier diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 4afcee7e..295492dd 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -88,6 +88,7 @@ class MMTkBarrierSetRuntime: public CHeapObj { virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const {}; /// Full arraycopy post-barrier virtual void object_reference_array_copy_post(oop* src, oop* dst, size_t count) const {}; + /// java.lang.Reference load barrier virtual void load_reference(DecoratorSet decorators, oop value) const {}; }; @@ -167,7 +168,6 @@ class MMTkBarrierSet : public BarrierSet { private: typedef BarrierSet::AccessBarrier Raw; public: - // Needed for weak references static oop oop_load_in_heap_at(oop base, ptrdiff_t offset) { oop value = Raw::oop_load_in_heap_at(base, offset); From a188d2430de03329efc7ea83874aec0d78cba594 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 6 Nov 2022 04:45:29 +0000 Subject: [PATCH 185/494] wip: final ref working --- mmtk/src/object_scanning.rs | 24 ++++++++++++------------ mmtk/src/reference_glue.rs | 33 +++++++++++---------------------- 2 files changed, 23 insertions(+), 34 deletions(-) diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index e283274c..fc26e28d 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -120,18 +120,11 @@ impl OopIterate for InstanceRefKlass { ReferenceType::None => { panic!("oop_iterate on InstanceRefKlass with reference_type as None") } - rt @ (ReferenceType::Weak | ReferenceType::Phantom | ReferenceType::Soft) => { + rt => { if !Self::discover_reference(oop, rt) { Self::process_ref_as_strong(oop, closure) } } - // Process these two types normally (as if they are strong refs) - // We will handle final reference later - // ReferenceType::Weak - // | ReferenceType::Phantom - ReferenceType::Other | ReferenceType::Final => { - Self::process_ref_as_strong(oop, closure) - } } } else { Self::process_ref_as_strong(oop, closure); @@ -141,7 +134,10 @@ impl OopIterate for InstanceRefKlass { impl InstanceRefKlass { #[inline] - fn should_discover_refs(rt: ReferenceType, disable_discovery: bool) -> bool { + fn should_discover_refs(mut rt: ReferenceType, disable_discovery: bool) -> bool { + if rt == ReferenceType::Other { + rt = ReferenceType::Weak; + } if disable_discovery { return false; } @@ -180,15 +176,19 @@ impl InstanceRefKlass { return false; } // Skip young referents + let reference: ObjectReference = oop.into(); + if mmtk::util::rc::count(reference) == 0 { + return false; + } if mmtk::util::rc::count(referent) == 0 { return false; } - // TODO: Do not discover if the referent is a nursery object. // Add to reference list - let reference: ObjectReference = oop.into(); if crate::VM_REF_PROCESSOR { - // crate::reference_glue::set_referent(reference, ObjectReference::NULL); + if rt == ReferenceType::Final { + println!("{:?} {:?} -> {:?}", rt, reference, referent); + } DISCOVERED_LISTS.get(rt).add(reference); } else { match rt { diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index a59ad639..6540e1fd 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -14,12 +14,7 @@ use mmtk::MMTK; #[inline(always)] pub fn set_referent(reff: ObjectReference, referent: ObjectReference) { let oop = Oop::from(reff); - mmtk::memory_manager::vm_write_field( - &SINGLETON, - reff, - InstanceRefKlass::referent_address(oop), - referent, - ); + unsafe { InstanceRefKlass::referent_address(oop).store(referent) } } #[inline(always)] @@ -41,7 +36,7 @@ fn get_next_reference(object: ObjectReference) -> ObjectReference { #[inline(always)] fn set_next_reference(object: ObjectReference, next: ObjectReference) { - mmtk::memory_manager::vm_write_field(&SINGLETON, object, get_next_reference_slot(object), next); + unsafe { get_next_reference_slot(object).store(next) } } pub struct VMReferenceGlue {} @@ -79,6 +74,7 @@ impl DiscoveredList { #[inline] pub fn add(&self, obj: ObjectReference) { let _ = mmtk::util::rc::inc(obj); + let _ = mmtk::util::rc::inc(get_referent(obj)); let oop = Oop::from(obj); let head = self.head.load(Ordering::Relaxed); let discovered_addr = unsafe { @@ -259,7 +255,6 @@ impl> GCWork for ProcessDiscoveredLis let forwarded = trace.trace_object(referent); assert!(forwarded.get_forwarded_object().is_none()); assert!(reference.get_forwarded_object().is_none()); - assert!(forwarded.is_reachable()); set_referent(reference, forwarded); // Remove from the discovered list return DiscoveredListIterationResult::Remove; @@ -291,7 +286,6 @@ impl> GCWork for ProcessDiscoveredLis .store(head, Ordering::SeqCst); } else { let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; - println!("swap_reference_pending_list {:?} {:?}\n", head, old_head); set_next_reference(tail, old_head); } } else { @@ -326,11 +320,10 @@ impl> GCWork for ResurrectFinalizable if let Some((head, tail)) = new_list { assert!(!head.is_null() && !tail.is_null()); - assert_eq!(tail, get_next_reference(tail)); + assert_eq!(ObjectReference::NULL, get_next_reference(tail)); DISCOVERED_LISTS.r#final[self.list_index] .head .store(ObjectReference::NULL, Ordering::SeqCst); - println!("swap_reference_pending_list\n"); let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; set_next_reference(tail, old_head); } @@ -353,23 +346,19 @@ fn iterate_list( loop { debug_assert!(!reference.is_null()); debug_assert!(reference.is_live()); - let old_ref = reference; + // Update reference forwarding pointer if let Some(forwarded) = reference.get_forwarded_object() { reference = forwarded; } assert!(reference.get_forwarded_object().is_none()); - assert_ne!(mmtk::util::rc::count(reference), 0); assert!(reference.is_reachable()); - let old_next_ref = get_next_reference(reference); - - let next_ref = old_next_ref.get_forwarded_object().unwrap_or(old_next_ref); - let end_of_list = { - old_ref == next_ref - || old_ref == old_next_ref - || reference == next_ref - || reference == old_next_ref - }; + // Update next_ref forwarding pointer + let next_ref = get_next_reference(reference); + let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); assert!(next_ref.get_forwarded_object().is_none()); + // Reaches the end of the list? + let end_of_list = next_ref == reference || next_ref.is_null(); + // Process reference let result = visitor(reference); // Remove `reference` from current list set_next_reference(reference, ObjectReference::NULL); From cbdee873e8cdf473ffa336775fa1745cb009ed90 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 6 Nov 2022 05:18:49 +0000 Subject: [PATCH 186/494] C1 reference load barrier --- openjdk/barriers/mmtkFieldBarrier.cpp | 32 ++++++++---------- openjdk/barriers/mmtkFieldBarrier.hpp | 9 ------ openjdk/mmtkBarrierSet.cpp | 4 +++ openjdk/mmtkBarrierSet.hpp | 5 ++- openjdk/mmtkBarrierSetAssembler_x86.cpp | 43 +++++++++++++++++++++++++ openjdk/mmtkBarrierSetAssembler_x86.hpp | 3 ++ openjdk/mmtkBarrierSetC1.cpp | 14 ++++++++ openjdk/mmtkBarrierSetC1.hpp | 20 ++++++++++++ 8 files changed, 102 insertions(+), 28 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index cc1d171a..20ffa4b7 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -18,10 +18,6 @@ void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, #endif } -void MMTkFieldBarrierSetRuntime::load_reference_call(void* ref) { - ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); -} - #define __ masm-> void MMTkFieldBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { @@ -33,7 +29,7 @@ void MMTkFieldBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet de if (on_oop && on_reference) { __ pusha(); __ mov(c_rarg0, dst); - __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkFieldBarrierSetRuntime::load_reference_call), 1); + __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); __ popa(); } } @@ -134,18 +130,18 @@ void MMTkFieldBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) BarrierSetC1::load_at_resolved(access, result); if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) { - Unimplemented(); - // // Register the value in the referent field with the pre-barrier - // LabelObj *Lcont_anonymous; - // if (is_anonymous) { - // Lcont_anonymous = new LabelObj(); - // generate_referent_check(access, Lcont_anonymous); - // } - // pre_barrier(access, LIR_OprFact::illegalOpr /* addr_opr */, - // result /* pre_val */, access.patch_emit_info() /* info */); - // if (is_anonymous) { - // __ branch_destination(Lcont_anonymous->label()); - // } + // Register the value in the referent field with the pre-barrier + LabelObj *Lcont_anonymous; + if (is_anonymous) { + Lcont_anonymous = new LabelObj(); + generate_referent_check(access, Lcont_anonymous); + } + auto slow = new MMTkC1ReferenceLoadBarrierStub(result, access.patch_emit_info()); + __ jump(slow); + __ branch_destination(slow->continuation()); + if (is_anonymous) { + __ branch_destination(Lcont_anonymous->label()); + } } } @@ -296,7 +292,7 @@ Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_ // TODO: Skip slow-call if concurrent marking is not in progress const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkFieldBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", load); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", load); kit->sync_kit(ideal); kit->insert_mem_bar(Op_MemBarVolatile); kit->final_sync(ideal); // Final sync IdealKit and GraphKit. diff --git a/openjdk/barriers/mmtkFieldBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp index f06c8d91..b03463f5 100644 --- a/openjdk/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -19,19 +19,10 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { public: // Interfaces called by `MMTkBarrierSet::AccessBarrier` - static void load_reference_call(void* ref); virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const override { object_reference_array_copy_pre_call((void*) src, (void*) dst, count); } - virtual bool is_slow_path_call(address call) const override { - return call == CAST_FROM_FN_PTR(address, object_reference_write_pre_call) - || call == CAST_FROM_FN_PTR(address, object_reference_write_post_call) - || call == CAST_FROM_FN_PTR(address, object_reference_write_slow_call) - || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call) - || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_post_call) - || call == CAST_FROM_FN_PTR(address, load_reference_call); - } virtual void load_reference(DecoratorSet decorators, oop value) const override { load_reference_call((void*) value); }; diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index 175c69a7..b24b5852 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -153,3 +153,7 @@ void MMTkBarrierSetRuntime::object_reference_array_copy_pre_call(void* src, void void MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, void* dst, size_t count) { ::mmtk_array_copy_post((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); } + +void MMTkBarrierSetRuntime::load_reference_call(void* ref) { + ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); +} \ No newline at end of file diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 295492dd..bb3f1594 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -61,6 +61,8 @@ MMTkAllocatorOffsets get_tlab_top_and_end_offsets(AllocatorSelector selector); class MMTkBarrierSetRuntime: public CHeapObj { public: + /// Weak ref load barrier + static void load_reference_call(void* ref); /// Generic pre-write barrier. Called by fast-paths. static void object_reference_write_pre_call(void* src, void* slot, void* target); /// Generic post-write barrier. Called by fast-paths. @@ -77,7 +79,8 @@ class MMTkBarrierSetRuntime: public CHeapObj { || call == CAST_FROM_FN_PTR(address, object_reference_write_post_call) || call == CAST_FROM_FN_PTR(address, object_reference_write_slow_call) || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call) - || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_post_call); + || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_post_call) + || call == CAST_FROM_FN_PTR(address, load_reference_call); } /// Full pre-barrier diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index f7998e6a..c9c24279 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -183,10 +183,53 @@ void MMTkBarrierSetAssembler::generate_c1_write_barrier_runtime_stub(StubAssembl __ epilogue(); } + +void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_runtime_stub(StubAssembler* sasm) const { + __ prologue("mmtk_ref_load_barrier", false); + + Address store_addr(rbp, 2*BytesPerWord); + + Label done, runtime; + + __ push(c_rarg0); + __ push(rax); + + __ load_parameter(0, c_rarg0); + + __ bind(runtime); + + __ save_live_registers_no_oop_map(true); + + __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); + + __ restore_live_registers(true); + + __ bind(done); + __ pop(rax); + __ pop(c_rarg0); + + __ epilogue(); +} + #undef __ #define __ ce->masm()-> +void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub) { + MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); + + __ bind(*stub->entry()); + assert(stub->val->is_register(), "Precondition."); + + Register val_reg = stub->val->as_register(); + + __ cmpptr(val_reg, (int32_t) NULL_WORD); + __ jcc(Assembler::equal, *stub->continuation()); + ce->store_parameter(stub->val->as_register(), 0); + __ call(RuntimeAddress(bs->_ref_load_barrier_c1_runtime_code_blob->code_begin())); + __ jmp(*stub->continuation()); +} + void MMTkBarrierSetAssembler::generate_c1_write_barrier_stub_call(LIR_Assembler* ce, MMTkC1BarrierStub* stub) { MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1(); __ bind(*stub->entry()); diff --git a/openjdk/mmtkBarrierSetAssembler_x86.hpp b/openjdk/mmtkBarrierSetAssembler_x86.hpp index 88a5f03e..dc60cab3 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.hpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.hpp @@ -6,6 +6,7 @@ class MMTkBarrierSetC1; class MMTkC1BarrierStub; +class MMTkC1ReferenceLoadBarrierStub; class LIR_Assembler; class StubAssembler; @@ -28,6 +29,7 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler { /// Generate C1 write barrier slow-call assembly code virtual void generate_c1_write_barrier_runtime_stub(StubAssembler* sasm) const; + virtual void generate_c1_ref_load_barrier_runtime_stub(StubAssembler* sasm) const; public: virtual void eden_allocate(MacroAssembler* masm, Register thread, Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Label& slow_case) override; @@ -39,5 +41,6 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler { /// Generate C1 write barrier slow-call stub static void generate_c1_write_barrier_stub_call(LIR_Assembler* ce, MMTkC1BarrierStub* stub); + static void generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub); }; #endif // MMTK_OPENJDK_MMTK_BARRIER_SET_ASSEMBLER_X86_HPP diff --git a/openjdk/mmtkBarrierSetC1.cpp b/openjdk/mmtkBarrierSetC1.cpp index b2c9a683..b962fde3 100644 --- a/openjdk/mmtkBarrierSetC1.cpp +++ b/openjdk/mmtkBarrierSetC1.cpp @@ -13,9 +13,23 @@ void MMTkBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) { }; MMTkBarrierCodeGenClosure write_code_gen_cl; _write_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "write_code_gen_cl", false, &write_code_gen_cl); + class MMTkRefLoadBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { + virtual OopMapSet* generate_code(StubAssembler* sasm) override { + MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); + bs->generate_c1_ref_load_barrier_runtime_stub(sasm); + return NULL; + } + }; + MMTkRefLoadBarrierCodeGenClosure write_code_gen_cl2; + _ref_load_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "write_code_gen_cl2", false, &write_code_gen_cl2); } void MMTkC1BarrierStub::emit_code(LIR_Assembler* ce) { MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); bs->generate_c1_write_barrier_stub_call(ce, this); } + +void MMTkC1ReferenceLoadBarrierStub::emit_code(LIR_Assembler* ce) { + MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); + bs->generate_c1_ref_load_barrier_stub_call(ce, this); +} diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index 99e81583..7ea22d6a 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -11,6 +11,7 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { protected: CodeBlob* _write_barrier_c1_runtime_code_blob; + CodeBlob* _ref_load_barrier_c1_runtime_code_blob; /// Full pre-barrier virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const {} @@ -94,4 +95,23 @@ struct MMTkC1BarrierStub: CodeStub { NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1BarrierStub"); }); }; +struct MMTkC1ReferenceLoadBarrierStub: CodeStub { + LIR_Opr val; + CodeEmitInfo* info; // Code patching info + + MMTkC1ReferenceLoadBarrierStub(LIR_Opr val, CodeEmitInfo* info = NULL): val(val), info(info) {} + + virtual void emit_code(LIR_Assembler* ce) override; + + virtual void visit(LIR_OpVisitState* visitor) override { + if (info != NULL) + visitor->do_slow_case(info); + else + visitor->do_slow_case(); + if (val != NULL) visitor->do_input(val); + } + + NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1ReferenceLoadBarrierStub"); }); +}; + #endif // MMTK_OPENJDK_MMTK_BARRIER_SET_C1_HPP From 797c3b3fdb97c62411e1e7f746ccc1a6aca765a6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 6 Nov 2022 05:42:13 +0000 Subject: [PATCH 187/494] don't call reference load barrier when concurrent marking is not active --- mmtk/src/collection.rs | 4 ++++ mmtk/src/lib.rs | 3 +++ openjdk/barriers/mmtkFieldBarrier.cpp | 31 ++++++++++++++++++++++----- openjdk/barriers/mmtkFieldBarrier.hpp | 2 +- openjdk/mmtk.h | 1 + 5 files changed, 35 insertions(+), 6 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index d408ec12..4f5d59e4 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -121,4 +121,8 @@ impl Collection for VMCollection { ((*UPCALLS).update_weak_processor)(); } } + + fn set_concurrent_marking_state(active: bool) { + unsafe { crate::CONCURRENT_MARKING_ACTIVE = if active { 1 } else { 0 } } + } } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 0eff4ce8..d5a12dc9 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -136,6 +136,9 @@ pub static DISABLE_ALLOCATION_FAST_PATH: i32 = cfg!(feature = "no_fast_alloc") a pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = std::mem::size_of::>(); +#[no_mangle] +pub static mut CONCURRENT_MARKING_ACTIVE: u8 = 0; + #[derive(Default)] pub struct OpenJDK; diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 20ffa4b7..e262fee3 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -27,10 +27,18 @@ void MMTkFieldBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet de bool on_reference = on_weak || on_phantom; BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); if (on_oop && on_reference) { + // Call slow-path only when concurrent marking is active + Label done; + Register tmp = rscratch1; + __ movptr(tmp, intptr_t(&CONCURRENT_MARKING_ACTIVE)); + __ movb(tmp, Address(tmp, 0)); + __ cmpptr(tmp, 1); + __ jcc(Assembler::notEqual, done); __ pusha(); __ mov(c_rarg0, dst); __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); __ popa(); + __ bind(done); } } @@ -137,7 +145,14 @@ void MMTkFieldBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) generate_referent_check(access, Lcont_anonymous); } auto slow = new MMTkC1ReferenceLoadBarrierStub(result, access.patch_emit_info()); - __ jump(slow); + // Call slow-path only when concurrent marking is active + LIR_Opr cm_flag_addr_opr = gen->new_pointer_register(); + __ move(LIR_OprFact::longConst(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), cm_flag_addr_opr); + LIR_Address* cm_flag_addr = new LIR_Address(cm_flag_addr_opr, T_BYTE); + LIR_Opr cm_flag = gen->new_register(T_INT); + __ move(cm_flag_addr, cm_flag); + __ cmp(lir_cond_equal, cm_flag, LIR_OprFact::intConst(1)); + __ branch(lir_cond_equal, T_BYTE, slow); __ branch_destination(slow->continuation()); if (is_anonymous) { __ branch_destination(Lcont_anonymous->label()); @@ -263,7 +278,6 @@ Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_ DecoratorSet decorators = access.decorators(); GraphKit* kit = access.kit(); - MMTkIdealKit ideal(kit, true); Node* adr = access.addr().node(); Node* obj = access.base(); @@ -290,9 +304,16 @@ Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_ return load; } - // TODO: Skip slow-call if concurrent marking is not in progress - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", load); + MMTkIdealKit ideal(kit, true); + // Call slow-path only when concurrent marking is active + Node* no_base = __ top(); + float unlikely = PROB_UNLIKELY(0.999); + Node* zero = __ ConI(0); + Node* cm_flag = __ load(__ ctrl(), __ ConP(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); + __ if_then(cm_flag, BoolTest::ne, zero, unlikely); { + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", load); + } __ end_if(); kit->sync_kit(ideal); kit->insert_mem_bar(Op_MemBarVolatile); kit->final_sync(ideal); // Final sync IdealKit and GraphKit. diff --git a/openjdk/barriers/mmtkFieldBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp index b03463f5..826b1a08 100644 --- a/openjdk/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -24,7 +24,7 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { object_reference_array_copy_pre_call((void*) src, (void*) dst, count); } virtual void load_reference(DecoratorSet decorators, oop value) const override { - load_reference_call((void*) value); + if (CONCURRENT_MARKING_ACTIVE == 1) load_reference_call((void*) value); }; }; diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 619e4b98..5667455e 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -23,6 +23,7 @@ extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; extern const uintptr_t GLOBAL_ALLOC_BIT_ADDRESS; extern const int DISABLE_ALLOCATION_FAST_PATH; extern const uintptr_t IMMIX_ALLOCATOR_SIZE; +extern uint8_t CONCURRENT_MARKING_ACTIVE; inline bool disable_fast_alloc() { return DISABLE_ALLOCATION_FAST_PATH != 0; From 053666a47c15f546e6b32ab4db252902c3bdc536 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 6 Nov 2022 06:19:54 +0000 Subject: [PATCH 188/494] cleanup --- mmtk/src/collection.rs | 12 ++------ mmtk/src/lib.rs | 2 -- mmtk/src/object_scanning.rs | 34 +++++---------------- mmtk/src/reference_glue.rs | 61 ++++++++++++++----------------------- 4 files changed, 33 insertions(+), 76 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 4f5d59e4..ca23f0f5 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -99,21 +99,15 @@ impl Collection for VMCollection { } fn process_weak_refs>(worker: &mut GCWorker) { - if crate::VM_REF_PROCESSOR { - DISCOVERED_LISTS.process_soft_weak_final_refs::(worker) - } + DISCOVERED_LISTS.process_soft_weak_final_refs::(worker) } fn process_final_refs>(worker: &mut GCWorker) { - if crate::VM_REF_PROCESSOR { - DISCOVERED_LISTS.resurrect_final_refs::(worker) - } + DISCOVERED_LISTS.resurrect_final_refs::(worker) } fn process_phantom_refs>(worker: &mut GCWorker) { - if crate::VM_REF_PROCESSOR { - DISCOVERED_LISTS.process_phantom_refs::(worker) - } + DISCOVERED_LISTS.process_phantom_refs::(worker) } fn update_weak_processor() { diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index d5a12dc9..4a890721 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -187,5 +187,3 @@ lazy_static! { /// A counter tracking the total size of the `CODE_CACHE_ROOTS`. static CODE_CACHE_ROOTS_SIZE: AtomicUsize = AtomicUsize::new(0); - -const VM_REF_PROCESSOR: bool = true; diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index fc26e28d..98402a6f 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -158,16 +158,9 @@ impl InstanceRefKlass { } #[inline] fn discover_reference(oop: Oop, rt: ReferenceType) -> bool { - use crate::api::{add_phantom_candidate, add_soft_candidate, add_weak_candidate}; // Do not discover new refs during reference processing. - if crate::VM_REF_PROCESSOR { - if !DISCOVERED_LISTS.allow_discover() { - return false; - } - } else { - if !crate::SINGLETON.reference_processors.allow_new_candidate() { - return false; - } + if !DISCOVERED_LISTS.allow_discover() { + return false; } // Do not discover if the referent is live. let referent: ObjectReference = unsafe { InstanceRefKlass::referent_address(oop).load() }; @@ -177,27 +170,14 @@ impl InstanceRefKlass { } // Skip young referents let reference: ObjectReference = oop.into(); - if mmtk::util::rc::count(reference) == 0 { - return false; - } - if mmtk::util::rc::count(referent) == 0 { + if !SINGLETON + .get_plan() + .should_process_reference(reference, referent) + { return false; } - // Add to reference list - if crate::VM_REF_PROCESSOR { - if rt == ReferenceType::Final { - println!("{:?} {:?} -> {:?}", rt, reference, referent); - } - DISCOVERED_LISTS.get(rt).add(reference); - } else { - match rt { - ReferenceType::Weak => add_weak_candidate(reference), - ReferenceType::Soft => add_soft_candidate(reference), - ReferenceType::Phantom => add_phantom_candidate(reference), - _ => unreachable!(), - } - } + DISCOVERED_LISTS.get(rt).add(reference, referent); true } } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 6540e1fd..188ab9ce 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -44,18 +44,15 @@ pub struct VMReferenceGlue {} impl ReferenceGlue for VMReferenceGlue { type FinalizableType = ObjectReference; - fn set_referent(_reff: ObjectReference, _referent: ObjectReference) { - unreachable!() + fn set_referent(reff: ObjectReference, referent: ObjectReference) { + let oop = Oop::from(reff); + unsafe { InstanceRefKlass::referent_address(oop).store(referent) }; } fn get_referent(object: ObjectReference) -> ObjectReference { let oop = Oop::from(object); unsafe { InstanceRefKlass::referent_address(oop).load::() } } - fn enqueue_references(_references: &[ObjectReference], _tls: VMWorkerThread) { - // unsafe { - // ((*UPCALLS).enqueue_references)(references.as_ptr(), references.len()); - // } - } + fn enqueue_references(_references: &[ObjectReference], _tls: VMWorkerThread) {} } pub struct DiscoveredList { @@ -72,15 +69,17 @@ impl DiscoveredList { } #[inline] - pub fn add(&self, obj: ObjectReference) { - let _ = mmtk::util::rc::inc(obj); - let _ = mmtk::util::rc::inc(get_referent(obj)); - let oop = Oop::from(obj); + pub fn add(&self, reference: ObjectReference, referent: ObjectReference) { + // Keep reference and referent alive during SATB + SINGLETON.get_plan().discover_reference(reference, referent); + // Add to the corresponding list + // Note that the list is a singly-linked list, and the tail object should point to itself. + let oop = Oop::from(reference); let head = self.head.load(Ordering::Relaxed); let discovered_addr = unsafe { InstanceRefKlass::discovered_address(oop).as_ref::>() }; - let next_discovered = if head.is_null() { obj } else { head }; + let next_discovered = if head.is_null() { reference } else { head }; debug_assert!(!next_discovered.is_null()); if discovered_addr .compare_exchange( @@ -91,10 +90,9 @@ impl DiscoveredList { ) .is_ok() { - // println!("add {:?} {:?}", self.rt, obj); - self.head.store(obj, Ordering::Relaxed); + self.head.store(reference, Ordering::Relaxed); } - debug_assert!(!get_next_reference(obj).is_null()); + debug_assert!(!get_next_reference(reference).is_null()); } } @@ -183,7 +181,6 @@ impl DiscoveredLists { &self, worker: &mut GCWorker, ) { - println!("process_soft_weak_final_refs"); self.allow_discover.store(false, Ordering::SeqCst); if !*SINGLETON.get_options().no_reference_types { self.process_lists::(worker, ReferenceType::Soft, &self.soft, true); @@ -198,7 +195,6 @@ impl DiscoveredLists { &self, worker: &mut GCWorker, ) { - println!("resurrect_final_refs"); assert!(!*SINGLETON.get_options().no_finalizer); let lists = &self.r#final; let mut packets = vec![]; @@ -222,7 +218,6 @@ impl DiscoveredLists { &self, worker: &mut GCWorker, ) { - println!("process_phantom_refs"); assert!(!*SINGLETON.get_options().no_reference_types); self.process_lists::(worker, ReferenceType::Phantom, &self.phantom, true); } @@ -253,8 +248,8 @@ impl> GCWork for ProcessDiscoveredLis } else if referent.is_reachable() || retain { // Keep this referent let forwarded = trace.trace_object(referent); - assert!(forwarded.get_forwarded_object().is_none()); - assert!(reference.get_forwarded_object().is_none()); + debug_assert!(forwarded.get_forwarded_object().is_none()); + debug_assert!(reference.get_forwarded_object().is_none()); set_referent(reference, forwarded); // Remove from the discovered list return DiscoveredListIterationResult::Remove; @@ -262,15 +257,6 @@ impl> GCWork for ProcessDiscoveredLis if self.rt != ReferenceType::Final { // Clear this referent set_referent(reference, ObjectReference::NULL); - println!( - " - [{:?}] {:?} {} => {:?} {} {} swpt", - self.rt, - reference, - mmtk::util::rc::count(reference), - referent, - mmtk::util::rc::count(referent), - referent.get_forwarded_object().is_none() - ); } // Keep the reference return DiscoveredListIterationResult::Enqueue; @@ -278,8 +264,8 @@ impl> GCWork for ProcessDiscoveredLis }); // Flush the list to the Universe::pending_list if let Some((head, tail)) = new_list { - assert!(!head.is_null() && !tail.is_null()); - assert_eq!(ObjectReference::NULL, get_next_reference(tail)); + debug_assert!(!head.is_null() && !tail.is_null()); + debug_assert_eq!(ObjectReference::NULL, get_next_reference(tail)); if self.rt == ReferenceType::Final { DISCOVERED_LISTS.r#final[self.list_index] .head @@ -313,14 +299,13 @@ impl> GCWork for ResurrectFinalizable let referent = get_referent(reference); let forwarded = trace.trace_object(referent); set_referent(reference, forwarded); - assert!(forwarded.get_forwarded_object().is_none()); - println!(" - [final] {:?} => {:?} forwarded", reference, forwarded); + debug_assert!(forwarded.get_forwarded_object().is_none()); return DiscoveredListIterationResult::Enqueue; }); if let Some((head, tail)) = new_list { - assert!(!head.is_null() && !tail.is_null()); - assert_eq!(ObjectReference::NULL, get_next_reference(tail)); + debug_assert!(!head.is_null() && !tail.is_null()); + debug_assert_eq!(ObjectReference::NULL, get_next_reference(tail)); DISCOVERED_LISTS.r#final[self.list_index] .head .store(ObjectReference::NULL, Ordering::SeqCst); @@ -350,12 +335,12 @@ fn iterate_list( if let Some(forwarded) = reference.get_forwarded_object() { reference = forwarded; } - assert!(reference.get_forwarded_object().is_none()); - assert!(reference.is_reachable()); + debug_assert!(reference.get_forwarded_object().is_none()); + debug_assert!(reference.is_reachable()); // Update next_ref forwarding pointer let next_ref = get_next_reference(reference); let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); - assert!(next_ref.get_forwarded_object().is_none()); + debug_assert!(next_ref.get_forwarded_object().is_none()); // Reaches the end of the list? let end_of_list = next_ref == reference || next_ref.is_null(); // Process reference From 6af64c61b8306d0824fe0aa7ede22411fc64fa10 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 6 Nov 2022 06:36:19 +0000 Subject: [PATCH 189/494] More load barrier performance tweaks --- openjdk/barriers/mmtkFieldBarrier.cpp | 120 +++++++++++++++++++++++--- openjdk/barriers/mmtkFieldBarrier.hpp | 2 +- 2 files changed, 107 insertions(+), 15 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index e262fee3..9888ea45 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -27,13 +27,17 @@ void MMTkFieldBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet de bool on_reference = on_weak || on_phantom; BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); if (on_oop && on_reference) { - // Call slow-path only when concurrent marking is active Label done; + // No slow-call if SATB is not active Register tmp = rscratch1; __ movptr(tmp, intptr_t(&CONCURRENT_MARKING_ACTIVE)); __ movb(tmp, Address(tmp, 0)); __ cmpptr(tmp, 1); __ jcc(Assembler::notEqual, done); + // No slow-call if dst is NULL + __ cmpptr(dst, 0); + __ jcc(Assembler::equal, done); + // Do slow-call __ pusha(); __ mov(c_rarg0, dst); __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); @@ -144,6 +148,8 @@ void MMTkFieldBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) Lcont_anonymous = new LabelObj(); generate_referent_check(access, Lcont_anonymous); } + assert(result->is_register(), "must be"); + assert(result->type() == T_OBJECT, "must be an object"); auto slow = new MMTkC1ReferenceLoadBarrierStub(result, access.patch_emit_info()); // Call slow-path only when concurrent marking is active LIR_Opr cm_flag_addr_opr = gen->new_pointer_register(); @@ -151,6 +157,7 @@ void MMTkFieldBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) LIR_Address* cm_flag_addr = new LIR_Address(cm_flag_addr_opr, T_BYTE); LIR_Opr cm_flag = gen->new_register(T_INT); __ move(cm_flag_addr, cm_flag); + // No slow-call if SATB is not active __ cmp(lir_cond_equal, cm_flag, LIR_OprFact::intConst(1)); __ branch(lir_cond_equal, T_BYTE, slow); __ branch_destination(slow->continuation()); @@ -274,6 +281,99 @@ void MMTkFieldBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } +static void reference_load_barrier(GraphKit* kit, Node* val, bool emit_barrier) { + MMTkIdealKit ideal(kit, true); + Node* no_base = __ top(); + float unlikely = PROB_UNLIKELY(0.999); + Node* zero = __ ConI(0); + Node* cm_flag = __ load(__ ctrl(), __ ConP(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); + // No slow-call if SATB is not active + __ if_then(cm_flag, BoolTest::ne, zero, unlikely); { + // No slow-call if dst is NULL + __ if_then(val, BoolTest::ne, kit->null()); { + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", val); + } __ end_if(); + } __ end_if(); + kit->sync_kit(ideal); + if (emit_barrier) kit->insert_mem_bar(Op_MemBarCPUOrder); + kit->final_sync(ideal); // Final sync IdealKit and GraphKit. +} + +static void reference_load_barrier_for_unknown_load(GraphKit* kit, Node* base_oop, Node* offset, Node* val, bool need_mem_bar) { + // We could be accessing the referent field of a reference object. If so, when G1 + // is enabled, we need to log the value in the referent field in an SATB buffer. + // This routine performs some compile time filters and generates suitable + // runtime filters that guard the pre-barrier code. + // Also add memory barrier for non volatile load from the referent field + // to prevent commoning of loads across safepoint. + + // Some compile time checks. + + // If offset is a constant, is it java_lang_ref_Reference::_reference_offset? + const TypeX* otype = offset->find_intptr_t_type(); + if (otype != NULL && otype->is_con() && + otype->get_con() != java_lang_ref_Reference::referent_offset) { + // Constant offset but not the reference_offset so just return + return; + } + + // We only need to generate the runtime guards for instances. + const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr(); + if (btype != NULL) { + if (btype->isa_aryptr()) { + // Array type so nothing to do + return; + } + + const TypeInstPtr* itype = btype->isa_instptr(); + if (itype != NULL) { + // Can the klass of base_oop be statically determined to be + // _not_ a sub-class of Reference and _not_ Object? + ciKlass* klass = itype->klass(); + if ( klass->is_loaded() && + !klass->is_subtype_of(kit->env()->Reference_klass()) && + !kit->env()->Object_klass()->is_subtype_of(klass)) { + return; + } + } + } + + float likely = PROB_LIKELY( 0.999); + float unlikely = PROB_UNLIKELY(0.999); + + IdealKit ideal(kit); + + Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset); + + __ if_then(offset, BoolTest::eq, referent_off, unlikely); { + // Update graphKit memory and control from IdealKit. + kit->sync_kit(ideal); + Node* ref_klass_con = kit->makecon(TypeKlassPtr::make(kit->env()->Reference_klass())); + Node* is_instof = kit->gen_instanceof(base_oop, ref_klass_con); + // Update IdealKit memory and control from graphKit. + __ sync_kit(kit); + Node* one = __ ConI(1); + // is_instof == 0 if base_oop == NULL + __ if_then(is_instof, BoolTest::eq, one, unlikely); { + // Update graphKit from IdeakKit. + kit->sync_kit(ideal); + // Use the pre-barrier to record the value in the referent field + reference_load_barrier(kit, val, false); + if (need_mem_bar) { + // Add memory barrier to prevent commoning reads from this field + // across safepoint since GC can change its value. + kit->insert_mem_bar(Op_MemBarCPUOrder); + } + // Update IdealKit from graphKit. + __ sync_kit(kit); + } __ end_if(); // _ref_type != ref_none + } __ end_if(); // offset == referent_offset + + // Final sync IdealKit and GraphKit. + kit->final_sync(ideal); +} + Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { DecoratorSet decorators = access.decorators(); @@ -304,19 +404,11 @@ Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_ return load; } - MMTkIdealKit ideal(kit, true); - // Call slow-path only when concurrent marking is active - Node* no_base = __ top(); - float unlikely = PROB_UNLIKELY(0.999); - Node* zero = __ ConI(0); - Node* cm_flag = __ load(__ ctrl(), __ ConP(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); - __ if_then(cm_flag, BoolTest::ne, zero, unlikely); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", load); - } __ end_if(); - kit->sync_kit(ideal); - kit->insert_mem_bar(Op_MemBarVolatile); - kit->final_sync(ideal); // Final sync IdealKit and GraphKit. + if (on_weak) { + reference_load_barrier(kit, load, true); + } else if (unknown) { + reference_load_barrier_for_unknown_load(kit, obj, offset, load, !need_cpu_mem_bar); + } return load; } diff --git a/openjdk/barriers/mmtkFieldBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp index 826b1a08..a319c629 100644 --- a/openjdk/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -24,7 +24,7 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { object_reference_array_copy_pre_call((void*) src, (void*) dst, count); } virtual void load_reference(DecoratorSet decorators, oop value) const override { - if (CONCURRENT_MARKING_ACTIVE == 1) load_reference_call((void*) value); + if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) load_reference_call((void*) value); }; }; From 7ddfe23c75a86d77887a537b45544cea1d9acc54 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 6 Nov 2022 10:04:53 +0000 Subject: [PATCH 190/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 85b7e269..7995a328 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eb8629b6e827684a443b1d1426ac012dcddeb213#eb8629b6e827684a443b1d1426ac012dcddeb213" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=fcdd8d6996e2153ef19b60b069c88bb106c6bb73#fcdd8d6996e2153ef19b60b069c88bb106c6bb73" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eb8629b6e827684a443b1d1426ac012dcddeb213#eb8629b6e827684a443b1d1426ac012dcddeb213" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=fcdd8d6996e2153ef19b60b069c88bb106c6bb73#fcdd8d6996e2153ef19b60b069c88bb106c6bb73" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 4a0baf8c..eaf2adf7 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "eb8629b6e827684a443b1d1426ac012dcddeb213" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "fcdd8d6996e2153ef19b60b069c88bb106c6bb73" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 5858bff06449540d946f53163976d9e5c6be9f6b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 7 Nov 2022 05:54:18 +0000 Subject: [PATCH 191/494] performace tweaks --- mmtk/Cargo.lock | 4 +-- mmtk/Cargo.toml | 2 +- mmtk/src/collection.rs | 4 +-- mmtk/src/lib.rs | 2 +- mmtk/src/object_scanning.rs | 8 +++-- mmtk/src/scanning.rs | 5 ++++ openjdk/mmtk.h | 2 +- openjdk/mmtkUpcalls.cpp | 58 ++++++++++++++++++++++--------------- 8 files changed, 52 insertions(+), 33 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 7995a328..af0e5fb6 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=fcdd8d6996e2153ef19b60b069c88bb106c6bb73#fcdd8d6996e2153ef19b60b069c88bb106c6bb73" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ae7dc0c976c219921130d5dd8cfa6e2549a33d8a#ae7dc0c976c219921130d5dd8cfa6e2549a33d8a" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=fcdd8d6996e2153ef19b60b069c88bb106c6bb73#fcdd8d6996e2153ef19b60b069c88bb106c6bb73" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ae7dc0c976c219921130d5dd8cfa6e2549a33d8a#ae7dc0c976c219921130d5dd8cfa6e2549a33d8a" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index eaf2adf7..edea8091 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "fcdd8d6996e2153ef19b60b069c88bb106c6bb73" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "ae7dc0c976c219921130d5dd8cfa6e2549a33d8a" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index ca23f0f5..b34d6109 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -110,9 +110,9 @@ impl Collection for VMCollection { DISCOVERED_LISTS.process_phantom_refs::(worker) } - fn update_weak_processor() { + fn update_weak_processor(lxr: bool) { unsafe { - ((*UPCALLS).update_weak_processor)(); + ((*UPCALLS).update_weak_processor)(lxr); } } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 4a890721..01f7fbaf 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -99,7 +99,7 @@ pub struct OpenJDK_Upcalls { pub number_of_mutators: extern "C" fn() -> usize, pub schedule_finalizer: extern "C" fn(), pub prepare_for_roots_re_scanning: extern "C" fn(), - pub update_weak_processor: extern "C" fn(), + pub update_weak_processor: extern "C" fn(lxr: bool), pub enqueue_references: extern "C" fn(objects: *const ObjectReference, len: usize), pub swap_reference_pending_list: extern "C" fn(objects: ObjectReference) -> ObjectReference, } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 98402a6f..277f2955 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -229,8 +229,12 @@ fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { #[inline(always)] pub fn is_obj_array(oop: Oop) -> bool { - let klass_id = oop.klass.id; - klass_id == KlassID::ObjArray + oop.klass.id == KlassID::ObjArray +} + +#[inline(always)] +pub fn is_val_array(oop: Oop) -> bool { + oop.klass.id == KlassID::TypeArray } #[inline(always)] diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 1a534722..bf0dcdf7 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -64,6 +64,11 @@ impl Scanning for VMScanning { crate::object_scanning::is_obj_array(unsafe { std::mem::transmute(o) }) } + #[inline(always)] + fn is_val_array(o: ObjectReference) -> bool { + crate::object_scanning::is_val_array(unsafe { std::mem::transmute(o) }) + } + fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { // unimplemented!() // TODO diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 5667455e..ddd81af4 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -192,7 +192,7 @@ typedef struct { size_t (*number_of_mutators)(); void (*schedule_finalizer)(); void (*prepare_for_roots_re_scanning)(); - void (*mmtk_update_weak_processor)(); + void (*mmtk_update_weak_processor)(bool lxr); void (*enqueue_references)(void** objects, size_t len); void* (*swap_reference_pending_list)(void* objects); } OpenJDK_Upcalls; diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 4aa46806..e3959dd8 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -59,43 +59,54 @@ class MMTkIsAliveClosure : public BoolObjectClosure { class MMTkForwardClosure : public BasicOopIterateClosure { public: - inline size_t read_forwarding_word(oop o) const { + inline static size_t read_forwarding_word(oop o) { return *((size_t*) (void*) o); } - inline oop extract_forwarding_pointer(size_t status) const { + inline static oop extract_forwarding_pointer(size_t status) { return (oop) (void*) (status << 8 >> 8); } - inline bool is_forwarded(size_t status) const { + inline static bool is_forwarded(size_t status) { return (status & (0xffull << 56)) != 0; } - virtual void do_oop(oop* slot) { + inline virtual void do_oop(oop* slot) { // *slot = (oop) mmtk_get_forwarded_ref((void*) *slot); - auto o = *slot; + const auto o = *slot; if (o == NULL) return; - auto status = read_forwarding_word(o); + const auto status = read_forwarding_word(o); if (is_forwarded(status)) { - o = extract_forwarding_pointer(status); - auto status = read_forwarding_word(o); - if (is_forwarded(status)) { - o = extract_forwarding_pointer(status); - } - *slot = o; + *slot = extract_forwarding_pointer(status); } } virtual void do_oop(narrowOop* o) {} virtual ReferenceIterationMode reference_iteration_mode() { return DO_FIELDS; } }; +class MMTkLXRFastIsAliveClosure : public BoolObjectClosure { +public: + inline virtual bool do_object_b(oop o) { + if (o == NULL) return false; + // is forwarded? + if (MMTkForwardClosure::is_forwarded(MMTkForwardClosure::read_forwarding_word(o))) return true; + // RC > 0? + const uintptr_t index = uintptr_t((void*)o) >> 4; + const uint8_t byte = *((uint8_t*) (uintptr_t(0xe0004000000) + (index >> 2))); + const uint8_t byte_mask = 0b11 << ((index & 0b11) << 1); + if ((byte & byte_mask) != 0) return true; + return false; + } +}; + static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure) { + log_debug(gc)("Requesting the VM to suspend all mutators..."); + MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_suspended, true); + log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); + ClassLoaderDataGraph::clear_claimed_marks(); CodeCache::gc_prologue(); #if COMPILER2_OR_JVMCI DerivedPointerTable::clear(); #endif - log_debug(gc)("Requesting the VM to suspend all mutators..."); - MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_suspended, true); - log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); mmtk_report_gc_start(); if (!scan_mutators_in_safepoint) { @@ -108,20 +119,19 @@ static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, M nmethod::oops_do_marking_prologue(); } -static void mmtk_update_weak_processor() { +static void mmtk_update_weak_processor(bool lxr) { HandleMark hm; - MMTkIsAliveClosure is_alive; MMTkForwardClosure forward; - WeakProcessor::weak_oops_do(&is_alive, &forward); -} - -static void mmtk_resume_mutators(void *tls) { - { - HandleMark hm; + if (lxr) { + MMTkLXRFastIsAliveClosure is_alive; + WeakProcessor::weak_oops_do(&is_alive, &forward); + } else { MMTkIsAliveClosure is_alive; - MMTkForwardClosure forward; WeakProcessor::weak_oops_do(&is_alive, &forward); } +} + +static void mmtk_resume_mutators(void *tls) { nmethod::oops_do_marking_epilogue(); // ClassLoaderDataGraph::purge(); // BiasedLocking::restore_marks(); From 43bb34dc7136f91431f7fc41c1e219db5883aa9a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 7 Nov 2022 10:41:04 +0000 Subject: [PATCH 192/494] Fix assembler load barrier --- openjdk/barriers/mmtkFieldBarrier.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 9888ea45..d9edabbf 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -30,9 +30,11 @@ void MMTkFieldBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet de Label done; // No slow-call if SATB is not active Register tmp = rscratch1; + Register tmp2 = rscratch2; __ movptr(tmp, intptr_t(&CONCURRENT_MARKING_ACTIVE)); - __ movb(tmp, Address(tmp, 0)); - __ cmpptr(tmp, 1); + __ xorq(tmp2, tmp2); + __ movb(tmp2, Address(tmp, 0)); + __ cmpptr(tmp2, 1); __ jcc(Assembler::notEqual, done); // No slow-call if dst is NULL __ cmpptr(dst, 0); From e189c7f432b0c687b19af80e5667eb47c7301b4a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 8 Nov 2022 12:07:22 +0000 Subject: [PATCH 193/494] Clone barrier workaround; Barrier validation --- mmtk/src/api.rs | 8 +++++++ mmtk/src/object_model.rs | 16 +++++++++++--- mmtk/src/reference_glue.rs | 8 +++++-- openjdk/barriers/mmtkFieldBarrier.cpp | 30 +++++++++++++++++++++++++++ openjdk/barriers/mmtkFieldBarrier.hpp | 4 ++++ openjdk/mmtk.h | 4 +++- openjdk/mmtkBarrierSet.cpp | 4 ++++ openjdk/mmtkBarrierSet.hpp | 6 +++++- openjdk/mmtkUpcalls.cpp | 11 ++++++++-- 9 files changed, 82 insertions(+), 9 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index b7fc68db..1ccfb3ac 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -317,6 +317,14 @@ pub extern "C" fn mmtk_load_reference(mutator: &'static mut Mutator, o: mutator.barrier().load_reference(o); } +#[no_mangle] +pub extern "C" fn mmtk_object_reference_clone_pre( + mutator: &'static mut Mutator, + obj: ObjectReference, +) { + mutator.barrier().object_reference_clone_pre(obj); +} + /// Full pre barrier #[no_mangle] pub extern "C" fn mmtk_object_reference_write_pre( diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 06b65657..482c3317 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -87,8 +87,18 @@ impl ObjectModel for VMObjectModel { } fn dump_object(object: ObjectReference) { - unsafe { - ((*UPCALLS).dump_object)(object); - } + use std::ffi::CStr; + let c_string = unsafe { ((*UPCALLS).dump_object_string)(std::mem::transmute(object)) }; + let c_str: &CStr = unsafe { CStr::from_ptr(c_string) }; + let s: &str = c_str.to_str().unwrap(); + println!("{}", s) + } + + fn dump_object_s(object: ObjectReference) -> String { + use std::ffi::CStr; + let c_string = unsafe { ((*UPCALLS).dump_object_string)(std::mem::transmute(object)) }; + let c_str: &CStr = unsafe { CStr::from_ptr(c_string) }; + let s: &str = c_str.to_str().unwrap(); + s.to_string() } } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 188ab9ce..597ac382 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -14,7 +14,9 @@ use mmtk::MMTK; #[inline(always)] pub fn set_referent(reff: ObjectReference, referent: ObjectReference) { let oop = Oop::from(reff); - unsafe { InstanceRefKlass::referent_address(oop).store(referent) } + let slot = InstanceRefKlass::referent_address(oop); + mmtk::plan::lxr::record_edge_for_validation(slot, referent); + unsafe { slot.store(referent) } } #[inline(always)] @@ -36,7 +38,9 @@ fn get_next_reference(object: ObjectReference) -> ObjectReference { #[inline(always)] fn set_next_reference(object: ObjectReference, next: ObjectReference) { - unsafe { get_next_reference_slot(object).store(next) } + let slot = get_next_reference_slot(object); + mmtk::plan::lxr::record_edge_for_validation(slot, next); + unsafe { slot.store(next) } } pub struct VMReferenceGlue {} diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index d9edabbf..daf25817 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -415,5 +415,35 @@ Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_ return load; } +void MMTkFieldBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { + if (!is_array) { + MMTkIdealKit ideal(kit); +#if MMTK_ENABLE_BARRIER_FASTPATH + Node* no_base = __ top(); + float unlikely = PROB_UNLIKELY(0.999); + + Node* zero = __ ConI(0); + Node* addr = __ CastPX(__ ctrl(), dst); + Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); + Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); + Node* shift = __ URShiftX(addr, __ ConI(3)); + shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); + Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); + + __ if_then(result, BoolTest::ne, zero, unlikely); { + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_clone_pre_call), "mmtk_barrier_call", dst); + } __ end_if(); +#else + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_clone_pre_call), "mmtk_barrier_call", dst); +#endif + kit->sync_kit(ideal); + kit->insert_mem_bar(Op_MemBarVolatile); + kit->final_sync(ideal); + } + BarrierSetC2::clone(kit, src, dst, size, is_array); +} + #undef __ \ No newline at end of file diff --git a/openjdk/barriers/mmtkFieldBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp index a319c629..c605fd27 100644 --- a/openjdk/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -26,6 +26,9 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { virtual void load_reference(DecoratorSet decorators, oop value) const override { if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) load_reference_call((void*) value); }; + virtual void clone_pre(DecoratorSet decorators, oop value) const override { + // object_reference_clone_pre_call((void*) value); + }; }; class MMTkFieldBarrierSetAssembler: public MMTkBarrierSetAssembler { @@ -56,6 +59,7 @@ class MMTkFieldBarrierSetC2: public MMTkBarrierSetC2 { return false; } virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const override; + virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const override; }; struct MMTkFieldBarrier: MMTkBarrierImpl< diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index ddd81af4..32a6e705 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -72,6 +72,8 @@ extern void mmtk_array_copy_pre(MMTk_Mutator mutator, void* src, void* dst, size /// Full array-copy post-barrier extern void mmtk_array_copy_post(MMTk_Mutator mutator, void* src, void* dst, size_t count); +extern void mmtk_object_reference_clone_pre(MMTk_Mutator mutator, void* obj); + extern void release_buffer(void** buffer, size_t len, size_t cap); extern bool is_in_mmtk_spaces(void* ref); @@ -174,7 +176,7 @@ typedef struct { int (*static_oop_field_count_offset) (); int (*referent_offset) (); int (*discovered_offset) (); - char* (*dump_object_string) (void* object); + const char* (*dump_object_string) (void* object); void (*scan_all_thread_roots)(EdgesClosure closure); void (*scan_thread_roots)(EdgesClosure closure, void* tls); void (*scan_universe_roots) (EdgesClosure closure); diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index b24b5852..6367d33a 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -156,4 +156,8 @@ void MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, voi void MMTkBarrierSetRuntime::load_reference_call(void* ref) { ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); +} + +void MMTkBarrierSetRuntime::object_reference_clone_pre_call(void* ref) { + ::mmtk_object_reference_clone_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); } \ No newline at end of file diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index bb3f1594..86ed42de 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -73,6 +73,7 @@ class MMTkBarrierSetRuntime: public CHeapObj { static void object_reference_array_copy_pre_call(void* src, void* dst, size_t count); /// Generic arraycopy pre-barrier. Called by fast-paths. static void object_reference_array_copy_post_call(void* src, void* dst, size_t count); + static void object_reference_clone_pre_call(void* obj); /// Check if the address is a slow-path function. virtual bool is_slow_path_call(address call) const { return call == CAST_FROM_FN_PTR(address, object_reference_write_pre_call) @@ -80,7 +81,8 @@ class MMTkBarrierSetRuntime: public CHeapObj { || call == CAST_FROM_FN_PTR(address, object_reference_write_slow_call) || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call) || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_post_call) - || call == CAST_FROM_FN_PTR(address, load_reference_call); + || call == CAST_FROM_FN_PTR(address, load_reference_call) + || call == CAST_FROM_FN_PTR(address, object_reference_clone_pre_call); } /// Full pre-barrier @@ -93,6 +95,7 @@ class MMTkBarrierSetRuntime: public CHeapObj { virtual void object_reference_array_copy_post(oop* src, oop* dst, size_t count) const {}; /// java.lang.Reference load barrier virtual void load_reference(DecoratorSet decorators, oop value) const {}; + virtual void clone_pre(DecoratorSet decorators, oop value) const {}; }; class MMTkBarrierC1; @@ -243,6 +246,7 @@ class MMTkBarrierSet : public BarrierSet { static void clone_in_heap(oop src, oop dst, size_t size) { // TODO: We don't need clone barriers at the moment. + runtime()->clone_pre(decorators, dst); Raw::clone(src, dst, size); } }; diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index e3959dd8..ce898848 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -353,9 +353,16 @@ static int discovered_offset() { return java_lang_ref_Reference::discovered_offset; } -static char* dump_object_string(void* object) { +char data[1024]; + +static const char* dump_object_string(void* object) { + HandleMark hm; + ResourceMark rm; + if (object == NULL) return NULL; oop o = (oop) object; - return o->print_value_string(); + const char* c = o->klass()->internal_name(); + strcpy(&data[0], c); + return &data[0]; } static void mmtk_schedule_finalizer() { From f28e378f13000c4e6ca4df10bf0dbeab514b8fb8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 8 Nov 2022 12:08:48 +0000 Subject: [PATCH 194/494] update mmtk-core --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index edea8091..4d169e9e 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "ae7dc0c976c219921130d5dd8cfa6e2549a33d8a" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7c8c10a32ce7f82c40baa3cfe4ef3cdf4acf824f" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 0e6f8f04867859d4a85dad8d85824266b7cb6513 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 8 Nov 2022 12:15:55 +0000 Subject: [PATCH 195/494] update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index af0e5fb6..59ae84ac 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ae7dc0c976c219921130d5dd8cfa6e2549a33d8a#ae7dc0c976c219921130d5dd8cfa6e2549a33d8a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3bd25c73bda6c8447ce308765cd190517a6872f6#3bd25c73bda6c8447ce308765cd190517a6872f6" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ae7dc0c976c219921130d5dd8cfa6e2549a33d8a#ae7dc0c976c219921130d5dd8cfa6e2549a33d8a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3bd25c73bda6c8447ce308765cd190517a6872f6#3bd25c73bda6c8447ce308765cd190517a6872f6" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 4d169e9e..7b88d667 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7c8c10a32ce7f82c40baa3cfe4ef3cdf4acf824f" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "3bd25c73bda6c8447ce308765cd190517a6872f6" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 81d90867df4c58a91032f7942ae718242592c46e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 14 Nov 2022 02:45:48 +0000 Subject: [PATCH 196/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 59ae84ac..bdac07d5 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3bd25c73bda6c8447ce308765cd190517a6872f6#3bd25c73bda6c8447ce308765cd190517a6872f6" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=025870dd48fe486c90e002c091bfc0d8016b1c6e#025870dd48fe486c90e002c091bfc0d8016b1c6e" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3bd25c73bda6c8447ce308765cd190517a6872f6#3bd25c73bda6c8447ce308765cd190517a6872f6" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=025870dd48fe486c90e002c091bfc0d8016b1c6e#025870dd48fe486c90e002c091bfc0d8016b1c6e" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 7b88d667..bc3742ce 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "3bd25c73bda6c8447ce308765cd190517a6872f6" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "025870dd48fe486c90e002c091bfc0d8016b1c6e" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From d7bc7e1c02d4dfa34db1b327f05516194efc079d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 14 Nov 2022 07:59:12 +0000 Subject: [PATCH 197/494] Reduce stop-all-mutators time --- openjdk/mmtkUpcalls.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index ce898848..0983d42c 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -101,14 +101,14 @@ static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, M MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_suspended, true); log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); - ClassLoaderDataGraph::clear_claimed_marks(); + mmtk_report_gc_start(); + + // ClassLoaderDataGraph::clear_claimed_marks(); CodeCache::gc_prologue(); #if COMPILER2_OR_JVMCI DerivedPointerTable::clear(); #endif - mmtk_report_gc_start(); - if (!scan_mutators_in_safepoint) { JavaThreadIteratorWithHandle jtiwh; while (JavaThread *cur = jtiwh.next()) { From d61ecc72962d002ab3afea227e0d5e98bd220f25 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 14 Nov 2022 12:04:20 +0000 Subject: [PATCH 198/494] Remove safepoint workers --- openjdk/mmtkHeap.cpp | 5 ----- openjdk/mmtkHeap.hpp | 2 -- 2 files changed, 7 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 9ee004ad..63fe7e66 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -67,11 +67,6 @@ MMTkHeap::MMTkHeap(MMTkCollectorPolicy* policy) : CollectedHeap(), _last_gc_time // , _par_state_string(StringTable::weak_storage()) { _heap = this; - unsigned int ncpus = (unsigned int) os::initial_active_processor_count(); - _safepoint_workers = new WorkGang("GC Thread", ncpus, - /* are_GC_task_threads */true, - /* are_ConcurrentGC_threads */false); - _safepoint_workers->initialize_workers(); } jint MMTkHeap::initialize() { diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index 35717b35..9e7af7a4 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -59,7 +59,6 @@ class MMTkHeap : public CollectedHeap { ContiguousSpace* _space; int _num_root_scan_tasks; MMTkVMCompanionThread* _companion_thread; - WorkGang* _safepoint_workers; public: MMTkHeap(MMTkCollectorPolicy* policy); @@ -180,7 +179,6 @@ class MMTkHeap : public CollectedHeap { // Default implementation does nothing. void print_tracing_info() const ; - inline virtual WorkGang* get_safepoint_workers() { return _safepoint_workers; } // An object is scavengable if its location may move during a scavenge. // (A scavenge is a GC which is not a full GC.) From 2e99e6ac0d5f0e35a7cbe026c107e6a4dcb15b5a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 15 Nov 2022 03:30:56 +0000 Subject: [PATCH 199/494] minor --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index bc3742ce..f96480f5 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -78,7 +78,7 @@ sanity = ["mmtk/sanity"] barrier_measurement = ["mmtk/barrier_measurement"] yield_and_roots_timer = ["mmtk/yield_and_roots_timer"] satb_timer = ["mmtk/satb_timer"] -instrumentation = ["mmtk/instrumentation"] +instrumentation = ["mmtk/instrumentation", "no_fast_alloc"] log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] log_work_packets = ["mmtk/log_work_packets"] From c71665d9ae72992eb742448628be52b772ede571 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 15 Nov 2022 03:31:53 +0000 Subject: [PATCH 200/494] Reduce allocation slow-path take-rate --- openjdk/mmtkBarrierSetAssembler_x86.cpp | 3 ++- openjdk/mmtkBarrierSetC2.cpp | 3 ++- openjdk/mmtkHeap.cpp | 35 ++++++++++++++++++++++--- openjdk/mmtkHeap.hpp | 1 + openjdk/mmtkMutator.cpp | 14 +++++++++- 5 files changed, 50 insertions(+), 6 deletions(-) diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index c9c24279..f5d375fe 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -31,6 +31,7 @@ #include "mmtkBarrierSetAssembler_x86.hpp" #include "mmtkBarrierSetC1.hpp" #include "mmtkMutator.hpp" +#include "mmtkHeap.hpp" #include "runtime/sharedRuntime.hpp" #include "utilities/macros.hpp" #include "c1/c1_LIRAssembler.hpp" @@ -55,7 +56,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa // fastpath, we only use default allocator Allocator allocator = AllocatorDefault; // We need to figure out which allocator we are using by querying MMTk. - AllocatorSelector selector = get_allocator_mapping(allocator); + AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; if (selector.tag == TAG_MARK_COMPACT) extra_header = MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; if (var_size_in_bytes == noreg) { diff --git a/openjdk/mmtkBarrierSetC2.cpp b/openjdk/mmtkBarrierSetC2.cpp index 30c1cdd3..55c9c0e6 100644 --- a/openjdk/mmtkBarrierSetC2.cpp +++ b/openjdk/mmtkBarrierSetC2.cpp @@ -27,6 +27,7 @@ #include "mmtkBarrierSet.hpp" #include "mmtkBarrierSetC2.hpp" #include "mmtkMutator.hpp" +#include "mmtkHeap.hpp" #include "opto/addnode.hpp" #include "opto/arraycopynode.hpp" #include "opto/callnode.hpp" @@ -84,7 +85,7 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, size_t extra_header = 0; // We always use the default allocator. // But we need to figure out which allocator we are using by querying MMTk. - AllocatorSelector selector = get_allocator_mapping(AllocatorDefault); + AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; if (selector.tag == TAG_MARK_COMPACT) extra_header = MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; // Check if allocation size is constant diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 63fe7e66..4d71edb0 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -109,6 +109,7 @@ jint MMTkHeap::initialize() { initialize_reserved_region(_start, _end); + default_allocator_selector = get_allocator_mapping(AllocatorDefault); MMTkBarrierSet* const barrier_set = new MMTkBarrierSet(reserved_region()); //barrier_set->initialize(); @@ -447,13 +448,41 @@ void MMTkHeap::scan_roots(OopClosure& cl) { WeakProcessor::oops_do(&cl); } +static inline HeapWord* alloc_fast(size_t bytes, Allocator allocator) { + // All allocations with size larger than max non-los bytes will get to this slowpath here. + // We will use LOS for those. + assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); + if (bytes >= MMTkMutatorContext::max_non_los_default_alloc_bytes) { + allocator = AllocatorLos; + } else if (allocator == AllocatorDefault) { + AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; + if (selector.tag == TAG_IMMIX) { + auto& allocator = Thread::current()->third_party_heap_mutator.allocators.immix[selector.index]; + auto cursor = uintptr_t(allocator.cursor); + auto limit = uintptr_t(allocator.limit); + if (cursor + bytes <= limit) { + allocator.cursor = (void*) (cursor + bytes); + return (HeapWord*) cursor; + } + } + } + + // FIXME: Proper use of slow-path api + HeapWord* o = (HeapWord*) ::alloc((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, bytes, HeapWordSize, 0, allocator); + // Post allocation hooks. Note that we can get a nullptr from mmtk core in the case of OOM. + // Hence, only call post allocation hooks if we have a proper object. + if (o != nullptr && allocator != AllocatorDefault) { + ::post_alloc((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, o, bytes, allocator); + } + return o; +} + HeapWord* MMTkHeap::mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) { - HeapWord* obj = Thread::current()->third_party_heap_mutator.alloc(size << LogHeapWordSize); - return obj; + return alloc_fast(size << LogHeapWordSize, AllocatorDefault); } HeapWord* MMTkHeap::mem_allocate_nonmove(size_t size, bool* gc_overhead_limit_was_exceeded) { - return Thread::current()->third_party_heap_mutator.alloc(size << LogHeapWordSize, AllocatorLos); + return alloc_fast(size << LogHeapWordSize, AllocatorLos); } /* diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index 9e7af7a4..6ca24026 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -60,6 +60,7 @@ class MMTkHeap : public CollectedHeap { int _num_root_scan_tasks; MMTkVMCompanionThread* _companion_thread; public: + AllocatorSelector default_allocator_selector; MMTkHeap(MMTkCollectorPolicy* policy); diff --git a/openjdk/mmtkMutator.cpp b/openjdk/mmtkMutator.cpp index 43139432..5113ef98 100644 --- a/openjdk/mmtkMutator.cpp +++ b/openjdk/mmtkMutator.cpp @@ -2,6 +2,7 @@ #include "precompiled.hpp" #include "mmtk.h" #include "mmtkMutator.hpp" +#include "mmtkHeap.hpp" size_t MMTkMutatorContext::max_non_los_default_alloc_bytes = 0; @@ -23,13 +24,24 @@ HeapWord* MMTkMutatorContext::alloc(size_t bytes, Allocator allocator) { assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); if (bytes >= MMTkMutatorContext::max_non_los_default_alloc_bytes) { allocator = AllocatorLos; + } else { + AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; + if (selector.tag == TAG_IMMIX) { + auto& allocator = allocators.immix[selector.index]; + auto cursor = uintptr_t(allocator.cursor); + auto limit = uintptr_t(allocator.limit); + if (cursor + bytes <= limit) { + allocator.cursor = (void*) (cursor + bytes); + return (HeapWord*) cursor; + } + } } // FIXME: Proper use of slow-path api HeapWord* o = (HeapWord*) ::alloc((MMTk_Mutator) this, bytes, HeapWordSize, 0, allocator); // Post allocation hooks. Note that we can get a nullptr from mmtk core in the case of OOM. // Hence, only call post allocation hooks if we have a proper object. - if (o != nullptr) { + if (o != nullptr && allocator != AllocatorDefault) { ::post_alloc((MMTk_Mutator) this, o, bytes, allocator); } return o; From 3d72caaf5e81eb170f1b3442278582b68adf2a57 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 15 Nov 2022 07:45:01 +0000 Subject: [PATCH 201/494] Optimize WeakProcessor --- openjdk/mmtkUpcalls.cpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 0983d42c..953bc143 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -57,7 +57,7 @@ class MMTkIsAliveClosure : public BoolObjectClosure { } }; -class MMTkForwardClosure : public BasicOopIterateClosure { +class MMTkForwardClosure : public OopClosure { public: inline static size_t read_forwarding_word(oop o) { return *((size_t*) (void*) o); @@ -68,7 +68,7 @@ class MMTkForwardClosure : public BasicOopIterateClosure { inline static bool is_forwarded(size_t status) { return (status & (0xffull << 56)) != 0; } - inline virtual void do_oop(oop* slot) { + inline void do_oop(oop* slot) { // *slot = (oop) mmtk_get_forwarded_ref((void*) *slot); const auto o = *slot; if (o == NULL) return; @@ -78,21 +78,18 @@ class MMTkForwardClosure : public BasicOopIterateClosure { } } virtual void do_oop(narrowOop* o) {} - virtual ReferenceIterationMode reference_iteration_mode() { return DO_FIELDS; } }; class MMTkLXRFastIsAliveClosure : public BoolObjectClosure { public: - inline virtual bool do_object_b(oop o) { + inline bool do_object_b(oop o) { if (o == NULL) return false; - // is forwarded? - if (MMTkForwardClosure::is_forwarded(MMTkForwardClosure::read_forwarding_word(o))) return true; // RC > 0? + // Note: forwarded objects should have zero RC count. const uintptr_t index = uintptr_t((void*)o) >> 4; const uint8_t byte = *((uint8_t*) (uintptr_t(0xe0004000000) + (index >> 2))); const uint8_t byte_mask = 0b11 << ((index & 0b11) << 1); - if ((byte & byte_mask) != 0) return true; - return false; + return (byte & byte_mask) != 0; } }; From b4ea6bf716aa46b9dfe935ce9c02e68f80b806d0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 15 Nov 2022 07:47:00 +0000 Subject: [PATCH 202/494] Optimize allocation: Use hotspot internal LOS check --- openjdk/mmtkBarrierSetAssembler_x86.cpp | 14 +------ openjdk/mmtkBarrierSetC2.cpp | 51 +++---------------------- openjdk/mmtkHeap.cpp | 36 ++--------------- openjdk/mmtkMutator.cpp | 29 -------------- openjdk/mmtkMutator.inline.hpp | 47 +++++++++++++++++++++++ 5 files changed, 57 insertions(+), 120 deletions(-) create mode 100644 openjdk/mmtkMutator.inline.hpp diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index f5d375fe..6c4814c9 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -51,7 +51,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa // I tried to set FastAllocateSizeLimit in MMTkHeap::initialize(). But there are still large objects allocated into the // default space. assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); - size_t max_non_los_bytes = MMTkMutatorContext::max_non_los_default_alloc_bytes; + size_t extra_header = 0; // fastpath, we only use default allocator Allocator allocator = AllocatorDefault; @@ -59,18 +59,6 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; if (selector.tag == TAG_MARK_COMPACT) extra_header = MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; - if (var_size_in_bytes == noreg) { - // constant alloc size. If it is larger than max_non_los_bytes, we directly go to slowpath. - if ((size_t)con_size_in_bytes > max_non_los_bytes - extra_header) { - __ jmp(slow_case); - return; - } - } else { - // var alloc size. We compare with max_non_los_bytes and conditionally jump to slowpath. - __ cmpptr(var_size_in_bytes, max_non_los_bytes - extra_header); - __ jcc(Assembler::aboveEqual, slow_case); - } - if (selector.tag == TAG_MALLOC || selector.tag == TAG_LARGE_OBJECT) { __ jmp(slow_case); return; diff --git a/openjdk/mmtkBarrierSetC2.cpp b/openjdk/mmtkBarrierSetC2.cpp index 55c9c0e6..ae6d3203 100644 --- a/openjdk/mmtkBarrierSetC2.cpp +++ b/openjdk/mmtkBarrierSetC2.cpp @@ -72,6 +72,8 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, if (tv >= 0) { always_slow = (tv == 1); initial_slow_test = NULL; + } else { + initial_slow_test = BoolNode::make_predicate(initial_slow_test, &x->_igvn); } // We will do MMTk size check. @@ -81,56 +83,13 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, // The max non-los bytes from MMTk assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); - size_t max_non_los_bytes = MMTkMutatorContext::max_non_los_default_alloc_bytes; + size_t extra_header = 0; // We always use the default allocator. // But we need to figure out which allocator we are using by querying MMTk. AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; if (selector.tag == TAG_MARK_COMPACT) extra_header = MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; - // Check if allocation size is constant - long const_size = x->_igvn.find_long_con(size_in_bytes, -1); - if (const_size >= 0) { - // Constant alloc size. We know it is non-negative, it is safe to cast to unsigned long and compare with size_t - if (((unsigned long)const_size) > max_non_los_bytes - extra_header) { - // We know at JIT time that we need to go to slowpath - always_slow = true; - initial_slow_test = NULL; - } - } else { - // Variable alloc size - - // Create a node for the constant and compare with size_in_bytes - Node *max_non_los_bytes_node = ConLNode::make((long)max_non_los_bytes - extra_header); - x->transform_later(max_non_los_bytes_node); - Node *mmtk_size_cmp = new CmpLNode(size_in_bytes, max_non_los_bytes_node); - x->transform_later(mmtk_size_cmp); - // Size Check: if size_in_bytes >= max_non_los_bytes - Node *mmtk_size_bool = new BoolNode(mmtk_size_cmp, BoolTest::ge); - x->transform_later(mmtk_size_bool); - - if (initial_slow_test == NULL) { - // If there is no previous initial_slow_test, we use the size check as initial_slow_test - initial_slow_test = BoolNode::make_predicate(mmtk_size_bool, &x->_igvn); - } else { - // If there is existing initial_slow_test, we combine the result by 'or' them together. - - // Conditionally move a value 1 or 0 depends on the size check. - // This is definitely not optimal. But to make things simple and to avoid changing the original - // control flow, it is much easier to implement with a cmov. And it should not affect performance much, - // as var size allocation is rare. - Node *mmtk_size_cmov = new CMoveINode(mmtk_size_bool, x->intcon(0), x->intcon(1), TypeInt::INT); - x->transform_later(mmtk_size_cmov); - - // Logical or the size check result (1 or 0) with initial_slow_test - Node* new_slow_test = new OrINode(mmtk_size_cmov, initial_slow_test); - x->transform_later(new_slow_test); - - // Use the or'd result as initial_slow_test - initial_slow_test = BoolNode::make_predicate(new_slow_test, &x->_igvn); - } - } - if (x->C->env()->dtrace_alloc_probes() || !MMTK_ENABLE_ALLOCATION_FASTPATH || disable_fast_alloc() // Malloc allocator has no fastpath || (selector.tag == TAG_MALLOC || selector.tag == TAG_LARGE_OBJECT)) { @@ -255,8 +214,8 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, // Slow-path does no I/O so just set it to the original I/O. result_phi_i_o->init_req(slow_result_path, i_o); - // i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem, - // old_eden_top, new_eden_top, length); + i_o = x->prefetch_allocation(i_o, needgc_false, contended_phi_rawmem, + old_eden_top, new_eden_top, length); // Name successful fast-path variables Node* fast_oop = old_eden_top; diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 4d71edb0..b5185e81 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -35,7 +35,7 @@ #include "memory/resourceArea.hpp" #include "mmtk.h" #include "mmtkHeap.hpp" -#include "mmtkMutator.hpp" +#include "mmtkMutator.inline.hpp" #include "mmtkUpcalls.hpp" #include "mmtkVMCompanionThread.hpp" #include "oops/oop.inline.hpp" @@ -88,6 +88,7 @@ jint MMTkHeap::initialize() { openjdk_gc_init(&mmtk_upcalls); // Cache the value here. It is a constant depending on the selected plan. The plan won't change from now, so value won't change. MMTkMutatorContext::max_non_los_default_alloc_bytes = get_max_non_los_default_alloc_bytes(); + // FastAllocateSizeLimit = MMTkMutatorContext::max_non_los_default_alloc_bytes; //ReservedSpace heap_rs = Universe::reserve_heap(mmtk_heap_size, _collector_policy->heap_alignment()); @@ -448,41 +449,12 @@ void MMTkHeap::scan_roots(OopClosure& cl) { WeakProcessor::oops_do(&cl); } -static inline HeapWord* alloc_fast(size_t bytes, Allocator allocator) { - // All allocations with size larger than max non-los bytes will get to this slowpath here. - // We will use LOS for those. - assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); - if (bytes >= MMTkMutatorContext::max_non_los_default_alloc_bytes) { - allocator = AllocatorLos; - } else if (allocator == AllocatorDefault) { - AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; - if (selector.tag == TAG_IMMIX) { - auto& allocator = Thread::current()->third_party_heap_mutator.allocators.immix[selector.index]; - auto cursor = uintptr_t(allocator.cursor); - auto limit = uintptr_t(allocator.limit); - if (cursor + bytes <= limit) { - allocator.cursor = (void*) (cursor + bytes); - return (HeapWord*) cursor; - } - } - } - - // FIXME: Proper use of slow-path api - HeapWord* o = (HeapWord*) ::alloc((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, bytes, HeapWordSize, 0, allocator); - // Post allocation hooks. Note that we can get a nullptr from mmtk core in the case of OOM. - // Hence, only call post allocation hooks if we have a proper object. - if (o != nullptr && allocator != AllocatorDefault) { - ::post_alloc((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, o, bytes, allocator); - } - return o; -} - HeapWord* MMTkHeap::mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) { - return alloc_fast(size << LogHeapWordSize, AllocatorDefault); + return Thread::current()->third_party_heap_mutator.alloc(size << LogHeapWordSize); } HeapWord* MMTkHeap::mem_allocate_nonmove(size_t size, bool* gc_overhead_limit_was_exceeded) { - return alloc_fast(size << LogHeapWordSize, AllocatorLos); + return Thread::current()->third_party_heap_mutator.alloc(size << LogHeapWordSize, AllocatorLos); } /* diff --git a/openjdk/mmtkMutator.cpp b/openjdk/mmtkMutator.cpp index 5113ef98..f308cc3d 100644 --- a/openjdk/mmtkMutator.cpp +++ b/openjdk/mmtkMutator.cpp @@ -18,35 +18,6 @@ bool MMTkMutatorContext::is_ready_to_bind() { return ::openjdk_is_gc_initialized(); } -HeapWord* MMTkMutatorContext::alloc(size_t bytes, Allocator allocator) { - // All allocations with size larger than max non-los bytes will get to this slowpath here. - // We will use LOS for those. - assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); - if (bytes >= MMTkMutatorContext::max_non_los_default_alloc_bytes) { - allocator = AllocatorLos; - } else { - AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; - if (selector.tag == TAG_IMMIX) { - auto& allocator = allocators.immix[selector.index]; - auto cursor = uintptr_t(allocator.cursor); - auto limit = uintptr_t(allocator.limit); - if (cursor + bytes <= limit) { - allocator.cursor = (void*) (cursor + bytes); - return (HeapWord*) cursor; - } - } - } - - // FIXME: Proper use of slow-path api - HeapWord* o = (HeapWord*) ::alloc((MMTk_Mutator) this, bytes, HeapWordSize, 0, allocator); - // Post allocation hooks. Note that we can get a nullptr from mmtk core in the case of OOM. - // Hence, only call post allocation hooks if we have a proper object. - if (o != nullptr && allocator != AllocatorDefault) { - ::post_alloc((MMTk_Mutator) this, o, bytes, allocator); - } - return o; -} - void MMTkMutatorContext::flush() { ::flush_mutator((MMTk_Mutator) this); } diff --git a/openjdk/mmtkMutator.inline.hpp b/openjdk/mmtkMutator.inline.hpp new file mode 100644 index 00000000..5730acfb --- /dev/null +++ b/openjdk/mmtkMutator.inline.hpp @@ -0,0 +1,47 @@ + +#ifndef MMTK_OPENJDK_MMTK_MUTATOR_INLINE_HPP +#define MMTK_OPENJDK_MMTK_MUTATOR_INLINE_HPP + +#include "mmtk.h" +#include "utilities/globalDefinitions.hpp" +#include "mmtkMutator.hpp" +#include "mmtkHeap.hpp" + + +inline HeapWord* MMTkMutatorContext::alloc(size_t bytes, Allocator allocator) { + // All allocations with size larger than max non-los bytes will get to this slowpath here. + // We will use LOS for those. + assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); + if (bytes >= MMTkMutatorContext::max_non_los_default_alloc_bytes) { + allocator = AllocatorLos; + } else { + AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; + if (selector.tag == TAG_IMMIX) { + auto& allocator = allocators.immix[selector.index]; + auto cursor = uintptr_t(allocator.cursor); + auto limit = uintptr_t(allocator.limit); + if (cursor + bytes <= limit) { + allocator.cursor = (void*) (cursor + bytes); + return (HeapWord*) cursor; + } else if (bytes > 256) { + auto large_cursor = uintptr_t(allocator.large_cursor); + auto large_limit = uintptr_t(allocator.large_limit); + if (large_cursor + bytes <= large_limit) { + allocator.large_cursor = (void*) (large_cursor + bytes); + return (HeapWord*) large_cursor; + } + } + } + } + + // FIXME: Proper use of slow-path api + HeapWord* o = (HeapWord*) ::alloc((MMTk_Mutator) this, bytes, HeapWordSize, 0, allocator); + // Post allocation hooks. Note that we can get a nullptr from mmtk core in the case of OOM. + // Hence, only call post allocation hooks if we have a proper object. + if (o != nullptr && allocator != AllocatorDefault) { + ::post_alloc((MMTk_Mutator) this, o, bytes, allocator); + } + return o; +} + +#endif // MMTK_OPENJDK_MMTK_MUTATOR_INLINE_HPP From 488a0027f37f55c70042068f711047aed7e717c5 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 16 Nov 2022 14:03:01 +0000 Subject: [PATCH 203/494] optimize WeakProcessor --- openjdk/mmtkUpcalls.cpp | 44 ++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 953bc143..767e3606 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -45,6 +45,13 @@ #include "jfr/jfr.hpp" #include "gc/shared/oopStorage.inline.hpp" #include "utilities/debug.hpp" +#include "classfile/systemDictionary.hpp" +#include "prims/jvmtiExport.hpp" +#include "runtime/jniHandles.hpp" +#include "utilities/macros.hpp" +#if INCLUDE_JFR +#include "jfr/jfr.hpp" +#endif // Note: This counter must be accessed using the Atomic class. static volatile size_t mmtk_start_the_world_count = 0; @@ -66,7 +73,7 @@ class MMTkForwardClosure : public OopClosure { return (oop) (void*) (status << 8 >> 8); } inline static bool is_forwarded(size_t status) { - return (status & (0xffull << 56)) != 0; + return (status >> 56) != 0; } inline void do_oop(oop* slot) { // *slot = (oop) mmtk_get_forwarded_ref((void*) *slot); @@ -82,15 +89,34 @@ class MMTkForwardClosure : public OopClosure { class MMTkLXRFastIsAliveClosure : public BoolObjectClosure { public: - inline bool do_object_b(oop o) { - if (o == NULL) return false; - // RC > 0? - // Note: forwarded objects should have zero RC count. - const uintptr_t index = uintptr_t((void*)o) >> 4; + static inline bool rc_live(oop o) { + const uintptr_t index = uintptr_t(o) >> 4; const uint8_t byte = *((uint8_t*) (uintptr_t(0xe0004000000) + (index >> 2))); const uint8_t byte_mask = 0b11 << ((index & 0b11) << 1); return (byte & byte_mask) != 0; } + + static inline bool is_forwarded(oop o) { + return (*(uint8_t*)(uintptr_t(o) + 7)) != 0; + } + + inline bool do_object_b(oop o) { + return o != NULL && (rc_live(o) || is_forwarded(o)); + } +}; + +class MMTkLXRFastUpdateClosure : public OopClosure { + public: + inline void do_oop(oop* slot) { + const auto o = *slot; + const auto status = MMTkForwardClosure::read_forwarding_word(o); + if (MMTkForwardClosure::is_forwarded(status)) { + *slot = MMTkForwardClosure::extract_forwarding_pointer(status); + } else if (!MMTkLXRFastIsAliveClosure::rc_live(o)) { + *slot = NULL; + } + } + virtual void do_oop(narrowOop* o) {} }; static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure) { @@ -121,7 +147,11 @@ static void mmtk_update_weak_processor(bool lxr) { MMTkForwardClosure forward; if (lxr) { MMTkLXRFastIsAliveClosure is_alive; - WeakProcessor::weak_oops_do(&is_alive, &forward); + MMTkLXRFastUpdateClosure fast_update; + JNIHandles::weak_global_handles()->weak_oops_do(&fast_update); + JvmtiExport::weak_oops_do(&is_alive, &forward); + SystemDictionary::vm_weak_oop_storage()->weak_oops_do(&fast_update); + JFR_ONLY(Jfr::weak_oops_do(&is_alive, &forward);) } else { MMTkIsAliveClosure is_alive; WeakProcessor::weak_oops_do(&is_alive, &forward); From 5ea3608c9fecdb4f79e061b66554f56cfa5e90a8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 16 Nov 2022 14:04:44 +0000 Subject: [PATCH 204/494] Update openjdk and mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index bdac07d5..5f5e0131 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=025870dd48fe486c90e002c091bfc0d8016b1c6e#025870dd48fe486c90e002c091bfc0d8016b1c6e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7203b160fcf0e600da2d4a4b2c680f0090cbe300#7203b160fcf0e600da2d4a4b2c680f0090cbe300" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=025870dd48fe486c90e002c091bfc0d8016b1c6e#025870dd48fe486c90e002c091bfc0d8016b1c6e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7203b160fcf0e600da2d4a4b2c680f0090cbe300#7203b160fcf0e600da2d4a4b2c680f0090cbe300" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index f96480f5..b4888658 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -25,7 +25,7 @@ codegen-units = 1 [package.metadata.openjdk] # Our CI matches the following line and extract mmtk/openjdk. If this line is updated, please check ci yaml files and make sure it works. openjdk_repo = "https://github.com/mmtk/openjdk.git" -openjdk_version = "6e3cdee5d77da998891b5b656fed0f0343ae10df" +openjdk_version = "b3c5f77563332565639abd027d5a375d0622b295" [dependencies] libc = "0.2" @@ -39,7 +39,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "025870dd48fe486c90e002c091bfc0d8016b1c6e" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7203b160fcf0e600da2d4a4b2c680f0090cbe300" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 7898dc0971a769becdbbbb021cf327f756c75eb6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 17 Nov 2022 02:17:14 +0000 Subject: [PATCH 205/494] Fix compiler error --- openjdk/mmtkUpcalls.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 767e3606..4419062f 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -90,14 +90,14 @@ class MMTkForwardClosure : public OopClosure { class MMTkLXRFastIsAliveClosure : public BoolObjectClosure { public: static inline bool rc_live(oop o) { - const uintptr_t index = uintptr_t(o) >> 4; + const uintptr_t index = uintptr_t((void*) o) >> 4; const uint8_t byte = *((uint8_t*) (uintptr_t(0xe0004000000) + (index >> 2))); const uint8_t byte_mask = 0b11 << ((index & 0b11) << 1); return (byte & byte_mask) != 0; } static inline bool is_forwarded(oop o) { - return (*(uint8_t*)(uintptr_t(o) + 7)) != 0; + return (*(uint8_t*)(uintptr_t((void*) o) + 7)) != 0; } inline bool do_object_b(oop o) { From c3e26a57d132d292c09c3c2c5b613516c04d5e2d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 17 Nov 2022 02:17:21 +0000 Subject: [PATCH 206/494] fix allocation --- openjdk/mmtkHeap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index b5185e81..f16e1afa 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -88,7 +88,7 @@ jint MMTkHeap::initialize() { openjdk_gc_init(&mmtk_upcalls); // Cache the value here. It is a constant depending on the selected plan. The plan won't change from now, so value won't change. MMTkMutatorContext::max_non_los_default_alloc_bytes = get_max_non_los_default_alloc_bytes(); - // FastAllocateSizeLimit = MMTkMutatorContext::max_non_los_default_alloc_bytes; + FastAllocateSizeLimit = MMTkMutatorContext::max_non_los_default_alloc_bytes; //ReservedSpace heap_rs = Universe::reserve_heap(mmtk_heap_size, _collector_policy->heap_alignment()); From b5fb35573e937278c2b5820ec65bfe0fd4b5126a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 17 Nov 2022 02:17:33 +0000 Subject: [PATCH 207/494] Fix root scanning bucket --- mmtk/src/scanning.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index bf0dcdf7..b191e934 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -111,7 +111,7 @@ impl Scanning for VMScanning { if !(Self::SCAN_MUTATORS_IN_SAFEPOINT && Self::SINGLE_THREAD_MUTATOR_SCANNING) { memory_manager::add_work_packet( &SINGLETON, - WorkBucketStage::Unconstrained, + WorkBucketStage::RCProcessIncs, ScanVMThreadRoots::new(factory), ); } From 7d8071abdf783d597be561019b889a3f299f2448 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 21 Nov 2022 23:29:40 +0000 Subject: [PATCH 208/494] Barrier Fix --- mmtk/src/api.rs | 1 + openjdk/barriers/mmtkFieldBarrier.cpp | 30 +++++++++++++++------------ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 1ccfb3ac..d297cfb1 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -34,6 +34,7 @@ pub extern "C" fn get_mmtk_version() -> *const c_char { #[no_mangle] pub extern "C" fn mmtk_active_barrier() -> *const c_char { + println!("{:?}", SINGLETON.get_plan().constraints().barrier); match SINGLETON.get_plan().constraints().barrier { BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index daf25817..6acd5a1f 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -252,11 +252,7 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op #define __ ideal. -void MMTkFieldBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const { - if (can_remove_barrier(kit, &kit->gvn(), src, slot, val, /* skip_const_null */ false)) return; - - MMTkIdealKit ideal(kit, true); - +static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* slot, Node* val) { #if MMTK_ENABLE_BARRIER_FASTPATH Node* no_base = __ top(); float unlikely = PROB_UNLIKELY(0.999); @@ -277,13 +273,22 @@ void MMTkFieldBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), "mmtk_barrier_call", src, slot, val); #endif +} + +void MMTkFieldBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const { + if (can_remove_barrier(kit, &kit->gvn(), src, slot, val, /* skip_const_null */ false)) return; + + MMTkIdealKit ideal(kit, true); + + insert_write_barrier_common(ideal, src, slot, val); + kit->sync_kit(ideal); kit->insert_mem_bar(Op_MemBarVolatile); kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } -static void reference_load_barrier(GraphKit* kit, Node* val, bool emit_barrier) { +static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool emit_barrier) { MMTkIdealKit ideal(kit, true); Node* no_base = __ top(); float unlikely = PROB_UNLIKELY(0.999); @@ -293,8 +298,7 @@ static void reference_load_barrier(GraphKit* kit, Node* val, bool emit_barrier) __ if_then(cm_flag, BoolTest::ne, zero, unlikely); { // No slow-call if dst is NULL __ if_then(val, BoolTest::ne, kit->null()); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", val); + insert_write_barrier_common(ideal, val, slot, val); } __ end_if(); } __ end_if(); kit->sync_kit(ideal); @@ -302,7 +306,7 @@ static void reference_load_barrier(GraphKit* kit, Node* val, bool emit_barrier) kit->final_sync(ideal); // Final sync IdealKit and GraphKit. } -static void reference_load_barrier_for_unknown_load(GraphKit* kit, Node* base_oop, Node* offset, Node* val, bool need_mem_bar) { +static void reference_load_barrier_for_unknown_load(GraphKit* kit, Node* base_oop, Node* offset, Node* slot, Node* val, bool need_mem_bar) { // We could be accessing the referent field of a reference object. If so, when G1 // is enabled, we need to log the value in the referent field in an SATB buffer. // This routine performs some compile time filters and generates suitable @@ -361,7 +365,7 @@ static void reference_load_barrier_for_unknown_load(GraphKit* kit, Node* base_oo // Update graphKit from IdeakKit. kit->sync_kit(ideal); // Use the pre-barrier to record the value in the referent field - reference_load_barrier(kit, val, false); + reference_load_barrier(kit, slot, val, false); if (need_mem_bar) { // Add memory barrier to prevent commoning reads from this field // across safepoint since GC can change its value. @@ -407,16 +411,16 @@ Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_ } if (on_weak) { - reference_load_barrier(kit, load, true); + reference_load_barrier(kit, adr, load, true); } else if (unknown) { - reference_load_barrier_for_unknown_load(kit, obj, offset, load, !need_cpu_mem_bar); + reference_load_barrier_for_unknown_load(kit, obj, offset, adr, load, !need_cpu_mem_bar); } return load; } void MMTkFieldBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { - if (!is_array) { + if (!is_array && dst != kit->just_allocated_object(kit->control())) { MMTkIdealKit ideal(kit); #if MMTK_ENABLE_BARRIER_FASTPATH Node* no_base = __ top(); From 9eeefc393a8eff0c1a5b75489af88ec2c8015ce8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 25 Nov 2022 11:30:36 +0000 Subject: [PATCH 209/494] Fix weak processor --- openjdk/mmtkUpcalls.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 4419062f..18739be0 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -97,10 +97,12 @@ class MMTkLXRFastIsAliveClosure : public BoolObjectClosure { } static inline bool is_forwarded(oop o) { - return (*(uint8_t*)(uintptr_t((void*) o) + 7)) != 0; + return MMTkForwardClosure::is_forwarded(MMTkForwardClosure::read_forwarding_word(o)); } inline bool do_object_b(oop o) { + const uintptr_t v = uintptr_t((void*) o); + if (v >= 0x220000000000ULL || v < 0x20000000000ULL) return false; return o != NULL && (rc_live(o) || is_forwarded(o)); } }; @@ -109,6 +111,11 @@ class MMTkLXRFastUpdateClosure : public OopClosure { public: inline void do_oop(oop* slot) { const auto o = *slot; + const uintptr_t v = uintptr_t((void*) o); + if (v >= 0x220000000000ULL || v < 0x20000000000ULL) { + *slot = NULL; + return; + } const auto status = MMTkForwardClosure::read_forwarding_word(o); if (MMTkForwardClosure::is_forwarded(status)) { *slot = MMTkForwardClosure::extract_forwarding_pointer(status); From 295dc634f466f1a5fd7330b698ad50e979874e83 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 12 Dec 2022 10:38:54 +0000 Subject: [PATCH 210/494] Squashed commit of the following: commit c12bc4c6e71165b492375dae78e7a74aefba1575 Author: Wenyu Zhao Date: Mon Dec 12 10:31:23 2022 +0000 update openjdk commit 7e25466ead990cf30f9cf347e16734512f4ce33e Author: Wenyu Zhao Date: Mon Dec 12 10:31:07 2022 +0000 cleanup commit ba1baec0e3faf6d2bfe1c890b2ffb86e833ca0a6 Author: Wenyu Zhao Date: Mon Dec 12 09:36:04 2022 +0000 cleanup commit 6d509e31df3535bd4b22ca089f4ec84a15e6e3db Author: Wenyu Zhao Date: Tue Dec 6 12:31:17 2022 +0000 rust side CLD scanning working commit b55d9b8de14bf595f183b7d39a70e86f6359b1f4 Author: Wenyu Zhao Date: Fri Dec 2 10:35:49 2022 +0000 lxr working commit 0b55c58f2a90abb57019ceba304bfc75145d395a Author: Wenyu Zhao Date: Tue Nov 29 03:45:08 2022 +0000 immix with class unloading working --- mmtk/Cargo.toml | 2 +- mmtk/src/abi.rs | 103 ++++++++- mmtk/src/collection.rs | 12 +- mmtk/src/gc_work.rs | 26 ++- mmtk/src/lib.rs | 27 ++- mmtk/src/object_scanning.rs | 108 ++++++---- openjdk/mmtk.h | 10 +- openjdk/mmtkHeap.cpp | 22 +- openjdk/mmtkHeap.hpp | 7 +- openjdk/mmtkParallelCleaning.cpp | 349 +++++++++++++++++++++++++++++++ openjdk/mmtkParallelCleaning.hpp | 158 ++++++++++++++ openjdk/mmtkRootsClosure.hpp | 21 +- openjdk/mmtkUpcalls.cpp | 47 ++++- 13 files changed, 815 insertions(+), 77 deletions(-) create mode 100644 openjdk/mmtkParallelCleaning.cpp create mode 100644 openjdk/mmtkParallelCleaning.hpp diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index b4888658..f561f34f 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -25,7 +25,7 @@ codegen-units = 1 [package.metadata.openjdk] # Our CI matches the following line and extract mmtk/openjdk. If this line is updated, please check ci yaml files and make sure it works. openjdk_repo = "https://github.com/mmtk/openjdk.git" -openjdk_version = "b3c5f77563332565639abd027d5a375d0622b295" +openjdk_version = "7e481ee4704870ac796f3830c48eb4018db4c512" [dependencies] libc = "0.2" diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 511487fa..52f129bb 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -1,11 +1,17 @@ +use crate::OpenJDKEdge; + use super::UPCALLS; +use atomic::Ordering; use mmtk::util::constants::*; use mmtk::util::conversions; use mmtk::util::ObjectReference; use mmtk::util::{Address, OpaquePointer}; +use mmtk::vm::EdgeVisitor; use std::ffi::CStr; use std::fmt; use std::ops::Range; +use std::sync::atomic::AtomicPtr; +use std::sync::atomic::AtomicU32; use std::{mem, slice}; #[repr(i32)] @@ -61,7 +67,7 @@ pub struct Klass { pub subklass: &'static Klass, pub next_sibling: &'static Klass, pub next_link: &'static Klass, - pub class_loader_data: OpaquePointer, // ClassLoaderData* + pub class_loader_data: &'static ClassLoaderData, pub modifier_flags: i32, pub access_flags: i32, // AccessFlags pub trace_id: u64, // JFR_ONLY(traceid _trace_id;) @@ -99,6 +105,11 @@ impl Klass { const fn layout_helper_header_size(lh: i32) -> i32 { (lh >> Self::LH_HEADER_SIZE_SHIFT) & Self::LH_HEADER_SIZE_MASK } + + #[inline(always)] + pub const fn is_instance_klass(&self) -> bool { + self.layout_helper > Self::LH_NEUTRAL_VALUE + } } #[repr(C)] @@ -171,6 +182,7 @@ pub enum ReferenceType { impl InstanceKlass { const HEADER_SIZE: usize = mem::size_of::() / BYTES_IN_WORD; const VTABLE_START_OFFSET: usize = Self::HEADER_SIZE * BYTES_IN_WORD; + const MISC_IS_ANONYMOUS: u16 = 1 << 5; #[inline(always)] fn start_of_vtable(&self) -> *const usize { unsafe { (self as *const _ as *const u8).add(Self::VTABLE_START_OFFSET) as _ } @@ -193,6 +205,10 @@ impl InstanceKlass { let count = self.nonstatic_oop_map_count(); unsafe { slice::from_raw_parts(start, count) } } + #[inline(always)] + pub fn is_anonymous(&self) -> bool { + self.misc_flags & Self::MISC_IS_ANONYMOUS != 0 + } } #[repr(C)] @@ -435,3 +451,88 @@ pub fn validate_memory_layouts() { }; assert_eq!(vm_checksum, binding_checksum); } + +#[repr(C)] +#[derive(Debug)] +struct Chunk { + #[cfg(debug_assertions)] + vptr: OpaquePointer, + data: [*mut OopDesc; 32], + size: AtomicU32, + next: *mut Chunk, +} + +#[repr(C)] +#[derive(Debug)] +struct ChunkedHandleList { + head: AtomicPtr, +} + +impl ChunkedHandleList { + unsafe fn oops_do_chunk( + &self, + chunk: &'static Chunk, + size: u32, + closure: &mut impl EdgeVisitor, + ) { + for i in 0..size { + if !chunk.data[i as usize].is_null() { + closure.visit_edge(Address::from_ref::<*mut OopDesc>(&chunk.data[i as usize])) + } + } + } + + fn oops_do(&self, closure: &mut impl EdgeVisitor) { + let head = self.head.load(Ordering::Acquire); + if !head.is_null() { + let head = unsafe { &*head }; + let size = head.size.load(Ordering::Acquire); + unsafe { self.oops_do_chunk(head, size, closure) }; + let mut c = head.next; + while !c.is_null() { + let chunk = unsafe { &*c }; + let size = chunk.size.load(Ordering::Relaxed); + unsafe { self.oops_do_chunk(chunk, size, closure) }; + c = chunk.next; + } + } + } +} + +#[repr(C)] +#[derive(Debug)] +pub struct ClassLoaderData { + #[cfg(debug_assertions)] + vptr: OpaquePointer, + holder: OpaquePointer, // WeakHandle + class_loader: OpaquePointer, // OopHandle + metaspace: OpaquePointer, // ClassLoaderMetaspace*volatile + metaspace_lock: OpaquePointer, // Mutex* + unloading: bool, // bool + is_anonymous: bool, // bool + modified_oops: bool, // bool + accumulated_modified_oops: bool, // bool + keep_alive: i16, + claimed: AtomicU32, + handles: ChunkedHandleList, +} + +impl ClassLoaderData { + #[inline] + fn claim(&self) -> bool { + if self.claimed.load(Ordering::Relaxed) == 1 { + return false; + } + self.claimed + .compare_exchange(0, 1, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + } + + #[inline] + pub fn oops_do>(&self, closure: &mut V) { + if closure.should_claim_clds() && !self.claim() { + return; + } + self.handles.oops_do(closure); + } +} diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index b34d6109..5590d0f5 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -36,8 +36,11 @@ impl Collection for VMCollection { /// the OpenJDK binding allows any MMTk GC thread to stop/start the world. const COORDINATOR_ONLY_STW: bool = false; - fn stop_all_mutators(tls: VMWorkerThread, mut mutator_visitor: F) - where + fn stop_all_mutators( + tls: VMWorkerThread, + mut mutator_visitor: F, + current_gc_should_unload_classes: bool, + ) where F: FnMut(&'static mut Mutator), { let scan_mutators_in_safepoint = @@ -48,14 +51,15 @@ impl Collection for VMCollection { tls, scan_mutators_in_safepoint, to_mutator_closure(&mut mutator_visitor), + current_gc_should_unload_classes, ); } } - fn resume_mutators(tls: VMWorkerThread) { + fn resume_mutators(tls: VMWorkerThread, lxr: bool, current_gc_should_unload_classes: bool) { DISCOVERED_LISTS.enable_discover(); unsafe { - ((*UPCALLS).resume_mutators)(tls); + ((*UPCALLS).resume_mutators)(tls, lxr, current_gc_should_unload_classes); } } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index b653762e..c0ce290e 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -36,12 +36,30 @@ scan_roots_work!(ScanJvmtiExportRoots, scan_jvmti_export_roots); scan_roots_work!(ScanAOTLoaderRoots, scan_aot_loader_roots); scan_roots_work!(ScanSystemDictionaryRoots, scan_system_dictionary_roots); scan_roots_work!(ScanStringTableRoots, scan_string_table_roots); -scan_roots_work!( - ScanClassLoaderDataGraphRoots, - scan_class_loader_data_graph_roots -); scan_roots_work!(ScanVMThreadRoots, scan_vm_thread_roots); +pub struct ScanClassLoaderDataGraphRoots> { + factory: F, +} + +impl> ScanClassLoaderDataGraphRoots { + pub fn new(factory: F) -> Self { + Self { factory } + } +} + +impl> GCWork for ScanClassLoaderDataGraphRoots { + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + unsafe { + ((*UPCALLS).scan_class_loader_data_graph_roots)( + to_edges_closure(&mut self.factory), + mmtk.get_plan() + .current_gc_should_scan_weak_classloader_roots(), + ); + } + } +} + pub struct ScanCodeCacheRoots> { factory: F, } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 01f7fbaf..776b6e4b 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -62,14 +62,22 @@ pub struct OpenJDK_Upcalls { tls: VMWorkerThread, scan_mutators_in_safepoint: bool, closure: MutatorClosure, + current_gc_should_unload_classes: bool, ), - pub resume_mutators: extern "C" fn(tls: VMWorkerThread), + pub resume_mutators: + extern "C" fn(tls: VMWorkerThread, lxr: bool, current_gc_should_unload_classes: bool), pub spawn_gc_thread: extern "C" fn(tls: VMThread, kind: libc::c_int, ctx: *mut libc::c_void), pub block_for_gc: extern "C" fn(), pub out_of_memory: extern "C" fn(tls: VMThread, err_kind: AllocationError), pub get_next_mutator: extern "C" fn() -> *mut Mutator, pub reset_mutator_iterator: extern "C" fn(), - pub scan_object: extern "C" fn(trace: *mut c_void, object: ObjectReference, tls: OpaquePointer), + pub scan_object: extern "C" fn( + trace: *mut c_void, + object: ObjectReference, + tls: OpaquePointer, + follow_clds: bool, + claim_clds: bool, + ), pub dump_object: extern "C" fn(object: ObjectReference), pub get_object_size: extern "C" fn(object: ObjectReference) -> usize, pub get_mmtk_mutator: extern "C" fn(tls: VMMutatorThread) -> *mut Mutator, @@ -93,7 +101,7 @@ pub struct OpenJDK_Upcalls { pub scan_system_dictionary_roots: extern "C" fn(closure: EdgesClosure), pub scan_code_cache_roots: extern "C" fn(closure: EdgesClosure), pub scan_string_table_roots: extern "C" fn(closure: EdgesClosure), - pub scan_class_loader_data_graph_roots: extern "C" fn(closure: EdgesClosure), + pub scan_class_loader_data_graph_roots: extern "C" fn(closure: EdgesClosure, scan_weak: bool), pub scan_weak_processor_roots: extern "C" fn(closure: EdgesClosure), pub scan_vm_thread_roots: extern "C" fn(closure: EdgesClosure), pub number_of_mutators: extern "C" fn() -> usize, @@ -102,7 +110,20 @@ pub struct OpenJDK_Upcalls { pub update_weak_processor: extern "C" fn(lxr: bool), pub enqueue_references: extern "C" fn(objects: *const ObjectReference, len: usize), pub swap_reference_pending_list: extern "C" fn(objects: ObjectReference) -> ObjectReference, + pub java_lang_class_klass_offset_in_bytes: extern "C" fn() -> usize, + pub java_lang_classloader_loader_data_offset: extern "C" fn() -> usize, + pub claim_cld: extern "C" fn(cld: *mut ()) -> bool, + pub scan_cld: + extern "C" fn(trace: *mut c_void, cld: *mut c_void, follow_clds: bool, claim_clds: bool), } + +lazy_static! { + pub static ref JAVA_LANG_CLASS_KLASS_OFFSET_IN_BYTES: usize = + unsafe { ((*UPCALLS).java_lang_class_klass_offset_in_bytes)() }; + pub static ref JAVA_LANG_CLASSLOADER_LOADER_DATA_OFFSET: usize = + unsafe { ((*UPCALLS).java_lang_classloader_loader_data_offset)() }; +} + thread_local! { pub static CURRENT_WORKER: RefCell>> = RefCell::new(None); } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 277f2955..948b4e9b 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -7,8 +7,8 @@ use mmtk::util::constants::*; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::EdgeVisitor; +use std::cell::UnsafeCell; use std::{mem, slice}; - trait OopIterate: Sized { fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor); } @@ -27,6 +27,9 @@ impl OopIterate for OopMapBlock { impl OopIterate for InstanceKlass { #[inline(always)] fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + if closure.should_follow_clds() { + do_klass(&oop.klass, closure); + } let oop_maps = self.nonstatic_oop_maps(); for map in oop_maps { map.oop_iterate(oop, closure) @@ -38,32 +41,21 @@ impl OopIterate for InstanceMirrorKlass { #[inline(always)] fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { self.instance_klass.oop_iterate(oop, closure); - // if (Devirtualizer::do_metadata(closure)) { - // Klass* klass = java_lang_Class::as_Klass(obj); - // // We'll get NULL for primitive mirrors. - // if (klass != NULL) { - // if (klass->is_instance_klass() && InstanceKlass::cast(klass)->is_anonymous()) { - // // An anonymous class doesn't have its own class loader, so when handling - // // the java mirror for an anonymous class we need to make sure its class - // // loader data is claimed, this is done by calling do_cld explicitly. - // // For non-anonymous classes the call to do_cld is made when the class - // // loader itself is handled. - // Devirtualizer::do_cld(closure, klass->class_loader_data()); - // } else { - // Devirtualizer::do_klass(closure, klass); - // } - // } else { - // // We would like to assert here (as below) that if klass has been NULL, then - // // this has been a mirror for a primitive type that we do not need to follow - // // as they are always strong roots. - // // However, we might get across a klass that just changed during CMS concurrent - // // marking if allocation occurred in the old generation. - // // This is benign here, as we keep alive all CLDs that were loaded during the - // // CMS concurrent phase in the class loading, i.e. they will be iterated over - // // and kept alive during remark. - // // assert(java_lang_Class::is_primitive(obj), "Sanity check"); - // } - // } + if closure.should_follow_clds() { + let klass = unsafe { + (oop.start() + *crate::JAVA_LANG_CLASS_KLASS_OFFSET_IN_BYTES).load::<*mut Klass>() + }; + if !klass.is_null() { + let klass = unsafe { &mut *klass }; + if klass.is_instance_klass() + && (unsafe { klass.cast::().is_anonymous() }) + { + do_cld(&klass.class_loader_data, closure) + } else { + do_klass(klass, closure) + } + } + } // static fields let start: *const Oop = Self::start_of_static_fields(oop).to_ptr::(); @@ -79,19 +71,24 @@ impl OopIterate for InstanceClassLoaderKlass { #[inline] fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { self.instance_klass.oop_iterate(oop, closure); - // if (Devirtualizer::do_metadata(closure)) { - // ClassLoaderData* cld = java_lang_ClassLoader::loader_data(obj); - // // cld can be null if we have a non-registered class loader. - // if (cld != NULL) { - // Devirtualizer::do_cld(closure, cld); - // } - // } + if closure.should_follow_clds() { + let cld = unsafe { + (oop.start() + *crate::JAVA_LANG_CLASSLOADER_LOADER_DATA_OFFSET) + .load::<*mut ClassLoaderData>() + }; + if !cld.is_null() { + do_cld(unsafe { &*cld }, closure); + } + } } } impl OopIterate for ObjArrayKlass { #[inline(always)] fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + if closure.should_follow_clds() { + do_klass(&oop.klass, closure); + } let array = unsafe { oop.as_array_oop() }; for oop in unsafe { array.data::(BasicType::T_OBJECT) } { closure.visit_edge(Address::from_ref(oop as &Oop)); @@ -183,14 +180,24 @@ impl InstanceRefKlass { } #[allow(unused)] -fn oop_iterate_slow(oop: Oop, closure: &mut impl EdgeVisitor, tls: OpaquePointer) { +fn oop_iterate_slow>(oop: Oop, closure: &mut V, tls: OpaquePointer) { unsafe { - ((*UPCALLS).scan_object)(closure as *mut _ as _, mem::transmute(oop), tls); + CLOSURE.with(|x| *x.get() = closure as *mut V as *mut u8); + ((*UPCALLS).scan_object)( + mem::transmute(scan_object_fn:: as *const unsafe extern "C" fn(edge: Address)), + mem::transmute(oop), + tls, + closure.should_follow_clds(), + closure.should_claim_clds(), + ); } } #[inline(always)] -fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { +fn oop_iterate>(oop: Oop, closure: &mut V) { + unsafe { + CLOSURE.with(|x| *x.get() = closure as *mut V as *mut u8); + } let klass_id = oop.klass.id; debug_assert!( klass_id as i32 >= 0 && (klass_id as i32) < 6, @@ -222,11 +229,26 @@ fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { KlassID::InstanceRef => { let instance_klass = unsafe { oop.klass.cast::() }; instance_klass.oop_iterate(oop, closure); - } // _ => oop_iterate_slow(oop, closure, tls), + } + // _ => oop_iterate_slow(oop, closure, OpaquePointer::UNINITIALIZED), _ => {} } } +fn do_cld>(cld: &ClassLoaderData, closure: &mut V) { + if !closure.should_follow_clds() { + return; + } + cld.oops_do(closure) +} + +fn do_klass>(klass: &Klass, closure: &mut V) { + if !closure.should_follow_clds() { + return; + } + do_cld(&klass.class_loader_data, closure) +} + #[inline(always)] pub fn is_obj_array(oop: Oop) -> bool { oop.klass.id == KlassID::ObjArray @@ -245,6 +267,16 @@ pub fn obj_array_data(oop: Oop) -> &'static [ObjectReference] { } } +thread_local! { + static CLOSURE: UnsafeCell<*mut u8> = UnsafeCell::new(std::ptr::null_mut()); +} + +pub unsafe extern "C" fn scan_object_fn>(edge: Address) { + let ptr: *mut u8 = CLOSURE.with(|x| *x.get()); + let closure = &mut *(ptr as *mut V); + closure.visit_edge(edge); +} + #[inline] pub fn scan_object( object: ObjectReference, diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 32a6e705..27d38308 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -157,14 +157,14 @@ struct EdgesClosure { * OpenJDK-specific */ typedef struct { - void (*stop_all_mutators) (void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure); - void (*resume_mutators) (void *tls); + void (*stop_all_mutators) (void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure, bool current_gc_should_unload_classes); + void (*resume_mutators) (void *tls, bool lxr, bool current_gc_should_unload_classes); void (*spawn_gc_thread) (void *tls, int kind, void *ctx); void (*block_for_gc) (); void (*out_of_memory) (void *tls, MMTkAllocationError err_kind); void* (*get_next_mutator) (); void (*reset_mutator_iterator) (); - void (*scan_object) (void* trace, void* object, void* tls); + void (*scan_object) (void* trace, void* object, void* tls, bool follow_clds, bool claim_clds); void (*dump_object) (void* object); size_t (*get_object_size) (void* object); void* (*get_mmtk_mutator) (void* tls); @@ -188,7 +188,7 @@ typedef struct { void (*scan_system_dictionary_roots) (EdgesClosure closure); void (*scan_code_cache_roots) (EdgesClosure closure); void (*scan_string_table_roots) (EdgesClosure closure); - void (*scan_class_loader_data_graph_roots) (EdgesClosure closure); + void (*scan_class_loader_data_graph_roots) (EdgesClosure closure, bool scan_weak); void (*scan_weak_processor_roots) (EdgesClosure closure); void (*scan_vm_thread_roots) (EdgesClosure closure); size_t (*number_of_mutators)(); @@ -197,6 +197,8 @@ typedef struct { void (*mmtk_update_weak_processor)(bool lxr); void (*enqueue_references)(void** objects, size_t len); void* (*swap_reference_pending_list)(void* objects); + size_t (*java_lang_class_klass_offset_in_bytes)(); + size_t (*java_lang_classloader_loader_data_offset)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index f16e1afa..c7a04a60 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -48,6 +48,7 @@ #include "services/memoryManager.hpp" #include "services/memTracker.hpp" #include "utilities/vmError.hpp" +#include "mmtkParallelCleaning.hpp" /* needed support from rust heap capacity @@ -121,6 +122,16 @@ jint MMTkHeap::initialize() { fprintf(stderr, "Failed to create thread"); guarantee(false, "panic"); } + if (ClassUnloading) { + printf("ClassUnloading Enabled\n"); + } else { + printf("ClassUnloading Disabled\n"); + } + + unsigned int ncpus = (unsigned int) os::initial_active_processor_count(); + _workers = new WorkGang("GC Thread", ncpus, /* are_GC_task_threads */true, /* are_ConcurrentGC_threads */false); + _workers->initialize_workers(); + os::start_thread(_companion_thread); // Set up the GCTaskManager // _mmtk_gc_task_manager = mmtkGCTaskManager::create(ParallelGCThreads); @@ -397,9 +408,9 @@ void MMTkHeap::scan_code_cache_roots(OopClosure& cl) { void MMTkHeap::scan_string_table_roots(OopClosure& cl) { StringTable::oops_do(&cl); } -void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl) { +void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, bool scan_weak) { CLDToOopClosure cld_cl(&cl, false); - ClassLoaderDataGraph::cld_do(&cld_cl); + ClassLoaderDataGraph::roots_cld_do(&cld_cl, ClassUnloading && !scan_weak ? NULL : &cld_cl); } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { ShouldNotReachHere(); @@ -457,6 +468,13 @@ HeapWord* MMTkHeap::mem_allocate_nonmove(size_t size, bool* gc_overhead_limit_wa return Thread::current()->third_party_heap_mutator.alloc(size << LogHeapWordSize, AllocatorLos); } +void MMTkHeap::complete_cleaning(BoolObjectClosure* is_alive, bool purged_classes) { + ResourceMark rm; + HandleMark hm; + uint num_workers = _workers->active_workers(); + mmtk::ParallelCleaningTask unlink_task(is_alive, num_workers, purged_classes); + _workers->run_task(&unlink_task); +} /* * files with prints currently: * collectedHeap.inline.hpp, mmtkHeap.cpp, diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index 6ca24026..295d0d0d 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -59,11 +59,14 @@ class MMTkHeap : public CollectedHeap { ContiguousSpace* _space; int _num_root_scan_tasks; MMTkVMCompanionThread* _companion_thread; + WorkGang* _workers; public: AllocatorSelector default_allocator_selector; MMTkHeap(MMTkCollectorPolicy* policy); + WorkGang* workers() const { return _workers; } + void schedule_finalizer(); inline static MMTkHeap* heap() { @@ -207,10 +210,12 @@ class MMTkHeap : public CollectedHeap { void scan_system_dictionary_roots(OopClosure& cl); void scan_code_cache_roots(OopClosure& cl); void scan_string_table_roots(OopClosure& cl); - void scan_class_loader_data_graph_roots(OopClosure& cl); + void scan_class_loader_data_graph_roots(OopClosure& cl, bool scan_weak); void scan_weak_processor_roots(OopClosure& cl); void scan_vm_thread_roots(OopClosure& cl); + void complete_cleaning(BoolObjectClosure* is_alive, bool class_unloading_occurred); + jlong _last_gc_time; }; diff --git a/openjdk/mmtkParallelCleaning.cpp b/openjdk/mmtkParallelCleaning.cpp new file mode 100644 index 00000000..3bb2d5df --- /dev/null +++ b/openjdk/mmtkParallelCleaning.cpp @@ -0,0 +1,349 @@ +/* + * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "classfile/symbolTable.hpp" +#include "classfile/stringTable.hpp" +#include "code/codeCache.hpp" +#include "memory/resourceArea.hpp" +#include "prims/resolvedMethodTable.hpp" +#include "logging/log.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/shared/gcTraceTime.hpp" +#include "gc/shared/gcTraceTime.inline.hpp" +#include "mmtkParallelCleaning.hpp" + +namespace mmtk { + +StringSymbolTableUnlinkTask::StringSymbolTableUnlinkTask(BoolObjectClosure* is_alive) : + AbstractGangTask("String/Symbol Unlinking"), + _is_alive(is_alive), + _par_state_string(StringTable::weak_storage()), + _strings_processed(0), _strings_removed(0), + _symbols_processed(0), _symbols_removed(0) { + + _initial_string_table_size = (int) StringTable::the_table()->table_size(); + _initial_symbol_table_size = SymbolTable::the_table()->table_size(); + // if (process_symbols) { + SymbolTable::clear_parallel_claimed_index(); + // } + // if (process_strings) { + StringTable::reset_dead_counter(); + // } +} + +StringSymbolTableUnlinkTask::~StringSymbolTableUnlinkTask() { + guarantee(!_process_symbols || SymbolTable::parallel_claimed_index() >= _initial_symbol_table_size, + "claim value %d after unlink less than initial symbol table size %d", + SymbolTable::parallel_claimed_index(), _initial_symbol_table_size); + + log_info(gc, stringtable)( + "Cleaned string and symbol table, " + "strings: " SIZE_FORMAT " processed, " SIZE_FORMAT " removed, " + "symbols: " SIZE_FORMAT " processed, " SIZE_FORMAT " removed", + strings_processed(), strings_removed(), + symbols_processed(), symbols_removed()); + + // if (_process_strings) { + StringTable::finish_dead_counter(); + // } +} + +void StringSymbolTableUnlinkTask::work(uint worker_id) { + int strings_processed = 0; + int strings_removed = 0; + int symbols_processed = 0; + int symbols_removed = 0; + // if (_process_strings) { + StringTable::possibly_parallel_unlink(&_par_state_string, _is_alive, &strings_processed, &strings_removed); + Atomic::add(strings_processed, &_strings_processed); + Atomic::add(strings_removed, &_strings_removed); + // } + // if (_process_symbols) { + SymbolTable::possibly_parallel_unlink(&symbols_processed, &symbols_removed); + Atomic::add(symbols_processed, &_symbols_processed); + Atomic::add(symbols_removed, &_symbols_removed); + // } +} + +size_t StringSymbolTableUnlinkTask::strings_processed() const { return (size_t)_strings_processed; } +size_t StringSymbolTableUnlinkTask::strings_removed() const { return (size_t)_strings_removed; } + +size_t StringSymbolTableUnlinkTask::symbols_processed() const { return (size_t)_symbols_processed; } +size_t StringSymbolTableUnlinkTask::symbols_removed() const { return (size_t)_symbols_removed; } + + +Monitor* CodeCacheUnloadingTask::_lock = new Monitor(Mutex::leaf, "Code Cache Unload lock", false, Monitor::_safepoint_check_never); + +CodeCacheUnloadingTask::CodeCacheUnloadingTask(uint num_workers, BoolObjectClosure* is_alive, bool unloading_occurred) : + _is_alive(is_alive), + _unloading_occurred(unloading_occurred), + _num_workers(num_workers), + _first_nmethod(NULL), + _claimed_nmethod(NULL), + _postponed_list(NULL), + _num_entered_barrier(0) +{ + CompiledMethod::increase_unloading_clock(); + // Get first alive nmethod + CompiledMethodIterator iter = CompiledMethodIterator(); + if(iter.next_alive()) { + _first_nmethod = iter.method(); + } + _claimed_nmethod = (volatile CompiledMethod*)_first_nmethod; +} + +CodeCacheUnloadingTask::~CodeCacheUnloadingTask() { + CodeCache::verify_clean_inline_caches(); + + CodeCache::set_needs_cache_clean(false); + guarantee(CodeCache::scavenge_root_nmethods() == NULL, "Must be"); + + CodeCache::verify_icholder_relocations(); +} + +void CodeCacheUnloadingTask::add_to_postponed_list(CompiledMethod* nm) { + CompiledMethod* old; + do { + old = (CompiledMethod*)_postponed_list; + nm->set_unloading_next(old); + } while ((CompiledMethod*)Atomic::cmpxchg(nm, &_postponed_list, old) != old); +} + +void CodeCacheUnloadingTask::clean_nmethod(CompiledMethod* nm) { + bool postponed = nm->do_unloading_parallel(_is_alive, _unloading_occurred); + + if (postponed) { + // This nmethod referred to an nmethod that has not been cleaned/unloaded yet. + add_to_postponed_list(nm); + } + + // Mark that this thread has been cleaned/unloaded. + // After this call, it will be safe to ask if this nmethod was unloaded or not. + nm->set_unloading_clock(CompiledMethod::global_unloading_clock()); +} + +void CodeCacheUnloadingTask::clean_nmethod_postponed(CompiledMethod* nm) { + nm->do_unloading_parallel_postponed(); +} + +void CodeCacheUnloadingTask::claim_nmethods(CompiledMethod** claimed_nmethods, int *num_claimed_nmethods) { + CompiledMethod* first; + CompiledMethodIterator last; + + do { + *num_claimed_nmethods = 0; + + first = (CompiledMethod*)_claimed_nmethod; + last = CompiledMethodIterator(first); + + if (first != NULL) { + + for (int i = 0; i < MaxClaimNmethods; i++) { + if (!last.next_alive()) { + break; + } + claimed_nmethods[i] = last.method(); + (*num_claimed_nmethods)++; + } + } + + } while ((CompiledMethod*)Atomic::cmpxchg(last.method(), &_claimed_nmethod, first) != first); +} + +CompiledMethod* CodeCacheUnloadingTask::claim_postponed_nmethod() { + CompiledMethod* claim; + CompiledMethod* next; + + do { + claim = (CompiledMethod*)_postponed_list; + if (claim == NULL) { + return NULL; + } + + next = claim->unloading_next(); + + } while ((CompiledMethod*)Atomic::cmpxchg(next, &_postponed_list, claim) != claim); + + return claim; +} + +// Mark that we're done with the first pass of nmethod cleaning. +void CodeCacheUnloadingTask::barrier_mark(uint worker_id) { + MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag); + _num_entered_barrier++; + if (_num_entered_barrier == _num_workers) { + ml.notify_all(); + } +} + +// See if we have to wait for the other workers to +// finish their first-pass nmethod cleaning work. +void CodeCacheUnloadingTask::barrier_wait(uint worker_id) { + if (_num_entered_barrier < _num_workers) { + MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag); + while (_num_entered_barrier < _num_workers) { + ml.wait(Mutex::_no_safepoint_check_flag, 0, false); + } + } +} + +// Cleaning and unloading of nmethods. Some work has to be postponed +// to the second pass, when we know which nmethods survive. +void CodeCacheUnloadingTask::work_first_pass(uint worker_id) { + // The first nmethods is claimed by the first worker. + if (worker_id == 0 && _first_nmethod != NULL) { + clean_nmethod(_first_nmethod); + _first_nmethod = NULL; + } + + int num_claimed_nmethods; + CompiledMethod* claimed_nmethods[MaxClaimNmethods]; + + while (true) { + claim_nmethods(claimed_nmethods, &num_claimed_nmethods); + + if (num_claimed_nmethods == 0) { + break; + } + + for (int i = 0; i < num_claimed_nmethods; i++) { + clean_nmethod(claimed_nmethods[i]); + } + } +} + +void CodeCacheUnloadingTask::work_second_pass(uint worker_id) { + CompiledMethod* nm; + // Take care of postponed nmethods. + while ((nm = claim_postponed_nmethod()) != NULL) { + clean_nmethod_postponed(nm); + } +} + +KlassCleaningTask::KlassCleaningTask(BoolObjectClosure* is_alive) : + _is_alive(is_alive), + _clean_klass_tree_claimed(0), + _klass_iterator() { +} + +bool KlassCleaningTask::claim_clean_klass_tree_task() { + if (_clean_klass_tree_claimed) { + return false; + } + + return Atomic::cmpxchg(1, &_clean_klass_tree_claimed, 0) == 0; +} + +InstanceKlass* KlassCleaningTask::claim_next_klass() { + Klass* klass; + do { + klass =_klass_iterator.next_klass(); + } while (klass != NULL && !klass->is_instance_klass()); + + // this can be null so don't call InstanceKlass::cast + return static_cast(klass); +} + +void KlassCleaningTask::clean_klass(InstanceKlass* ik) { + ik->clean_weak_instanceklass_links(); +} + +void KlassCleaningTask::work() { + ResourceMark rm; + + // One worker will clean the subklass/sibling klass tree. + if (claim_clean_klass_tree_task()) { + Klass::clean_subklass_tree(); + } + + // All workers will help cleaning the classes, + InstanceKlass* klass; + while ((klass = claim_next_klass()) != NULL) { + clean_klass(klass); + } +} + +bool ResolvedMethodCleaningTask::claim_resolved_method_task() { + if (_resolved_method_task_claimed) { + return false; + } + return Atomic::cmpxchg(1, &_resolved_method_task_claimed, 0) == 0; +} + +// These aren't big, one thread can do it all. +void ResolvedMethodCleaningTask::work() { + if (claim_resolved_method_task()) { + ResolvedMethodTable::unlink(); + } +} + +ParallelCleaningTask::ParallelCleaningTask(BoolObjectClosure* is_alive, + uint num_workers, + bool unloading_occurred) : + AbstractGangTask("Parallel Cleaning"), + _string_symbol_task(is_alive), + _code_cache_task(num_workers, is_alive, unloading_occurred), + _klass_cleaning_task(is_alive), + _resolved_method_cleaning_task(is_alive) + // _phase(phase) +{ +} + +// The parallel work done by all worker threads. +void ParallelCleaningTask::work(uint worker_id) { + { + // Do first pass of code cache cleaning. + _code_cache_task.work_first_pass(worker_id); + + // Let the threads mark that the first pass is done. + _code_cache_task.barrier_mark(worker_id); + } + + { + // Clean the Strings and Symbols. + _string_symbol_task.work(worker_id); + } + + { + // Clean unreferenced things in the ResolvedMethodTable + _resolved_method_cleaning_task.work(); + } + + { + // Wait for all workers to finish the first code cache cleaning pass. + _code_cache_task.barrier_wait(worker_id); + + // Do the second code cache cleaning work, which realize on + // the liveness information gathered during the first pass. + _code_cache_task.work_second_pass(worker_id); + } + + { + // Clean all klasses that were not unloaded. + _klass_cleaning_task.work(); + } +} + +} \ No newline at end of file diff --git a/openjdk/mmtkParallelCleaning.hpp b/openjdk/mmtkParallelCleaning.hpp new file mode 100644 index 00000000..c8b680eb --- /dev/null +++ b/openjdk/mmtkParallelCleaning.hpp @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef MMTK_OPENJDK_MMTK_PARALLELCLEANING_HPP +#define MMTK_OPENJDK_MMTK_PARALLELCLEANING_HPP + +#include "gc/shared/oopStorageParState.hpp" +#include "gc/shared/workgroup.hpp" + +namespace mmtk { + +class StringSymbolTableUnlinkTask : public AbstractGangTask { +private: + BoolObjectClosure* _is_alive; + OopStorage::ParState _par_state_string; + int _initial_string_table_size; + int _initial_symbol_table_size; + + bool _process_strings; + volatile int _strings_processed; + volatile int _strings_removed; + + bool _process_symbols; + volatile int _symbols_processed; + volatile int _symbols_removed; + +public: + StringSymbolTableUnlinkTask(BoolObjectClosure* is_alive); + ~StringSymbolTableUnlinkTask(); + + void work(uint worker_id); + + size_t strings_processed() const; + size_t strings_removed() const; + + size_t symbols_processed() const; + size_t symbols_removed() const; +}; + +class CodeCacheUnloadingTask { +private: + static Monitor* _lock; + + BoolObjectClosure* const _is_alive; + const bool _unloading_occurred; + const uint _num_workers; + + // Variables used to claim nmethods. + CompiledMethod* _first_nmethod; + volatile CompiledMethod* _claimed_nmethod; + + // The list of nmethods that need to be processed by the second pass. + volatile CompiledMethod* _postponed_list; + volatile uint _num_entered_barrier; + + public: + CodeCacheUnloadingTask(uint num_workers, BoolObjectClosure* is_alive, bool unloading_occurred); + ~CodeCacheUnloadingTask(); + + private: + void add_to_postponed_list(CompiledMethod* nm); + + void clean_nmethod(CompiledMethod* nm); + + void clean_nmethod_postponed(CompiledMethod* nm); + + static const int MaxClaimNmethods = 16; + + void claim_nmethods(CompiledMethod** claimed_nmethods, int *num_claimed_nmethods); + + CompiledMethod* claim_postponed_nmethod(); + + public: + // Mark that we're done with the first pass of nmethod cleaning. + void barrier_mark(uint worker_id); + + // See if we have to wait for the other workers to + // finish their first-pass nmethod cleaning work. + void barrier_wait(uint worker_id); + + // Cleaning and unloading of nmethods. Some work has to be postponed + // to the second pass, when we know which nmethods survive. + void work_first_pass(uint worker_id); + + void work_second_pass(uint worker_id); +}; + +class KlassCleaningTask : public StackObj { + BoolObjectClosure* _is_alive; + volatile int _clean_klass_tree_claimed; + ClassLoaderDataGraphKlassIteratorAtomic _klass_iterator; + + public: + KlassCleaningTask(BoolObjectClosure* is_alive); + + private: + bool claim_clean_klass_tree_task(); + + InstanceKlass* claim_next_klass(); + +public: + + void clean_klass(InstanceKlass* ik); + + void work(); +}; + +class ResolvedMethodCleaningTask : public StackObj { + BoolObjectClosure* _is_alive; + volatile int _resolved_method_task_claimed; +public: + ResolvedMethodCleaningTask(BoolObjectClosure* is_alive) : + _is_alive(is_alive), _resolved_method_task_claimed(0) {} + + bool claim_resolved_method_task(); + void work(); +}; + +// To minimize the remark pause times, the tasks below are done in parallel. +class ParallelCleaningTask : public AbstractGangTask { +private: + StringSymbolTableUnlinkTask _string_symbol_task; + CodeCacheUnloadingTask _code_cache_task; + KlassCleaningTask _klass_cleaning_task; + ResolvedMethodCleaningTask _resolved_method_cleaning_task; + +public: + // The constructor is run in the VMThread. + ParallelCleaningTask(BoolObjectClosure* is_alive, uint num_workers, bool unloading_occurred); + + // The parallel work done by all worker threads. + void work(uint worker_id); +}; + +} + +#endif // MMTK_OPENJDK_MMTK_PARALLELCLEANING_HPP diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index c4f97896..41262b59 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -6,6 +6,7 @@ #include "oops/oop.hpp" #include "oops/oop.inline.hpp" #include "utilities/globalDefinitions.hpp" +#include "classfile/classLoaderData.inline.hpp" #define ROOTS_BUFFER_SIZE 4096 @@ -95,17 +96,18 @@ class MMTkRootsClosure2 : public OopClosure { }; class MMTkScanObjectClosure : public BasicOopIterateClosure { - void* _trace; - CLDToOopClosure follow_cld_closure; + void (*_trace)(void*); + bool _follow_clds; + bool _claim_clds; template void do_oop_work(T* p) { // oop ref = (void*) oopDesc::decode_heap_oop(oopDesc::load_heap_oop(p)); - // process_edge(_trace, (void*) p); + _trace((void*) p); } public: - MMTkScanObjectClosure(void* trace): _trace(trace), follow_cld_closure(this, false) {} + MMTkScanObjectClosure(void* trace, bool follow_clds, bool claim_clds): _trace((void (*)(void*)) trace), _follow_clds(follow_clds), _claim_clds(claim_clds) {} virtual void do_oop(oop* p) { do_oop_work(p); } virtual void do_oop(narrowOop* p) { @@ -114,18 +116,17 @@ class MMTkScanObjectClosure : public BasicOopIterateClosure { } virtual bool do_metadata() { - return true; + return _follow_clds; } virtual void do_klass(Klass* k) { - // follow_cld_closure.do_cld(k->class_loader_data()); - // oop op = k->klass_holder(); - // oop new_op = (oop) trace_root_object(_trace, op); - // guarantee(new_op == op, "trace_root_object returned a different value %p -> %p", op, new_op); + if (!_follow_clds) return; + do_cld(k->class_loader_data()); } virtual void do_cld(ClassLoaderData* cld) { - follow_cld_closure.do_cld(cld); + if (!_follow_clds) return; + cld->oops_do(this, _claim_clds); } virtual ReferenceIterationMode reference_iteration_mode() { return DO_FIELDS; } diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 18739be0..cdd798f1 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -126,14 +126,15 @@ class MMTkLXRFastUpdateClosure : public OopClosure { virtual void do_oop(narrowOop* o) {} }; -static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure) { +static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure, bool current_gc_should_unload_classes) { log_debug(gc)("Requesting the VM to suspend all mutators..."); MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_suspended, true); log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); mmtk_report_gc_start(); - - // ClassLoaderDataGraph::clear_claimed_marks(); + if (ClassUnloading && current_gc_should_unload_classes) { + ClassLoaderDataGraph::clear_claimed_marks(); + } CodeCache::gc_prologue(); #if COMPILER2_OR_JVMCI DerivedPointerTable::clear(); @@ -165,9 +166,23 @@ static void mmtk_update_weak_processor(bool lxr) { } } -static void mmtk_resume_mutators(void *tls) { +static void mmtk_resume_mutators(void *tls, bool lxr, bool current_gc_should_unload_classes) { nmethod::oops_do_marking_epilogue(); - // ClassLoaderDataGraph::purge(); + if (ClassUnloading && current_gc_should_unload_classes) { + // Unload classes and purge SystemDictionary. + auto purged_classes = SystemDictionary::do_unloading(NULL, false /* Defer cleaning */); + if (lxr) { + MMTkLXRFastIsAliveClosure is_alive; + MMTkHeap::heap()->complete_cleaning(&is_alive, purged_classes); + } else { + MMTkIsAliveClosure is_alive; + MMTkHeap::heap()->complete_cleaning(&is_alive, purged_classes); + } + ClassLoaderDataGraph::purge(); + // Resize and verify metaspace + MetaspaceGC::compute_new_size(); + MetaspaceUtils::verify_metrics(); + } // BiasedLocking::restore_marks(); CodeCache::gc_epilogue(); JvmtiExport::gc_epilogue(); @@ -323,8 +338,8 @@ static void mmtk_scan_thread_roots(EdgesClosure closure, void* tls) { thread->oops_do(&cl, NULL); } -static void mmtk_scan_object(void* trace, void* object, void* tls) { - MMTkScanObjectClosure cl(trace); +static void mmtk_scan_object(void* trace, void* object, void* tls, bool follow_clds, bool claim_clds) { + MMTkScanObjectClosure cl(trace, follow_clds, claim_clds); ((oop) object)->oop_iterate(&cl); } @@ -412,7 +427,7 @@ static void mmtk_scan_aot_loader_roots(EdgesClosure closure) { MMTkRootsClosure2 static void mmtk_scan_system_dictionary_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_system_dictionary_roots(cl); } static void mmtk_scan_code_cache_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } static void mmtk_scan_string_table_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_string_table_roots(cl); } -static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl); } +static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, bool scan_weak) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl, scan_weak); } static void mmtk_scan_weak_processor_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_weak_processor_roots(cl); } static void mmtk_scan_vm_thread_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } @@ -452,6 +467,18 @@ static void* mmtk_swap_reference_pending_list(void* object) { return Universe::swap_reference_pending_list((oop) object); } +static size_t mmtk_java_lang_class_klass_offset_in_bytes() { + auto v = java_lang_Class::klass_offset_in_bytes(); + guarantee(v != 0 && v != -1, "checking"); + return v; +} + +static size_t mmtk_java_lang_classloader_loader_data_offset() { + auto v = java_lang_ClassLoader::loader_data_offset(); + guarantee(v != 0 && v != -1, "checking"); + return v; +} + OpenJDK_Upcalls mmtk_upcalls = { mmtk_stop_all_mutators, mmtk_resume_mutators, @@ -492,5 +519,7 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_prepare_for_roots_re_scanning, mmtk_update_weak_processor, mmtk_enqueue_references, - mmtk_swap_reference_pending_list + mmtk_swap_reference_pending_list, + mmtk_java_lang_class_klass_offset_in_bytes, + mmtk_java_lang_classloader_loader_data_offset, }; From 321beb32769a05266ba3b66aee04e2cb34f191d3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 12 Dec 2022 10:39:31 +0000 Subject: [PATCH 211/494] update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 5f5e0131..966d8c05 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7203b160fcf0e600da2d4a4b2c680f0090cbe300#7203b160fcf0e600da2d4a4b2c680f0090cbe300" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=d7c97990dcfd1d60d3e52a1e15ea2511b8734e17#d7c97990dcfd1d60d3e52a1e15ea2511b8734e17" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7203b160fcf0e600da2d4a4b2c680f0090cbe300#7203b160fcf0e600da2d4a4b2c680f0090cbe300" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=d7c97990dcfd1d60d3e52a1e15ea2511b8734e17#d7c97990dcfd1d60d3e52a1e15ea2511b8734e17" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index f561f34f..aae62da2 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7203b160fcf0e600da2d4a4b2c680f0090cbe300" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "d7c97990dcfd1d60d3e52a1e15ea2511b8734e17" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 5be7777b097cc1901e191599c639319b10b26c94 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 13 Dec 2022 09:22:27 +0000 Subject: [PATCH 212/494] wip --- mmtk/src/lib.rs | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 2bcee86c..41fe4f5c 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -187,29 +187,14 @@ impl OpenJDKEdge { impl Edge for OpenJDKEdge { /// Load object reference from the edge. #[inline(always)] - fn load(&self) -> ObjectReference { - if cfg!(debug_assertions) { - if !ROOT { - if *USE_COMPRESSED_OOPS { - assert!(self.is_compressed()) - } else { - assert!(self.0.as_usize() & Self::MASK == 0) - } - } else { - if !*USE_COMPRESSED_OOPS { - assert!(self.0.as_usize() & Self::MASK == 0) - } - } - } - if ROOT && COMPRESSED { + fn load(&self) -> ObjectReference { + if COMPRESSED { let slot = self.untagged_address(); if self.is_compressed() { decompress(unsafe { slot.load::() }) } else { unsafe { slot.load::() } } - } else if COMPRESSED { - decompress(unsafe { self.0.load::() }) } else { unsafe { self.0.load::() } } @@ -217,16 +202,14 @@ impl Edge for OpenJDKEdge { /// Store the object reference `object` into the edge. #[inline(always)] - fn store(&self, object: ObjectReference) { - if ROOT && COMPRESSED { + fn store(&self, object: ObjectReference) { + if COMPRESSED { let slot = self.untagged_address(); if self.is_compressed() { unsafe { slot.store(compress(object)) } } else { unsafe { slot.store(object) } } - } else if COMPRESSED { - unsafe { self.0.store(compress(object)) } } else { unsafe { self.0.store(object) } } From f1243bb4b2fd577ac01530a6621423f04ed5fd43 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 13 Dec 2022 10:04:24 +0000 Subject: [PATCH 213/494] Use openjdk command line args --- mmtk/src/abi.rs | 4 ++-- mmtk/src/api.rs | 4 ++-- mmtk/src/lib.rs | 19 +++++++++---------- mmtk/src/object_model.rs | 2 +- mmtk/src/object_scanning.rs | 2 +- openjdk/mmtkHeap.cpp | 10 +++++----- openjdk/thirdPartyHeapArguments.cpp | 9 +++++---- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index f00c7607..9db1254b 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -294,7 +294,7 @@ impl OopDesc { #[inline(always)] pub fn klass(&self) -> &'static Klass { // self.klass - if *crate::USE_COMPRESSED_OOPS { + if crate::use_compressed_oops() { lazy_static! { static ref COMPRESSED_KLASS_BASE: Address = unsafe { ((*UPCALLS).compressed_klass_base)() }; @@ -397,7 +397,7 @@ pub type ArrayOop = &'static ArrayOopDesc; impl ArrayOopDesc { fn length_offset() -> usize { - if *crate::USE_COMPRESSED_OOPS { + if crate::use_compressed_oops() { mem::size_of::() + mem::size_of::() } else { mem::size_of::() diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 082ebdc4..1d41487b 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -212,8 +212,8 @@ pub extern "C" fn handle_user_collection_request(tls: VMMutatorThread) { } #[no_mangle] -pub extern "C" fn mmtk_use_compressed_ptrs() -> bool { - *crate::USE_COMPRESSED_OOPS +pub extern "C" fn mmtk_use_compressed_ptrs() { + unsafe { crate::USE_COMPRESSED_OOPS = true } } #[no_mangle] diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 2bcee86c..fa5d473b 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -117,12 +117,11 @@ pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS: uintptr_t = pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t = crate::mmtk::util::metadata::side_metadata::ALLOC_SIDE_METADATA_ADDR.as_usize(); -lazy_static! { - static ref USE_COMPRESSED_OOPS: bool = { - std::env::var("MMTK_COMPRESSED_PTRS") - .map(|s| s.trim() != "0" || s.trim() != "false") - .unwrap_or(false) - }; +static mut USE_COMPRESSED_OOPS: bool = false; + +#[inline(always)] +fn use_compressed_oops() -> bool { + unsafe { USE_COMPRESSED_OOPS } } static mut BASE: Address = Address::ZERO; @@ -190,13 +189,13 @@ impl Edge for OpenJDKEdge { fn load(&self) -> ObjectReference { if cfg!(debug_assertions) { if !ROOT { - if *USE_COMPRESSED_OOPS { + if use_compressed_oops() { assert!(self.is_compressed()) } else { assert!(self.0.as_usize() & Self::MASK == 0) } } else { - if !*USE_COMPRESSED_OOPS { + if !use_compressed_oops() { assert!(self.0.as_usize() & Self::MASK == 0) } } @@ -320,14 +319,14 @@ lazy_static! { pub static ref BUILDER: Mutex = Mutex::new(MMTKBuilder::new()); pub static ref SINGLETON: MMTK = { let mut builder = BUILDER.lock().unwrap(); - if *USE_COMPRESSED_OOPS { + if use_compressed_oops() { builder.set_option("use_35bit_address_space", "true"); builder.set_option("use_35bit_address_space", "true"); } assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); - if *USE_COMPRESSED_OOPS { + if use_compressed_oops() { initialize_compressed_oops(); } *ret diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index ff69e7a6..ae4e8d1b 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -93,6 +93,6 @@ impl ObjectModel for VMObjectModel { } fn compressed_pointers_enabled() -> bool { - *crate::USE_COMPRESSED_OOPS + crate::use_compressed_oops() } } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 012fc652..15c05c5e 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -82,7 +82,7 @@ impl OopIterate for InstanceMirrorKlass { // static fields let start = Self::start_of_static_fields(oop); let len = Self::static_oop_field_count(oop); - if *crate::USE_COMPRESSED_OOPS { + if crate::use_compressed_oops() { let start: *const NarrowOop = start.to_ptr::(); let slice = unsafe { slice::from_raw_parts(start, len as _) }; for narrow_oop in slice { diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 83c2fba5..62c84dd5 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -71,12 +71,12 @@ MMTkHeap::MMTkHeap(MMTkCollectorPolicy* policy) : CollectedHeap(), _last_gc_time jint MMTkHeap::initialize() { assert(!UseTLAB , "should disable UseTLAB"); - auto compressed_oops = mmtk_use_compressed_ptrs(); - if (compressed_oops) { - assert(UseCompressedOops , "should enable CompressedOops"); + if (UseCompressedOops) { + mmtk_use_compressed_ptrs(); + } + if (UseCompressedOops) { assert(UseCompressedClassPointers , "should enable UseCompressedClassPointers"); } else { - assert(!UseCompressedOops , "should disable CompressedOops"); assert(!UseCompressedClassPointers , "should disable UseCompressedClassPointers"); } const size_t heap_size = collector_policy()->max_heap_byte_size(); @@ -112,7 +112,7 @@ jint MMTkHeap::initialize() { _start = (HeapWord*) starting_heap_address(); _end = (HeapWord*) last_heap_address(); printf("start: %p, end: %p\n", _start, _end); - if (compressed_oops) { + if (UseCompressedOops) { Universe::set_narrow_oop_base((address) mmtk_narrow_oop_base()); Universe::set_narrow_oop_shift(mmtk_narrow_oop_shift()); printf("narrow_oop_mode: %s\n", Universe::narrow_oop_mode_to_string(Universe::narrow_oop_mode())); diff --git a/openjdk/thirdPartyHeapArguments.cpp b/openjdk/thirdPartyHeapArguments.cpp index cd019347..c3121b03 100644 --- a/openjdk/thirdPartyHeapArguments.cpp +++ b/openjdk/thirdPartyHeapArguments.cpp @@ -43,10 +43,11 @@ void ThirdPartyHeapArguments::initialize() { GCArguments::initialize(); assert(UseThirdPartyHeap , "Error, should UseThirdPartyHeap"); FLAG_SET_DEFAULT(UseTLAB, false); - auto compressed_oops = mmtk_use_compressed_ptrs(); - printf("MMTk Compressed Pointers: %s\n", compressed_oops ? "enabled" : "disabled"); - FLAG_SET_DEFAULT(UseCompressedOops, compressed_oops); - FLAG_SET_DEFAULT(UseCompressedClassPointers, compressed_oops); + if (UseCompressedOops) mmtk_use_compressed_ptrs(); + // auto compressed_oops = mmtk_use_compressed_ptrs(); + printf("MMTk Compressed Pointers: %s\n", UseCompressedOops ? "enabled" : "disabled"); + // FLAG_SET_DEFAULT(UseCompressedOops, compressed_oops); + FLAG_SET_DEFAULT(UseCompressedClassPointers, UseCompressedOops); } CollectedHeap* ThirdPartyHeapArguments::create_heap() { From 0c0fd714832a5bfd0a37e56cf99b26ea713cd871 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 14 Dec 2022 09:45:00 +0000 Subject: [PATCH 214/494] Fix class pointer check --- mmtk/src/abi.rs | 17 +++++++++++++++++ mmtk/src/api.rs | 4 +++- mmtk/src/object_model.rs | 7 +++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 5b952591..8fc1ff3e 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -318,6 +318,23 @@ impl OopDesc { unsafe { mem::transmute(self) } } #[inline(always)] + pub(crate) fn klass_ptr(&self) -> Address { + // self.klass + if crate::use_compressed_oops() { + lazy_static! { + static ref COMPRESSED_KLASS_BASE: Address = + unsafe { ((*UPCALLS).compressed_klass_base)() }; + static ref COMPRESSED_KLASS_SHIFT: usize = + unsafe { ((*UPCALLS).compressed_klass_shift)() }; + } + let compressed = unsafe { self.klass.narrow_klass }; + let addr = *COMPRESSED_KLASS_BASE + ((compressed as usize) << *COMPRESSED_KLASS_SHIFT); + addr + } else { + unsafe { Address::from_ref(self.klass.klass) } + } + } + #[inline(always)] pub fn klass(&self) -> &'static Klass { // self.klass if crate::use_compressed_oops() { diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 83ef9b57..eea39517 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -442,9 +442,11 @@ pub extern "C" fn get_finalized_object() -> ObjectReference { /// Note: only call this method after the liveness tracing and before gc release. #[no_mangle] pub extern "C" fn mmtk_is_live(object: ObjectReference) -> usize { - if object.is_null() || !object.to_address().is_mapped() || !object.class_is_valid() { + if object.is_null() { return 0; } + debug_assert!(object.to_address().is_mapped()); + debug_assert!(object.class_is_valid::()); object.is_live() as _ } diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index bdd1c32d..8ff8bf7b 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -102,7 +102,14 @@ impl ObjectModel for VMObjectModel { s.to_string() } + #[inline(always)] fn compressed_pointers_enabled() -> bool { crate::use_compressed_oops() } + + #[inline(always)] + fn get_class_pointer(object: ObjectReference) -> Address { + let oop: Oop = unsafe { std::mem::transmute(object) }; + oop.klass_ptr() + } } From 6da70f8942603ba6a410663d284c1290296d6014 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 16 Dec 2022 14:10:03 +0000 Subject: [PATCH 215/494] Fix pointer width --- mmtk/src/abi.rs | 6 ------ mmtk/src/api.rs | 16 +++++++++++++--- mmtk/src/lib.rs | 35 ++++++++++++++++++++++++++--------- 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 8fc1ff3e..c78ffc5d 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -480,12 +480,6 @@ impl ArrayOopDesc { pub unsafe fn data(&self, ty: BasicType) -> &[T] { slice::from_raw_parts(self.base(ty).to_ptr(), self.length() as _) } - #[inline(always)] - pub fn data_range(&self) -> Range
{ - let base = self.base(BasicType::T_OBJECT); - let len = self.length() as usize; - base..(base + (len << BYTES_IN_ADDRESS)) - } } #[repr(C)] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index eea39517..54f4a7c6 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -11,7 +11,7 @@ use mmtk::plan::BarrierSelector; use mmtk::scheduler::GCController; use mmtk::scheduler::GCWorker; use mmtk::util::alloc::AllocatorSelector; -use mmtk::util::constants::LOG_BYTES_IN_ADDRESS; +use mmtk::util::constants::{LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_INT}; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::AllocationSemantics; @@ -390,7 +390,12 @@ pub extern "C" fn mmtk_array_copy_pre( dst: Address, count: usize, ) { - let bytes = count << LOG_BYTES_IN_ADDRESS; + let bytes = count + << if crate::use_compressed_oops() { + LOG_BYTES_IN_INT + } else { + LOG_BYTES_IN_ADDRESS + }; mutator.barrier().memory_region_copy_pre( OpenJDKEdgeRange { start: OpenJDKEdge(src), @@ -411,7 +416,12 @@ pub extern "C" fn mmtk_array_copy_post( dst: Address, count: usize, ) { - let bytes = count << LOG_BYTES_IN_ADDRESS; + let bytes = count + << if crate::use_compressed_oops() { + LOG_BYTES_IN_INT + } else { + LOG_BYTES_IN_ADDRESS + }; mutator.barrier().memory_region_copy_post( OpenJDKEdgeRange { start: OpenJDKEdge(src), diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 01d85373..58b41f21 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -15,7 +15,9 @@ use std::sync::Mutex; use libc::{c_char, c_void, uintptr_t}; use mmtk::scheduler::GCWorker; use mmtk::util::alloc::AllocationError; -use mmtk::util::constants::{BYTES_IN_ADDRESS, LOG_BYTES_IN_ADDRESS}; +use mmtk::util::constants::{ + BYTES_IN_ADDRESS, BYTES_IN_INT, LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_INT, +}; use mmtk::util::heap::layout::vm_layout_constants::VM_LAYOUT_CONSTANTS; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; @@ -261,12 +263,12 @@ impl Edge for OpenJDKEdge { #[inline(always)] fn to_address(&self) -> Address { - self.0 + self.untagged_address() } #[inline(always)] fn from_address(a: Address) -> Self { - OpenJDKEdge(a) + unreachable!(); } } @@ -280,6 +282,7 @@ pub struct OpenJDKEdgeRange { pub struct AddressRangeIterator { cursor: Address, limit: Address, + width: usize, } impl Iterator for AddressRangeIterator { @@ -291,7 +294,7 @@ impl Iterator for AddressRangeIterator { None } else { let edge = self.cursor; - self.cursor += BYTES_IN_ADDRESS; + self.cursor += self.width; Some(OpenJDKEdge(edge)) } } @@ -306,6 +309,11 @@ impl MemorySlice for OpenJDKEdgeRange { AddressRangeIterator { cursor: self.start.0, limit: self.end.0, + width: if crate::use_compressed_oops() { + BYTES_IN_INT + } else { + BYTES_IN_ADDRESS + }, } } @@ -328,11 +336,20 @@ impl MemorySlice for OpenJDKEdgeRange { "bytes are not a multiple of words" ); // Raw memory copy - unsafe { - let words = tgt.bytes() >> LOG_BYTES_IN_ADDRESS; - let src = src.start().to_ptr::(); - let tgt = tgt.start().to_mut_ptr::(); - std::ptr::copy(src, tgt, words) + if crate::use_compressed_oops() { + unsafe { + let words = tgt.bytes() >> LOG_BYTES_IN_INT; + let src = src.start().to_ptr::(); + let tgt = tgt.start().to_mut_ptr::(); + std::ptr::copy(src, tgt, words) + } + } else { + unsafe { + let words = tgt.bytes() >> LOG_BYTES_IN_ADDRESS; + let src = src.start().to_ptr::(); + let tgt = tgt.start().to_mut_ptr::(); + std::ptr::copy(src, tgt, words) + } } } } From e254baf36314f107c0b3b7f2d238edbdab5ac3b6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 17 Dec 2022 09:28:04 +0000 Subject: [PATCH 216/494] Fix barrier when fast-path is disabled --- openjdk/barriers/mmtkFieldBarrier.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 6acd5a1f..6a91325b 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -86,10 +86,12 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma __ bind(done); #else + __ pusha(); __ movptr(c_rarg0, dst.base()); __ lea(c_rarg1, dst); __ movptr(c_rarg2, val == noreg ? (int32_t) NULL_WORD : val); __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); + __ popa(); #endif } From 2768793833e79e2f61ab0edb69dee9786335db9d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 17 Dec 2022 09:28:42 +0000 Subject: [PATCH 217/494] Remove conservative heap range check --- openjdk/mmtkUpcalls.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 6f23de80..efe55742 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -109,7 +109,7 @@ class MMTkLXRFastIsAliveClosure : public BoolObjectClosure { inline virtual bool do_object_b(oop o) { const uintptr_t v = uintptr_t((void*) o); - if (v >= 0x220000000000ULL || v < 0x20000000000ULL) return false; + // if (v >= 0x220000000000ULL || v < 0x20000000000ULL) return false; return o != NULL && (rc_live(o) || is_forwarded(o)); } }; @@ -119,10 +119,10 @@ class MMTkLXRFastUpdateClosure : public OopClosure { inline virtual void do_oop(oop* slot) { const auto o = *slot; const uintptr_t v = uintptr_t((void*) o); - if (v >= 0x220000000000ULL || v < 0x20000000000ULL) { - *slot = NULL; - return; - } + // if (v >= 0x220000000000ULL || v < 0x20000000000ULL) { + // *slot = NULL; + // return; + // } const auto status = MMTkForwardClosure::read_forwarding_word(o); if (MMTkForwardClosure::is_forwarded(status)) { *slot = MMTkForwardClosure::extract_forwarding_pointer(status); @@ -135,10 +135,10 @@ class MMTkLXRFastUpdateClosure : public OopClosure { if (CompressedOops::is_null(heap_oop)) return; oop o = CompressedOops::decode_not_null(heap_oop); const uintptr_t v = uintptr_t((void*) o); - if (v >= 0x220000000000ULL || v < 0x20000000000ULL) { - RawAccess<>::oop_store(slot, CompressedOops::encode(oop(NULL))); - return; - } + // if (v >= 0x220000000000ULL || v < 0x20000000000ULL) { + // RawAccess<>::oop_store(slot, CompressedOops::encode(oop(NULL))); + // return; + // } const auto status = MMTkForwardClosure::read_forwarding_word(o); if (MMTkForwardClosure::is_forwarded(status)) { RawAccess<>::oop_store(slot, CompressedOops::encode_not_null(MMTkForwardClosure::extract_forwarding_pointer(status))); From ea79a70a9a21575975eced13e970aa8e93b92e22 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 18 Dec 2022 02:38:32 +0000 Subject: [PATCH 218/494] Fix `MemorySlice` iteration --- mmtk/src/abi.rs | 10 +++++- mmtk/src/api.rs | 2 +- mmtk/src/lib.rs | 71 +++++++++++++++++++++++++++++++++---- mmtk/src/object_scanning.rs | 4 +-- mmtk/src/scanning.rs | 2 +- 5 files changed, 78 insertions(+), 11 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index c78ffc5d..0a0d3e74 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -9,7 +9,6 @@ use mmtk::util::{Address, OpaquePointer}; use mmtk::vm::EdgeVisitor; use std::ffi::CStr; use std::fmt; -use std::ops::Range; use std::sync::atomic::AtomicPtr; use std::sync::atomic::AtomicU32; use std::{mem, slice}; @@ -480,6 +479,15 @@ impl ArrayOopDesc { pub unsafe fn data(&self, ty: BasicType) -> &[T] { slice::from_raw_parts(self.base(ty).to_ptr(), self.length() as _) } + #[inline(always)] + pub unsafe fn slice(&self, ty: BasicType) -> crate::OpenJDKEdgeRange { + let base = self.base(ty); + let start = crate::OpenJDKEdge(base); + let end = crate::OpenJDKEdge( + base + ((self.length() as usize) << if crate::use_compressed_oops() { 2 } else { 3 }), + ); + crate::OpenJDKEdgeRange { start, end } + } } #[repr(C)] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 54f4a7c6..1b30f4af 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -217,7 +217,7 @@ pub extern "C" fn handle_user_collection_request(tls: VMMutatorThread) { #[no_mangle] pub extern "C" fn mmtk_use_compressed_ptrs() { - unsafe { crate::USE_COMPRESSED_OOPS = true } + crate::init_compressed_oop_constants() } #[no_mangle] diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 58b41f21..3c8cb748 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -165,12 +165,32 @@ pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = pub static mut CONCURRENT_MARKING_ACTIVE: u8 = 0; static mut USE_COMPRESSED_OOPS: bool = false; +static mut LOG_BYTES_IN_FIELD: usize = LOG_BYTES_IN_ADDRESS as _; +static mut BYTES_IN_FIELD: usize = BYTES_IN_ADDRESS as _; + +fn init_compressed_oop_constants() { + unsafe { + USE_COMPRESSED_OOPS = true; + LOG_BYTES_IN_FIELD = LOG_BYTES_IN_INT as _; + BYTES_IN_FIELD = BYTES_IN_INT as _; + } +} #[inline(always)] fn use_compressed_oops() -> bool { unsafe { USE_COMPRESSED_OOPS } } +#[inline(always)] +fn log_bytes_in_field() -> usize { + unsafe { LOG_BYTES_IN_FIELD } +} + +#[inline(always)] +fn bytes_in_field() -> usize { + unsafe { BYTES_IN_FIELD } +} + static mut BASE: Address = Address::ZERO; static mut SHIFT: usize = 0; @@ -268,7 +288,7 @@ impl Edge for OpenJDKEdge { #[inline(always)] fn from_address(a: Address) -> Self { - unreachable!(); + Self(a) } } @@ -300,20 +320,54 @@ impl Iterator for AddressRangeIterator { } } +pub struct ChunkIterator { + cursor: Address, + limit: Address, + step: usize, +} + +impl Iterator for ChunkIterator { + type Item = OpenJDKEdgeRange; + + #[inline] + fn next(&mut self) -> Option { + if self.cursor >= self.limit { + None + } else { + let start = self.cursor; + let mut end = start + self.step; + if end > self.limit { + end = self.limit; + } + self.cursor = end; + Some(OpenJDKEdgeRange { + start: OpenJDKEdge(start), + end: OpenJDKEdge(end), + }) + } + } +} + impl MemorySlice for OpenJDKEdgeRange { type Edge = OpenJDKEdge; type EdgeIterator = AddressRangeIterator; + type ChunkIterator = ChunkIterator; #[inline] fn iter_edges(&self) -> Self::EdgeIterator { AddressRangeIterator { cursor: self.start.0, limit: self.end.0, - width: if crate::use_compressed_oops() { - BYTES_IN_INT - } else { - BYTES_IN_ADDRESS - }, + width: crate::bytes_in_field(), + } + } + + #[inline] + fn chunks(&self, chunk_size: usize) -> Self::ChunkIterator { + ChunkIterator { + cursor: self.start.0, + limit: self.end.0, + step: chunk_size << crate::log_bytes_in_field(), } } @@ -327,6 +381,11 @@ impl MemorySlice for OpenJDKEdgeRange { self.end.0 - self.start.0 } + #[inline] + fn len(&self) -> usize { + (self.end.0 - self.start.0) >> crate::log_bytes_in_field() + } + #[inline] fn copy(src: &Self, tgt: &Self) { debug_assert_eq!(src.bytes(), tgt.bytes()); diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 5baef3d7..da7a0843 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -311,10 +311,10 @@ pub fn is_val_array(oop: Oop) -> bool { } #[inline(always)] -pub fn obj_array_data(oop: Oop) -> &'static [ObjectReference] { +pub fn obj_array_data(oop: Oop) -> crate::OpenJDKEdgeRange { unsafe { let array = oop.as_array_oop(); - array.data::(BasicType::T_OBJECT) + array.slice(BasicType::T_OBJECT) } } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 11cfe67a..ebd1a4d7 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -56,7 +56,7 @@ impl Scanning for VMScanning { } #[inline(always)] - fn obj_array_data(o: ObjectReference) -> &'static [ObjectReference] { + fn obj_array_data(o: ObjectReference) -> crate::OpenJDKEdgeRange { crate::object_scanning::obj_array_data(unsafe { std::mem::transmute(o) }) } From 065bacd5a9999bede0c789b2259fe50d2236117b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 18 Dec 2022 02:51:37 +0000 Subject: [PATCH 219/494] Barrier fast-path for compressed-ptrs --- openjdk/barriers/mmtkFieldBarrier.cpp | 34 +++++++-------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 6a91325b..49d5058f 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -6,8 +6,8 @@ constexpr int kUnloggedValue = 1; void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, oop target) const { #if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = ((intptr_t) (void*) slot); - const volatile uint8_t * meta_addr = (const volatile uint8_t *) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); - intptr_t shift = (addr >> 3) & 0b111; + const volatile uint8_t * meta_addr = (const volatile uint8_t *) (SIDE_METADATA_BASE_ADDRESS + (addr >> (UseCompressedOops ? 5 : 6))); + intptr_t shift = (addr >> (UseCompressedOops ? 2 : 3)) & 0b111; uint8_t byte_val = *meta_addr; if (((byte_val >> shift) & 1) == kUnloggedValue) { // MMTkObjectBarrierSetRuntime::object_reference_write_pre_slow()((void*) src); @@ -59,12 +59,12 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma // tmp5 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); __ lea(tmp3, dst); - __ shrptr(tmp3, 6); + __ shrptr(tmp3, UseCompressedOops ? 5 : 6); __ movptr(tmp5, SIDE_METADATA_BASE_ADDRESS); __ movb(tmp5, Address(tmp5, tmp3)); // tmp3 = (obj >> 3) & 7 __ lea(tmp3, dst); - __ shrptr(tmp3, 3); + __ shrptr(tmp3, UseCompressedOops ? 2 : 3); __ andptr(tmp3, 7); // tmp5 = tmp5 >> tmp3 __ movptr(tmp4, rcx); @@ -223,7 +223,7 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op // uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); LIR_Opr offset = gen->new_pointer_register(); __ move(addr, offset); - __ unsigned_shift_right(offset, 6, offset); + __ unsigned_shift_right(offset, UseCompressedOops ? 5 : 6, offset); LIR_Opr base = gen->new_pointer_register(); __ move(LIR_OprFact::longConst(SIDE_METADATA_BASE_ADDRESS), base); LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); @@ -233,7 +233,7 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op // intptr_t shift = (addr >> 3) & 0b111; LIR_Opr shift = gen->new_register(T_INT); __ move(addr, shift); - __ unsigned_shift_right(shift, 3, shift); + __ unsigned_shift_right(shift, UseCompressedOops ? 2 : 3, shift); __ logical_and(shift, LIR_OprFact::intConst(0b111), shift); // if (((byte_val >> shift) & 1) == 1) slow; LIR_Opr result = byte_val; @@ -261,9 +261,9 @@ static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* sl Node* zero = __ ConI(0); Node* addr = __ CastPX(__ ctrl(), slot); - Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); + Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(UseCompressedOops ? 5 : 6))); Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); - Node* shift = __ URShiftX(addr, __ ConI(3)); + Node* shift = __ URShiftX(addr, __ ConI(UseCompressedOops ? 2 : 3)); shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); @@ -424,26 +424,8 @@ Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_ void MMTkFieldBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { if (!is_array && dst != kit->just_allocated_object(kit->control())) { MMTkIdealKit ideal(kit); -#if MMTK_ENABLE_BARRIER_FASTPATH - Node* no_base = __ top(); - float unlikely = PROB_UNLIKELY(0.999); - - Node* zero = __ ConI(0); - Node* addr = __ CastPX(__ ctrl(), dst); - Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(6))); - Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); - Node* shift = __ URShiftX(addr, __ ConI(3)); - shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); - Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); - - __ if_then(result, BoolTest::ne, zero, unlikely); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_clone_pre_call), "mmtk_barrier_call", dst); - } __ end_if(); -#else const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_clone_pre_call), "mmtk_barrier_call", dst); -#endif kit->sync_kit(ideal); kit->insert_mem_bar(Op_MemBarVolatile); kit->final_sync(ideal); From 688655a3a9edb4e4d12460f7db2a8c14a2f49c39 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 18 Dec 2022 02:53:05 +0000 Subject: [PATCH 220/494] Update mmtk-core; Compressed pointers working! --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 966d8c05..bb184855 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=d7c97990dcfd1d60d3e52a1e15ea2511b8734e17#d7c97990dcfd1d60d3e52a1e15ea2511b8734e17" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=22106c5dce4740c7149938878217c4e1d91335f2#22106c5dce4740c7149938878217c4e1d91335f2" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=d7c97990dcfd1d60d3e52a1e15ea2511b8734e17#d7c97990dcfd1d60d3e52a1e15ea2511b8734e17" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=22106c5dce4740c7149938878217c4e1d91335f2#22106c5dce4740c7149938878217c4e1d91335f2" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index aae62da2..c94931f8 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "d7c97990dcfd1d60d3e52a1e15ea2511b8734e17" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "22106c5dce4740c7149938878217c4e1d91335f2" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From f9139887d7b9503d3468ec03a40f07b29fef7c7f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 18 Dec 2022 06:24:30 +0000 Subject: [PATCH 221/494] Switch `GLOBAL_LOG_BIT_SPEC` dynamically --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/lib.rs | 12 ++++++++---- mmtk/src/object_model.rs | 2 ++ mmtk/src/vm_metadata/constants.rs | 3 +++ openjdk/barriers/mmtkFieldBarrier.cpp | 12 ++++++++---- openjdk/mmtk.h | 2 +- openjdk/mmtkBarrierSet.hpp | 2 ++ 8 files changed, 27 insertions(+), 12 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index bb184855..ce01d33f 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=22106c5dce4740c7149938878217c4e1d91335f2#22106c5dce4740c7149938878217c4e1d91335f2" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ce20f915bcbe4891ca14e3ec3ceec3ceeaba42c7#ce20f915bcbe4891ca14e3ec3ceec3ceeaba42c7" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=22106c5dce4740c7149938878217c4e1d91335f2#22106c5dce4740c7149938878217c4e1d91335f2" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ce20f915bcbe4891ca14e3ec3ceec3ceeaba42c7#ce20f915bcbe4891ca14e3ec3ceec3ceeaba42c7" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index c94931f8..2173a83b 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "22106c5dce4740c7149938878217c4e1d91335f2" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "ce20f915bcbe4891ca14e3ec3ceec3ceeaba42c7" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 3c8cb748..6b0d1033 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -142,14 +142,18 @@ pub fn current_worker() -> &'static mut GCWorker { pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); -#[no_mangle] -pub static GLOBAL_SIDE_METADATA_BASE_ADDRESS: uintptr_t = - crate::mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_BASE_ADDRESS.as_usize(); - #[no_mangle] pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS: uintptr_t = crate::mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize(); +#[no_mangle] +pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS_COMPRESSED: uintptr_t = + crate::vm_metadata::LOGGING_SIDE_METADATA_SPEC + .as_spec() + .extract_side_spec() + .upper_bound_address_for_contiguous() + .as_usize(); + #[no_mangle] pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t = crate::mmtk::util::metadata::side_metadata::ALLOC_SIDE_METADATA_ADDR.as_usize(); diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 8ff8bf7b..6b38a133 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -10,6 +10,8 @@ pub struct VMObjectModel {} impl ObjectModel for VMObjectModel { const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = vm_metadata::LOGGING_SIDE_METADATA_SPEC; + const GLOBAL_LOG_BIT_SPEC_COMPRESSED: VMGlobalLogBitSpecCompressed = + vm_metadata::LOGGING_SIDE_METADATA_SPEC_COMPRESSED; const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = vm_metadata::FORWARDING_POINTER_METADATA_SPEC; diff --git a/mmtk/src/vm_metadata/constants.rs b/mmtk/src/vm_metadata/constants.rs index 2d4cf193..642c7ce3 100644 --- a/mmtk/src/vm_metadata/constants.rs +++ b/mmtk/src/vm_metadata/constants.rs @@ -13,6 +13,9 @@ pub(crate) const FORWARDING_POINTER_OFFSET: isize = 0; /// 1 bit per object pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); +pub(crate) const LOGGING_SIDE_METADATA_SPEC_COMPRESSED: VMGlobalLogBitSpecCompressed = + VMGlobalLogBitSpecCompressed::side_after(LOGGING_SIDE_METADATA_SPEC.as_spec()); + // Global MetadataSpecs - End // PolicySpecific MetadataSpecs - Start diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 49d5058f..89b76560 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -3,10 +3,14 @@ constexpr int kUnloggedValue = 1; +static inline intptr_t side_metadata_base_address() { + return UseCompressedOops ? SIDE_METADATA_BASE_ADDRESS_COMPRESSED : SIDE_METADATA_BASE_ADDRESS; +} + void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, oop target) const { #if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = ((intptr_t) (void*) slot); - const volatile uint8_t * meta_addr = (const volatile uint8_t *) (SIDE_METADATA_BASE_ADDRESS + (addr >> (UseCompressedOops ? 5 : 6))); + const volatile uint8_t * meta_addr = (const volatile uint8_t *) (side_metadata_base_address() + (addr >> (UseCompressedOops ? 5 : 6))); intptr_t shift = (addr >> (UseCompressedOops ? 2 : 3)) & 0b111; uint8_t byte_val = *meta_addr; if (((byte_val >> shift) & 1) == kUnloggedValue) { @@ -60,7 +64,7 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma // tmp5 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); __ lea(tmp3, dst); __ shrptr(tmp3, UseCompressedOops ? 5 : 6); - __ movptr(tmp5, SIDE_METADATA_BASE_ADDRESS); + __ movptr(tmp5, side_metadata_base_address()); __ movb(tmp5, Address(tmp5, tmp3)); // tmp3 = (obj >> 3) & 7 __ lea(tmp3, dst); @@ -225,7 +229,7 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op __ move(addr, offset); __ unsigned_shift_right(offset, UseCompressedOops ? 5 : 6, offset); LIR_Opr base = gen->new_pointer_register(); - __ move(LIR_OprFact::longConst(SIDE_METADATA_BASE_ADDRESS), base); + __ move(LIR_OprFact::longConst(side_metadata_base_address()), base); LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); // uint8_t byte_val = *meta_addr; LIR_Opr byte_val = gen->new_register(T_INT); @@ -261,7 +265,7 @@ static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* sl Node* zero = __ ConI(0); Node* addr = __ CastPX(__ ctrl(), slot); - Node* meta_addr = __ AddP(no_base, __ ConP(SIDE_METADATA_BASE_ADDRESS), __ URShiftX(addr, __ ConI(UseCompressedOops ? 5 : 6))); + Node* meta_addr = __ AddP(no_base, __ ConP(side_metadata_base_address()), __ URShiftX(addr, __ ConI(UseCompressedOops ? 5 : 6))); Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); Node* shift = __ URShiftX(addr, __ ConI(UseCompressedOops ? 2 : 3)); shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 6bee848c..b720b4cc 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -18,8 +18,8 @@ typedef enum { MmapOutOfMemory, } MMTkAllocationError; -extern const uintptr_t GLOBAL_SIDE_METADATA_BASE_ADDRESS; extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; +extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS_COMPRESSED; extern const uintptr_t GLOBAL_ALLOC_BIT_ADDRESS; extern const int DISABLE_ALLOCATION_FAST_PATH; extern const uintptr_t IMMIX_ALLOCATOR_SIZE; diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 86ed42de..8da99aad 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -41,6 +41,8 @@ const intptr_t SIDE_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; +const intptr_t SIDE_METADATA_BASE_ADDRESS_COMPRESSED = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS_COMPRESSED; + const intptr_t ALLOC_BIT_BASE_ADDRESS = GLOBAL_ALLOC_BIT_ADDRESS; struct MMTkAllocatorOffsets { From 559aabf41de5b9705dcf721b85b0fb451645f6e5 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 18 Dec 2022 07:16:26 +0000 Subject: [PATCH 222/494] Reduce compressed pointer check --- mmtk/Cargo.lock | 4 +- mmtk/Cargo.toml | 2 +- mmtk/src/abi.rs | 66 ++++++++++++---------- mmtk/src/api.rs | 15 +---- mmtk/src/collection.rs | 18 +++++- mmtk/src/object_model.rs | 21 +++++-- mmtk/src/object_scanning.rs | 54 +++++++++--------- mmtk/src/reference_glue.rs | 108 +++++++++++++----------------------- mmtk/src/scanning.rs | 12 ++-- 9 files changed, 148 insertions(+), 152 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index ce01d33f..06ae0f31 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ce20f915bcbe4891ca14e3ec3ceec3ceeaba42c7#ce20f915bcbe4891ca14e3ec3ceec3ceeaba42c7" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=bd20f327f99224332bd3bee5423eecade57ec95e#bd20f327f99224332bd3bee5423eecade57ec95e" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ce20f915bcbe4891ca14e3ec3ceec3ceeaba42c7#ce20f915bcbe4891ca14e3ec3ceec3ceeaba42c7" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=bd20f327f99224332bd3bee5423eecade57ec95e#bd20f327f99224332bd3bee5423eecade57ec95e" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 2173a83b..619609ec 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "ce20f915bcbe4891ca14e3ec3ceec3ceeaba42c7" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "bd20f327f99224332bd3bee5423eecade57ec95e" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 0a0d3e74..ddeb6df7 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -317,9 +317,9 @@ impl OopDesc { unsafe { mem::transmute(self) } } #[inline(always)] - pub(crate) fn klass_ptr(&self) -> Address { + pub(crate) fn klass_ptr(&self) -> Address { // self.klass - if crate::use_compressed_oops() { + if COMPRESSED { lazy_static! { static ref COMPRESSED_KLASS_BASE: Address = unsafe { ((*UPCALLS).compressed_klass_base)() }; @@ -334,9 +334,9 @@ impl OopDesc { } } #[inline(always)] - pub fn klass(&self) -> &'static Klass { + pub fn klass(&self) -> &'static Klass { // self.klass - if crate::use_compressed_oops() { + if COMPRESSED { lazy_static! { static ref COMPRESSED_KLASS_BASE: Address = unsafe { ((*UPCALLS).compressed_klass_base)() }; @@ -407,8 +407,8 @@ impl OopDesc { /// Calculate object instance size #[inline(always)] - pub unsafe fn size(&self) -> usize { - let klass = self.klass(); + pub unsafe fn size(&self) -> usize { + let klass = self.klass::(); let lh = klass.layout_helper; // The (scalar) instance size is pre-recorded in the TIB? if lh > Klass::LH_NEUTRAL_VALUE { @@ -420,7 +420,7 @@ impl OopDesc { } else if lh <= Klass::LH_NEUTRAL_VALUE { if lh < Klass::LH_NEUTRAL_VALUE { // Calculate array size - let array_length = self.as_array_oop().length(); + let array_length = self.as_array_oop().length::(); let mut size_in_bytes: usize = (array_length as usize) << Klass::layout_helper_log2_element_size(lh); size_in_bytes += Klass::layout_helper_header_size(lh) as usize; @@ -440,8 +440,8 @@ pub struct ArrayOopDesc(OopDesc); pub type ArrayOop = &'static ArrayOopDesc; impl ArrayOopDesc { - fn length_offset() -> usize { - if crate::use_compressed_oops() { + fn length_offset() -> usize { + if COMPRESSED { mem::size_of::() + mem::size_of::() } else { mem::size_of::() @@ -453,9 +453,11 @@ impl ArrayOopDesc { } #[inline(always)] - fn header_size(ty: BasicType) -> usize { - let typesize_in_bytes = - conversions::raw_align_up(Self::length_offset() + BYTES_IN_INT, BYTES_IN_LONG); + fn header_size(ty: BasicType) -> usize { + let typesize_in_bytes = conversions::raw_align_up( + Self::length_offset::() + BYTES_IN_INT, + BYTES_IN_LONG, + ); if Self::element_type_should_be_aligned(ty) { conversions::raw_align_up(typesize_in_bytes / BYTES_IN_WORD, BYTES_IN_LONG) } else { @@ -463,12 +465,15 @@ impl ArrayOopDesc { } } #[inline(always)] - fn length(&self) -> i32 { - unsafe { *((self as *const _ as *const u8).add(Self::length_offset()) as *const i32) } + fn length(&self) -> i32 { + unsafe { + *((self as *const _ as *const u8).add(Self::length_offset::()) + as *const i32) + } } #[inline(always)] - fn base(&self, ty: BasicType) -> Address { - let base_offset_in_bytes = Self::header_size(ty) * BYTES_IN_WORD; + fn base(&self, ty: BasicType) -> Address { + let base_offset_in_bytes = Self::header_size::(ty) * BYTES_IN_WORD; Address::from_ptr(unsafe { (self as *const _ as *const u8).add(base_offset_in_bytes) }) } // This provides an easy way to access the array data in Rust. However, the array data @@ -476,15 +481,18 @@ impl ArrayOopDesc { // 1. matches the actual Java type // 2. matches the argument, BasicType `ty` #[inline(always)] - pub unsafe fn data(&self, ty: BasicType) -> &[T] { - slice::from_raw_parts(self.base(ty).to_ptr(), self.length() as _) + pub unsafe fn data(&self, ty: BasicType) -> &[T] { + slice::from_raw_parts( + self.base::(ty).to_ptr(), + self.length::() as _, + ) } #[inline(always)] - pub unsafe fn slice(&self, ty: BasicType) -> crate::OpenJDKEdgeRange { - let base = self.base(ty); + pub unsafe fn slice(&self, ty: BasicType) -> crate::OpenJDKEdgeRange { + let base = self.base::(ty); let start = crate::OpenJDKEdge(base); let end = crate::OpenJDKEdge( - base + ((self.length() as usize) << if crate::use_compressed_oops() { 2 } else { 3 }), + base + ((self.length::() as usize) << if COMPRESSED { 2 } else { 3 }), ); crate::OpenJDKEdgeRange { start, end } } @@ -528,17 +536,17 @@ struct ChunkedHandleList { } impl ChunkedHandleList { - unsafe fn oops_do_chunk( + unsafe fn oops_do_chunk, const COMPRESSED: bool>( &self, chunk: &'static Chunk, size: u32, - closure: &mut impl EdgeVisitor, + closure: &mut V, ) { for i in 0..size { if !chunk.data[i as usize].is_null() { let mut word = Address::from_ref::<*mut OopDesc>(&chunk.data[i as usize]).as_usize(); - if crate::use_compressed_oops() { + if COMPRESSED { word = word | (1usize << 63); } closure.visit_edge(OpenJDKEdge(Address::from_usize(word))) @@ -546,17 +554,17 @@ impl ChunkedHandleList { } } - fn oops_do(&self, closure: &mut impl EdgeVisitor) { + fn oops_do, const COMPRESSED: bool>(&self, closure: &mut V) { let head = self.head.load(Ordering::Acquire); if !head.is_null() { let head = unsafe { &*head }; let size = head.size.load(Ordering::Acquire); - unsafe { self.oops_do_chunk(head, size, closure) }; + unsafe { self.oops_do_chunk::<_, COMPRESSED>(head, size, closure) }; let mut c = head.next; while !c.is_null() { let chunk = unsafe { &*c }; let size = chunk.size.load(Ordering::Relaxed); - unsafe { self.oops_do_chunk(chunk, size, closure) }; + unsafe { self.oops_do_chunk::<_, COMPRESSED>(chunk, size, closure) }; c = chunk.next; } } @@ -593,10 +601,10 @@ impl ClassLoaderData { } #[inline] - pub fn oops_do>(&self, closure: &mut V) { + pub fn oops_do, const COMPRESSED: bool>(&self, closure: &mut V) { if closure.should_claim_clds() && !self.claim() { return; } - self.handles.oops_do(closure); + self.handles.oops_do::<_, COMPRESSED>(closure); } } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 1b30f4af..794f3e2f 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -11,7 +11,6 @@ use mmtk::plan::BarrierSelector; use mmtk::scheduler::GCController; use mmtk::scheduler::GCWorker; use mmtk::util::alloc::AllocatorSelector; -use mmtk::util::constants::{LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_INT}; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::AllocationSemantics; @@ -390,12 +389,7 @@ pub extern "C" fn mmtk_array_copy_pre( dst: Address, count: usize, ) { - let bytes = count - << if crate::use_compressed_oops() { - LOG_BYTES_IN_INT - } else { - LOG_BYTES_IN_ADDRESS - }; + let bytes = count << crate::log_bytes_in_field(); mutator.barrier().memory_region_copy_pre( OpenJDKEdgeRange { start: OpenJDKEdge(src), @@ -416,12 +410,7 @@ pub extern "C" fn mmtk_array_copy_post( dst: Address, count: usize, ) { - let bytes = count - << if crate::use_compressed_oops() { - LOG_BYTES_IN_INT - } else { - LOG_BYTES_IN_ADDRESS - }; + let bytes = count << crate::log_bytes_in_field(); mutator.barrier().memory_region_copy_post( OpenJDKEdgeRange { start: OpenJDKEdge(src), diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 5590d0f5..629b25c2 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -103,15 +103,27 @@ impl Collection for VMCollection { } fn process_weak_refs>(worker: &mut GCWorker) { - DISCOVERED_LISTS.process_soft_weak_final_refs::(worker) + if crate::use_compressed_oops() { + DISCOVERED_LISTS.process_soft_weak_final_refs::(worker) + } else { + DISCOVERED_LISTS.process_soft_weak_final_refs::(worker) + } } fn process_final_refs>(worker: &mut GCWorker) { - DISCOVERED_LISTS.resurrect_final_refs::(worker) + if crate::use_compressed_oops() { + DISCOVERED_LISTS.resurrect_final_refs::(worker) + } else { + DISCOVERED_LISTS.resurrect_final_refs::(worker) + } } fn process_phantom_refs>(worker: &mut GCWorker) { - DISCOVERED_LISTS.process_phantom_refs::(worker) + if crate::use_compressed_oops() { + DISCOVERED_LISTS.process_phantom_refs::(worker) + } else { + DISCOVERED_LISTS.process_phantom_refs::(worker) + } } fn update_weak_processor(lxr: bool) { diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 6b38a133..df92e944 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -20,13 +20,17 @@ impl ObjectModel for VMObjectModel { const LOCAL_MARK_BIT_SPEC: VMLocalMarkBitSpec = vm_metadata::MARKING_METADATA_SPEC; const LOCAL_LOS_MARK_NURSERY_SPEC: VMLocalLOSMarkNurserySpec = vm_metadata::LOS_METADATA_SPEC; - #[inline] + #[inline(always)] fn copy( from: ObjectReference, copy: CopySemantics, copy_context: &mut GCWorkerCopyContext, ) -> ObjectReference { - let bytes = unsafe { Oop::from(from).size() }; + let bytes = if crate::use_compressed_oops() { + unsafe { Oop::from(from).size::() } + } else { + unsafe { Oop::from(from).size::() } + }; let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::(), 0, copy); // Copy let src = from.to_address(); @@ -59,8 +63,13 @@ impl ObjectModel for VMObjectModel { unsafe { to.to_object_reference() } } + #[inline(always)] fn get_current_size(object: ObjectReference) -> usize { - unsafe { Oop::from(object).size() } + if crate::use_compressed_oops() { + unsafe { Oop::from(object).size::() } + } else { + unsafe { Oop::from(object).size::() } + } } fn get_size_when_copied(object: ObjectReference) -> usize { @@ -112,6 +121,10 @@ impl ObjectModel for VMObjectModel { #[inline(always)] fn get_class_pointer(object: ObjectReference) -> Address { let oop: Oop = unsafe { std::mem::transmute(object) }; - oop.klass_ptr() + if crate::use_compressed_oops() { + oop.klass_ptr::() + } else { + oop.klass_ptr::() + } } } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index da7a0843..918f662d 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -41,7 +41,7 @@ impl OopIterate for InstanceKlass { closure: &mut V, ) { if closure.should_follow_clds() { - do_klass(oop.klass(), closure); + do_klass::<_, COMPRESSED>(oop.klass::(), closure); } let oop_maps = self.nonstatic_oop_maps(); for map in oop_maps { @@ -68,9 +68,9 @@ impl OopIterate for InstanceMirrorKlass { if klass.is_instance_klass() && (unsafe { klass.cast::().is_anonymous() }) { - do_cld(&klass.class_loader_data, closure) + do_cld::<_, COMPRESSED>(&klass.class_loader_data, closure) } else { - do_klass(klass, closure) + do_klass::<_, COMPRESSED>(klass, closure) } } } @@ -78,7 +78,7 @@ impl OopIterate for InstanceMirrorKlass { // static fields let start = Self::start_of_static_fields(oop); let len = Self::static_oop_field_count(oop); - if crate::use_compressed_oops() { + if COMPRESSED { let start: *const NarrowOop = start.to_ptr::(); let slice = unsafe { slice::from_raw_parts(start, len as _) }; for narrow_oop in slice { @@ -109,7 +109,7 @@ impl OopIterate for InstanceClassLoaderKlass { .load::<*mut ClassLoaderData>() }; if !cld.is_null() { - do_cld(unsafe { &*cld }, closure); + do_cld::<_, COMPRESSED>(unsafe { &*cld }, closure); } } } @@ -123,15 +123,15 @@ impl OopIterate for ObjArrayKlass { closure: &mut V, ) { if closure.should_follow_clds() { - do_klass(oop.klass(), closure); + do_klass::<_, COMPRESSED>(oop.klass::(), closure); } let array = unsafe { oop.as_array_oop() }; if COMPRESSED { - for narrow_oop in unsafe { array.data::(BasicType::T_OBJECT) } { + for narrow_oop in unsafe { array.data::(BasicType::T_OBJECT) } { closure.visit_edge(OpenJDKEdge(narrow_oop.slot())); } } else { - for oop in unsafe { array.data::(BasicType::T_OBJECT) } { + for oop in unsafe { array.data::(BasicType::T_OBJECT) } { closure.visit_edge(OpenJDKEdge(Address::from_ref(oop as &Oop))); } } @@ -249,7 +249,7 @@ fn oop_iterate_slow>(oop: Oop, closure: &mut V, tls: #[inline(always)] fn oop_iterate, const COMPRESSED: bool>(oop: Oop, closure: &mut V) { - let klass_id = oop.klass().id; + let klass_id = oop.klass::().id; debug_assert!( klass_id as i32 >= 0 && (klass_id as i32) < 6, "Invalid klass-id: {:x} for oop: {:x}", @@ -258,27 +258,28 @@ fn oop_iterate, const COMPRESSED: bool>(oop: Oop, cl ); match klass_id { KlassID::Instance => { - let instance_klass = unsafe { oop.klass().cast::() }; + let instance_klass = unsafe { oop.klass::().cast::() }; instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceClassLoader => { - let instance_klass = unsafe { oop.klass().cast::() }; + let instance_klass = + unsafe { oop.klass::().cast::() }; instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceMirror => { - let instance_klass = unsafe { oop.klass().cast::() }; + let instance_klass = unsafe { oop.klass::().cast::() }; instance_klass.oop_iterate::(oop, closure); } KlassID::ObjArray => { - let array_klass = unsafe { oop.klass().cast::() }; + let array_klass = unsafe { oop.klass::().cast::() }; array_klass.oop_iterate::(oop, closure); } // KlassID::TypeArray => { - // let array_klass = unsafe { oop.klass().cast::() }; + // let array_klass = unsafe { oop.klass::().cast::() }; // array_klass.oop_iterate::(oop, closure); // } KlassID::InstanceRef => { - let instance_klass = unsafe { oop.klass().cast::() }; + let instance_klass = unsafe { oop.klass::().cast::() }; instance_klass.oop_iterate::(oop, closure); } // _ => oop_iterate_slow(oop, closure, OpaquePointer::UNINITIALIZED), @@ -286,35 +287,38 @@ fn oop_iterate, const COMPRESSED: bool>(oop: Oop, cl } } -fn do_cld>(cld: &ClassLoaderData, closure: &mut V) { +fn do_cld, const COMPRESSED: bool>( + cld: &ClassLoaderData, + closure: &mut V, +) { if !closure.should_follow_clds() { return; } - cld.oops_do(closure) + cld.oops_do::<_, COMPRESSED>(closure) } -fn do_klass>(klass: &Klass, closure: &mut V) { +fn do_klass, const COMPRESSED: bool>(klass: &Klass, closure: &mut V) { if !closure.should_follow_clds() { return; } - do_cld(&klass.class_loader_data, closure) + do_cld::<_, COMPRESSED>(&klass.class_loader_data, closure) } #[inline(always)] -pub fn is_obj_array(oop: Oop) -> bool { - oop.klass().id == KlassID::ObjArray +pub fn is_obj_array(oop: Oop) -> bool { + oop.klass::().id == KlassID::ObjArray } #[inline(always)] -pub fn is_val_array(oop: Oop) -> bool { - oop.klass().id == KlassID::TypeArray +pub fn is_val_array(oop: Oop) -> bool { + oop.klass::().id == KlassID::TypeArray } #[inline(always)] -pub fn obj_array_data(oop: Oop) -> crate::OpenJDKEdgeRange { +pub fn obj_array_data(oop: Oop) -> crate::OpenJDKEdgeRange { unsafe { let array = oop.as_array_oop(); - array.slice(BasicType::T_OBJECT) + array.slice::(BasicType::T_OBJECT) } } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 09747ae7..c08fb1b9 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -84,7 +84,7 @@ impl DiscoveredList { let oop = Oop::from(reference); let head = self.head.load(Ordering::Relaxed); let addr = InstanceRefKlass::discovered_address(oop); - if crate::use_compressed_oops() { + if COMPRESSED { let discovered_addr = unsafe { addr.to_address().as_ref::>() }; let next_discovered = if head.is_null() { reference } else { head }; debug_assert!(!next_discovered.is_null()); @@ -168,7 +168,7 @@ impl DiscoveredLists { self.get_by_rt_and_index(rt, id) } - pub fn process_lists>( + pub fn process_lists, const COMPRESSED: bool>( &self, worker: &mut GCWorker, rt: ReferenceType, @@ -184,7 +184,7 @@ impl DiscoveredLists { if head.is_null() { continue; } - let w = ProcessDiscoveredList { + let w = ProcessDiscoveredList::<_, COMPRESSED> { list_index: i, head, rt, @@ -201,21 +201,24 @@ impl DiscoveredLists { ) { } - pub fn process_soft_weak_final_refs>( + pub fn process_soft_weak_final_refs< + E: ProcessEdgesWork, + const COMPRESSED: bool, + >( &self, worker: &mut GCWorker, ) { self.allow_discover.store(false, Ordering::SeqCst); if !*SINGLETON.get_options().no_reference_types { - self.process_lists::(worker, ReferenceType::Soft, &self.soft, true); - self.process_lists::(worker, ReferenceType::Weak, &self.weak, true); + self.process_lists::(worker, ReferenceType::Soft, &self.soft, true); + self.process_lists::(worker, ReferenceType::Weak, &self.weak, true); } if !*SINGLETON.get_options().no_finalizer { - self.process_lists::(worker, ReferenceType::Final, &self.r#final, false); + self.process_lists::(worker, ReferenceType::Final, &self.r#final, false); } } - pub fn resurrect_final_refs>( + pub fn resurrect_final_refs, const COMPRESSED: bool>( &self, worker: &mut GCWorker, ) { @@ -228,7 +231,7 @@ impl DiscoveredLists { if head.is_null() { continue; } - let w = ResurrectFinalizables { + let w = ResurrectFinalizables::<_, COMPRESSED> { list_index: i, head, _p: PhantomData::, @@ -238,12 +241,12 @@ impl DiscoveredLists { worker.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets); } - pub fn process_phantom_refs>( + pub fn process_phantom_refs, const COMPRESSED: bool>( &self, worker: &mut GCWorker, ) { assert!(!*SINGLETON.get_options().no_reference_types); - self.process_lists::(worker, ReferenceType::Phantom, &self.phantom, true); + self.process_lists::(worker, ReferenceType::Phantom, &self.phantom, true); } } @@ -252,24 +255,22 @@ lazy_static! { pub static ref DISCOVERED_LISTS: DiscoveredLists = DiscoveredLists::new(); } -pub struct ProcessDiscoveredList> { +pub struct ProcessDiscoveredList, const COMPRESSED: bool> { list_index: usize, head: ObjectReference, rt: ReferenceType, _p: PhantomData, } -impl> GCWork for ProcessDiscoveredList { +impl, const COMPRESSED: bool> GCWork + for ProcessDiscoveredList +{ fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { let mut trace = E::new(vec![], false, mmtk); trace.set_worker(worker); let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); - let new_list = iterate_list(self.head, |reference| { - let referent = if crate::use_compressed_oops() { - get_referent::(reference) - } else { - get_referent::(reference) - }; + let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { + let referent = get_referent::(reference); if referent.is_null() { // Remove from the discovered list return DiscoveredListIterationResult::Remove; @@ -278,21 +279,13 @@ impl> GCWork for ProcessDiscoveredLis let forwarded = trace.trace_object(referent); debug_assert!(forwarded.get_forwarded_object().is_none()); debug_assert!(reference.get_forwarded_object().is_none()); - if crate::use_compressed_oops() { - set_referent::(reference, forwarded); - } else { - set_referent::(reference, forwarded); - } + set_referent::(reference, forwarded); // Remove from the discovered list return DiscoveredListIterationResult::Remove; } else { if self.rt != ReferenceType::Final { // Clear this referent - if crate::use_compressed_oops() { - set_referent::(reference, ObjectReference::NULL); - } else { - set_referent::(reference, ObjectReference::NULL); - } + set_referent::(reference, ObjectReference::NULL); // set_referent(reference, ObjectReference::NULL); } // Keep the reference @@ -309,11 +302,7 @@ impl> GCWork for ProcessDiscoveredLis .store(head, Ordering::SeqCst); } else { let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; - if crate::use_compressed_oops() { - set_next_reference::(tail, old_head); - } else { - set_next_reference::(tail, old_head); - } + set_next_reference::(tail, old_head); } } else { if self.rt == ReferenceType::Final { @@ -326,28 +315,22 @@ impl> GCWork for ProcessDiscoveredLis } } -pub struct ResurrectFinalizables> { +pub struct ResurrectFinalizables, const COMPRESSED: bool> { list_index: usize, head: ObjectReference, _p: PhantomData, } -impl> GCWork for ResurrectFinalizables { +impl, const COMPRESSED: bool> GCWork + for ResurrectFinalizables +{ fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { let mut trace = E::new(vec![], false, mmtk); trace.set_worker(worker); - let new_list = iterate_list(self.head, |reference| { - let referent = if crate::use_compressed_oops() { - get_referent::(reference) - } else { - get_referent::(reference) - }; + let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { + let referent = get_referent::(reference); let forwarded = trace.trace_object(referent); - if crate::use_compressed_oops() { - set_referent::(reference, forwarded); - } else { - set_referent::(reference, forwarded); - } + set_referent::(reference, forwarded); debug_assert!(forwarded.get_forwarded_object().is_none()); return DiscoveredListIterationResult::Enqueue; }); @@ -359,11 +342,7 @@ impl> GCWork for ResurrectFinalizable .head .store(ObjectReference::NULL, Ordering::SeqCst); let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; - if crate::use_compressed_oops() { - set_next_reference::(tail, old_head); - } else { - set_next_reference::(tail, old_head); - } + set_next_reference::(tail, old_head); } trace.flush(); } @@ -374,9 +353,12 @@ enum DiscoveredListIterationResult { Enqueue, } -fn iterate_list( +fn iterate_list< + F: FnMut(ObjectReference) -> DiscoveredListIterationResult, + const COMPRESSED: bool, +>( head: ObjectReference, - mut visitor: impl FnMut(ObjectReference) -> DiscoveredListIterationResult, + mut visitor: F, ) -> Option<(ObjectReference, ObjectReference)> { let mut new_head: Option = None; let mut new_tail: Option = None; @@ -391,11 +373,7 @@ fn iterate_list( debug_assert!(reference.get_forwarded_object().is_none()); debug_assert!(reference.is_reachable()); // Update next_ref forwarding pointer - let next_ref = if crate::use_compressed_oops() { - get_next_reference::(reference) - } else { - get_next_reference::(reference) - }; + let next_ref = get_next_reference::(reference); let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); debug_assert!(next_ref.get_forwarded_object().is_none()); // Reaches the end of the list? @@ -403,21 +381,13 @@ fn iterate_list( // Process reference let result = visitor(reference); // Remove `reference` from current list - if crate::use_compressed_oops() { - set_next_reference::(reference, ObjectReference::NULL); - } else { - set_next_reference::(reference, ObjectReference::NULL); - } + set_next_reference::(reference, ObjectReference::NULL); match result { DiscoveredListIterationResult::Remove => {} DiscoveredListIterationResult::Enqueue => { // Add to new list if let Some(new_head) = new_head { - if crate::use_compressed_oops() { - set_next_reference::(reference, new_head) - } else { - set_next_reference::(reference, new_head) - }; + set_next_reference::(reference, new_head); } else { new_tail = Some(reference); } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index ebd1a4d7..6cdda4e1 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -56,18 +56,18 @@ impl Scanning for VMScanning { } #[inline(always)] - fn obj_array_data(o: ObjectReference) -> crate::OpenJDKEdgeRange { - crate::object_scanning::obj_array_data(unsafe { std::mem::transmute(o) }) + fn obj_array_data(o: ObjectReference) -> crate::OpenJDKEdgeRange { + crate::object_scanning::obj_array_data::(unsafe { std::mem::transmute(o) }) } #[inline(always)] - fn is_obj_array(o: ObjectReference) -> bool { - crate::object_scanning::is_obj_array(unsafe { std::mem::transmute(o) }) + fn is_obj_array(o: ObjectReference) -> bool { + crate::object_scanning::is_obj_array::(unsafe { std::mem::transmute(o) }) } #[inline(always)] - fn is_val_array(o: ObjectReference) -> bool { - crate::object_scanning::is_val_array(unsafe { std::mem::transmute(o) }) + fn is_val_array(o: ObjectReference) -> bool { + crate::object_scanning::is_val_array::(unsafe { std::mem::transmute(o) }) } fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { From 9814cdb6a4c0d168b0d51dad79d6a2f257ecadf3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 19 Dec 2022 00:55:08 +0000 Subject: [PATCH 223/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 06ae0f31..78a9fa39 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=bd20f327f99224332bd3bee5423eecade57ec95e#bd20f327f99224332bd3bee5423eecade57ec95e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=70843b1b56379e3bad94b106e8d31c895856db8a#70843b1b56379e3bad94b106e8d31c895856db8a" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=bd20f327f99224332bd3bee5423eecade57ec95e#bd20f327f99224332bd3bee5423eecade57ec95e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=70843b1b56379e3bad94b106e8d31c895856db8a#70843b1b56379e3bad94b106e8d31c895856db8a" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 619609ec..c1cae45e 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "bd20f327f99224332bd3bee5423eecade57ec95e" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "70843b1b56379e3bad94b106e8d31c895856db8a" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From f3a86ab7da3ce15d4a0eded8d38c98ef9e7ba327 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 19 Dec 2022 02:54:37 +0000 Subject: [PATCH 224/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 78a9fa39..5d63df9b 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -442,7 +442,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=70843b1b56379e3bad94b106e8d31c895856db8a#70843b1b56379e3bad94b106e8d31c895856db8a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=269348e11cf8492f7a4349f3a74adb0975b8d64a#269348e11cf8492f7a4349f3a74adb0975b8d64a" dependencies = [ "atomic", "atomic-traits", @@ -469,7 +469,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.15.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=70843b1b56379e3bad94b106e8d31c895856db8a#70843b1b56379e3bad94b106e8d31c895856db8a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=269348e11cf8492f7a4349f3a74adb0975b8d64a#269348e11cf8492f7a4349f3a74adb0975b8d64a" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index c1cae45e..ea21819c 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "70843b1b56379e3bad94b106e8d31c895856db8a" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "269348e11cf8492f7a4349f3a74adb0975b8d64a" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From e81972c1c35597e8bf1e6e1d7582a67dffdc2338 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 19 Dec 2022 02:56:06 +0000 Subject: [PATCH 225/494] update openjdk --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index ea21819c..15341b47 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -25,7 +25,7 @@ codegen-units = 1 [package.metadata.openjdk] # Our CI matches the following line and extract mmtk/openjdk. If this line is updated, please check ci yaml files and make sure it works. openjdk_repo = "https://github.com/mmtk/openjdk.git" -openjdk_version = "7e481ee4704870ac796f3830c48eb4018db4c512" +openjdk_version = "7fc4fa696755701f16f24e668f23192809b8058f" [dependencies] libc = "0.2" From f808b4581996981ad15421f0fbdb1a16019162d2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 19 Dec 2022 09:22:51 +0000 Subject: [PATCH 226/494] cleanup --- openjdk/barriers/mmtkObjectBarrier.cpp | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index d056d1f2..bcc977b9 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -23,7 +23,7 @@ void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* if (can_remove_barrier(decorators, val, /* skip_const_null */ true)) return; Register obj = dst.base(); #if MMTK_ENABLE_BARRIER_FASTPATH -Label done; + Label done; Register tmp3 = rscratch1; Register tmp4 = rscratch2; @@ -98,14 +98,6 @@ void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access, LIR_ } assert(src->is_register(), "must be a register at this point"); if (!slot->is_register()) { - // LIR_Opr reg = gen->new_pointer_register(); - // if (slot->is_constant()) { - // __ move(slot, reg); - // } else { - // __ leal(slot, reg); - // } - // slot = reg; - LIR_Address* address = slot->as_address_ptr(); LIR_Opr ptr = gen->new_pointer_register(); if (!address->index()->is_valid() && address->disp() == 0) { From 8fc967c0dd24f816668759e3b654d8757bfaeb1a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 19 Dec 2022 09:34:12 +0000 Subject: [PATCH 227/494] Update mmtk-core and openjdk --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 9d8067ef..1433ec97 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -447,7 +447,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=98e85d27ffd3d5dd8da0e9d02769a9a3c767014f#98e85d27ffd3d5dd8da0e9d02769a9a3c767014f" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b912e691156edcd38b07d79009c8ff601613ef4f#b912e691156edcd38b07d79009c8ff601613ef4f" dependencies = [ "atomic", "atomic-traits", @@ -475,7 +475,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=98e85d27ffd3d5dd8da0e9d02769a9a3c767014f#98e85d27ffd3d5dd8da0e9d02769a9a3c767014f" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b912e691156edcd38b07d79009c8ff601613ef4f#b912e691156edcd38b07d79009c8ff601613ef4f" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index e84bbf60..5c9d105d 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -26,7 +26,7 @@ codegen-units = 1 [package.metadata.openjdk] # Our CI matches the following line and extract mmtk/openjdk. If this line is updated, please check ci yaml files and make sure it works. openjdk_repo = "https://github.com/mmtk/openjdk.git" -openjdk_version = "7fc4fa696755701f16f24e668f23192809b8058f" +openjdk_version = "29eaca621b13650f073b167b7a286196c702e848" [dependencies] libc = "0.2" @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "98e85d27ffd3d5dd8da0e9d02769a9a3c767014f" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "b912e691156edcd38b07d79009c8ff601613ef4f" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From fb99ec141c3d1e259c22b9806a5b8e91b9ff5018 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 21 Dec 2022 12:04:56 +1100 Subject: [PATCH 228/494] Rever commit `b4ea6bf` "Optimize allocation: Use hotspot internal LOS check" --- openjdk/mmtkBarrierSetAssembler_x86.cpp | 14 ++++++- openjdk/mmtkBarrierSetC2.cpp | 51 ++++++++++++++++++++++--- openjdk/mmtkHeap.cpp | 36 +++++++++++++++-- openjdk/mmtkMutator.cpp | 29 ++++++++++++++ openjdk/mmtkMutator.inline.hpp | 47 ----------------------- 5 files changed, 120 insertions(+), 57 deletions(-) delete mode 100644 openjdk/mmtkMutator.inline.hpp diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index 15e756ed..01ec636e 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -51,7 +51,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa // I tried to set FastAllocateSizeLimit in MMTkHeap::initialize(). But there are still large objects allocated into the // default space. assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); - + size_t max_non_los_bytes = MMTkMutatorContext::max_non_los_default_alloc_bytes; size_t extra_header = 0; // fastpath, we only use default allocator Allocator allocator = AllocatorDefault; @@ -59,6 +59,18 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; if (selector.tag == TAG_MARK_COMPACT) extra_header = MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; + if (var_size_in_bytes == noreg) { + // constant alloc size. If it is larger than max_non_los_bytes, we directly go to slowpath. + if ((size_t)con_size_in_bytes > max_non_los_bytes - extra_header) { + __ jmp(slow_case); + return; + } + } else { + // var alloc size. We compare with max_non_los_bytes and conditionally jump to slowpath. + __ cmpptr(var_size_in_bytes, max_non_los_bytes - extra_header); + __ jcc(Assembler::aboveEqual, slow_case); + } + if (selector.tag == TAG_MALLOC || selector.tag == TAG_LARGE_OBJECT || selector.tag == TAG_FREE_LIST) { __ jmp(slow_case); return; diff --git a/openjdk/mmtkBarrierSetC2.cpp b/openjdk/mmtkBarrierSetC2.cpp index de0ea33a..41c5c3a1 100644 --- a/openjdk/mmtkBarrierSetC2.cpp +++ b/openjdk/mmtkBarrierSetC2.cpp @@ -72,8 +72,6 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, if (tv >= 0) { always_slow = (tv == 1); initial_slow_test = NULL; - } else { - initial_slow_test = BoolNode::make_predicate(initial_slow_test, &x->_igvn); } // We will do MMTk size check. @@ -83,13 +81,56 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, // The max non-los bytes from MMTk assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); - + size_t max_non_los_bytes = MMTkMutatorContext::max_non_los_default_alloc_bytes; size_t extra_header = 0; // We always use the default allocator. // But we need to figure out which allocator we are using by querying MMTk. AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; if (selector.tag == TAG_MARK_COMPACT) extra_header = MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; + // Check if allocation size is constant + long const_size = x->_igvn.find_long_con(size_in_bytes, -1); + if (const_size >= 0) { + // Constant alloc size. We know it is non-negative, it is safe to cast to unsigned long and compare with size_t + if (((unsigned long)const_size) > max_non_los_bytes - extra_header) { + // We know at JIT time that we need to go to slowpath + always_slow = true; + initial_slow_test = NULL; + } + } else { + // Variable alloc size + + // Create a node for the constant and compare with size_in_bytes + Node *max_non_los_bytes_node = ConLNode::make((long)max_non_los_bytes - extra_header); + x->transform_later(max_non_los_bytes_node); + Node *mmtk_size_cmp = new CmpLNode(size_in_bytes, max_non_los_bytes_node); + x->transform_later(mmtk_size_cmp); + // Size Check: if size_in_bytes >= max_non_los_bytes + Node *mmtk_size_bool = new BoolNode(mmtk_size_cmp, BoolTest::ge); + x->transform_later(mmtk_size_bool); + + if (initial_slow_test == NULL) { + // If there is no previous initial_slow_test, we use the size check as initial_slow_test + initial_slow_test = BoolNode::make_predicate(mmtk_size_bool, &x->_igvn); + } else { + // If there is existing initial_slow_test, we combine the result by 'or' them together. + + // Conditionally move a value 1 or 0 depends on the size check. + // This is definitely not optimal. But to make things simple and to avoid changing the original + // control flow, it is much easier to implement with a cmov. And it should not affect performance much, + // as var size allocation is rare. + Node *mmtk_size_cmov = new CMoveINode(mmtk_size_bool, x->intcon(0), x->intcon(1), TypeInt::INT); + x->transform_later(mmtk_size_cmov); + + // Logical or the size check result (1 or 0) with initial_slow_test + Node* new_slow_test = new OrINode(mmtk_size_cmov, initial_slow_test); + x->transform_later(new_slow_test); + + // Use the or'd result as initial_slow_test + initial_slow_test = BoolNode::make_predicate(new_slow_test, &x->_igvn); + } + } + if (x->C->env()->dtrace_alloc_probes() || !MMTK_ENABLE_ALLOCATION_FASTPATH || disable_fast_alloc() // Malloc allocator has no fastpath || (selector.tag == TAG_MALLOC || selector.tag == TAG_LARGE_OBJECT || selector.tag == TAG_FREE_LIST)) { @@ -214,8 +255,8 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, // Slow-path does no I/O so just set it to the original I/O. result_phi_i_o->init_req(slow_result_path, i_o); - i_o = x->prefetch_allocation(i_o, needgc_false, contended_phi_rawmem, - old_eden_top, new_eden_top, length); + // i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem, + // old_eden_top, new_eden_top, length); // Name successful fast-path variables Node* fast_oop = old_eden_top; diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 39693c47..46df9173 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -35,7 +35,7 @@ #include "memory/resourceArea.hpp" #include "mmtk.h" #include "mmtkHeap.hpp" -#include "mmtkMutator.inline.hpp" +#include "mmtkMutator.hpp" #include "mmtkUpcalls.hpp" #include "mmtkVMCompanionThread.hpp" #include "oops/oop.inline.hpp" @@ -95,7 +95,6 @@ jint MMTkHeap::initialize() { openjdk_gc_init(&mmtk_upcalls); // Cache the value here. It is a constant depending on the selected plan. The plan won't change from now, so value won't change. MMTkMutatorContext::max_non_los_default_alloc_bytes = get_max_non_los_default_alloc_bytes(); - FastAllocateSizeLimit = MMTkMutatorContext::max_non_los_default_alloc_bytes; //ReservedSpace heap_rs = Universe::reserve_heap(mmtk_heap_size, _collector_policy->heap_alignment()); @@ -472,12 +471,41 @@ void MMTkHeap::scan_roots(OopClosure& cl) { WeakProcessor::oops_do(&cl); } +static inline HeapWord* alloc_fast(size_t bytes, Allocator allocator) { + // All allocations with size larger than max non-los bytes will get to this slowpath here. + // We will use LOS for those. + assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); + if (bytes >= MMTkMutatorContext::max_non_los_default_alloc_bytes) { + allocator = AllocatorLos; + } else if (allocator == AllocatorDefault) { + AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; + if (selector.tag == TAG_IMMIX) { + auto& allocator = Thread::current()->third_party_heap_mutator.allocators.immix[selector.index]; + auto cursor = uintptr_t(allocator.cursor); + auto limit = uintptr_t(allocator.limit); + if (cursor + bytes <= limit) { + allocator.cursor = (void*) (cursor + bytes); + return (HeapWord*) cursor; + } + } + } + + // FIXME: Proper use of slow-path api + HeapWord* o = (HeapWord*) ::alloc((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, bytes, HeapWordSize, 0, allocator); + // Post allocation hooks. Note that we can get a nullptr from mmtk core in the case of OOM. + // Hence, only call post allocation hooks if we have a proper object. + if (o != nullptr && allocator != AllocatorDefault) { + ::post_alloc((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, o, bytes, allocator); + } + return o; +} + HeapWord* MMTkHeap::mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) { - return Thread::current()->third_party_heap_mutator.alloc(size << LogHeapWordSize); + return alloc_fast(size << LogHeapWordSize, AllocatorDefault); } HeapWord* MMTkHeap::mem_allocate_nonmove(size_t size, bool* gc_overhead_limit_was_exceeded) { - return Thread::current()->third_party_heap_mutator.alloc(size << LogHeapWordSize, AllocatorLos); + return alloc_fast(size << LogHeapWordSize, AllocatorLos); } void MMTkHeap::complete_cleaning(BoolObjectClosure* is_alive, bool purged_classes) { diff --git a/openjdk/mmtkMutator.cpp b/openjdk/mmtkMutator.cpp index c8931ad4..c78c35df 100644 --- a/openjdk/mmtkMutator.cpp +++ b/openjdk/mmtkMutator.cpp @@ -22,6 +22,35 @@ bool MMTkMutatorContext::is_ready_to_bind() { return ::openjdk_is_gc_initialized(); } +HeapWord* MMTkMutatorContext::alloc(size_t bytes, Allocator allocator) { + // All allocations with size larger than max non-los bytes will get to this slowpath here. + // We will use LOS for those. + assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); + if (bytes >= MMTkMutatorContext::max_non_los_default_alloc_bytes) { + allocator = AllocatorLos; + } else { + AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; + if (selector.tag == TAG_IMMIX) { + auto& allocator = allocators.immix[selector.index]; + auto cursor = uintptr_t(allocator.cursor); + auto limit = uintptr_t(allocator.limit); + if (cursor + bytes <= limit) { + allocator.cursor = (void*) (cursor + bytes); + return (HeapWord*) cursor; + } + } + } + + // FIXME: Proper use of slow-path api + HeapWord* o = (HeapWord*) ::alloc((MMTk_Mutator) this, bytes, HeapWordSize, 0, allocator); + // Post allocation hooks. Note that we can get a nullptr from mmtk core in the case of OOM. + // Hence, only call post allocation hooks if we have a proper object. + if (o != nullptr && allocator != AllocatorDefault) { + ::post_alloc((MMTk_Mutator) this, o, bytes, allocator); + } + return o; +} + void MMTkMutatorContext::flush() { ::flush_mutator((MMTk_Mutator) this); } diff --git a/openjdk/mmtkMutator.inline.hpp b/openjdk/mmtkMutator.inline.hpp deleted file mode 100644 index 5730acfb..00000000 --- a/openjdk/mmtkMutator.inline.hpp +++ /dev/null @@ -1,47 +0,0 @@ - -#ifndef MMTK_OPENJDK_MMTK_MUTATOR_INLINE_HPP -#define MMTK_OPENJDK_MMTK_MUTATOR_INLINE_HPP - -#include "mmtk.h" -#include "utilities/globalDefinitions.hpp" -#include "mmtkMutator.hpp" -#include "mmtkHeap.hpp" - - -inline HeapWord* MMTkMutatorContext::alloc(size_t bytes, Allocator allocator) { - // All allocations with size larger than max non-los bytes will get to this slowpath here. - // We will use LOS for those. - assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); - if (bytes >= MMTkMutatorContext::max_non_los_default_alloc_bytes) { - allocator = AllocatorLos; - } else { - AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; - if (selector.tag == TAG_IMMIX) { - auto& allocator = allocators.immix[selector.index]; - auto cursor = uintptr_t(allocator.cursor); - auto limit = uintptr_t(allocator.limit); - if (cursor + bytes <= limit) { - allocator.cursor = (void*) (cursor + bytes); - return (HeapWord*) cursor; - } else if (bytes > 256) { - auto large_cursor = uintptr_t(allocator.large_cursor); - auto large_limit = uintptr_t(allocator.large_limit); - if (large_cursor + bytes <= large_limit) { - allocator.large_cursor = (void*) (large_cursor + bytes); - return (HeapWord*) large_cursor; - } - } - } - } - - // FIXME: Proper use of slow-path api - HeapWord* o = (HeapWord*) ::alloc((MMTk_Mutator) this, bytes, HeapWordSize, 0, allocator); - // Post allocation hooks. Note that we can get a nullptr from mmtk core in the case of OOM. - // Hence, only call post allocation hooks if we have a proper object. - if (o != nullptr && allocator != AllocatorDefault) { - ::post_alloc((MMTk_Mutator) this, o, bytes, allocator); - } - return o; -} - -#endif // MMTK_OPENJDK_MMTK_MUTATOR_INLINE_HPP From aa289f51d3eb1c486d405fac3fb458169c9e69a1 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 21 Dec 2022 12:05:08 +1100 Subject: [PATCH 229/494] Update openjdk --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 5c9d105d..e1e538d2 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -26,7 +26,7 @@ codegen-units = 1 [package.metadata.openjdk] # Our CI matches the following line and extract mmtk/openjdk. If this line is updated, please check ci yaml files and make sure it works. openjdk_repo = "https://github.com/mmtk/openjdk.git" -openjdk_version = "29eaca621b13650f073b167b7a286196c702e848" +openjdk_version = "02a4d3bc271210a033a43f44f5c8c749006e0b12" [dependencies] libc = "0.2" From 623f2908cb5b7c1cdef8b121a5b257afce668760 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 23 Dec 2022 00:44:53 +0000 Subject: [PATCH 230/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- openjdk/mmtkMutator.hpp | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 1433ec97..ca2956fc 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -447,7 +447,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b912e691156edcd38b07d79009c8ff601613ef4f#b912e691156edcd38b07d79009c8ff601613ef4f" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7286d16d9b768f73c7f5b27e5313e99185d909b4#7286d16d9b768f73c7f5b27e5313e99185d909b4" dependencies = [ "atomic", "atomic-traits", @@ -475,7 +475,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b912e691156edcd38b07d79009c8ff601613ef4f#b912e691156edcd38b07d79009c8ff601613ef4f" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7286d16d9b768f73c7f5b27e5313e99185d909b4#7286d16d9b768f73c7f5b27e5313e99185d909b4" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index e1e538d2..1b0033bb 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "b912e691156edcd38b07d79009c8ff601613ef4f" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7286d16d9b768f73c7f5b27e5313e99185d909b4" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp index 5b890fed..6248a958 100644 --- a/openjdk/mmtkMutator.hpp +++ b/openjdk/mmtkMutator.hpp @@ -57,6 +57,7 @@ struct ImmixAllocator { uint8_t line_opt_tag; uintptr_t line_opt; uintptr_t mutator_recycled_blocks; + uintptr_t mutator_recycled_lines; uint8_t retry; }; From 120cf0cb78a3914973e5faf966a2119d8aa421e3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 4 Jan 2023 13:51:39 +0000 Subject: [PATCH 231/494] Fix C2 write barrier by adding a dependency to the slow-call node. Not sure how this can work though.... since `slot` is already a dependency. --- openjdk/barriers/mmtkFieldBarrier.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 89b76560..eaad0fbb 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -274,10 +274,18 @@ static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* sl __ if_then(result, BoolTest::ne, zero, unlikely); { const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), "mmtk_barrier_call", src, slot, val); + // Looks like this is necessary + // See https://github.com/mmtk/openjdk/blob/c82e5c44adced4383162826c2c3933a83cfb139b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp#L288-L291 + Node* call = __ ctrl()->in(0); + call->add_req(slot); } __ end_if(); #else const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), "mmtk_barrier_call", src, slot, val); + // Looks like this is necessary + // See https://github.com/mmtk/openjdk/blob/c82e5c44adced4383162826c2c3933a83cfb139b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp#L288-L291 + Node* call = __ ctrl()->in(0); + call->add_req(slot); #endif } @@ -288,10 +296,7 @@ void MMTkFieldBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, insert_write_barrier_common(ideal, src, slot, val); - kit->sync_kit(ideal); - kit->insert_mem_bar(Op_MemBarVolatile); - - kit->final_sync(ideal); // Final sync IdealKit and GraphKit. + kit->final_sync(ideal); } static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool emit_barrier) { From 0b931b2a354dd7a7b91a02b6c24c7fe5703f4535 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 11 Jan 2023 00:56:00 +0000 Subject: [PATCH 232/494] Update dependencies to match Google3 --- mmtk/Cargo.lock | 221 +++++++++++++++++++++++++++----------------- mmtk/Cargo.toml | 10 +- mmtk/rust-toolchain | 2 +- 3 files changed, 143 insertions(+), 90 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index ca2956fc..0504fb3c 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -31,19 +31,19 @@ dependencies = [ [[package]] name = "atomic-traits" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e609bfb99c65d95ebabff6df4641de32b4492ae10a6d77b628a03936ac6ad8d2" +checksum = "b29ec3788e96fb4fdb275ccb9d62811f2fa903d76c5eb4dd6fe7d09a7ed5871f" dependencies = [ - "cfg-if 0.1.10", + "cfg-if", "rustc_version", ] [[package]] name = "atomic_refcell" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b5e5f48b927f04e952dedc932f31995a65a0bf65ec971c74436e51bf6e970d" +checksum = "857253367827bd9d0fd973f0ef15506a96e79e41b0ad7aa691203a4e3214f6c8" [[package]] name = "atty" @@ -51,7 +51,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] @@ -93,9 +93,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "built" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f346b6890a0dfa7266974910e7df2d5088120dd54721b9b0e5aae1ae5e05715" +checksum = "5b9c056b9ed43aee5e064b683aa1ec783e19c6acec7559e3ae931b7490472fbe" dependencies = [ "cargo-lock", "git2", @@ -103,11 +103,11 @@ dependencies = [ [[package]] name = "cargo-lock" -version = "7.1.0" +version = "8.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c408da54db4c50d4693f7e649c299bc9de9c23ead86249e5368830bb32a734b" +checksum = "031718ddb8f78aa5def78a09e90defe30151d1f6c672f937af4dd916429ed996" dependencies = [ - "semver 1.0.14", + "semver 1.0.16", "serde", "toml", "url", @@ -115,9 +115,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.77" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4" +checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" dependencies = [ "jobserver", ] @@ -131,12 +131,6 @@ dependencies = [ "nom", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.0" @@ -175,7 +169,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", @@ -189,7 +183,7 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", ] @@ -199,7 +193,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-epoch", "crossbeam-utils", ] @@ -211,7 +205,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" dependencies = [ "autocfg", - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", "memoffset", "scopeguard", @@ -223,7 +217,7 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "crossbeam-utils", ] @@ -233,7 +227,7 @@ version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -250,18 +244,18 @@ checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" [[package]] name = "enum-map" -version = "2.1.0" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0348b2a57c82f98b9dbd8098b1abb2416f221823d3e50cbe24eaebdd16896826" +checksum = "50c25992259941eb7e57b936157961b217a4fc8597829ddef0596d6c3cd86e1a" dependencies = [ "enum-map-derive", ] [[package]] name = "enum-map-derive" -version = "0.8.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a63b7a0ddec6f38dcec5e36257750b7a8fcaf4227e12ceb306e341d63634da05" +checksum = "2a4da76b3b6116d758c7ba93f7ec6a35d2e2cf24feda76c6e38a375f4d5c59f2" dependencies = [ "proc-macro2", "quote", @@ -292,9 +286,9 @@ dependencies = [ [[package]] name = "git2" -version = "0.13.25" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6" +checksum = "2994bee4a3a6a51eb90c218523be382fd7ea09b16380b9312e9dbe955ff7c7d1" dependencies = [ "bitflags", "libc", @@ -305,15 +299,18 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "heck" -version = "0.4.0" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "hermit-abi" @@ -324,6 +321,15 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + [[package]] name = "humantime" version = "2.1.0" @@ -372,15 +378,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.137" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libgit2-sys" -version = "0.12.26+1.3.0" +version = "0.14.1+1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494" +checksum = "4a07fb2692bc3593bda59de45a502bb3071659f2c515e28c71e728306b038e17" dependencies = [ "cc", "libc", @@ -390,11 +396,11 @@ dependencies = [ [[package]] name = "libloading" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "winapi", ] @@ -426,7 +432,7 @@ version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -447,13 +453,13 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7286d16d9b768f73c7f5b27e5313e99185d909b4#7286d16d9b768f73c7f5b27e5313e99185d909b4" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7000642765495fd8649b4709be512bcfd031a4e6#7000642765495fd8649b4709be512bcfd031a4e6" dependencies = [ "atomic", "atomic-traits", "atomic_refcell", "built", - "cfg-if 1.0.0", + "cfg-if", "crossbeam", "downcast-rs", "enum-map", @@ -475,7 +481,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7286d16d9b768f73c7f5b27e5313e99185d909b4#7286d16d9b768f73c7f5b27e5313e99185d909b4" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7000642765495fd8649b4709be512bcfd031a4e6#7000642765495fd8649b4709be512bcfd031a4e6" dependencies = [ "proc-macro-error", "proc-macro2", @@ -517,19 +523,19 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "peeking_take_while" @@ -552,6 +558,16 @@ dependencies = [ "libc", ] +[[package]] +name = "pest" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f6e86fb9e7026527a0d46bc308b841d73170ef8f443e1807f6ef88526a816d4" +dependencies = [ + "thiserror", + "ucd-trie", +] + [[package]] name = "pfm" version = "0.1.0-beta.1" @@ -604,27 +620,27 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -645,18 +661,18 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc_version" -version = "0.2.3" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" dependencies = [ - "semver 0.9.0", + "semver 0.11.0", ] [[package]] name = "rustversion" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" +checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" [[package]] name = "scopeguard" @@ -666,42 +682,45 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "semver" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" dependencies = [ "semver-parser", ] [[package]] name = "semver" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" +checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" dependencies = [ "serde", ] [[package]] name = "semver-parser" -version = "0.7.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] [[package]] name = "serde" -version = "1.0.148" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.148" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -731,15 +750,15 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "strum" -version = "0.24.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" [[package]] name = "strum_macros" -version = "0.24.3" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" dependencies = [ "heck", "proc-macro2", @@ -750,9 +769,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.104" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", @@ -777,13 +796,35 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thiserror" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thread-priority" -version = "0.2.4" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbb8681124ddad2b07e677d7a99f564c9edbd289cc53cdc0010fd110cb4818ec" +checksum = "978519ae4c6891352f964b88da4ab5a3a9b74a40247cda5baee145dae6cd3b71" dependencies = [ + "cfg-if", "libc", + "log", "winapi", ] @@ -804,13 +845,19 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "toml" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +checksum = "1333c76748e868a4d9d1017b5ab53171dfd095f70c712fdb4653a406547f598f" dependencies = [ "serde", ] +[[package]] +name = "ucd-trie" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" + [[package]] name = "unicode-bidi" version = "0.3.8" @@ -819,9 +866,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-normalization" @@ -832,6 +879,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" + [[package]] name = "unicode-width" version = "0.1.10" diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 1b0033bb..117f6863 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -29,10 +29,10 @@ openjdk_repo = "https://github.com/mmtk/openjdk.git" openjdk_version = "02a4d3bc271210a033a43f44f5c8c749006e0b12" [dependencies] -libc = "0.2" -lazy_static = "1.1" -spin = {version = "0.9.3", features = ["std"]} -once_cell = "1.10.0" +libc = "0.2.138" +lazy_static = "1.4.0" +spin = {version = "0.9.4", features = ["std"]} +once_cell = "1.16.0" atomic = "0.5.1" # Be very careful to commit any changes to the following mmtk dependency, as our CI scripts (including mmtk-core CI) # rely on matching these lines to modify them: e.g. comment out the git dependency and use the local path. @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7286d16d9b768f73c7f5b27e5313e99185d909b4" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7000642765495fd8649b4709be512bcfd031a4e6" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/rust-toolchain b/mmtk/rust-toolchain index bb120e87..91951fd8 100644 --- a/mmtk/rust-toolchain +++ b/mmtk/rust-toolchain @@ -1 +1 @@ -1.59.0 +1.61.0 From 182435fbc9263d72b771346408e8ee0bd7c9a622 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 11 Jan 2023 06:09:13 +0000 Subject: [PATCH 233/494] Fix weak-ref discovered-list forwarding --- mmtk/src/reference_glue.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index c08fb1b9..4fb3ecc7 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -270,6 +270,7 @@ impl, const COMPRESSED: bool> GCWork trace.set_worker(worker); let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { + let reference = trace.trace_object(reference); let referent = get_referent::(reference); if referent.is_null() { // Remove from the discovered list @@ -289,7 +290,7 @@ impl, const COMPRESSED: bool> GCWork // set_referent(reference, ObjectReference::NULL); } // Keep the reference - return DiscoveredListIterationResult::Enqueue; + return DiscoveredListIterationResult::Enqueue(reference); } }); // Flush the list to the Universe::pending_list @@ -328,11 +329,12 @@ impl, const COMPRESSED: bool> GCWork let mut trace = E::new(vec![], false, mmtk); trace.set_worker(worker); let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { + let reference = trace.trace_object(reference); let referent = get_referent::(reference); let forwarded = trace.trace_object(referent); set_referent::(reference, forwarded); debug_assert!(forwarded.get_forwarded_object().is_none()); - return DiscoveredListIterationResult::Enqueue; + return DiscoveredListIterationResult::Enqueue(reference); }); if let Some((head, tail)) = new_list { @@ -350,7 +352,7 @@ impl, const COMPRESSED: bool> GCWork enum DiscoveredListIterationResult { Remove, - Enqueue, + Enqueue(ObjectReference), } fn iterate_list< @@ -384,7 +386,7 @@ fn iterate_list< set_next_reference::(reference, ObjectReference::NULL); match result { DiscoveredListIterationResult::Remove => {} - DiscoveredListIterationResult::Enqueue => { + DiscoveredListIterationResult::Enqueue(reference) => { // Add to new list if let Some(new_head) = new_head { set_next_reference::(reference, new_head); From 689acad419bb66ffcee85b79e94e727b546d8d64 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 11 Jan 2023 06:52:11 +0000 Subject: [PATCH 234/494] Add weak processor heap range check --- mmtk/src/lib.rs | 10 ++++++++++ openjdk/mmtk.h | 2 ++ openjdk/mmtkUpcalls.cpp | 19 +++++++++++-------- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 12012569..3969fcbf 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -172,6 +172,12 @@ pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = #[no_mangle] pub static mut CONCURRENT_MARKING_ACTIVE: u8 = 0; +#[no_mangle] +pub static mut HEAP_START: Address = Address::ZERO; + +#[no_mangle] +pub static mut HEAP_END: Address = Address::ZERO; + static mut USE_COMPRESSED_OOPS: bool = false; static mut LOG_BYTES_IN_FIELD: usize = LOG_BYTES_IN_ADDRESS as _; static mut BYTES_IN_FIELD: usize = BYTES_IN_ADDRESS as _; @@ -455,6 +461,10 @@ lazy_static! { if use_compressed_oops() { initialize_compressed_oops(); } + unsafe { + HEAP_START = VM_LAYOUT_CONSTANTS.heap_start; + HEAP_END = VM_LAYOUT_CONSTANTS.heap_end; + } *ret }; } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 46c5acac..c483e7f0 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -30,6 +30,8 @@ inline bool disable_fast_alloc() { } extern const size_t MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; extern const uintptr_t FREE_LIST_ALLOCATOR_SIZE; +extern uintptr_t HEAP_START; +extern uintptr_t HEAP_END; extern const char* get_mmtk_version(); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index efe55742..1a471b2c 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -115,14 +115,17 @@ class MMTkLXRFastIsAliveClosure : public BoolObjectClosure { }; class MMTkLXRFastUpdateClosure : public OopClosure { + uintptr_t heap_start = HEAP_START; + uintptr_t heap_end = HEAP_END; + public: inline virtual void do_oop(oop* slot) { const auto o = *slot; const uintptr_t v = uintptr_t((void*) o); - // if (v >= 0x220000000000ULL || v < 0x20000000000ULL) { - // *slot = NULL; - // return; - // } + if (v >= heap_end || v < heap_start) { + *slot = NULL; + return; + } const auto status = MMTkForwardClosure::read_forwarding_word(o); if (MMTkForwardClosure::is_forwarded(status)) { *slot = MMTkForwardClosure::extract_forwarding_pointer(status); @@ -135,10 +138,10 @@ class MMTkLXRFastUpdateClosure : public OopClosure { if (CompressedOops::is_null(heap_oop)) return; oop o = CompressedOops::decode_not_null(heap_oop); const uintptr_t v = uintptr_t((void*) o); - // if (v >= 0x220000000000ULL || v < 0x20000000000ULL) { - // RawAccess<>::oop_store(slot, CompressedOops::encode(oop(NULL))); - // return; - // } + if (v >= heap_end || v < heap_start) { + RawAccess<>::oop_store(slot, CompressedOops::encode(oop(NULL))); + return; + } const auto status = MMTkForwardClosure::read_forwarding_word(o); if (MMTkForwardClosure::is_forwarded(status)) { RawAccess<>::oop_store(slot, CompressedOops::encode_not_null(MMTkForwardClosure::extract_forwarding_pointer(status))); From 348209f7ae68d06b4fefd2bb4835607b0f0be2e3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 11 Jan 2023 07:17:45 +0000 Subject: [PATCH 235/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 0504fb3c..31c8ee0b 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7000642765495fd8649b4709be512bcfd031a4e6#7000642765495fd8649b4709be512bcfd031a4e6" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe#e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe" dependencies = [ "atomic", "atomic-traits", @@ -481,7 +481,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7000642765495fd8649b4709be512bcfd031a4e6#7000642765495fd8649b4709be512bcfd031a4e6" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe#e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 117f6863..3144a041 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7000642765495fd8649b4709be512bcfd031a4e6" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 8074f68fa31dcadf866d9c9f6ad727586811e93a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 11 Jan 2023 08:17:25 +0000 Subject: [PATCH 236/494] Prevent large root edges packets --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/gc_work.rs | 15 ++++++++++----- mmtk/src/scanning.rs | 4 +--- openjdk/mmtkRootsClosure.hpp | 2 -- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 31c8ee0b..33ddb65c 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe#e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=66c34dcce6e5b403940f4f7e509ccba08cafba2c#66c34dcce6e5b403940f4f7e509ccba08cafba2c" dependencies = [ "atomic", "atomic-traits", @@ -481,7 +481,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe#e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=66c34dcce6e5b403940f4f7e509ccba08cafba2c#66c34dcce6e5b403940f4f7e509ccba08cafba2c" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 3144a041..55b97c25 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "66c34dcce6e5b403940f4f7e509ccba08cafba2c" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 3bbe1311..3e6cd446 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -1,5 +1,3 @@ -use std::sync::atomic::Ordering; - use crate::scanning::to_edges_closure; use crate::{OpenJDK, OpenJDKEdge, UPCALLS}; use mmtk::scheduler::*; @@ -73,14 +71,21 @@ impl> ScanCodeCacheRoots { impl> GCWork for ScanCodeCacheRoots { fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { // Collect all the cached roots - let mut edges = Vec::with_capacity(crate::CODE_CACHE_ROOTS_SIZE.load(Ordering::Relaxed)); + let mut edges = Vec::with_capacity(F::BUFFER_SIZE); for roots in (*crate::CODE_CACHE_ROOTS.lock().unwrap()).values() { for r in roots { - edges.push(OpenJDKEdge(*r)) + edges.push(OpenJDKEdge(*r)); + if edges.len() >= F::BUFFER_SIZE { + self.factory + .create_process_edge_roots_work(std::mem::take(&mut edges)); + edges.reserve(F::BUFFER_SIZE); + } } } // Create work packet - self.factory.create_process_edge_roots_work(edges); + if !edges.is_empty() { + self.factory.create_process_edge_roots_work(edges); + } // Use the following code to scan CodeCache directly, instead of scanning the "remembered set". // unsafe { // ((*UPCALLS).scan_code_cache_roots)(create_process_edges_work:: as _); diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index b269b9ce..bb3d2d19 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -11,8 +11,6 @@ use mmtk::MutatorContext; pub struct VMScanning {} -const WORK_PACKET_CAPACITY: usize = 4096; - extern "C" fn report_edges_and_renew_buffer>( ptr: *mut Address, length: usize, @@ -28,7 +26,7 @@ extern "C" fn report_edges_and_renew_buffer>( let (ptr, _, capacity) = { // TODO: Use Vec::into_raw_parts() when the method is available. use std::mem::ManuallyDrop; - let new_vec = Vec::with_capacity(WORK_PACKET_CAPACITY); + let new_vec = Vec::with_capacity(F::BUFFER_SIZE); let mut me = ManuallyDrop::new(new_vec); (me.as_mut_ptr(), me.len(), me.capacity()) }; diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index f4bb26a1..a059e2f6 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -8,8 +8,6 @@ #include "utilities/globalDefinitions.hpp" #include "classfile/classLoaderData.inline.hpp" -#define ROOTS_BUFFER_SIZE 4096 - class MMTkRootsClosure2 : public OopClosure { EdgesClosure _edges_closure; void** _buffer; From 2b385b992cd66a556ddc8bb92dc82a1ebed67b98 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 11 Jan 2023 08:21:29 +0000 Subject: [PATCH 237/494] Prevent large root edges packet --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/gc_work.rs | 15 ++++++++++----- mmtk/src/scanning.rs | 4 +--- openjdk/mmtkRootsClosure.hpp | 2 -- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 31c8ee0b..33ddb65c 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe#e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=66c34dcce6e5b403940f4f7e509ccba08cafba2c#66c34dcce6e5b403940f4f7e509ccba08cafba2c" dependencies = [ "atomic", "atomic-traits", @@ -481,7 +481,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe#e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=66c34dcce6e5b403940f4f7e509ccba08cafba2c#66c34dcce6e5b403940f4f7e509ccba08cafba2c" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 3144a041..55b97c25 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e37fa8a6e5f62b2673df588654a07b8ed5c2c2fe" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "66c34dcce6e5b403940f4f7e509ccba08cafba2c" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 3bbe1311..3e6cd446 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -1,5 +1,3 @@ -use std::sync::atomic::Ordering; - use crate::scanning::to_edges_closure; use crate::{OpenJDK, OpenJDKEdge, UPCALLS}; use mmtk::scheduler::*; @@ -73,14 +71,21 @@ impl> ScanCodeCacheRoots { impl> GCWork for ScanCodeCacheRoots { fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { // Collect all the cached roots - let mut edges = Vec::with_capacity(crate::CODE_CACHE_ROOTS_SIZE.load(Ordering::Relaxed)); + let mut edges = Vec::with_capacity(F::BUFFER_SIZE); for roots in (*crate::CODE_CACHE_ROOTS.lock().unwrap()).values() { for r in roots { - edges.push(OpenJDKEdge(*r)) + edges.push(OpenJDKEdge(*r)); + if edges.len() >= F::BUFFER_SIZE { + self.factory + .create_process_edge_roots_work(std::mem::take(&mut edges)); + edges.reserve(F::BUFFER_SIZE); + } } } // Create work packet - self.factory.create_process_edge_roots_work(edges); + if !edges.is_empty() { + self.factory.create_process_edge_roots_work(edges); + } // Use the following code to scan CodeCache directly, instead of scanning the "remembered set". // unsafe { // ((*UPCALLS).scan_code_cache_roots)(create_process_edges_work:: as _); diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index b269b9ce..bb3d2d19 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -11,8 +11,6 @@ use mmtk::MutatorContext; pub struct VMScanning {} -const WORK_PACKET_CAPACITY: usize = 4096; - extern "C" fn report_edges_and_renew_buffer>( ptr: *mut Address, length: usize, @@ -28,7 +26,7 @@ extern "C" fn report_edges_and_renew_buffer>( let (ptr, _, capacity) = { // TODO: Use Vec::into_raw_parts() when the method is available. use std::mem::ManuallyDrop; - let new_vec = Vec::with_capacity(WORK_PACKET_CAPACITY); + let new_vec = Vec::with_capacity(F::BUFFER_SIZE); let mut me = ManuallyDrop::new(new_vec); (me.as_mut_ptr(), me.len(), me.capacity()) }; diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index f4bb26a1..a059e2f6 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -8,8 +8,6 @@ #include "utilities/globalDefinitions.hpp" #include "classfile/classLoaderData.inline.hpp" -#define ROOTS_BUFFER_SIZE 4096 - class MMTkRootsClosure2 : public OopClosure { EdgesClosure _edges_closure; void** _buffer; From 2c3f3a2ba7e8994dcc1802e5583e9ed5a4b92d58 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 13 Jan 2023 04:09:34 +0000 Subject: [PATCH 238/494] Update Cargo.lock --- mmtk/Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 33ddb65c..4ddf64ce 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -560,9 +560,9 @@ dependencies = [ [[package]] name = "pest" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6e86fb9e7026527a0d46bc308b841d73170ef8f443e1807f6ef88526a816d4" +checksum = "4257b4a04d91f7e9e6290be5d3da4804dd5784fafde3a497d73eb2b4a158c30a" dependencies = [ "thiserror", "ucd-trie", From 92508c1fadf12c80499b56a3ed3655502e1193ce Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 14 Jan 2023 22:46:28 +0000 Subject: [PATCH 239/494] disable soft-ref load barrier --- openjdk/barriers/mmtkFieldBarrier.cpp | 15 +++++++++++++++ openjdk/barriers/mmtkFieldBarrier.hpp | 4 +--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index eaad0fbb..1af1a0f6 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -1,12 +1,21 @@ #include "mmtkFieldBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" +#define SOFT_REFERENCE_LOAD_BARRIER false + constexpr int kUnloggedValue = 1; static inline intptr_t side_metadata_base_address() { return UseCompressedOops ? SIDE_METADATA_BASE_ADDRESS_COMPRESSED : SIDE_METADATA_BASE_ADDRESS; } + +void MMTkFieldBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const override { +#if SOFT_REFERENCE_LOAD_BARRIER + if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) load_reference_call((void*) value); +#endif +}; + void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, oop target) const { #if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = ((intptr_t) (void*) slot); @@ -30,6 +39,7 @@ void MMTkFieldBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet de bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool on_reference = on_weak || on_phantom; BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); +#if SOFT_REFERENCE_LOAD_BARRIER if (on_oop && on_reference) { Label done; // No slow-call if SATB is not active @@ -50,6 +60,7 @@ void MMTkFieldBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet de __ popa(); __ bind(done); } +#endif } void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const { @@ -149,6 +160,7 @@ void MMTkFieldBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) BarrierSetC1::load_at_resolved(access, result); +#if SOFT_REFERENCE_LOAD_BARRIER if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) { // Register the value in the referent field with the pre-barrier LabelObj *Lcont_anonymous; @@ -173,6 +185,7 @@ void MMTkFieldBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) __ branch_destination(Lcont_anonymous->label()); } } +#endif } void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const { @@ -421,11 +434,13 @@ Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_ return load; } +#if SOFT_REFERENCE_LOAD_BARRIER if (on_weak) { reference_load_barrier(kit, adr, load, true); } else if (unknown) { reference_load_barrier_for_unknown_load(kit, obj, offset, adr, load, !need_cpu_mem_bar); } +#endif return load; } diff --git a/openjdk/barriers/mmtkFieldBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp index c605fd27..5a0ee931 100644 --- a/openjdk/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -23,9 +23,7 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const override { object_reference_array_copy_pre_call((void*) src, (void*) dst, count); } - virtual void load_reference(DecoratorSet decorators, oop value) const override { - if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) load_reference_call((void*) value); - }; + virtual void load_reference(DecoratorSet decorators, oop value) const override; virtual void clone_pre(DecoratorSet decorators, oop value) const override { // object_reference_clone_pre_call((void*) value); }; From cb8a3daa3faff4700e2a91f92baff49a282de55f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 14 Jan 2023 22:48:45 +0000 Subject: [PATCH 240/494] Fix compiler error --- openjdk/barriers/mmtkFieldBarrier.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 1af1a0f6..7bf9c7ac 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -10,7 +10,7 @@ static inline intptr_t side_metadata_base_address() { } -void MMTkFieldBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const override { +void MMTkFieldBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const { #if SOFT_REFERENCE_LOAD_BARRIER if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) load_reference_call((void*) value); #endif From e8238cb71c95206b255d34379ee832db5f1030cb Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 24 Jan 2023 00:19:49 +0000 Subject: [PATCH 241/494] Enable soft-reference read barrier --- openjdk/barriers/mmtkFieldBarrier.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 7bf9c7ac..6c795efe 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -1,7 +1,7 @@ #include "mmtkFieldBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" -#define SOFT_REFERENCE_LOAD_BARRIER false +#define SOFT_REFERENCE_LOAD_BARRIER true constexpr int kUnloggedValue = 1; From 1556bcbccbdb96ac22f1aabdbdc307d832ea08b5 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 24 Jan 2023 01:38:13 +0000 Subject: [PATCH 242/494] Update mmtk-core --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 55b97c25..a7d7203c 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "66c34dcce6e5b403940f4f7e509ccba08cafba2c" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "443daac1ec377c31c3d6b8c643a7a93dcd814f51" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From b4f890006c9ace4f27a56af86f74e711ecf54487 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 24 Jan 2023 19:58:14 +0000 Subject: [PATCH 243/494] Update Cargo.lock --- mmtk/Cargo.lock | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 4ddf64ce..7d88a254 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=66c34dcce6e5b403940f4f7e509ccba08cafba2c#66c34dcce6e5b403940f4f7e509ccba08cafba2c" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=443daac1ec377c31c3d6b8c643a7a93dcd814f51#443daac1ec377c31c3d6b8c643a7a93dcd814f51" dependencies = [ "atomic", "atomic-traits", @@ -475,13 +475,12 @@ dependencies = [ "spin", "strum", "strum_macros", - "thread-priority", ] [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=66c34dcce6e5b403940f4f7e509ccba08cafba2c#66c34dcce6e5b403940f4f7e509ccba08cafba2c" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=443daac1ec377c31c3d6b8c643a7a93dcd814f51#443daac1ec377c31c3d6b8c643a7a93dcd814f51" dependencies = [ "proc-macro-error", "proc-macro2", @@ -816,18 +815,6 @@ dependencies = [ "syn", ] -[[package]] -name = "thread-priority" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978519ae4c6891352f964b88da4ab5a3a9b74a40247cda5baee145dae6cd3b71" -dependencies = [ - "cfg-if", - "libc", - "log", - "winapi", -] - [[package]] name = "tinyvec" version = "1.6.0" From b1aa5249ad41c14de5a0644cb1232dabd82ced13 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 25 Jan 2023 02:02:26 +0000 Subject: [PATCH 244/494] Update mmtk-core; Fix overwritten remset slot during mature evacuation --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/lib.rs | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 7d88a254..c408fe77 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=443daac1ec377c31c3d6b8c643a7a93dcd814f51#443daac1ec377c31c3d6b8c643a7a93dcd814f51" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=11138a94a6dcfc758b3b0bdc7ab62c761e41ed2b#11138a94a6dcfc758b3b0bdc7ab62c761e41ed2b" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=443daac1ec377c31c3d6b8c643a7a93dcd814f51#443daac1ec377c31c3d6b8c643a7a93dcd814f51" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=11138a94a6dcfc758b3b0bdc7ab62c761e41ed2b#11138a94a6dcfc758b3b0bdc7ab62c761e41ed2b" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index a7d7203c..d2aa5831 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "443daac1ec377c31c3d6b8c643a7a93dcd814f51" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "11138a94a6dcfc758b3b0bdc7ab62c761e41ed2b" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 3969fcbf..e3114ec1 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -7,7 +7,7 @@ extern crate spin; use std::cell::RefCell; use std::collections::HashMap; use std::ptr::null_mut; -use std::sync::atomic::AtomicUsize; +use std::sync::atomic::{AtomicU32, AtomicUsize}; use std::sync::Mutex; use libc::{c_char, c_void, uintptr_t}; @@ -295,6 +295,38 @@ impl Edge for OpenJDKEdge { } } + fn compare_exchange( + &self, + old_object: ObjectReference, + new_object: ObjectReference, + success: Ordering, + failure: Ordering, + ) -> Result { + if COMPRESSED { + let old_value = compress(old_object); + let new_value = compress(new_object); + let slot = self.untagged_address(); + unsafe { + match slot.compare_exchange::(old_value, new_value, success, failure) { + Ok(v) => Ok(decompress(v)), + Err(v) => Err(decompress(v)), + } + } + } else { + unsafe { + match self.0.compare_exchange::( + old_object.to_address::().as_usize(), + new_object.to_address::().as_usize(), + success, + failure, + ) { + Ok(v) => Ok(ObjectReference::from_raw_address(Address::from_usize(v))), + Err(v) => Err(ObjectReference::from_raw_address(Address::from_usize(v))), + } + } + } + } + #[inline(always)] fn to_address(&self) -> Address { self.untagged_address() From 94e511b79991417b2c91efad3b3c72c06f5ee9c7 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 26 Jan 2023 18:32:35 +0000 Subject: [PATCH 245/494] Refactor logs --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 4 ++-- mmtk/src/api.rs | 6 +++++- openjdk/mmtk.h | 2 ++ openjdk/mmtkHeap.cpp | 16 ++++++++++------ openjdk/thirdPartyHeapArguments.cpp | 1 - 6 files changed, 21 insertions(+), 12 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index c408fe77..9a505209 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=11138a94a6dcfc758b3b0bdc7ab62c761e41ed2b#11138a94a6dcfc758b3b0bdc7ab62c761e41ed2b" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=094a3eeccab308f57035a6d59d53350e583e5681#094a3eeccab308f57035a6d59d53350e583e5681" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=11138a94a6dcfc758b3b0bdc7ab62c761e41ed2b#11138a94a6dcfc758b3b0bdc7ab62c761e41ed2b" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=094a3eeccab308f57035a6d59d53350e583e5681#094a3eeccab308f57035a6d59d53350e583e5681" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index d2aa5831..443cef95 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -26,7 +26,7 @@ codegen-units = 1 [package.metadata.openjdk] # Our CI matches the following line and extract mmtk/openjdk. If this line is updated, please check ci yaml files and make sure it works. openjdk_repo = "https://github.com/mmtk/openjdk.git" -openjdk_version = "02a4d3bc271210a033a43f44f5c8c749006e0b12" +openjdk_version = "c558e3eeea9b16fa9f481d1fdd18fead2f462e19" [dependencies] libc = "0.2.138" @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "11138a94a6dcfc758b3b0bdc7ab62c761e41ed2b" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "094a3eeccab308f57035a6d59d53350e583e5681" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 4f161aeb..b8d964cf 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -35,7 +35,6 @@ pub extern "C" fn get_mmtk_version() -> *const c_char { #[no_mangle] pub extern "C" fn mmtk_active_barrier() -> *const c_char { - println!("{:?}", SINGLETON.get_plan().constraints().barrier); match SINGLETON.get_plan().constraints().barrier { BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), @@ -505,3 +504,8 @@ pub extern "C" fn mmtk_unregister_nmethod(nm: Address) { ); } } + +#[no_mangle] +pub extern "C" fn mmtk_verbose() -> usize { + *crate::SINGLETON.options.verbose +} \ No newline at end of file diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index c483e7f0..2f71ada5 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -128,6 +128,8 @@ extern size_t mmtk_add_nmethod_oop(void* object); extern size_t mmtk_register_nmethod(void* nm); extern size_t mmtk_unregister_nmethod(void* nm); +extern size_t mmtk_verbose(); + /** * VM Accounting */ diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 46df9173..ba55a175 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -112,11 +112,13 @@ jint MMTkHeap::initialize() { _start = (HeapWord*) starting_heap_address(); _end = (HeapWord*) last_heap_address(); - printf("start: %p, end: %p\n", _start, _end); + if (mmtk_verbose() > 0) printf("MMTk Heap: start=%p, end=%p\n", _start, _end); + if (mmtk_verbose() > 0) + printf("MMTk Compressed Pointers: %s\n", UseCompressedOops ? "enabled" : "disabled"); if (UseCompressedOops) { Universe::set_narrow_oop_base((address) mmtk_narrow_oop_base()); Universe::set_narrow_oop_shift(mmtk_narrow_oop_shift()); - printf("narrow_oop_mode: %s\n", Universe::narrow_oop_mode_to_string(Universe::narrow_oop_mode())); + if (mmtk_verbose() > 0) printf("narrow_oop_mode: %s\n", Universe::narrow_oop_mode_to_string(Universe::narrow_oop_mode())); } initialize_reserved_region(_start, _end); @@ -132,10 +134,12 @@ jint MMTkHeap::initialize() { fprintf(stderr, "Failed to create thread"); guarantee(false, "panic"); } - if (ClassUnloading) { - printf("ClassUnloading Enabled\n"); - } else { - printf("ClassUnloading Disabled\n"); + if (mmtk_verbose() > 0) { + if (ClassUnloading) { + printf("ClassUnloading Enabled\n"); + } else { + printf("ClassUnloading Disabled\n"); + } } unsigned int ncpus = (unsigned int) os::initial_active_processor_count(); diff --git a/openjdk/thirdPartyHeapArguments.cpp b/openjdk/thirdPartyHeapArguments.cpp index c3121b03..f2530ded 100644 --- a/openjdk/thirdPartyHeapArguments.cpp +++ b/openjdk/thirdPartyHeapArguments.cpp @@ -45,7 +45,6 @@ void ThirdPartyHeapArguments::initialize() { FLAG_SET_DEFAULT(UseTLAB, false); if (UseCompressedOops) mmtk_use_compressed_ptrs(); // auto compressed_oops = mmtk_use_compressed_ptrs(); - printf("MMTk Compressed Pointers: %s\n", UseCompressedOops ? "enabled" : "disabled"); // FLAG_SET_DEFAULT(UseCompressedOops, compressed_oops); FLAG_SET_DEFAULT(UseCompressedClassPointers, UseCompressedOops); } From 965e31a34b5aae51194c4962cd4daee4c3382f49 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 29 Jan 2023 01:34:17 +0000 Subject: [PATCH 246/494] Fix missing array copy type conversion check --- openjdk/mmtkBarrierSet.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 8da99aad..b93ac45e 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -239,7 +239,7 @@ class MMTkBarrierSet : public BarrierSet { T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); runtime()->object_reference_array_copy_pre((oop*) src, (oop*) dst, length); - bool result = Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, + bool result = Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); runtime()->object_reference_array_copy_post((oop*) src, (oop*) dst, length); @@ -249,7 +249,7 @@ class MMTkBarrierSet : public BarrierSet { static void clone_in_heap(oop src, oop dst, size_t size) { // TODO: We don't need clone barriers at the moment. runtime()->clone_pre(decorators, dst); - Raw::clone(src, dst, size); + Raw::clone_in_heap(src, dst, size); } }; From 2e69cbde782750ccd80fbf2039592833c3eaad32 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 29 Jan 2023 01:34:40 +0000 Subject: [PATCH 247/494] Fix null pointer segfault in C1 barrier --- openjdk/barriers/mmtkFieldBarrier.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 6c795efe..01f2fcf0 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -203,6 +203,10 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op src = reg; } assert(src->is_register(), "must be a register at this point"); + LabelObj *Ldone = new LabelObj(); + __ cmp(lir_cond_equal, src, LIR_OprFact::oopConst(NULL)); + __ branch(lir_cond_equal, T_OBJECT, Ldone->label()); + if (!slot->is_register() && !needs_patching) { LIR_Address* address = slot->as_address_ptr(); LIR_Opr ptr = gen->new_pointer_register(); @@ -264,6 +268,7 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op #endif __ branch_destination(slow->continuation()); + __ branch_destination(Ldone->label()); } #undef __ From 8d582b4111d3828d8f799418842c2ffcb56d5200 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 30 Jan 2023 03:34:38 +0000 Subject: [PATCH 248/494] Fix CLD root scannign and full gc class unloading --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/gc_work.rs | 36 +++++++++++++++++++++++++++++++++++- mmtk/src/lib.rs | 2 +- openjdk/mmtk.h | 2 +- openjdk/mmtkHeap.cpp | 5 +++-- openjdk/mmtkHeap.hpp | 2 +- openjdk/mmtkUpcalls.cpp | 6 +++++- 8 files changed, 49 insertions(+), 10 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 9a505209..ae85de47 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=094a3eeccab308f57035a6d59d53350e583e5681#094a3eeccab308f57035a6d59d53350e583e5681" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8894b855d90a8eaf1a0712571ec48fae72478990#8894b855d90a8eaf1a0712571ec48fae72478990" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=094a3eeccab308f57035a6d59d53350e583e5681#094a3eeccab308f57035a6d59d53350e583e5681" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8894b855d90a8eaf1a0712571ec48fae72478990#8894b855d90a8eaf1a0712571ec48fae72478990" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 443cef95..415d6698 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "094a3eeccab308f57035a6d59d53350e583e5681" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8894b855d90a8eaf1a0712571ec48fae72478990" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 3e6cd446..b79c37ba 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -3,6 +3,9 @@ use crate::{OpenJDK, OpenJDKEdge, UPCALLS}; use mmtk::scheduler::*; use mmtk::vm::RootsWorkFactory; use mmtk::MMTK; +use crate::Address; +use crate::NewBuffer; +use crate::EdgesClosure; macro_rules! scan_roots_work { ($struct_name: ident, $func_name: ident) => { @@ -46,11 +49,42 @@ impl> ScanClassLoaderDataGraphRoots { } } + + +extern "C" fn report_edges_and_renew_buffer_cld, const WEAK: bool>( + ptr: *mut Address, + length: usize, + capacity: usize, + factory_ptr: *mut libc::c_void, +) -> NewBuffer { + if !ptr.is_null() { + let ptr = ptr as *mut OpenJDKEdge; + let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; + let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; + factory.create_process_edge_roots_work_for_cld_roots(buf, WEAK); + } + let (ptr, _, capacity) = { + // TODO: Use Vec::into_raw_parts() when the method is available. + use std::mem::ManuallyDrop; + let new_vec = Vec::with_capacity(F::BUFFER_SIZE); + let mut me = ManuallyDrop::new(new_vec); + (me.as_mut_ptr(), me.len(), me.capacity()) + }; + NewBuffer { ptr, capacity } +} + +fn to_edges_closure_cld, const WEAK: bool>(factory: &mut F) -> EdgesClosure { + EdgesClosure { + func: report_edges_and_renew_buffer_cld::, + data: factory as *mut F as *mut libc::c_void, + } +} impl> GCWork for ScanClassLoaderDataGraphRoots { fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { unsafe { ((*UPCALLS).scan_class_loader_data_graph_roots)( - to_edges_closure(&mut self.factory), + to_edges_closure_cld::(&mut self.factory), + to_edges_closure_cld::(&mut self.factory), mmtk.get_plan() .current_gc_should_scan_weak_classloader_roots(), ); diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index e3114ec1..9e9428ca 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -105,7 +105,7 @@ pub struct OpenJDK_Upcalls { pub scan_system_dictionary_roots: extern "C" fn(closure: EdgesClosure), pub scan_code_cache_roots: extern "C" fn(closure: EdgesClosure), pub scan_string_table_roots: extern "C" fn(closure: EdgesClosure), - pub scan_class_loader_data_graph_roots: extern "C" fn(closure: EdgesClosure, scan_weak: bool), + pub scan_class_loader_data_graph_roots: extern "C" fn(closure: EdgesClosure, weak_closure: EdgesClosure, scan_weak: bool), pub scan_weak_processor_roots: extern "C" fn(closure: EdgesClosure), pub scan_vm_thread_roots: extern "C" fn(closure: EdgesClosure), pub number_of_mutators: extern "C" fn() -> usize, diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 2f71ada5..c1ff8947 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -194,7 +194,7 @@ typedef struct { void (*scan_system_dictionary_roots) (EdgesClosure closure); void (*scan_code_cache_roots) (EdgesClosure closure); void (*scan_string_table_roots) (EdgesClosure closure); - void (*scan_class_loader_data_graph_roots) (EdgesClosure closure, bool scan_weak); + void (*scan_class_loader_data_graph_roots) (EdgesClosure closure, EdgesClosure weak_closure, bool scan_weak); void (*scan_weak_processor_roots) (EdgesClosure closure); void (*scan_vm_thread_roots) (EdgesClosure closure); size_t (*number_of_mutators)(); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index ba55a175..374355e2 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -423,9 +423,10 @@ void MMTkHeap::scan_code_cache_roots(OopClosure& cl) { void MMTkHeap::scan_string_table_roots(OopClosure& cl) { StringTable::oops_do(&cl); } -void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, bool scan_weak) { +void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& weak_cl, bool scan_weak) { CLDToOopClosure cld_cl(&cl, false); - ClassLoaderDataGraph::roots_cld_do(&cld_cl, ClassUnloading && !scan_weak ? NULL : &cld_cl); + CLDToOopClosure weak_cld_cl(&weak_cl, false); + ClassLoaderDataGraph::roots_cld_do(&cld_cl, ClassUnloading && !scan_weak ? NULL : &weak_cld_cl); } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { ShouldNotReachHere(); diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index 295d0d0d..65f2fbb5 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -210,7 +210,7 @@ class MMTkHeap : public CollectedHeap { void scan_system_dictionary_roots(OopClosure& cl); void scan_code_cache_roots(OopClosure& cl); void scan_string_table_roots(OopClosure& cl); - void scan_class_loader_data_graph_roots(OopClosure& cl, bool scan_weak); + void scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& weak_cl, bool scan_weak); void scan_weak_processor_roots(OopClosure& cl); void scan_vm_thread_roots(OopClosure& cl); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 1a471b2c..abfa0a39 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -452,7 +452,11 @@ static void mmtk_scan_aot_loader_roots(EdgesClosure closure) { MMTkRootsClosure2 static void mmtk_scan_system_dictionary_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_system_dictionary_roots(cl); } static void mmtk_scan_code_cache_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } static void mmtk_scan_string_table_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_string_table_roots(cl); } -static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, bool scan_weak) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl, scan_weak); } +static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, EdgesClosure weak_closure, bool scan_weak) { + MMTkRootsClosure2 cl(closure); + MMTkRootsClosure2 weak_cl(weak_closure); + MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl, weak_cl, scan_weak); +} static void mmtk_scan_weak_processor_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_weak_processor_roots(cl); } static void mmtk_scan_vm_thread_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } From ec9d70b186d7f50fcbd333f73715bb5821b41bad Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 30 Jan 2023 19:42:58 +0000 Subject: [PATCH 249/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index ae85de47..75baf600 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8894b855d90a8eaf1a0712571ec48fae72478990#8894b855d90a8eaf1a0712571ec48fae72478990" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=1b1e60df17967b6976e0aadfea859ac62eaf9593#1b1e60df17967b6976e0aadfea859ac62eaf9593" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8894b855d90a8eaf1a0712571ec48fae72478990#8894b855d90a8eaf1a0712571ec48fae72478990" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=1b1e60df17967b6976e0aadfea859ac62eaf9593#1b1e60df17967b6976e0aadfea859ac62eaf9593" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 415d6698..bb29e61c 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8894b855d90a8eaf1a0712571ec48fae72478990" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "1b1e60df17967b6976e0aadfea859ac62eaf9593" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 98e322773a728347e8c09b5cfd1d4aefe32c8f34 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 31 Jan 2023 00:04:34 +0000 Subject: [PATCH 250/494] C1 barrier code patch workaround --- openjdk/mmtkBarrierSetAssembler_x86.cpp | 15 ++++++++++++--- openjdk/mmtkBarrierSetAssembler_x86.hpp | 2 +- openjdk/mmtkBarrierSetC1.cpp | 14 ++++++++++---- openjdk/mmtkBarrierSetC1.hpp | 1 + 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index 01ec636e..8b8896a2 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -147,7 +147,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa #define __ sasm-> -void MMTkBarrierSetAssembler::generate_c1_write_barrier_runtime_stub(StubAssembler* sasm) const { +void MMTkBarrierSetAssembler::generate_c1_write_barrier_runtime_stub(StubAssembler* sasm, bool do_code_patch) const { __ prologue("mmtk_write_barrier", false); Address store_addr(rbp, 4*BytesPerWord); @@ -167,11 +167,17 @@ void MMTkBarrierSetAssembler::generate_c1_write_barrier_runtime_stub(StubAssembl __ save_live_registers_no_oop_map(true); + if (do_code_patch) { + // We don't know the field offset when a code patching is required. + // As a temporary fix, we apply field barrier to all fields in this object. + __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_clone_pre_call), c_rarg0); + } else { #if MMTK_ENABLE_BARRIER_FASTPATH - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); + __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), c_rarg0, c_rarg1, c_rarg2); #else __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); #endif + } __ restore_live_registers(true); @@ -237,6 +243,7 @@ void MMTkBarrierSetAssembler::generate_c1_write_barrier_stub_call(LIR_Assembler* // For pre-barriers, stub->slot may not be a resolved address. // Manually patch the address and goes to the slow-path unconditionally. + address runtime_address; if (stub->patch_code != lir_patch_none) { // Patch assert(stub->scratch->is_single_cpu(), "must be"); @@ -249,14 +256,16 @@ void MMTkBarrierSetAssembler::generate_c1_write_barrier_stub_call(LIR_Assembler* __ lea(stub->scratch->as_register(), from_addr); // Store parameter ce->store_parameter(stub->scratch->as_pointer_register(), 1); + runtime_address = (bs->_write_barrier_c1_runtime_code_blob_with_patch_fix->code_begin()); } else { // Store parameter ce->store_parameter(stub->slot->as_pointer_register(), 1); + runtime_address = (bs->_write_barrier_c1_runtime_code_blob->code_begin()); } ce->store_parameter(stub->src->as_pointer_register(), 0); ce->store_parameter(stub->new_val->as_pointer_register(), 2); - __ call(RuntimeAddress(bs->_write_barrier_c1_runtime_code_blob->code_begin())); + __ call(RuntimeAddress(runtime_address)); __ jmp(*stub->continuation()); } diff --git a/openjdk/mmtkBarrierSetAssembler_x86.hpp b/openjdk/mmtkBarrierSetAssembler_x86.hpp index dc60cab3..c30dbed6 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.hpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.hpp @@ -28,7 +28,7 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler { } /// Generate C1 write barrier slow-call assembly code - virtual void generate_c1_write_barrier_runtime_stub(StubAssembler* sasm) const; + virtual void generate_c1_write_barrier_runtime_stub(StubAssembler* sasm, bool do_code_patch) const; virtual void generate_c1_ref_load_barrier_runtime_stub(StubAssembler* sasm) const; public: diff --git a/openjdk/mmtkBarrierSetC1.cpp b/openjdk/mmtkBarrierSetC1.cpp index b962fde3..079ab8d8 100644 --- a/openjdk/mmtkBarrierSetC1.cpp +++ b/openjdk/mmtkBarrierSetC1.cpp @@ -5,14 +5,20 @@ void MMTkBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) { class MMTkBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { + bool _do_code_patch; virtual OopMapSet* generate_code(StubAssembler* sasm) override { MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); - bs->generate_c1_write_barrier_runtime_stub(sasm); + bs->generate_c1_write_barrier_runtime_stub(sasm, _do_code_patch); return NULL; } + public: + MMTkBarrierCodeGenClosure(bool do_code_patch): _do_code_patch(do_code_patch) {} }; - MMTkBarrierCodeGenClosure write_code_gen_cl; + MMTkBarrierCodeGenClosure write_code_gen_cl(false); _write_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "write_code_gen_cl", false, &write_code_gen_cl); + MMTkBarrierCodeGenClosure write_code_gen_cl_patch_fix(true); + _write_barrier_c1_runtime_code_blob_with_patch_fix = Runtime1::generate_blob(buffer_blob, -1, "write_code_gen_cl_patch_fix", false, &write_code_gen_cl_patch_fix); + class MMTkRefLoadBarrierCodeGenClosure : public StubAssemblerCodeGenClosure { virtual OopMapSet* generate_code(StubAssembler* sasm) override { MMTkBarrierSetAssembler* bs = (MMTkBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); @@ -20,8 +26,8 @@ void MMTkBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) { return NULL; } }; - MMTkRefLoadBarrierCodeGenClosure write_code_gen_cl2; - _ref_load_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "write_code_gen_cl2", false, &write_code_gen_cl2); + MMTkRefLoadBarrierCodeGenClosure load_code_gen_cl; + _ref_load_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1, "load_code_gen_cl", false, &load_code_gen_cl); } void MMTkC1BarrierStub::emit_code(LIR_Assembler* ce) { diff --git a/openjdk/mmtkBarrierSetC1.hpp b/openjdk/mmtkBarrierSetC1.hpp index 7ea22d6a..0043579b 100644 --- a/openjdk/mmtkBarrierSetC1.hpp +++ b/openjdk/mmtkBarrierSetC1.hpp @@ -11,6 +11,7 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { protected: CodeBlob* _write_barrier_c1_runtime_code_blob; + CodeBlob* _write_barrier_c1_runtime_code_blob_with_patch_fix; CodeBlob* _ref_load_barrier_c1_runtime_code_blob; /// Full pre-barrier From 028f1aea5a2ca8d8402f1631b6553f2f9d2510e2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 31 Jan 2023 00:45:12 +0000 Subject: [PATCH 251/494] minor --- openjdk/mmtkBarrierSetAssembler_x86.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index 8b8896a2..ddd29daf 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -173,7 +173,7 @@ void MMTkBarrierSetAssembler::generate_c1_write_barrier_runtime_stub(StubAssembl __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_clone_pre_call), c_rarg0); } else { #if MMTK_ENABLE_BARRIER_FASTPATH - __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), c_rarg0, c_rarg1, c_rarg2); + __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); #else __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); #endif From 59729ee234a5038b5b1087396c1dc5bc82ad84ae Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 31 Jan 2023 01:16:13 +0000 Subject: [PATCH 252/494] cleanup --- openjdk/mmtkBarrierSetAssembler_x86.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index ddd29daf..5768620b 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -256,11 +256,11 @@ void MMTkBarrierSetAssembler::generate_c1_write_barrier_stub_call(LIR_Assembler* __ lea(stub->scratch->as_register(), from_addr); // Store parameter ce->store_parameter(stub->scratch->as_pointer_register(), 1); - runtime_address = (bs->_write_barrier_c1_runtime_code_blob_with_patch_fix->code_begin()); + runtime_address = bs->_write_barrier_c1_runtime_code_blob_with_patch_fix->code_begin(); } else { // Store parameter ce->store_parameter(stub->slot->as_pointer_register(), 1); - runtime_address = (bs->_write_barrier_c1_runtime_code_blob->code_begin()); + runtime_address = bs->_write_barrier_c1_runtime_code_blob->code_begin(); } ce->store_parameter(stub->src->as_pointer_register(), 0); From 0ece2d7b2997f54f1421926757b88ab033271029 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 31 Jan 2023 19:51:34 +0000 Subject: [PATCH 253/494] Disable MMTkFinalizerThread --- openjdk/mmtkHeap.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 374355e2..5a800a54 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -158,7 +158,7 @@ const char* MMTkHeap::version() { } void MMTkHeap::schedule_finalizer() { - MMTkFinalizerThread::instance->schedule(); + // MMTkFinalizerThread::instance->schedule(); } void MMTkHeap::post_initialize() { @@ -168,7 +168,7 @@ void MMTkHeap::post_initialize() { void MMTkHeap::enable_collection() { // Initialize finalizer thread before enable_collection(). // Otherwise it is possible that we schedule finalizer (during a GC) before the finalizer thread is ready. - MMTkFinalizerThread::initialize(); + // MMTkFinalizerThread::initialize(); ::initialize_collection(0); } From 3c10aacc0a8e0ffa25a71998a3fc947e5e17d6ab Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 2 Feb 2023 00:08:34 +0000 Subject: [PATCH 254/494] Remove `guarantee(false)` in unsupported interfaces --- openjdk/mmtkHeap.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 5a800a54..e158d3b5 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -298,13 +298,13 @@ GrowableArray MMTkHeap::memory_pools() {//may cause error // Iterate over all objects, calling "cl.do_object" on each. void MMTkHeap::object_iterate(ObjectClosure* cl) { //No need to implement.Traced whole path.Only other heaps call it. - guarantee(false, "object iterate not supported"); + // guarantee(false, "object iterate not supported"); } // Similar to object_iterate() except iterates only // over live objects. void MMTkHeap::safe_object_iterate(ObjectClosure* cl) { //not sure..many dependencies from vm - guarantee(false, "safe object iterate not supported"); + // guarantee(false, "safe object iterate not supported"); } HeapWord* MMTkHeap::block_start(const void* addr) const {//OK @@ -350,7 +350,7 @@ void MMTkHeap::print_on(outputStream* st) const {} // Print all GC threads (other than the VM thread) // used by this heap. -void MMTkHeap::print_gc_threads_on(outputStream* st) const {guarantee(false, "print gc threads on not supported");} +void MMTkHeap::print_gc_threads_on(outputStream* st) const {} // Iterator for all GC threads (other than VM thread) void MMTkHeap::gc_threads_do(ThreadClosure* tc) const { From b7f6d1c9f663fec27d7246f4479b31d08685414c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 2 Feb 2023 21:22:59 +0000 Subject: [PATCH 255/494] Support metaspace gc triggered by the VM thread --- mmtk/Cargo.lock | 4 +-- mmtk/Cargo.toml | 2 +- openjdk/mmtkHeap.cpp | 7 ++++- openjdk/mmtkHeap.hpp | 6 +++-- openjdk/mmtkVMCompanionThread.cpp | 45 ++++++++++++++++++++++++++----- openjdk/mmtkVMCompanionThread.hpp | 5 ++++ 6 files changed, 57 insertions(+), 12 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 75baf600..6fd4e582 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=1b1e60df17967b6976e0aadfea859ac62eaf9593#1b1e60df17967b6976e0aadfea859ac62eaf9593" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f1bf381f512b2336a063abd95704092acce91aba#f1bf381f512b2336a063abd95704092acce91aba" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=1b1e60df17967b6976e0aadfea859ac62eaf9593#1b1e60df17967b6976e0aadfea859ac62eaf9593" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f1bf381f512b2336a063abd95704092acce91aba#f1bf381f512b2336a063abd95704092acce91aba" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index bb29e61c..ebc9cb72 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "1b1e60df17967b6976e0aadfea859ac62eaf9593" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f1bf381f512b2336a063abd95704092acce91aba" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index e158d3b5..cc53ef49 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -273,8 +273,13 @@ void MMTkHeap::collect(GCCause::Cause cause) {//later when gc is implemented in // Perform a full collection void MMTkHeap::do_full_collection(bool clear_all_soft_refs) {//later when gc is implemented in rust // guarantee(false, "do full collection not supported"); + handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator); +} - // handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator); +void MMTkHeap::collect_as_vm_thread(GCCause::Cause cause) { + MMTkHeap::heap()->companion_thread()->vm_thread_requires_gc_pause(); + handle_user_collection_request(NULL); + MMTkHeap::heap()->companion_thread()->block_vm_thread(); } diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index 65f2fbb5..b38317cf 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -130,10 +130,12 @@ class MMTkHeap : public CollectedHeap { // mark to be thus strictly sequenced after the stores. bool card_mark_must_follow_store() const; - void collect(GCCause::Cause cause); + virtual void collect(GCCause::Cause cause) override; // Perform a full collection - void do_full_collection(bool clear_all_soft_refs); + virtual void do_full_collection(bool clear_all_soft_refs) override; + + virtual void collect_as_vm_thread(GCCause::Cause cause) override; // Return the CollectorPolicy for the heap diff --git a/openjdk/mmtkVMCompanionThread.cpp b/openjdk/mmtkVMCompanionThread.cpp index 4f99efba..101bdd67 100644 --- a/openjdk/mmtkVMCompanionThread.cpp +++ b/openjdk/mmtkVMCompanionThread.cpp @@ -60,12 +60,22 @@ void MMTkVMCompanionThread::run() { // Let the VM thread stop the world. log_trace(gc)("MMTkVMCompanionThread: Letting VMThread execute VM op..."); - VM_MMTkSTWOperation op(this); - // VMThread::execute() is blocking. The companion thread will be blocked - // here waiting for the VM thread to execute op, and the VM thread will - // be blocked in reach_suspended_and_wait_for_resume() until a GC thread - // calls request(_threads_resumed). - VMThread::execute(&op); + if (_vm_thread_requires_gc_pause) { + MutexLockerEx locker(_lock, Mutex::_no_safepoint_check_flag); + _vm_thread_requires_gc_pause = false; + _vm_thread_suspend_for_gc = true; + _lock->notify_all(); + while (_vm_thread_suspend_for_gc) { + _lock->wait(Mutex::_no_safepoint_check_flag); + } + } else { + VM_MMTkSTWOperation op(this); + // VMThread::execute() is blocking. The companion thread will be blocked + // here waiting for the VM thread to execute op, and the VM thread will + // be blocked in reach_suspended_and_wait_for_resume() until a GC thread + // calls request(_threads_resumed). + VMThread::execute(&op); + } // Tell the waiter thread that the world has resumed. log_trace(gc)("MMTkVMCompanionThread: Notifying threads resumption..."); @@ -106,6 +116,29 @@ void MMTkVMCompanionThread::request(stw_state desired_state, bool wait_until_rea } } +void MMTkVMCompanionThread::vm_thread_requires_gc_pause() { + MutexLockerEx locker(_lock, Mutex::_no_safepoint_check_flag); + _vm_thread_requires_gc_pause = true; +} + +void MMTkVMCompanionThread::block_vm_thread() { + { + MutexLockerEx locker(_lock, Mutex::_no_safepoint_check_flag); + while (!_vm_thread_suspend_for_gc) { + _lock->wait(Mutex::_no_safepoint_check_flag); + } + } + + VM_MMTkSTWOperation op(this); + VMThread::execute(&op); + + { + MutexLockerEx locker(_lock, Mutex::_no_safepoint_check_flag); + _vm_thread_suspend_for_gc = false; + _lock->notify_all(); + } +} + // Wait until the desired state is reached. Usually called after calling the // request method. Supposed to be called by a GC thread. void MMTkVMCompanionThread::wait_for_reached(stw_state desired_state) { diff --git a/openjdk/mmtkVMCompanionThread.hpp b/openjdk/mmtkVMCompanionThread.hpp index 335eaa85..c62aa622 100644 --- a/openjdk/mmtkVMCompanionThread.hpp +++ b/openjdk/mmtkVMCompanionThread.hpp @@ -50,6 +50,8 @@ class MMTkVMCompanionThread: public NamedThread { Monitor* _lock; stw_state _desired_state; stw_state _reached_state; + bool _vm_thread_requires_gc_pause = false; + bool _vm_thread_suspend_for_gc = false; public: // Constructor @@ -62,6 +64,9 @@ class MMTkVMCompanionThread: public NamedThread { void request(stw_state desired_state, bool wait_until_reached); void wait_for_reached(stw_state reached_state); + void vm_thread_requires_gc_pause(); + void block_vm_thread(); + // Interface for the VM_MMTkSTWOperation void reach_suspended_and_wait_for_resume(); }; From e3d117269014ec84fae20677ec7d222edeb15f73 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 2 Feb 2023 21:49:10 +0000 Subject: [PATCH 256/494] Update README-legacy permission --- README-legacy.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 README-legacy.md diff --git a/README-legacy.md b/README-legacy.md old mode 100755 new mode 100644 From 66c0a8326daca0126924065cd6dca421e03d1eaa Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 6 Feb 2023 07:29:46 +0000 Subject: [PATCH 257/494] cargo fmt --- mmtk/src/api.rs | 2 +- mmtk/src/gc_work.rs | 17 ++++++++++------- mmtk/src/lib.rs | 3 ++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index b8d964cf..1ac3ee76 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -508,4 +508,4 @@ pub extern "C" fn mmtk_unregister_nmethod(nm: Address) { #[no_mangle] pub extern "C" fn mmtk_verbose() -> usize { *crate::SINGLETON.options.verbose -} \ No newline at end of file +} diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index b79c37ba..0d621b6b 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -1,11 +1,11 @@ use crate::scanning::to_edges_closure; +use crate::Address; +use crate::EdgesClosure; +use crate::NewBuffer; use crate::{OpenJDK, OpenJDKEdge, UPCALLS}; use mmtk::scheduler::*; use mmtk::vm::RootsWorkFactory; use mmtk::MMTK; -use crate::Address; -use crate::NewBuffer; -use crate::EdgesClosure; macro_rules! scan_roots_work { ($struct_name: ident, $func_name: ident) => { @@ -49,9 +49,10 @@ impl> ScanClassLoaderDataGraphRoots { } } - - -extern "C" fn report_edges_and_renew_buffer_cld, const WEAK: bool>( +extern "C" fn report_edges_and_renew_buffer_cld< + F: RootsWorkFactory, + const WEAK: bool, +>( ptr: *mut Address, length: usize, capacity: usize, @@ -73,7 +74,9 @@ extern "C" fn report_edges_and_renew_buffer_cld NewBuffer { ptr, capacity } } -fn to_edges_closure_cld, const WEAK: bool>(factory: &mut F) -> EdgesClosure { +fn to_edges_closure_cld, const WEAK: bool>( + factory: &mut F, +) -> EdgesClosure { EdgesClosure { func: report_edges_and_renew_buffer_cld::, data: factory as *mut F as *mut libc::c_void, diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 9e9428ca..62357e33 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -105,7 +105,8 @@ pub struct OpenJDK_Upcalls { pub scan_system_dictionary_roots: extern "C" fn(closure: EdgesClosure), pub scan_code_cache_roots: extern "C" fn(closure: EdgesClosure), pub scan_string_table_roots: extern "C" fn(closure: EdgesClosure), - pub scan_class_loader_data_graph_roots: extern "C" fn(closure: EdgesClosure, weak_closure: EdgesClosure, scan_weak: bool), + pub scan_class_loader_data_graph_roots: + extern "C" fn(closure: EdgesClosure, weak_closure: EdgesClosure, scan_weak: bool), pub scan_weak_processor_roots: extern "C" fn(closure: EdgesClosure), pub scan_vm_thread_roots: extern "C" fn(closure: EdgesClosure), pub number_of_mutators: extern "C" fn() -> usize, From 5eac79317245691740405eefccb78338dbf6044b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 6 Feb 2023 08:08:11 +0000 Subject: [PATCH 258/494] Reduce thread_local --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/api.rs | 1 - mmtk/src/lib.rs | 9 +-------- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 6fd4e582..caa62ff6 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f1bf381f512b2336a063abd95704092acce91aba#f1bf381f512b2336a063abd95704092acce91aba" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e1b6b9327eb286680f9eeba0d7a6a625c152675e#e1b6b9327eb286680f9eeba0d7a6a625c152675e" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f1bf381f512b2336a063abd95704092acce91aba#f1bf381f512b2336a063abd95704092acce91aba" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e1b6b9327eb286680f9eeba0d7a6a625c152675e#e1b6b9327eb286680f9eeba0d7a6a625c152675e" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index ebc9cb72..6e3ffe1e 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f1bf381f512b2336a063abd95704092acce91aba" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e1b6b9327eb286680f9eeba0d7a6a625c152675e" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 1ac3ee76..e8842e5b 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -183,7 +183,6 @@ pub extern "C" fn start_control_collector( // We trust the worker pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] pub extern "C" fn start_worker(tls: VMWorkerThread, worker: *mut GCWorker) { - crate::CURRENT_WORKER.with(|x| x.replace(Some(unsafe { &mut *worker }))); let mut worker = unsafe { Box::from_raw(worker) }; memory_manager::start_worker::(&SINGLETON, tls, &mut worker) } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 62357e33..22a1a7d1 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -129,16 +129,9 @@ lazy_static! { unsafe { ((*UPCALLS).java_lang_classloader_loader_data_offset)() }; } -thread_local! { - pub static CURRENT_WORKER: RefCell>> = RefCell::new(None); -} - #[inline(always)] pub fn current_worker() -> &'static mut GCWorker { - CURRENT_WORKER.with(|x| { - let ptr = x.borrow().unwrap(); - unsafe { &mut *ptr } - }) + GCWorker::::current() } pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); From cbaccabe37ad0c181ccea125f248911f3c22fc6f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 6 Feb 2023 08:08:18 +0000 Subject: [PATCH 259/494] Update rust-toolchain --- mmtk/rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/rust-toolchain b/mmtk/rust-toolchain index 91951fd8..65ee0959 100644 --- a/mmtk/rust-toolchain +++ b/mmtk/rust-toolchain @@ -1 +1 @@ -1.61.0 +1.67.0 From a30ef0962f904b0627f9668d1cc1fc3285683fd6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 6 Feb 2023 19:55:09 +0000 Subject: [PATCH 260/494] Remoev implicit pointer casting --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/abi.rs | 8 ++++---- mmtk/src/lib.rs | 1 - 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index caa62ff6..2cb3ce36 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e1b6b9327eb286680f9eeba0d7a6a625c152675e#e1b6b9327eb286680f9eeba0d7a6a625c152675e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a5c96367d1cff4dcd8912165639c2e0d42a303e9#a5c96367d1cff4dcd8912165639c2e0d42a303e9" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e1b6b9327eb286680f9eeba0d7a6a625c152675e#e1b6b9327eb286680f9eeba0d7a6a625c152675e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a5c96367d1cff4dcd8912165639c2e0d42a303e9#a5c96367d1cff4dcd8912165639c2e0d42a303e9" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 6e3ffe1e..e6bef36f 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e1b6b9327eb286680f9eeba0d7a6a625c152675e" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "a5c96367d1cff4dcd8912165639c2e0d42a303e9" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index ddeb6df7..989a5263 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -87,7 +87,7 @@ impl Klass { pub const LH_HEADER_SIZE_MASK: i32 = (1 << BITS_IN_BYTE) - 1; #[inline(always)] pub unsafe fn cast<'a, T>(&self) -> &'a T { - &*(self as *const _ as usize as *const T) + &*(self as *const Self as *const T) } /// Force slow-path for instance size calculation? #[inline(always)] @@ -184,7 +184,7 @@ impl InstanceKlass { const MISC_IS_ANONYMOUS: u16 = 1 << 5; #[inline(always)] fn start_of_vtable(&self) -> *const usize { - unsafe { (self as *const _ as *const u8).add(Self::VTABLE_START_OFFSET) as _ } + unsafe { (self as *const Self as *const u8).add(Self::VTABLE_START_OFFSET) as _ } } #[inline(always)] fn start_of_itable(&self) -> *const usize { @@ -467,14 +467,14 @@ impl ArrayOopDesc { #[inline(always)] fn length(&self) -> i32 { unsafe { - *((self as *const _ as *const u8).add(Self::length_offset::()) + *((self as *const Self as *const u8).add(Self::length_offset::()) as *const i32) } } #[inline(always)] fn base(&self, ty: BasicType) -> Address { let base_offset_in_bytes = Self::header_size::(ty) * BYTES_IN_WORD; - Address::from_ptr(unsafe { (self as *const _ as *const u8).add(base_offset_in_bytes) }) + Address::from_ptr(unsafe { (self as *const Self as *const u8).add(base_offset_in_bytes) }) } // This provides an easy way to access the array data in Rust. However, the array data // is Java types, so we have to map Java types to Rust types. The caller needs to guarantee: diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 22a1a7d1..e6de2814 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -4,7 +4,6 @@ extern crate atomic; extern crate once_cell; extern crate spin; -use std::cell::RefCell; use std::collections::HashMap; use std::ptr::null_mut; use std::sync::atomic::{AtomicU32, AtomicUsize}; From a34791c9533b97330c7311bcaa3237f99e62641c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 10 Feb 2023 06:18:56 +0000 Subject: [PATCH 261/494] Ignore system gc when the `DisableExplicitGC` flag is set --- openjdk/mmtkHeap.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index cc53ef49..efcd41e0 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -266,17 +266,18 @@ bool MMTkHeap::card_mark_must_follow_store() const { //OK } void MMTkHeap::collect(GCCause::Cause cause) {//later when gc is implemented in rust + if (DisableExplicitGC) return; handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator); - // guarantee(false, "collect not supported"); } // Perform a full collection void MMTkHeap::do_full_collection(bool clear_all_soft_refs) {//later when gc is implemented in rust - // guarantee(false, "do full collection not supported"); + if (DisableExplicitGC) return; handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } void MMTkHeap::collect_as_vm_thread(GCCause::Cause cause) { + if (DisableExplicitGC) return; MMTkHeap::heap()->companion_thread()->vm_thread_requires_gc_pause(); handle_user_collection_request(NULL); MMTkHeap::heap()->companion_thread()->block_vm_thread(); From 9ddac162fe6936ab2445d243cdb4703579ea4637 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 10 Feb 2023 06:29:49 +0000 Subject: [PATCH 262/494] Remove inline pragmas --- mmtk/src/abi.rs | 34 ---------------------------------- mmtk/src/lib.rs | 16 ---------------- mmtk/src/object_model.rs | 8 -------- mmtk/src/object_scanning.rs | 15 --------------- mmtk/src/reference_glue.rs | 10 ---------- mmtk/src/scanning.rs | 4 ---- 6 files changed, 87 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 989a5263..3f39e9e1 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -85,27 +85,22 @@ impl Klass { pub const LH_LOG2_ELEMENT_SIZE_MASK: i32 = BITS_IN_LONG as i32 - 1; pub const LH_HEADER_SIZE_SHIFT: i32 = BITS_IN_BYTE as i32 * 2; pub const LH_HEADER_SIZE_MASK: i32 = (1 << BITS_IN_BYTE) - 1; - #[inline(always)] pub unsafe fn cast<'a, T>(&self) -> &'a T { &*(self as *const Self as *const T) } /// Force slow-path for instance size calculation? - #[inline(always)] const fn layout_helper_needs_slow_path(lh: i32) -> bool { (lh & Self::LH_INSTANCE_SLOW_PATH_BIT) != 0 } /// Get log2 array element size - #[inline(always)] const fn layout_helper_log2_element_size(lh: i32) -> i32 { (lh >> Self::LH_LOG2_ELEMENT_SIZE_SHIFT) & Self::LH_LOG2_ELEMENT_SIZE_MASK } /// Get array header size - #[inline(always)] const fn layout_helper_header_size(lh: i32) -> i32 { (lh >> Self::LH_HEADER_SIZE_SHIFT) & Self::LH_HEADER_SIZE_MASK } - #[inline(always)] pub const fn is_instance_klass(&self) -> bool { self.layout_helper > Self::LH_NEUTRAL_VALUE } @@ -182,29 +177,24 @@ impl InstanceKlass { const HEADER_SIZE: usize = mem::size_of::() / BYTES_IN_WORD; const VTABLE_START_OFFSET: usize = Self::HEADER_SIZE * BYTES_IN_WORD; const MISC_IS_ANONYMOUS: u16 = 1 << 5; - #[inline(always)] fn start_of_vtable(&self) -> *const usize { unsafe { (self as *const Self as *const u8).add(Self::VTABLE_START_OFFSET) as _ } } - #[inline(always)] fn start_of_itable(&self) -> *const usize { unsafe { self.start_of_vtable().add(self.klass.vtable_len as _) } } - #[inline(always)] fn nonstatic_oop_map_count(&self) -> usize { let oop_map_block_size = mem::size_of::(); let oop_map_block_size_up = mmtk::util::conversions::raw_align_up(oop_map_block_size, BYTES_IN_WORD); self.nonstatic_oop_map_size as usize / (oop_map_block_size_up >> LOG_BYTES_IN_WORD) } - #[inline(always)] pub fn nonstatic_oop_maps(&self) -> &'static [OopMapBlock] { let start_of_itable = self.start_of_itable(); let start = unsafe { start_of_itable.add(self.itable_len as _) as *const OopMapBlock }; let count = self.nonstatic_oop_map_count(); unsafe { slice::from_raw_parts(start, count) } } - #[inline(always)] pub fn is_anonymous(&self) -> bool { self.misc_flags & Self::MISC_IS_ANONYMOUS != 0 } @@ -216,7 +206,6 @@ pub struct InstanceMirrorKlass { } impl InstanceMirrorKlass { - #[inline(always)] fn offset_of_static_fields() -> usize { lazy_static! { pub static ref OFFSET_OF_STATIC_FIELDS: usize = @@ -224,7 +213,6 @@ impl InstanceMirrorKlass { } *OFFSET_OF_STATIC_FIELDS } - #[inline(always)] fn static_oop_field_count_offset() -> i32 { lazy_static! { pub static ref STATIC_OOP_FIELD_COUNT_OFFSET: i32 = @@ -232,11 +220,9 @@ impl InstanceMirrorKlass { } *STATIC_OOP_FIELD_COUNT_OFFSET } - #[inline(always)] pub fn start_of_static_fields(oop: Oop) -> Address { Address::from_ref(oop) + Self::offset_of_static_fields() } - #[inline(always)] pub fn static_oop_field_count(oop: Oop) -> usize { let offset = Self::static_oop_field_count_offset(); unsafe { oop.get_field_address(offset).load::() as _ } @@ -275,25 +261,21 @@ pub struct InstanceRefKlass { } impl InstanceRefKlass { - #[inline(always)] fn referent_offset() -> i32 { lazy_static! { pub static ref REFERENT_OFFSET: i32 = unsafe { ((*UPCALLS).referent_offset)() }; } *REFERENT_OFFSET } - #[inline(always)] fn discovered_offset() -> i32 { lazy_static! { pub static ref DISCOVERED_OFFSET: i32 = unsafe { ((*UPCALLS).discovered_offset)() }; } *DISCOVERED_OFFSET } - #[inline(always)] pub fn referent_address(oop: Oop) -> OpenJDKEdge { OpenJDKEdge(oop.get_field_address(Self::referent_offset())) } - #[inline(always)] pub fn discovered_address(oop: Oop) -> OpenJDKEdge { OpenJDKEdge(oop.get_field_address(Self::discovered_offset())) } @@ -312,11 +294,9 @@ pub struct OopDesc { } impl OopDesc { - #[inline(always)] pub fn start(&self) -> Address { unsafe { mem::transmute(self) } } - #[inline(always)] pub(crate) fn klass_ptr(&self) -> Address { // self.klass if COMPRESSED { @@ -333,7 +313,6 @@ impl OopDesc { unsafe { Address::from_ref(self.klass.klass) } } } - #[inline(always)] pub fn klass(&self) -> &'static Klass { // self.klass if COMPRESSED { @@ -375,38 +354,32 @@ impl NarrowOop { /// Convert ObjectReference to Oop impl From for &OopDesc { - #[inline(always)] fn from(o: ObjectReference) -> Self { unsafe { mem::transmute(o) } } } impl Into for &OopDesc { - #[inline(always)] fn into(self) -> ObjectReference { unsafe { mem::transmute::<&OopDesc, _>(self) } } } impl OopDesc { - #[inline(always)] pub unsafe fn as_array_oop(&self) -> ArrayOop { &*(self as *const OopDesc as *const ArrayOopDesc) } - #[inline(always)] pub fn get_field_address(&self, offset: i32) -> Address { Address::from_ref(self) + offset as isize } /// Slow-path for calculating object instance size - #[inline(always)] unsafe fn size_slow(&self) -> usize { ((*UPCALLS).get_object_size)(self.into()) } /// Calculate object instance size - #[inline(always)] pub unsafe fn size(&self) -> usize { let klass = self.klass::(); let lh = klass.layout_helper; @@ -452,7 +425,6 @@ impl ArrayOopDesc { ty == BasicType::T_DOUBLE || ty == BasicType::T_LONG } - #[inline(always)] fn header_size(ty: BasicType) -> usize { let typesize_in_bytes = conversions::raw_align_up( Self::length_offset::() + BYTES_IN_INT, @@ -464,14 +436,12 @@ impl ArrayOopDesc { typesize_in_bytes / BYTES_IN_WORD } } - #[inline(always)] fn length(&self) -> i32 { unsafe { *((self as *const Self as *const u8).add(Self::length_offset::()) as *const i32) } } - #[inline(always)] fn base(&self, ty: BasicType) -> Address { let base_offset_in_bytes = Self::header_size::(ty) * BYTES_IN_WORD; Address::from_ptr(unsafe { (self as *const Self as *const u8).add(base_offset_in_bytes) }) @@ -480,14 +450,12 @@ impl ArrayOopDesc { // is Java types, so we have to map Java types to Rust types. The caller needs to guarantee: // 1. matches the actual Java type // 2. matches the argument, BasicType `ty` - #[inline(always)] pub unsafe fn data(&self, ty: BasicType) -> &[T] { slice::from_raw_parts( self.base::(ty).to_ptr(), self.length::() as _, ) } - #[inline(always)] pub unsafe fn slice(&self, ty: BasicType) -> crate::OpenJDKEdgeRange { let base = self.base::(ty); let start = crate::OpenJDKEdge(base); @@ -590,7 +558,6 @@ pub struct ClassLoaderData { } impl ClassLoaderData { - #[inline] fn claim(&self) -> bool { if self.claimed.load(Ordering::Relaxed) == 1 { return false; @@ -600,7 +567,6 @@ impl ClassLoaderData { .is_ok() } - #[inline] pub fn oops_do, const COMPRESSED: bool>(&self, closure: &mut V) { if closure.should_claim_clds() && !self.claim() { return; diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index e6de2814..ea9c34ad 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -128,7 +128,6 @@ lazy_static! { unsafe { ((*UPCALLS).java_lang_classloader_loader_data_offset)() }; } -#[inline(always)] pub fn current_worker() -> &'static mut GCWorker { GCWorker::::current() } @@ -183,17 +182,14 @@ fn init_compressed_oop_constants() { } } -#[inline(always)] fn use_compressed_oops() -> bool { unsafe { USE_COMPRESSED_OOPS } } -#[inline(always)] fn log_bytes_in_field() -> usize { unsafe { LOG_BYTES_IN_FIELD } } -#[inline(always)] fn bytes_in_field() -> usize { unsafe { BYTES_IN_FIELD } } @@ -259,7 +255,6 @@ impl OpenJDKEdge { impl Edge for OpenJDKEdge { /// Load object reference from the edge. - #[inline(always)] fn load(&self) -> ObjectReference { if COMPRESSED { let slot = self.untagged_address(); @@ -274,7 +269,6 @@ impl Edge for OpenJDKEdge { } /// Store the object reference `object` into the edge. - #[inline(always)] fn store(&self, object: ObjectReference) { if COMPRESSED { let slot = self.untagged_address(); @@ -320,12 +314,10 @@ impl Edge for OpenJDKEdge { } } - #[inline(always)] fn to_address(&self) -> Address { self.untagged_address() } - #[inline(always)] fn from_address(a: Address) -> Self { Self(a) } @@ -347,7 +339,6 @@ pub struct AddressRangeIterator { impl Iterator for AddressRangeIterator { type Item = OpenJDKEdge; - #[inline] fn next(&mut self) -> Option { if self.cursor >= self.limit { None @@ -368,7 +359,6 @@ pub struct ChunkIterator { impl Iterator for ChunkIterator { type Item = OpenJDKEdgeRange; - #[inline] fn next(&mut self) -> Option { if self.cursor >= self.limit { None @@ -392,7 +382,6 @@ impl MemorySlice for OpenJDKEdgeRange { type EdgeIterator = AddressRangeIterator; type ChunkIterator = ChunkIterator; - #[inline] fn iter_edges(&self) -> Self::EdgeIterator { AddressRangeIterator { cursor: self.start.0, @@ -401,7 +390,6 @@ impl MemorySlice for OpenJDKEdgeRange { } } - #[inline] fn chunks(&self, chunk_size: usize) -> Self::ChunkIterator { ChunkIterator { cursor: self.start.0, @@ -410,22 +398,18 @@ impl MemorySlice for OpenJDKEdgeRange { } } - #[inline] fn start(&self) -> Address { self.start.0 } - #[inline] fn bytes(&self) -> usize { self.end.0 - self.start.0 } - #[inline] fn len(&self) -> usize { (self.end.0 - self.start.0) >> crate::log_bytes_in_field() } - #[inline] fn copy(src: &Self, tgt: &Self) { debug_assert_eq!(src.bytes(), tgt.bytes()); debug_assert_eq!( diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index b217fe5a..6f336a92 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -23,7 +23,6 @@ impl ObjectModel for VMObjectModel { const UNIFIED_OBJECT_REFERENCE_ADDRESS: bool = true; const OBJECT_REF_OFFSET_LOWER_BOUND: isize = 0; - #[inline(always)] fn copy( from: ObjectReference, copy: CopySemantics, @@ -66,7 +65,6 @@ impl ObjectModel for VMObjectModel { ObjectReference::from_raw_address(to) } - #[inline(always)] fn get_current_size(object: ObjectReference) -> usize { if crate::use_compressed_oops() { unsafe { Oop::from(object).size::() } @@ -92,22 +90,18 @@ impl ObjectModel for VMObjectModel { unimplemented!() } - #[inline(always)] fn ref_to_object_start(object: ObjectReference) -> Address { object.to_raw_address() } - #[inline(always)] fn ref_to_address(object: ObjectReference) -> Address { object.to_raw_address() } - #[inline(always)] fn ref_to_header(object: ObjectReference) -> Address { object.to_raw_address() } - #[inline(always)] fn address_to_ref(address: Address) -> ObjectReference { ObjectReference::from_raw_address(address) } @@ -128,12 +122,10 @@ impl ObjectModel for VMObjectModel { s.to_string() } - #[inline(always)] fn compressed_pointers_enabled() -> bool { crate::use_compressed_oops() } - #[inline(always)] fn get_class_pointer(object: ObjectReference) -> Address { let oop: Oop = unsafe { std::mem::transmute(object) }; if crate::use_compressed_oops() { diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 918f662d..3257ea11 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -18,7 +18,6 @@ trait OopIterate: Sized { } impl OopIterate for OopMapBlock { - #[inline(always)] fn oop_iterate, const COMPRESSED: bool>( &self, oop: Oop, @@ -34,7 +33,6 @@ impl OopIterate for OopMapBlock { } impl OopIterate for InstanceKlass { - #[inline(always)] fn oop_iterate, const COMPRESSED: bool>( &self, oop: Oop, @@ -51,7 +49,6 @@ impl OopIterate for InstanceKlass { } impl OopIterate for InstanceMirrorKlass { - #[inline] fn oop_iterate, const COMPRESSED: bool>( &self, oop: Oop, @@ -95,7 +92,6 @@ impl OopIterate for InstanceMirrorKlass { } impl OopIterate for InstanceClassLoaderKlass { - #[inline] fn oop_iterate, const COMPRESSED: bool>( &self, oop: Oop, @@ -116,7 +112,6 @@ impl OopIterate for InstanceClassLoaderKlass { } impl OopIterate for ObjArrayKlass { - #[inline(always)] fn oop_iterate, const COMPRESSED: bool>( &self, oop: Oop, @@ -139,7 +134,6 @@ impl OopIterate for ObjArrayKlass { } impl OopIterate for TypeArrayKlass { - #[inline(always)] fn oop_iterate, const COMPRESSED: bool>( &self, _oop: Oop, @@ -151,7 +145,6 @@ impl OopIterate for TypeArrayKlass { } impl OopIterate for InstanceRefKlass { - #[inline(always)] fn oop_iterate, const COMPRESSED: bool>( &self, oop: Oop, @@ -181,7 +174,6 @@ impl OopIterate for InstanceRefKlass { } impl InstanceRefKlass { - #[inline] fn should_discover_refs(mut rt: ReferenceType, disable_discovery: bool) -> bool { if rt == ReferenceType::Other { rt = ReferenceType::Weak; @@ -197,14 +189,12 @@ impl InstanceRefKlass { } true } - #[inline] fn process_ref_as_strong(oop: Oop, closure: &mut impl EdgeVisitor) { let referent_addr = Self::referent_address(oop); closure.visit_edge(referent_addr); let discovered_addr = Self::discovered_address(oop); closure.visit_edge(discovered_addr); } - #[inline] fn discover_reference(oop: Oop, rt: ReferenceType) -> bool { // Do not discover new refs during reference processing. if !DISCOVERED_LISTS.allow_discover() { @@ -247,7 +237,6 @@ fn oop_iterate_slow>(oop: Oop, closure: &mut V, tls: } } -#[inline(always)] fn oop_iterate, const COMPRESSED: bool>(oop: Oop, closure: &mut V) { let klass_id = oop.klass::().id; debug_assert!( @@ -304,17 +293,14 @@ fn do_klass, const COMPRESSED: bool>(klass: &Klass, do_cld::<_, COMPRESSED>(&klass.class_loader_data, closure) } -#[inline(always)] pub fn is_obj_array(oop: Oop) -> bool { oop.klass::().id == KlassID::ObjArray } -#[inline(always)] pub fn is_val_array(oop: Oop) -> bool { oop.klass::().id == KlassID::TypeArray } -#[inline(always)] pub fn obj_array_data(oop: Oop) -> crate::OpenJDKEdgeRange { unsafe { let array = oop.as_array_oop(); @@ -332,7 +318,6 @@ pub unsafe extern "C" fn scan_object_fn>(edge: Addre closure.visit_edge(OpenJDKEdge(edge)); } -#[inline] pub fn scan_object, const COMPRESSED: bool>( object: ObjectReference, closure: &mut C, diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 4fb3ecc7..f625411e 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -12,7 +12,6 @@ use mmtk::vm::edge_shape::Edge; use mmtk::vm::ReferenceGlue; use mmtk::MMTK; -#[inline(always)] pub fn set_referent(reff: ObjectReference, referent: ObjectReference) { let oop = Oop::from(reff); let slot = InstanceRefKlass::referent_address(oop); @@ -20,24 +19,20 @@ pub fn set_referent(reff: ObjectReference, referent: Obj slot.store::(referent) } -#[inline(always)] fn get_referent(object: ObjectReference) -> ObjectReference { let oop = Oop::from(object); InstanceRefKlass::referent_address(oop).load::() } -#[inline(always)] fn get_next_reference_slot(object: ObjectReference) -> crate::OpenJDKEdge { let oop = Oop::from(object); InstanceRefKlass::discovered_address(oop) } -#[inline(always)] fn get_next_reference(object: ObjectReference) -> ObjectReference { get_next_reference_slot(object).load::() } -#[inline(always)] fn set_next_reference(object: ObjectReference, next: ObjectReference) { let slot = get_next_reference_slot(object); mmtk::plan::lxr::record_edge_for_validation(slot, next); @@ -71,7 +66,6 @@ impl DiscoveredList { } } - #[inline] pub fn add( &self, reference: ObjectReference, @@ -141,17 +135,14 @@ impl DiscoveredLists { } } - #[inline] pub fn enable_discover(&self) { self.allow_discover.store(true, Ordering::SeqCst) } - #[inline(always)] pub fn allow_discover(&self) -> bool { self.allow_discover.load(Ordering::SeqCst) } - #[inline(always)] pub fn get_by_rt_and_index(&self, rt: ReferenceType, index: usize) -> &DiscoveredList { match rt { ReferenceType::Soft => &self.soft[index], @@ -162,7 +153,6 @@ impl DiscoveredLists { } } - #[inline(always)] pub fn get(&self, rt: ReferenceType) -> &DiscoveredList { let id = mmtk::scheduler::worker::current_worker_ordinal().unwrap(); self.get_by_rt_and_index(rt, id) diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index bb3d2d19..809b7c5a 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -44,7 +44,6 @@ impl Scanning for VMScanning { const SCAN_MUTATORS_IN_SAFEPOINT: bool = false; const SINGLE_THREAD_MUTATOR_SCANNING: bool = false; - #[inline] fn scan_object, const COMPRESSED: bool>( tls: VMWorkerThread, object: ObjectReference, @@ -53,17 +52,14 @@ impl Scanning for VMScanning { crate::object_scanning::scan_object::(object, edge_visitor, tls); } - #[inline(always)] fn obj_array_data(o: ObjectReference) -> crate::OpenJDKEdgeRange { crate::object_scanning::obj_array_data::(unsafe { std::mem::transmute(o) }) } - #[inline(always)] fn is_obj_array(o: ObjectReference) -> bool { crate::object_scanning::is_obj_array::(unsafe { std::mem::transmute(o) }) } - #[inline(always)] fn is_val_array(o: ObjectReference) -> bool { crate::object_scanning::is_val_array::(unsafe { std::mem::transmute(o) }) } From 285cb3d501cec97ba568f9e734a17dc391f37a4c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 10 Feb 2023 20:29:17 +0000 Subject: [PATCH 263/494] Abandon partial reference discovery --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/collection.rs | 8 ++++++++ mmtk/src/reference_glue.rs | 28 ++++++++++++++++++++++------ 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 2cb3ce36..b762c3b1 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a5c96367d1cff4dcd8912165639c2e0d42a303e9#a5c96367d1cff4dcd8912165639c2e0d42a303e9" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ab97b107e793ade723e7078c2a2d13da3503604a#ab97b107e793ade723e7078c2a2d13da3503604a" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a5c96367d1cff4dcd8912165639c2e0d42a303e9#a5c96367d1cff4dcd8912165639c2e0d42a303e9" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ab97b107e793ade723e7078c2a2d13da3503604a#ab97b107e793ade723e7078c2a2d13da3503604a" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index e6bef36f..009d70db 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "a5c96367d1cff4dcd8912165639c2e0d42a303e9" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "ab97b107e793ade723e7078c2a2d13da3503604a" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index f7cd5297..a0d82b8e 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -137,4 +137,12 @@ impl Collection for VMCollection { fn set_concurrent_marking_state(active: bool) { unsafe { crate::CONCURRENT_MARKING_ACTIVE = if active { 1 } else { 0 } } } + + fn abandon_partial_discovery() { + if crate::use_compressed_oops() { + DISCOVERED_LISTS.abandon_partial_discovery::() + } else { + DISCOVERED_LISTS.abandon_partial_discovery::() + } + } } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 4fb3ecc7..75f35c55 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -118,6 +118,16 @@ impl DiscoveredList { debug_assert!(!get_next_reference::(reference).is_null()); } } + + fn abandon_partial_discovery(&self) { + let mut reference = self.head.load(Ordering::SeqCst); + while !reference.is_null() { + let next_ref = get_next_reference::(reference); + set_next_reference::(reference, ObjectReference::NULL); + reference = next_ref; + } + self.head.store(ObjectReference::NULL, Ordering::SeqCst); + } } pub struct DiscoveredLists { @@ -248,6 +258,12 @@ impl DiscoveredLists { assert!(!*SINGLETON.get_options().no_reference_types); self.process_lists::(worker, ReferenceType::Phantom, &self.phantom, true); } + + pub fn abandon_partial_discovery(&self) { + for list in &self.r#final { + list.abandon_partial_discovery::(); + } + } } lazy_static! { @@ -297,13 +313,13 @@ impl, const COMPRESSED: bool> GCWork if let Some((head, tail)) = new_list { debug_assert!(!head.is_null() && !tail.is_null()); // debug_assert_eq!(ObjectReference::NULL, get_next_reference(tail)); - if self.rt == ReferenceType::Final { - DISCOVERED_LISTS.r#final[self.list_index] - .head - .store(head, Ordering::SeqCst); - } else { + if self.rt != ReferenceType::Final { let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; set_next_reference::(tail, old_head); + } else { + DISCOVERED_LISTS.r#final[self.list_index] + .head + .store(head, Ordering::SeqCst) } } else { if self.rt == ReferenceType::Final { @@ -342,7 +358,7 @@ impl, const COMPRESSED: bool> GCWork // debug_assert_eq!(ObjectReference::NULL, get_next_reference(tail)); DISCOVERED_LISTS.r#final[self.list_index] .head - .store(ObjectReference::NULL, Ordering::SeqCst); + .store(head, Ordering::SeqCst); let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; set_next_reference::(tail, old_head); } From 07e2650136852d311cac4c28562b079b0b9ac60a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 11 Feb 2023 21:12:13 +0000 Subject: [PATCH 264/494] wip: fix soft ref processing --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/collection.rs | 8 ------- mmtk/src/object_scanning.rs | 3 +++ mmtk/src/reference_glue.rs | 47 +++++++++++++++++-------------------- 5 files changed, 28 insertions(+), 36 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index b762c3b1..37686dd7 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ab97b107e793ade723e7078c2a2d13da3503604a#ab97b107e793ade723e7078c2a2d13da3503604a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=9308035f0ce47bde0e66932178a57dcdc7d1a755#9308035f0ce47bde0e66932178a57dcdc7d1a755" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ab97b107e793ade723e7078c2a2d13da3503604a#ab97b107e793ade723e7078c2a2d13da3503604a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=9308035f0ce47bde0e66932178a57dcdc7d1a755#9308035f0ce47bde0e66932178a57dcdc7d1a755" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 009d70db..ddf1972e 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "ab97b107e793ade723e7078c2a2d13da3503604a" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "9308035f0ce47bde0e66932178a57dcdc7d1a755" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index a0d82b8e..f7cd5297 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -137,12 +137,4 @@ impl Collection for VMCollection { fn set_concurrent_marking_state(active: bool) { unsafe { crate::CONCURRENT_MARKING_ACTIVE = if active { 1 } else { 0 } } } - - fn abandon_partial_discovery() { - if crate::use_compressed_oops() { - DISCOVERED_LISTS.abandon_partial_discovery::() - } else { - DISCOVERED_LISTS.abandon_partial_discovery::() - } - } } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 918f662d..0b51ada4 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -225,6 +225,9 @@ impl InstanceRefKlass { { return false; } + if rt == ReferenceType::Final && DISCOVERED_LISTS.is_discovered::(reference) { + return false; + } // Add to reference list DISCOVERED_LISTS .get(rt) diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 75f35c55..8e708294 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -113,21 +113,11 @@ impl DiscoveredList { ) .is_ok() { - self.head.store(reference, Ordering::Relaxed); + self.head.store(reference, Ordering::SeqCst); } debug_assert!(!get_next_reference::(reference).is_null()); } } - - fn abandon_partial_discovery(&self) { - let mut reference = self.head.load(Ordering::SeqCst); - while !reference.is_null() { - let next_ref = get_next_reference::(reference); - set_next_reference::(reference, ObjectReference::NULL); - reference = next_ref; - } - self.head.store(ObjectReference::NULL, Ordering::SeqCst); - } } pub struct DiscoveredLists { @@ -161,6 +151,10 @@ impl DiscoveredLists { self.allow_discover.load(Ordering::SeqCst) } + pub fn is_discovered(&self, reference: ObjectReference) -> bool { + !get_next_reference::(reference).is_null() + } + #[inline(always)] pub fn get_by_rt_and_index(&self, rt: ReferenceType, index: usize) -> &DiscoveredList { match rt { @@ -258,12 +252,6 @@ impl DiscoveredLists { assert!(!*SINGLETON.get_options().no_reference_types); self.process_lists::(worker, ReferenceType::Phantom, &self.phantom, true); } - - pub fn abandon_partial_discovery(&self) { - for list in &self.r#final { - list.abandon_partial_discovery::(); - } - } } lazy_static! { @@ -285,7 +273,7 @@ impl, const COMPRESSED: bool> GCWork let mut trace = E::new(vec![], false, mmtk); trace.set_worker(worker); let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); - let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { + let new_list = iterate_list::<_, COMPRESSED>(self.head, true, |reference| { let reference = trace.trace_object(reference); let referent = get_referent::(reference); if referent.is_null() { @@ -313,13 +301,13 @@ impl, const COMPRESSED: bool> GCWork if let Some((head, tail)) = new_list { debug_assert!(!head.is_null() && !tail.is_null()); // debug_assert_eq!(ObjectReference::NULL, get_next_reference(tail)); - if self.rt != ReferenceType::Final { - let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; - set_next_reference::(tail, old_head); - } else { + if self.rt == ReferenceType::Final { DISCOVERED_LISTS.r#final[self.list_index] .head - .store(head, Ordering::SeqCst) + .store(head, Ordering::SeqCst); + } else { + let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; + set_next_reference::(tail, old_head); } } else { if self.rt == ReferenceType::Final { @@ -344,7 +332,7 @@ impl, const COMPRESSED: bool> GCWork fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { let mut trace = E::new(vec![], false, mmtk); trace.set_worker(worker); - let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { + let new_list = iterate_list::<_, COMPRESSED>(self.head, false, |reference| { let reference = trace.trace_object(reference); let referent = get_referent::(reference); let forwarded = trace.trace_object(referent); @@ -358,7 +346,7 @@ impl, const COMPRESSED: bool> GCWork // debug_assert_eq!(ObjectReference::NULL, get_next_reference(tail)); DISCOVERED_LISTS.r#final[self.list_index] .head - .store(head, Ordering::SeqCst); + .store(ObjectReference::NULL, Ordering::SeqCst); let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; set_next_reference::(tail, old_head); } @@ -376,6 +364,7 @@ fn iterate_list< const COMPRESSED: bool, >( head: ObjectReference, + no_null: bool, mut visitor: F, ) -> Option<(ObjectReference, ObjectReference)> { let mut new_head: Option = None; @@ -395,6 +384,14 @@ fn iterate_list< let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); debug_assert!(next_ref.get_forwarded_object().is_none()); // Reaches the end of the list? + if no_null { + debug_assert!( + !next_ref.is_null(), + "Invalid ref {:?} next={:?}", + reference, + next_ref + ); + } let end_of_list = next_ref == reference || next_ref.is_null(); // Process reference let result = visitor(reference); From 5abc257d2d452bac4b66d993f03b96beb6b29baa Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 12 Feb 2023 05:43:12 +0000 Subject: [PATCH 265/494] Fix soft ref processing --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/reference_glue.rs | 30 ++++++++++++++++++------------ 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 37686dd7..b6ec4f1c 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -453,7 +453,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=9308035f0ce47bde0e66932178a57dcdc7d1a755#9308035f0ce47bde0e66932178a57dcdc7d1a755" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=aaef4f8db0e278e10de394292558c5e6e408545b#aaef4f8db0e278e10de394292558c5e6e408545b" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=9308035f0ce47bde0e66932178a57dcdc7d1a755#9308035f0ce47bde0e66932178a57dcdc7d1a755" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=aaef4f8db0e278e10de394292558c5e6e408545b#aaef4f8db0e278e10de394292558c5e6e408545b" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index ddf1972e..44c353f2 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "9308035f0ce47bde0e66932178a57dcdc7d1a755" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "aaef4f8db0e278e10de394292558c5e6e408545b" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 8e708294..20b7211c 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -115,7 +115,15 @@ impl DiscoveredList { { self.head.store(reference, Ordering::SeqCst); } - debug_assert!(!get_next_reference::(reference).is_null()); + debug_assert!( + !get_next_reference::(reference).is_null(), + "discover ref {:?}->{:?} next={:?} real-next={:?} bin={:?}", + reference, + referent, + next_discovered, + get_next_reference::(reference), + mmtk::scheduler::worker::current_worker_ordinal().unwrap() + ); } } } @@ -273,7 +281,14 @@ impl, const COMPRESSED: bool> GCWork let mut trace = E::new(vec![], false, mmtk); trace.set_worker(worker); let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); - let new_list = iterate_list::<_, COMPRESSED>(self.head, true, |reference| { + let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { + debug_assert!( + !get_next_reference::(reference).is_null(), + "next can't be null. ref={:?} {:?} bin={}", + reference, + self.rt, + self.list_index + ); let reference = trace.trace_object(reference); let referent = get_referent::(reference); if referent.is_null() { @@ -332,7 +347,7 @@ impl, const COMPRESSED: bool> GCWork fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { let mut trace = E::new(vec![], false, mmtk); trace.set_worker(worker); - let new_list = iterate_list::<_, COMPRESSED>(self.head, false, |reference| { + let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { let reference = trace.trace_object(reference); let referent = get_referent::(reference); let forwarded = trace.trace_object(referent); @@ -364,7 +379,6 @@ fn iterate_list< const COMPRESSED: bool, >( head: ObjectReference, - no_null: bool, mut visitor: F, ) -> Option<(ObjectReference, ObjectReference)> { let mut new_head: Option = None; @@ -384,14 +398,6 @@ fn iterate_list< let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); debug_assert!(next_ref.get_forwarded_object().is_none()); // Reaches the end of the list? - if no_null { - debug_assert!( - !next_ref.is_null(), - "Invalid ref {:?} next={:?}", - reference, - next_ref - ); - } let end_of_list = next_ref == reference || next_ref.is_null(); // Process reference let result = visitor(reference); From 24d0465ffedf7457abf5f47fdb25e56fe349d6c4 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 14 Feb 2023 02:42:19 +0000 Subject: [PATCH 266/494] Scan modified CLDs --- openjdk/mmtkHeap.cpp | 4 ++++ openjdk/mmtkRootsClosure.hpp | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index efcd41e0..51b25f80 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -49,6 +49,7 @@ #include "services/memTracker.hpp" #include "utilities/vmError.hpp" #include "mmtkParallelCleaning.hpp" +#include "mmtkRootsClosure.hpp" /* needed support from rust heap capacity @@ -433,6 +434,9 @@ void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& we CLDToOopClosure cld_cl(&cl, false); CLDToOopClosure weak_cld_cl(&weak_cl, false); ClassLoaderDataGraph::roots_cld_do(&cld_cl, ClassUnloading && !scan_weak ? NULL : &weak_cld_cl); + // Scan modified CLDs + MMTkScanCLDClosure cld_closure(&cl); + ClassLoaderDataGraph::cld_do(&cld_closure); } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { ShouldNotReachHere(); diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index a059e2f6..023b73e3 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -56,6 +56,19 @@ class MMTkRootsClosure2 : public OopClosure { virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } }; +class MMTkScanCLDClosure: public CLDClosure { + private: + OopClosure* _oop_closure; + protected: + public: + MMTkScanCLDClosure(OopClosure* c) : _oop_closure(c) { } + void do_cld(ClassLoaderData* cld) { + if (cld->has_modified_oops()) { + cld->oops_do(_oop_closure, false, /*clear_modified_oops*/true); + } + } +}; + class MMTkScanObjectClosure : public BasicOopIterateClosure { void (*_trace)(void*); bool _follow_clds; From 6b9e2b13231cacec53541ea79cc3710fdef6e7dd Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 15 Feb 2023 19:37:20 +0000 Subject: [PATCH 267/494] WIP: Fix openjdk weak handles --- mmtk/Cargo.lock | 25 ++++++----------- mmtk/Cargo.toml | 2 +- mmtk/src/gc_work.rs | 52 ++++++++++++++++++++++++++++++++++++ mmtk/src/scanning.rs | 1 + openjdk/mmtkHeap.cpp | 3 ++- openjdk/mmtkRootsClosure.hpp | 45 +++++++++++++++++++++++++++++++ openjdk/mmtkUpcalls.cpp | 8 ++---- 7 files changed, 111 insertions(+), 25 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index b6ec4f1c..d9eeb626 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -305,12 +305,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "heck" -version = "0.3.3" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -453,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=aaef4f8db0e278e10de394292558c5e6e408545b#aaef4f8db0e278e10de394292558c5e6e408545b" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e34dce39a4afaa057d8a28be4b708d66def60d04#e34dce39a4afaa057d8a28be4b708d66def60d04" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=aaef4f8db0e278e10de394292558c5e6e408545b#aaef4f8db0e278e10de394292558c5e6e408545b" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e34dce39a4afaa057d8a28be4b708d66def60d04#e34dce39a4afaa057d8a28be4b708d66def60d04" dependencies = [ "proc-macro-error", "proc-macro2", @@ -749,15 +746,15 @@ checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" [[package]] name = "strum" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" [[package]] name = "strum_macros" -version = "0.23.1" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ "heck", "proc-macro2", @@ -866,12 +863,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-segmentation" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" - [[package]] name = "unicode-width" version = "0.1.10" diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 44c353f2..dd146ab8 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "aaef4f8db0e278e10de394292558c5e6e408545b" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e34dce39a4afaa057d8a28be4b708d66def60d04" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 0d621b6b..eff6e58f 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -4,6 +4,7 @@ use crate::EdgesClosure; use crate::NewBuffer; use crate::{OpenJDK, OpenJDKEdge, UPCALLS}; use mmtk::scheduler::*; +use mmtk::util::ObjectReference; use mmtk::vm::RootsWorkFactory; use mmtk::MMTK; @@ -129,3 +130,54 @@ impl> GCWork for ScanCodeCacheRoots // } } } + +extern "C" fn report_edges_and_renew_buffer_weakref>( + ptr: *mut Address, + length: usize, + capacity: usize, + factory_ptr: *mut libc::c_void, +) -> NewBuffer { + if !ptr.is_null() { + let buf = unsafe { + Vec::::from_raw_parts(ptr as *mut ObjectReference, length, capacity) + }; + let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; + // println!("weakref nodes {:?}", buf); + factory.create_process_node_roots_work(buf); + } + let (ptr, _, capacity) = { + // TODO: Use Vec::into_raw_parts() when the method is available. + use std::mem::ManuallyDrop; + let new_vec = Vec::with_capacity(F::BUFFER_SIZE); + let mut me = ManuallyDrop::new(new_vec); + (me.as_mut_ptr(), me.len(), me.capacity()) + }; + NewBuffer { ptr, capacity } +} + +fn to_edges_closure_weakref>(factory: &mut F) -> EdgesClosure { + EdgesClosure { + func: report_edges_and_renew_buffer_weakref::, + data: factory as *mut F as *mut libc::c_void, + } +} + +pub struct ScaWeakProcessorRoots> { + factory: F, +} + +impl> ScaWeakProcessorRoots { + pub fn new(factory: F) -> Self { + Self { factory } + } +} + +impl> GCWork for ScaWeakProcessorRoots { + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { + unsafe { + ((*UPCALLS).scan_weak_processor_roots)(to_edges_closure_weakref::<_>( + &mut self.factory, + )); + } + } +} diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 809b7c5a..a2c341ad 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -101,6 +101,7 @@ impl Scanning for VMScanning { Box::new(ScanCodeCacheRoots::new(factory.clone())) as _, Box::new(ScanStringTableRoots::new(factory.clone())) as _, Box::new(ScanClassLoaderDataGraphRoots::new(factory.clone())) as _, + Box::new(ScaWeakProcessorRoots::new(factory.clone())) as _, ], ); if !(Self::SCAN_MUTATORS_IN_SAFEPOINT && Self::SINGLE_THREAD_MUTATOR_SCANNING) { diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 51b25f80..a2cdc936 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -439,7 +439,8 @@ void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& we ClassLoaderDataGraph::cld_do(&cld_closure); } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { - ShouldNotReachHere(); + ResourceMark rm; + WeakProcessor::oops_do(&cl); } void MMTkHeap::scan_vm_thread_roots(OopClosure& cl) { ResourceMark rm; diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 023b73e3..1a8a2929 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -56,6 +56,51 @@ class MMTkRootsClosure2 : public OopClosure { virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } }; +class MMTkCollectRootObjects : public OopClosure { + EdgesClosure _edges_closure; + void** _buffer; + size_t _cap; + size_t _cursor; + + template + void do_oop_work(T* p, bool narrow) { + T heap_oop = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(heap_oop)) { + _buffer[_cursor++] = (void*) CompressedOops::decode(heap_oop); + if (_cursor >= _cap) { + flush(); + } + } + } + + void flush() { + if (_cursor > 0) { + NewBuffer buf = _edges_closure.invoke(_buffer, _cursor, _cap); + _buffer = buf.buf; + _cap = buf.cap; + _cursor = 0; + } + } + +public: + + MMTkCollectRootObjects(EdgesClosure edges_closure): _edges_closure(edges_closure), _cursor(0) { + NewBuffer buf = edges_closure.invoke(NULL, 0, 0); + _buffer = buf.buf; + _cap = buf.cap; + } + + ~MMTkCollectRootObjects() { + if (_cursor > 0) flush(); + if (_buffer != NULL) { + release_buffer(_buffer, _cursor, _cap); + } + } + + virtual void do_oop(oop* p) { do_oop_work(p, false); } + virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } +}; + class MMTkScanCLDClosure: public CLDClosure { private: OopClosure* _oop_closure; diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index abfa0a39..bb21b6d7 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -180,11 +180,7 @@ static void mmtk_update_weak_processor(bool lxr) { MMTkForwardClosure forward; if (lxr) { MMTkLXRFastIsAliveClosure is_alive; - MMTkLXRFastUpdateClosure fast_update; - JNIHandles::weak_global_handles()->weak_oops_do(&fast_update); - JvmtiExport::weak_oops_do(&is_alive, &forward); - SystemDictionary::vm_weak_oop_storage()->weak_oops_do(&fast_update); - JFR_ONLY(Jfr::weak_oops_do(&is_alive, &forward);) + WeakProcessor::weak_oops_do(&is_alive, &forward); } else { MMTkIsAliveClosure is_alive; WeakProcessor::weak_oops_do(&is_alive, &forward); @@ -457,7 +453,7 @@ static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, EdgesC MMTkRootsClosure2 weak_cl(weak_closure); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl, weak_cl, scan_weak); } -static void mmtk_scan_weak_processor_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_weak_processor_roots(cl); } +static void mmtk_scan_weak_processor_roots(EdgesClosure closure) { MMTkCollectRootObjects cl(closure); MMTkHeap::heap()->scan_weak_processor_roots(cl); } static void mmtk_scan_vm_thread_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } static size_t mmtk_number_of_mutators() { From e27c3bece2f44e28df894ad43195689e1ca185c7 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 Feb 2023 13:41:14 -0800 Subject: [PATCH 268/494] Fix edge compare exchange when compressed pointer is enabled --- mmtk/src/lib.rs | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index ea9c34ad..316a2784 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -290,13 +290,28 @@ impl Edge for OpenJDKEdge { failure: Ordering, ) -> Result { if COMPRESSED { - let old_value = compress(old_object); - let new_value = compress(new_object); - let slot = self.untagged_address(); - unsafe { - match slot.compare_exchange::(old_value, new_value, success, failure) { - Ok(v) => Ok(decompress(v)), - Err(v) => Err(decompress(v)), + if self.is_compressed() { + let old_value = compress(old_object); + let new_value = compress(new_object); + let slot = self.untagged_address(); + unsafe { + match slot.compare_exchange::(old_value, new_value, success, failure) { + Ok(v) => Ok(decompress(v)), + Err(v) => Err(decompress(v)), + } + } + } else { + let slot = self.untagged_address(); + unsafe { + match slot.compare_exchange::( + old_object.to_address::().as_usize(), + new_object.to_address::().as_usize(), + success, + failure, + ) { + Ok(v) => Ok(ObjectReference::from_raw_address(Address::from_usize(v))), + Err(v) => Err(ObjectReference::from_raw_address(Address::from_usize(v))), + } } } } else { @@ -318,6 +333,10 @@ impl Edge for OpenJDKEdge { self.untagged_address() } + fn raw_address(&self) -> Address { + self.0 + } + fn from_address(a: Address) -> Self { Self(a) } From a9ce54d96ab9cd7ff55dbd59fa59284f148d8eff Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 Feb 2023 13:56:04 -0800 Subject: [PATCH 269/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index d9eeb626..d7bdce83 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e34dce39a4afaa057d8a28be4b708d66def60d04#e34dce39a4afaa057d8a28be4b708d66def60d04" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=fea095c029ef446df05095da0c3178697b502781#fea095c029ef446df05095da0c3178697b502781" dependencies = [ "atomic", "atomic-traits", @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e34dce39a4afaa057d8a28be4b708d66def60d04#e34dce39a4afaa057d8a28be4b708d66def60d04" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=fea095c029ef446df05095da0c3178697b502781#fea095c029ef446df05095da0c3178697b502781" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index dd146ab8..2e1cd617 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e34dce39a4afaa057d8a28be4b708d66def60d04" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "fea095c029ef446df05095da0c3178697b502781" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 4454e22901e9f61d456ddbbcc86e36352c2178ae Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 Feb 2023 13:57:30 -0800 Subject: [PATCH 270/494] cargo fmt; update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/lib.rs | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index d7bdce83..6a92a6a9 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=fea095c029ef446df05095da0c3178697b502781#fea095c029ef446df05095da0c3178697b502781" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=5902a205051a0765e5174ec454d1072369d149b6#5902a205051a0765e5174ec454d1072369d149b6" dependencies = [ "atomic", "atomic-traits", @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=fea095c029ef446df05095da0c3178697b502781#fea095c029ef446df05095da0c3178697b502781" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=5902a205051a0765e5174ec454d1072369d149b6#5902a205051a0765e5174ec454d1072369d149b6" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 2e1cd617..4b2c97c5 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "fea095c029ef446df05095da0c3178697b502781" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "5902a205051a0765e5174ec454d1072369d149b6" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 316a2784..d739a018 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -295,7 +295,8 @@ impl Edge for OpenJDKEdge { let new_value = compress(new_object); let slot = self.untagged_address(); unsafe { - match slot.compare_exchange::(old_value, new_value, success, failure) { + match slot.compare_exchange::(old_value, new_value, success, failure) + { Ok(v) => Ok(decompress(v)), Err(v) => Err(decompress(v)), } From fb7cbb168fdf575b74b754ebcec72970a10c741f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 2 Mar 2023 12:28:09 -0800 Subject: [PATCH 271/494] Fix C2 weakref load barrier --- openjdk/barriers/mmtkFieldBarrier.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 01f2fcf0..c9d8eae2 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -327,7 +327,8 @@ static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool em __ if_then(cm_flag, BoolTest::ne, zero, unlikely); { // No slow-call if dst is NULL __ if_then(val, BoolTest::ne, kit->null()); { - insert_write_barrier_common(ideal, val, slot, val); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", val); } __ end_if(); } __ end_if(); kit->sync_kit(ideal); @@ -451,14 +452,14 @@ Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_ } void MMTkFieldBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { - if (!is_array && dst != kit->just_allocated_object(kit->control())) { + // if (!is_array && dst != kit->just_allocated_object(kit->control())) { MMTkIdealKit ideal(kit); const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_clone_pre_call), "mmtk_barrier_call", dst); kit->sync_kit(ideal); kit->insert_mem_bar(Op_MemBarVolatile); kit->final_sync(ideal); - } + // } BarrierSetC2::clone(kit, src, dst, size, is_array); } From 030671378c6b1eca408528852acd57ec0f989a6d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 2 Mar 2023 12:29:01 -0800 Subject: [PATCH 272/494] update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 6a92a6a9..61458fda 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=5902a205051a0765e5174ec454d1072369d149b6#5902a205051a0765e5174ec454d1072369d149b6" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f9a1ee30e15f0817a552e9b054fb0e4ed195fabe#f9a1ee30e15f0817a552e9b054fb0e4ed195fabe" dependencies = [ "atomic", "atomic-traits", @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=5902a205051a0765e5174ec454d1072369d149b6#5902a205051a0765e5174ec454d1072369d149b6" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f9a1ee30e15f0817a552e9b054fb0e4ed195fabe#f9a1ee30e15f0817a552e9b054fb0e4ed195fabe" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 4b2c97c5..8fbc2294 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "5902a205051a0765e5174ec454d1072369d149b6" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f9a1ee30e15f0817a552e9b054fb0e4ed195fabe" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 7c93757a062002061964603a36e23f0b59041d35 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 2 Mar 2023 14:26:43 -0800 Subject: [PATCH 273/494] Filter out invalid values in C2 write barrier --- openjdk/mmtkBarrierSetC2.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openjdk/mmtkBarrierSetC2.hpp b/openjdk/mmtkBarrierSetC2.hpp index d8052d9a..28949434 100644 --- a/openjdk/mmtkBarrierSetC2.hpp +++ b/openjdk/mmtkBarrierSetC2.hpp @@ -54,7 +54,10 @@ class MMTkBarrierSetC2: public BarrierSetC2 { virtual void object_reference_write_post(GraphKit* kit, Node* src, Node* slot, Node* val) const {} virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const { - if (access.is_oop()) object_reference_write_pre(access.kit(), access.base(), access.addr().node(), val.node()); + DecoratorSet decorators = access.decorators(); + bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; + bool in_heap = (decorators & IN_HEAP) != 0; + if (access.is_oop() && (in_heap || anonymous)) object_reference_write_pre(access.kit(), access.base(), access.addr().node(), val.node()); Node* store = BarrierSetC2::store_at_resolved(access, val); if (access.is_oop()) object_reference_write_post(access.kit(), access.base(), access.addr().node(), val.node()); return store; From 2b45e86d1b19b0166c4b6e0c8cd4e0cbc5c2c85e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 7 Mar 2023 10:44:12 -0800 Subject: [PATCH 274/494] Fix missing array copy barrier --- openjdk/barriers/mmtkFieldBarrier.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index c9d8eae2..2bc4cb0d 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -111,8 +111,8 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma } void MMTkFieldBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { - const bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; - if ((type == T_OBJECT || type == T_ARRAY) && !dest_uninitialized) { + // const bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; + if ((type == T_OBJECT || type == T_ARRAY)/* && !dest_uninitialized */) { // Label slow, done; // // Bailout if count is zero // __ cmpptr(count, 0); From d06fb82a89c29e36f6cc330d6e51e8680b1410ba Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 7 Mar 2023 10:46:40 -0800 Subject: [PATCH 275/494] Add JRT_LEAF wrapper --- openjdk/mmtkBarrierSet.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index c0ab9da3..1df9024e 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -135,30 +135,30 @@ bool MMTkBarrierSet::is_slow_path_call(address call) { return runtime()->is_slow_path_call(call); } -void MMTkBarrierSetRuntime::object_reference_write_pre_call(void* src, void* slot, void* target) { +JRT_LEAF(void, MMTkBarrierSetRuntime::object_reference_write_pre_call(void* src, void* slot, void* target)) ::mmtk_object_reference_write_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, target); -} +JRT_END -void MMTkBarrierSetRuntime::object_reference_write_post_call(void* src, void* slot, void* target) { +JRT_LEAF(void, MMTkBarrierSetRuntime::object_reference_write_post_call(void* src, void* slot, void* target)) ::mmtk_object_reference_write_post((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, target); -} +JRT_END -void MMTkBarrierSetRuntime::object_reference_write_slow_call(void* src, void* slot, void* target) { +JRT_LEAF(void, MMTkBarrierSetRuntime::object_reference_write_slow_call(void* src, void* slot, void* target)) ::mmtk_object_reference_write_slow((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, target); -} +JRT_END -void MMTkBarrierSetRuntime::object_reference_array_copy_pre_call(void* src, void* dst, size_t count) { +JRT_LEAF(void, MMTkBarrierSetRuntime::object_reference_array_copy_pre_call(void* src, void* dst, size_t count)) ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); -} +JRT_END -void MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, void* dst, size_t count) { +JRT_LEAF(void, MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, void* dst, size_t count)) ::mmtk_array_copy_post((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); -} +JRT_END -void MMTkBarrierSetRuntime::load_reference_call(void* ref) { +JRT_LEAF(void, MMTkBarrierSetRuntime::load_reference_call(void* ref)) ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); -} +JRT_END -void MMTkBarrierSetRuntime::object_reference_clone_pre_call(void* ref) { +JRT_LEAF(void, MMTkBarrierSetRuntime::object_reference_clone_pre_call(void* ref)) ::mmtk_object_reference_clone_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); -} \ No newline at end of file +JRT_END \ No newline at end of file From 4cd413dbd3b89ff25cfe44b889995c5f09e88993 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 7 Mar 2023 15:28:27 -0800 Subject: [PATCH 276/494] Fix concurrent marker crash if the object is killed and overwritten after marking and before scanning --- mmtk/Cargo.lock | 4 +-- mmtk/Cargo.toml | 2 +- mmtk/src/object_scanning.rs | 55 ++++++++++++++++++++++++------------- mmtk/src/scanning.rs | 14 ++++++++++ 4 files changed, 53 insertions(+), 22 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 61458fda..27983d95 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f9a1ee30e15f0817a552e9b054fb0e4ed195fabe#f9a1ee30e15f0817a552e9b054fb0e4ed195fabe" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=777a205f24e0ab3f5c464218f95a88a1c1e2be43#777a205f24e0ab3f5c464218f95a88a1c1e2be43" dependencies = [ "atomic", "atomic-traits", @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f9a1ee30e15f0817a552e9b054fb0e4ed195fabe#f9a1ee30e15f0817a552e9b054fb0e4ed195fabe" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=777a205f24e0ab3f5c464218f95a88a1c1e2be43#777a205f24e0ab3f5c464218f95a88a1c1e2be43" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 8fbc2294..9244bf4f 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f9a1ee30e15f0817a552e9b054fb0e4ed195fabe" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "777a205f24e0ab3f5c464218f95a88a1c1e2be43" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index b11ce61e..269b51a3 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -1,14 +1,14 @@ -use crate::reference_glue::DISCOVERED_LISTS; -use crate::SINGLETON; - use super::abi::*; use super::{OpenJDKEdge, UPCALLS}; +use crate::reference_glue::DISCOVERED_LISTS; +use crate::SINGLETON; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::edge_shape::Edge; use mmtk::vm::EdgeVisitor; use std::cell::UnsafeCell; use std::{mem, slice}; + trait OopIterate: Sized { fn oop_iterate, const COMPRESSED: bool>( &self, @@ -240,9 +240,18 @@ fn oop_iterate_slow>(oop: Oop, closure: &mut V, tls: } } -fn oop_iterate, const COMPRESSED: bool>(oop: Oop, closure: &mut V) { - let klass_id = oop.klass::().id; - debug_assert!( +fn oop_iterate, const COMPRESSED: bool>( + oop: Oop, + closure: &mut V, + klass: Option
, +) { + let klass = if let Some(klass) = klass { + unsafe { &*(klass.as_usize() as *const Klass) } + } else { + oop.klass::() + }; + let klass_id = klass.id; + assert!( klass_id as i32 >= 0 && (klass_id as i32) < 6, "Invalid klass-id: {:x} for oop: {:x}", klass_id as i32, @@ -250,32 +259,31 @@ fn oop_iterate, const COMPRESSED: bool>(oop: Oop, cl ); match klass_id { KlassID::Instance => { - let instance_klass = unsafe { oop.klass::().cast::() }; + let instance_klass = unsafe { klass.cast::() }; instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceClassLoader => { - let instance_klass = - unsafe { oop.klass::().cast::() }; + let instance_klass = unsafe { klass.cast::() }; instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceMirror => { - let instance_klass = unsafe { oop.klass::().cast::() }; + let instance_klass = unsafe { klass.cast::() }; instance_klass.oop_iterate::(oop, closure); } KlassID::ObjArray => { - let array_klass = unsafe { oop.klass::().cast::() }; + let array_klass = unsafe { klass.cast::() }; array_klass.oop_iterate::(oop, closure); } - // KlassID::TypeArray => { - // let array_klass = unsafe { oop.klass::().cast::() }; - // array_klass.oop_iterate::(oop, closure); - // } + KlassID::TypeArray => { + // let array_klass = unsafe { oop.klass::().cast::() }; + // array_klass.oop_iterate::(oop, closure); + } KlassID::InstanceRef => { - let instance_klass = unsafe { oop.klass::().cast::() }; + let instance_klass = unsafe { klass.cast::() }; instance_klass.oop_iterate::(oop, closure); } - // _ => oop_iterate_slow(oop, closure, OpaquePointer::UNINITIALIZED), - _ => {} + #[allow(unreachable_patterns)] + _ => unreachable!(), // _ => oop_iterate_slow(oop, closure, OpaquePointer::UNINITIALIZED), } } @@ -326,5 +334,14 @@ pub fn scan_object, const COMPRESSED: bool>( closure: &mut C, _tls: VMWorkerThread, ) { - unsafe { oop_iterate::(mem::transmute(object), closure) } + unsafe { oop_iterate::(mem::transmute(object), closure, None) } +} + +pub fn scan_object_with_klass, const COMPRESSED: bool>( + object: ObjectReference, + closure: &mut C, + _tls: VMWorkerThread, + klass: Address, +) { + unsafe { oop_iterate::(mem::transmute(object), closure, Some(klass)) } } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index a2c341ad..3e55abc2 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -52,6 +52,20 @@ impl Scanning for VMScanning { crate::object_scanning::scan_object::(object, edge_visitor, tls); } + fn scan_object_with_klass, const COMPRESSED: bool>( + tls: VMWorkerThread, + object: ObjectReference, + edge_visitor: &mut EV, + klass: Address, + ) { + crate::object_scanning::scan_object_with_klass::( + object, + edge_visitor, + tls, + klass, + ); + } + fn obj_array_data(o: ObjectReference) -> crate::OpenJDKEdgeRange { crate::object_scanning::obj_array_data::(unsafe { std::mem::transmute(o) }) } From 8229ba4e409a4717423d82187b877529030a8f5b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 7 Mar 2023 16:09:58 -0800 Subject: [PATCH 277/494] update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 27983d95..64034e32 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=777a205f24e0ab3f5c464218f95a88a1c1e2be43#777a205f24e0ab3f5c464218f95a88a1c1e2be43" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=518239d5ae0ff30486313b13decbcf57d7b1222e#518239d5ae0ff30486313b13decbcf57d7b1222e" dependencies = [ "atomic", "atomic-traits", @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=777a205f24e0ab3f5c464218f95a88a1c1e2be43#777a205f24e0ab3f5c464218f95a88a1c1e2be43" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=518239d5ae0ff30486313b13decbcf57d7b1222e#518239d5ae0ff30486313b13decbcf57d7b1222e" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 9244bf4f..87a9576a 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "777a205f24e0ab3f5c464218f95a88a1c1e2be43" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "518239d5ae0ff30486313b13decbcf57d7b1222e" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 668226b829904857fabf5f3d04466a08338d053b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 7 Mar 2023 16:49:30 -0800 Subject: [PATCH 278/494] Fix clone and array copy barrier --- openjdk/barriers/mmtkFieldBarrier.cpp | 6 +++--- openjdk/barriers/mmtkFieldBarrier.hpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 2bc4cb0d..914e26fc 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -452,14 +452,14 @@ Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_ } void MMTkFieldBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { - // if (!is_array && dst != kit->just_allocated_object(kit->control())) { + if (dst != kit->just_allocated_object(kit->control())) { MMTkIdealKit ideal(kit); - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); + const TypeFunc* tf = __ func_type(TypeOopPtr::NOTNULL); Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_clone_pre_call), "mmtk_barrier_call", dst); kit->sync_kit(ideal); kit->insert_mem_bar(Op_MemBarVolatile); kit->final_sync(ideal); - // } + } BarrierSetC2::clone(kit, src, dst, size, is_array); } diff --git a/openjdk/barriers/mmtkFieldBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp index 5a0ee931..dc6d83d4 100644 --- a/openjdk/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -25,7 +25,7 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { } virtual void load_reference(DecoratorSet decorators, oop value) const override; virtual void clone_pre(DecoratorSet decorators, oop value) const override { - // object_reference_clone_pre_call((void*) value); + object_reference_clone_pre_call((void*) value); }; }; @@ -54,7 +54,7 @@ class MMTkFieldBarrierSetC2: public MMTkBarrierSetC2 { virtual void object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const override; public: virtual bool array_copy_requires_gc_barriers(BasicType type) const override { - return false; + return true; } virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const override; virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const override; From bf4d2b4fc7c5b46096b7aff97535b6f0f199458d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 8 Mar 2023 10:53:18 -0800 Subject: [PATCH 279/494] Add missing `on_slowpath_allocation_exit` barrier --- mmtk/src/lib.rs | 4 ++++ openjdk/barriers/mmtkFieldBarrier.cpp | 17 +++++++---------- openjdk/barriers/mmtkFieldBarrier.hpp | 1 + openjdk/mmtk.h | 8 ++++++++ openjdk/mmtkBarrierSet.hpp | 18 +++++++++++++----- 5 files changed, 33 insertions(+), 15 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index d739a018..0d6bb6a2 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -146,6 +146,10 @@ pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS_COMPRESSED: uintptr_t = .upper_bound_address_for_contiguous() .as_usize(); +#[no_mangle] +pub static RC_TABLE_BASE_ADDRESS: uintptr_t = + mmtk::util::rc::RC_TABLE.get_absolute_offset().as_usize(); + #[no_mangle] pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t = mmtk::util::metadata::side_metadata::ALLOC_SIDE_METADATA_ADDR.as_usize(); diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 914e26fc..3d809852 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -31,6 +31,12 @@ void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, #endif } +void MMTkFieldBarrierSetRuntime::on_slowpath_allocation_exit(oop new_obj) const { + if (mmtk_get_rc(new_obj) != 0) { + object_reference_clone_pre_call((void*) new_obj); + } +} + #define __ masm-> void MMTkFieldBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { @@ -111,8 +117,7 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma } void MMTkFieldBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { - // const bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; - if ((type == T_OBJECT || type == T_ARRAY)/* && !dest_uninitialized */) { + if (type == T_OBJECT || type == T_ARRAY) { // Label slow, done; // // Bailout if count is zero // __ cmpptr(count, 0); @@ -452,14 +457,6 @@ Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_ } void MMTkFieldBarrierSetC2::clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const { - if (dst != kit->just_allocated_object(kit->control())) { - MMTkIdealKit ideal(kit); - const TypeFunc* tf = __ func_type(TypeOopPtr::NOTNULL); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_clone_pre_call), "mmtk_barrier_call", dst); - kit->sync_kit(ideal); - kit->insert_mem_bar(Op_MemBarVolatile); - kit->final_sync(ideal); - } BarrierSetC2::clone(kit, src, dst, size, is_array); } diff --git a/openjdk/barriers/mmtkFieldBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp index dc6d83d4..08311412 100644 --- a/openjdk/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -27,6 +27,7 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { virtual void clone_pre(DecoratorSet decorators, oop value) const override { object_reference_clone_pre_call((void*) value); }; + virtual void on_slowpath_allocation_exit(oop new_obj) const override; }; class MMTkFieldBarrierSetAssembler: public MMTkBarrierSetAssembler { diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index c1ff8947..3614462a 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -20,6 +20,7 @@ typedef enum { extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS_COMPRESSED; +extern const uintptr_t RC_TABLE_BASE_ADDRESS; extern const uintptr_t GLOBAL_ALLOC_BIT_ADDRESS; extern const int DISABLE_ALLOCATION_FAST_PATH; extern const uintptr_t IMMIX_ALLOCATOR_SIZE; @@ -242,6 +243,13 @@ extern void add_phantom_candidate(void* ref, void* referent); extern void mmtk_harness_begin_impl(); extern void mmtk_harness_end_impl(); +inline uint8_t mmtk_get_rc(oop o) { + const uintptr_t index = uintptr_t((void*) o) >> 4; + const uint8_t byte = *((uint8_t*) (RC_TABLE_BASE_ADDRESS + (index >> 2))); + auto v = byte >> ((index & 0b11) << 1); + return v & 0b11; +} + #ifdef __cplusplus } #endif diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index b93ac45e..40116dd7 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -97,7 +97,12 @@ class MMTkBarrierSetRuntime: public CHeapObj { virtual void object_reference_array_copy_post(oop* src, oop* dst, size_t count) const {}; /// java.lang.Reference load barrier virtual void load_reference(DecoratorSet decorators, oop value) const {}; + /// Object clone pre-barrier virtual void clone_pre(DecoratorSet decorators, oop value) const {}; + /// Called at the end of every C2 slowpath allocation. + /// Deoptimization can happen after C2 slowpath allocation, and the newly allocated object can be promoted. + /// So this callback is requierd for any generational collectors. + virtual void on_slowpath_allocation_exit(oop new_obj) const {}; }; class MMTkBarrierC1; @@ -132,7 +137,7 @@ class MMTkBarrierSet : public BarrierSet { MMTkBarrierSetRuntime* _runtime; protected: - virtual void write_ref_array_work(MemRegion mr) ; + virtual void write_ref_array_work(MemRegion mr); public: MMTkBarrierSet(MemRegion whole_heap); @@ -141,9 +146,12 @@ class MMTkBarrierSet : public BarrierSet { return ((MMTkBarrierSet*) BarrierSet::barrier_set())->_runtime; } - virtual void on_thread_destroy(Thread* thread); - virtual void on_thread_attach(JavaThread* thread); - virtual void on_thread_detach(JavaThread* thread); + virtual void on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) override { + runtime()->on_slowpath_allocation_exit(new_obj); + } + virtual void on_thread_destroy(Thread* thread) override; + virtual void on_thread_attach(JavaThread* thread) override; + virtual void on_thread_detach(JavaThread* thread) override; // Inform the BarrierSet that the the covered heap region that starts // with "base" has been changed to have the given size (possibly from 0, @@ -155,7 +163,7 @@ class MMTkBarrierSet : public BarrierSet { virtual bool is_aligned(HeapWord* addr); // Print a description of the memory for the barrier set - virtual void print_on(outputStream* st) const; + virtual void print_on(outputStream* st) const override; // The AccessBarrier of a BarrierSet subclass is called by the Access API From eae38848dbeedf54e53a39495b1ad9c9f8170396 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 8 Mar 2023 10:57:45 -0800 Subject: [PATCH 280/494] Set `array_copy_requires_gc_barriers` to false --- openjdk/barriers/mmtkFieldBarrier.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp index 08311412..609355ed 100644 --- a/openjdk/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -55,7 +55,7 @@ class MMTkFieldBarrierSetC2: public MMTkBarrierSetC2 { virtual void object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const override; public: virtual bool array_copy_requires_gc_barriers(BasicType type) const override { - return true; + return false; } virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const override; virtual void clone(GraphKit* kit, Node* src, Node* dst, Node* size, bool is_array) const override; From 1aaf1da9188ce1680c96ce01bd4fa399dea22ec0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 8 Mar 2023 11:48:45 -0800 Subject: [PATCH 281/494] Fix compiler errors and warnings --- openjdk/barriers/mmtkFieldBarrier.cpp | 2 +- openjdk/mmtk.h | 2 +- openjdk/mmtkHeap.hpp | 70 +++++++++++++-------------- 3 files changed, 37 insertions(+), 37 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 3d809852..1fb43433 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -32,7 +32,7 @@ void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, } void MMTkFieldBarrierSetRuntime::on_slowpath_allocation_exit(oop new_obj) const { - if (mmtk_get_rc(new_obj) != 0) { + if (mmtk_get_rc((void*) new_obj) != 0) { object_reference_clone_pre_call((void*) new_obj); } } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 3614462a..aafb1a49 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -243,7 +243,7 @@ extern void add_phantom_candidate(void* ref, void* referent); extern void mmtk_harness_begin_impl(); extern void mmtk_harness_end_impl(); -inline uint8_t mmtk_get_rc(oop o) { +inline uint8_t mmtk_get_rc(void* o) { const uintptr_t index = uintptr_t((void*) o) >> 4; const uint8_t byte = *((uint8_t*) (RC_TABLE_BASE_ADDRESS + (index >> 2))); auto v = byte >> ((index & 0b11) << 1); diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index b38317cf..4618377c 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -75,8 +75,8 @@ class MMTkHeap : public CollectedHeap { static HeapWord* allocate_from_tlab(Klass* klass, Thread* thread, size_t size); - jint initialize(); - void enable_collection(); + virtual jint initialize() override; + virtual void enable_collection() override; virtual HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded); HeapWord* mem_allocate_nonmove(size_t size, bool* gc_overhead_limit_was_exceeded); @@ -86,33 +86,33 @@ class MMTkHeap : public CollectedHeap { } - Name kind() const { + virtual Name kind() const override { return CollectedHeap::ThirdPartyHeap; } - const char* name() const { + virtual const char* name() const override { return "MMTk"; } static const char* version(); - size_t capacity() const; - size_t used() const; + virtual size_t capacity() const override; + virtual size_t used() const override; - bool is_maximal_no_gc() const; + virtual bool is_maximal_no_gc() const override; - size_t max_capacity() const; - bool is_in(const void* p) const; - bool is_in_reserved(const void* p) const; - bool supports_tlab_allocation() const; + virtual size_t max_capacity() const override; + virtual bool is_in(const void* p) const override; + virtual bool is_in_reserved(const void* p) const override; + virtual bool supports_tlab_allocation() const override; - bool supports_inline_contig_alloc() const { + virtual bool supports_inline_contig_alloc() const override { return MMTK_ENABLE_ALLOCATION_FASTPATH && !disable_fast_alloc(); } // The amount of space available for thread-local allocation buffers. - size_t tlab_capacity(Thread *thr) const; + virtual size_t tlab_capacity(Thread *thr) const override; // The amount of used space for thread-local allocation buffers for the given thread. - size_t tlab_used(Thread *thr) const; + virtual size_t tlab_used(Thread *thr) const override; void new_collector_thread() { _n_workers += 1; @@ -139,65 +139,65 @@ class MMTkHeap : public CollectedHeap { // Return the CollectorPolicy for the heap - CollectorPolicy* collector_policy() const ; + virtual CollectorPolicy* collector_policy() const override; - SoftRefPolicy* soft_ref_policy(); + virtual SoftRefPolicy* soft_ref_policy() override; - GrowableArray memory_managers() ; - GrowableArray memory_pools(); + virtual GrowableArray memory_managers() override; + virtual GrowableArray memory_pools() override; // Iterate over all objects, calling "cl.do_object" on each. - void object_iterate(ObjectClosure* cl); + virtual void object_iterate(ObjectClosure* cl) override; // Similar to object_iterate() except iterates only // over live objects. - void safe_object_iterate(ObjectClosure* cl) ; + virtual void safe_object_iterate(ObjectClosure* cl) override; - HeapWord* block_start(const void* addr) const ; + virtual HeapWord* block_start(const void* addr) const override; - size_t block_size(const HeapWord* addr) const ; + virtual size_t block_size(const HeapWord* addr) const override; - bool block_is_obj(const HeapWord* addr) const; + virtual bool block_is_obj(const HeapWord* addr) const override; - jlong millis_since_last_gc() ; + virtual jlong millis_since_last_gc() ; - void prepare_for_verify() ; + virtual void prepare_for_verify() ; private: - void initialize_serviceability() ; + virtual void initialize_serviceability() ; public: // Print heap information on the given outputStream. - void print_on(outputStream* st) const ; + virtual void print_on(outputStream* st) const override; // Print all GC threads (other than the VM thread) // used by this heap. - void print_gc_threads_on(outputStream* st) const; + virtual void print_gc_threads_on(outputStream* st) const override; // Iterator for all GC threads (other than VM thread) - void gc_threads_do(ThreadClosure* tc) const; + virtual void gc_threads_do(ThreadClosure* tc) const override; // Print any relevant tracing info that flags imply. // Default implementation does nothing. - void print_tracing_info() const ; + virtual void print_tracing_info() const override; // An object is scavengable if its location may move during a scavenge. // (A scavenge is a GC which is not a full GC.) - inline bool is_scavengable(oop obj) { return true; } + virtual inline bool is_scavengable(oop obj) override { return true; } // Registering and unregistering an nmethod (compiled code) with the heap. // Override with specific mechanism for each specialized heap type. - virtual void register_nmethod(nmethod* nm); - virtual void unregister_nmethod(nmethod* nm); + virtual void register_nmethod(nmethod* nm) override; + virtual void unregister_nmethod(nmethod* nm) override; // Heap verification - void verify(VerifyOption option); + virtual void verify(VerifyOption option) override; - void post_initialize(); + virtual void post_initialize() override; void scan_roots(OopClosure& cl); From 5bef7fb12d6016f89e49b32e1505dee4f56f57ba Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 8 Mar 2023 12:02:44 -0800 Subject: [PATCH 282/494] Fix warnings --- openjdk/mmtkHeap.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index 4618377c..629aa3a9 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -78,7 +78,7 @@ class MMTkHeap : public CollectedHeap { virtual jint initialize() override; virtual void enable_collection() override; - virtual HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded); + virtual HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) override; HeapWord* mem_allocate_nonmove(size_t size, bool* gc_overhead_limit_was_exceeded); MMTkVMCompanionThread* companion_thread() const { @@ -159,14 +159,14 @@ class MMTkHeap : public CollectedHeap { virtual bool block_is_obj(const HeapWord* addr) const override; - virtual jlong millis_since_last_gc() ; + virtual jlong millis_since_last_gc() override; - virtual void prepare_for_verify() ; + virtual void prepare_for_verify() override; private: - virtual void initialize_serviceability() ; + virtual void initialize_serviceability() override; public: From e7358e72ea03eea5a28d64f2e41a6d5bbc9368a5 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 8 Mar 2023 12:44:39 -0800 Subject: [PATCH 283/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 64034e32..3805f22d 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=518239d5ae0ff30486313b13decbcf57d7b1222e#518239d5ae0ff30486313b13decbcf57d7b1222e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=49033052f21880bcf9114b472b8f3b1099a48929#49033052f21880bcf9114b472b8f3b1099a48929" dependencies = [ "atomic", "atomic-traits", @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=518239d5ae0ff30486313b13decbcf57d7b1222e#518239d5ae0ff30486313b13decbcf57d7b1222e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=49033052f21880bcf9114b472b8f3b1099a48929#49033052f21880bcf9114b472b8f3b1099a48929" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 87a9576a..e14be978 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "518239d5ae0ff30486313b13decbcf57d7b1222e" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "49033052f21880bcf9114b472b8f3b1099a48929" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From c1e9835aad39a9816de2f37ec81dddacb61deb45 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 9 Mar 2023 11:52:58 -0800 Subject: [PATCH 284/494] rename variables and functions; support no-class-unloading --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/collection.rs | 6 ++++++ mmtk/src/gc_work.rs | 2 +- mmtk/src/lib.rs | 1 + openjdk/mmtk.h | 1 + openjdk/mmtkHeap.cpp | 22 +++++++++++++++------- openjdk/mmtkHeap.hpp | 2 +- openjdk/mmtkRootsClosure.hpp | 8 ++++++-- openjdk/mmtkUpcalls.cpp | 9 +++++++-- 10 files changed, 41 insertions(+), 16 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 3805f22d..186f631a 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=49033052f21880bcf9114b472b8f3b1099a48929#49033052f21880bcf9114b472b8f3b1099a48929" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=564cee90e199278ef6a5f730845e109207f02071#564cee90e199278ef6a5f730845e109207f02071" dependencies = [ "atomic", "atomic-traits", @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=49033052f21880bcf9114b472b8f3b1099a48929#49033052f21880bcf9114b472b8f3b1099a48929" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=564cee90e199278ef6a5f730845e109207f02071#564cee90e199278ef6a5f730845e109207f02071" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index e14be978..f81f0549 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "49033052f21880bcf9114b472b8f3b1099a48929" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "564cee90e199278ef6a5f730845e109207f02071" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index f7cd5297..fbbd2dae 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -134,6 +134,12 @@ impl Collection for VMCollection { } } + fn clear_cld_claimed_marks() { + unsafe { + ((*UPCALLS).clear_claimed_marks)(); + } + } + fn set_concurrent_marking_state(active: bool) { unsafe { crate::CONCURRENT_MARKING_ACTIVE = if active { 1 } else { 0 } } } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index eff6e58f..283d1629 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -90,7 +90,7 @@ impl> GCWork for ScanClassLoaderDataGr to_edges_closure_cld::(&mut self.factory), to_edges_closure_cld::(&mut self.factory), mmtk.get_plan() - .current_gc_should_scan_weak_classloader_roots(), + .current_gc_should_scan_all_classloader_strong_roots(), ); } } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 0d6bb6a2..88340ebf 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -119,6 +119,7 @@ pub struct OpenJDK_Upcalls { pub compressed_klass_base: extern "C" fn() -> Address, pub compressed_klass_shift: extern "C" fn() -> usize, pub nmethod_fix_relocation: extern "C" fn(Address), + pub clear_claimed_marks: extern "C" fn(), } lazy_static! { diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index aafb1a49..be7d4855 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -209,6 +209,7 @@ typedef struct { void* (*compressed_klass_base)(); size_t (*compressed_klass_shift)(); void (*nmethod_fix_relocation)(void* nmethod); + void (*clear_claimed_marks)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index a2cdc936..871844ac 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -430,13 +430,21 @@ void MMTkHeap::scan_code_cache_roots(OopClosure& cl) { void MMTkHeap::scan_string_table_roots(OopClosure& cl) { StringTable::oops_do(&cl); } -void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& weak_cl, bool scan_weak) { - CLDToOopClosure cld_cl(&cl, false); - CLDToOopClosure weak_cld_cl(&weak_cl, false); - ClassLoaderDataGraph::roots_cld_do(&cld_cl, ClassUnloading && !scan_weak ? NULL : &weak_cld_cl); - // Scan modified CLDs - MMTkScanCLDClosure cld_closure(&cl); - ClassLoaderDataGraph::cld_do(&cld_closure); +void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& weak_cl, bool scan_all_strong_roots) { + if (!ClassUnloading) { + CLDToOopClosure cld_cl(&cl, false); + CLDToOopClosure weak_cld_cl(&weak_cl, false); + ClassLoaderDataGraph::roots_cld_do(&cld_cl, &weak_cld_cl); + } else if (scan_all_strong_roots) { + // At the start of full heap trace, we want to scan all the strong CLD roots + all the modified CLDs. + MMTkScanCLDClosure cld_cl(&cl); + MMTkScanCLDClosure weak_cld_cl(&cl); + ClassLoaderDataGraph::roots_cld_do(&cld_cl, &weak_cld_cl); + } else { + // Normal RC pause: We simply apply increments to modified CLD roots. No decrements will be applied on these roots. + MMTkScanCLDClosure cld_cl(&cl); + ClassLoaderDataGraph::cld_do(&cld_cl); + } } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { ResourceMark rm; diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index 629aa3a9..36e6a8dc 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -212,7 +212,7 @@ class MMTkHeap : public CollectedHeap { void scan_system_dictionary_roots(OopClosure& cl); void scan_code_cache_roots(OopClosure& cl); void scan_string_table_roots(OopClosure& cl); - void scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& weak_cl, bool scan_weak); + void scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& weak_cl, bool scan_all_strong_roots); void scan_weak_processor_roots(OopClosure& cl); void scan_vm_thread_roots(OopClosure& cl); diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 1a8a2929..76a8e513 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -101,6 +101,7 @@ class MMTkCollectRootObjects : public OopClosure { virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } }; +template class MMTkScanCLDClosure: public CLDClosure { private: OopClosure* _oop_closure; @@ -108,8 +109,11 @@ class MMTkScanCLDClosure: public CLDClosure { public: MMTkScanCLDClosure(OopClosure* c) : _oop_closure(c) { } void do_cld(ClassLoaderData* cld) { - if (cld->has_modified_oops()) { - cld->oops_do(_oop_closure, false, /*clear_modified_oops*/true); + if (MODIFIED_ONLY) { + if (cld->has_modified_oops()) cld->oops_do(_oop_closure, false, /*clear_modified_oops*/true); + } else { + if (cld->has_modified_oops() || !WEAK) + cld->oops_do(_oop_closure, false, /*clear_modified_oops*/true); } } }; diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index bb21b6d7..c47139e4 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -175,6 +175,10 @@ static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, M nmethod::oops_do_marking_prologue(); } +static void mmtk_clear_claimed_marks() { + ClassLoaderDataGraph::clear_claimed_marks(); +} + static void mmtk_update_weak_processor(bool lxr) { HandleMark hm; MMTkForwardClosure forward; @@ -448,10 +452,10 @@ static void mmtk_scan_aot_loader_roots(EdgesClosure closure) { MMTkRootsClosure2 static void mmtk_scan_system_dictionary_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_system_dictionary_roots(cl); } static void mmtk_scan_code_cache_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } static void mmtk_scan_string_table_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_string_table_roots(cl); } -static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, EdgesClosure weak_closure, bool scan_weak) { +static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, EdgesClosure weak_closure, bool scank_all_strong_roots) { MMTkRootsClosure2 cl(closure); MMTkRootsClosure2 weak_cl(weak_closure); - MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl, weak_cl, scan_weak); + MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl, weak_cl, scank_all_strong_roots); } static void mmtk_scan_weak_processor_roots(EdgesClosure closure) { MMTkCollectRootObjects cl(closure); MMTkHeap::heap()->scan_weak_processor_roots(cl); } static void mmtk_scan_vm_thread_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } @@ -562,4 +566,5 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_compressed_klass_base, mmtk_compressed_klass_shift, nmethod_fix_relocation, + mmtk_clear_claimed_marks, }; From ce70ebcb4df52e829e2bb9852bb149a89716611b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 9 Mar 2023 12:03:11 -0800 Subject: [PATCH 285/494] Fix JRTLeafVerifier checks --- openjdk/barriers/mmtkFieldBarrier.cpp | 10 +++++----- openjdk/barriers/mmtkFieldBarrier.hpp | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 1fb43433..fb7aad6b 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -12,7 +12,8 @@ static inline intptr_t side_metadata_base_address() { void MMTkFieldBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const { #if SOFT_REFERENCE_LOAD_BARRIER - if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) load_reference_call((void*) value); + if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) + ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); #endif }; @@ -23,17 +24,16 @@ void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, intptr_t shift = (addr >> (UseCompressedOops ? 2 : 3)) & 0b111; uint8_t byte_val = *meta_addr; if (((byte_val >> shift) & 1) == kUnloggedValue) { - // MMTkObjectBarrierSetRuntime::object_reference_write_pre_slow()((void*) src); - object_reference_write_slow_call((void*) src, (void*) slot, (void*) target); + ::mmtk_object_reference_write_slow((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) slot, (void*) target); } #else - object_reference_write_pre_call((void*) src, (void*) slot, (void*) target); + ::object_reference_write_pre_call((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) slot, (void*) target); #endif } void MMTkFieldBarrierSetRuntime::on_slowpath_allocation_exit(oop new_obj) const { if (mmtk_get_rc((void*) new_obj) != 0) { - object_reference_clone_pre_call((void*) new_obj); + ::mmtk_object_reference_clone_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) new_obj); } } diff --git a/openjdk/barriers/mmtkFieldBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp index 609355ed..d0488e74 100644 --- a/openjdk/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -21,11 +21,11 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { // Interfaces called by `MMTkBarrierSet::AccessBarrier` virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const override { - object_reference_array_copy_pre_call((void*) src, (void*) dst, count); + ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) dst, count); } virtual void load_reference(DecoratorSet decorators, oop value) const override; virtual void clone_pre(DecoratorSet decorators, oop value) const override { - object_reference_clone_pre_call((void*) value); + ::mmtk_object_reference_clone_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); }; virtual void on_slowpath_allocation_exit(oop new_obj) const override; }; From 4ab2e64eb0eabaca3d44ba3c8e380c00723101db Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 14 Mar 2023 11:22:04 -0700 Subject: [PATCH 286/494] print logs to stderr --- openjdk/mmtkHeap.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 871844ac..806b4156 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -113,13 +113,13 @@ jint MMTkHeap::initialize() { _start = (HeapWord*) starting_heap_address(); _end = (HeapWord*) last_heap_address(); - if (mmtk_verbose() > 0) printf("MMTk Heap: start=%p, end=%p\n", _start, _end); + if (mmtk_verbose() > 0) fprintf(stderr, "MMTk Heap: start=%p, end=%p\n", _start, _end); if (mmtk_verbose() > 0) - printf("MMTk Compressed Pointers: %s\n", UseCompressedOops ? "enabled" : "disabled"); + fprintf(stderr, "MMTk Compressed Pointers: %s\n", UseCompressedOops ? "enabled" : "disabled"); if (UseCompressedOops) { Universe::set_narrow_oop_base((address) mmtk_narrow_oop_base()); Universe::set_narrow_oop_shift(mmtk_narrow_oop_shift()); - if (mmtk_verbose() > 0) printf("narrow_oop_mode: %s\n", Universe::narrow_oop_mode_to_string(Universe::narrow_oop_mode())); + if (mmtk_verbose() > 0) fprintf(stderr, "narrow_oop_mode: %s\n", Universe::narrow_oop_mode_to_string(Universe::narrow_oop_mode())); } initialize_reserved_region(_start, _end); @@ -137,9 +137,9 @@ jint MMTkHeap::initialize() { } if (mmtk_verbose() > 0) { if (ClassUnloading) { - printf("ClassUnloading Enabled\n"); + fprintf(stderr, "ClassUnloading Enabled\n"); } else { - printf("ClassUnloading Disabled\n"); + fprintf(stderr, "ClassUnloading Disabled\n"); } } From e5a74dfab18913e0055292bb3662de714ccb04fc Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 14 Mar 2023 11:25:43 -0700 Subject: [PATCH 287/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 186f631a..0fe69a7f 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=564cee90e199278ef6a5f730845e109207f02071#564cee90e199278ef6a5f730845e109207f02071" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a9dcf07b4f0428af8272e9f271f949c2030a835e#a9dcf07b4f0428af8272e9f271f949c2030a835e" dependencies = [ "atomic", "atomic-traits", @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=564cee90e199278ef6a5f730845e109207f02071#564cee90e199278ef6a5f730845e109207f02071" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a9dcf07b4f0428af8272e9f271f949c2030a835e#a9dcf07b4f0428af8272e9f271f949c2030a835e" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index f81f0549..abb62034 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "564cee90e199278ef6a5f730845e109207f02071" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "a9dcf07b4f0428af8272e9f271f949c2030a835e" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From b7b65c390bea8761c4c341849ad632cf73b6e2b0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 15 Mar 2023 00:05:30 -0700 Subject: [PATCH 288/494] Remove reference from discovered list before forwarding --- mmtk/src/reference_glue.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 0b0ca093..d9e293d9 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -389,10 +389,13 @@ fn iterate_list< debug_assert!(next_ref.get_forwarded_object().is_none()); // Reaches the end of the list? let end_of_list = next_ref == reference || next_ref.is_null(); - // Process reference - let result = visitor(reference); // Remove `reference` from current list set_next_reference::(reference, ObjectReference::NULL); + if let Some(forwarded_ref) = reference.get_forwarded_object() { + set_next_reference::(forwarded_ref, ObjectReference::NULL); + } + // Process reference + let result = visitor(reference); match result { DiscoveredListIterationResult::Remove => {} DiscoveredListIterationResult::Enqueue(reference) => { From 55fabd87e3074f52beb63e4d9126dd0414ad50d9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 15 Mar 2023 12:56:18 -0700 Subject: [PATCH 289/494] Remove hardcoded address --- openjdk/mmtkUpcalls.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index c47139e4..24e79302 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -98,7 +98,7 @@ class MMTkLXRFastIsAliveClosure : public BoolObjectClosure { public: static inline bool rc_live(oop o) { const uintptr_t index = uintptr_t((void*) o) >> 4; - const uint8_t byte = *((uint8_t*) (uintptr_t(0xe0004000000) + (index >> 2))); + const uint8_t byte = *((uint8_t*) (RC_TABLE_BASE_ADDRESS + (index >> 2))); const uint8_t byte_mask = 0b11 << ((index & 0b11) << 1); return (byte & byte_mask) != 0; } From 9a808e633f17eecba0ec03cea9e756957802e33e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 15 Mar 2023 22:02:06 -0700 Subject: [PATCH 290/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 0fe69a7f..4b9bfe41 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a9dcf07b4f0428af8272e9f271f949c2030a835e#a9dcf07b4f0428af8272e9f271f949c2030a835e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8e62833a1707469460eb0412e74a0b4d7ce2d0ba#8e62833a1707469460eb0412e74a0b4d7ce2d0ba" dependencies = [ "atomic", "atomic-traits", @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a9dcf07b4f0428af8272e9f271f949c2030a835e#a9dcf07b4f0428af8272e9f271f949c2030a835e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8e62833a1707469460eb0412e74a0b4d7ce2d0ba#8e62833a1707469460eb0412e74a0b4d7ce2d0ba" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index abb62034..e26bdefb 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "a9dcf07b4f0428af8272e9f271f949c2030a835e" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8e62833a1707469460eb0412e74a0b4d7ce2d0ba" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 190e7b43106e468f3bb4b27ce2e5dbb41294853e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 16 Mar 2023 23:01:18 +0000 Subject: [PATCH 291/494] Fix debug assertions --- mmtk/src/reference_glue.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index d9e293d9..922c74e7 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -273,8 +273,8 @@ impl, const COMPRESSED: bool> GCWork let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { debug_assert!( - !get_next_reference::(reference).is_null(), - "next can't be null. ref={:?} {:?} bin={}", + get_next_reference::(reference).is_null(), + "next must be null. ref={:?} {:?} bin={}", reference, self.rt, self.list_index @@ -376,7 +376,7 @@ fn iterate_list< let mut reference = head; loop { debug_assert!(!reference.is_null()); - debug_assert!(reference.is_live()); + // debug_assert!(reference.is_live()); // Update reference forwarding pointer if let Some(forwarded) = reference.get_forwarded_object() { reference = forwarded; From 6279e32378100526b53445c04dbf882afdf20392 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 20 Mar 2023 10:31:14 -0700 Subject: [PATCH 292/494] Fix incorrect class unloading liveness check --- openjdk/mmtkUpcalls.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 24e79302..75a865cc 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -196,13 +196,8 @@ static void mmtk_resume_mutators(void *tls, bool lxr, bool current_gc_should_unl if (ClassUnloading && current_gc_should_unload_classes) { // Unload classes and purge SystemDictionary. auto purged_classes = SystemDictionary::do_unloading(NULL, false /* Defer cleaning */); - if (lxr) { - MMTkLXRFastIsAliveClosure is_alive; - MMTkHeap::heap()->complete_cleaning(&is_alive, purged_classes); - } else { - MMTkIsAliveClosure is_alive; - MMTkHeap::heap()->complete_cleaning(&is_alive, purged_classes); - } + MMTkIsAliveClosure is_alive; + MMTkHeap::heap()->complete_cleaning(&is_alive, purged_classes); ClassLoaderDataGraph::purge(); // Resize and verify metaspace MetaspaceGC::compute_new_size(); From af5867a5db9b6df143cb5102310bb093d484eb85 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 20 Mar 2023 10:31:48 -0700 Subject: [PATCH 293/494] update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 4b9bfe41..61010554 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8e62833a1707469460eb0412e74a0b4d7ce2d0ba#8e62833a1707469460eb0412e74a0b4d7ce2d0ba" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b3377359852489b2c021670c3c4ca04bf68740d6#b3377359852489b2c021670c3c4ca04bf68740d6" dependencies = [ "atomic", "atomic-traits", @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8e62833a1707469460eb0412e74a0b4d7ce2d0ba#8e62833a1707469460eb0412e74a0b4d7ce2d0ba" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b3377359852489b2c021670c3c4ca04bf68740d6#b3377359852489b2c021670c3c4ca04bf68740d6" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index e26bdefb..47cde05f 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8e62833a1707469460eb0412e74a0b4d7ce2d0ba" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "b3377359852489b2c021670c3c4ca04bf68740d6" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From c6813254c2d0ab00ec8f2905c1016fe2c09b9a69 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 21 Mar 2023 16:48:25 -0700 Subject: [PATCH 294/494] Update is_gc_active --- openjdk/mmtkHeap.hpp | 4 ++++ openjdk/mmtkUpcalls.cpp | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index 36e6a8dc..fe6d6ebb 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -69,6 +69,10 @@ class MMTkHeap : public CollectedHeap { void schedule_finalizer(); + void set_is_gc_active(bool is_gc_active) { + _is_gc_active = is_gc_active; + } + inline static MMTkHeap* heap() { return _heap; } diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 75a865cc..1e1a5eda 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -155,6 +155,7 @@ static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, M log_debug(gc)("Requesting the VM to suspend all mutators..."); MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_suspended, true); log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); + MMTkHeap::heap()->set_is_gc_active(true); mmtk_report_gc_start(); if (ClassUnloading && current_gc_should_unload_classes) { @@ -214,7 +215,8 @@ static void mmtk_resume_mutators(void *tls, bool lxr, bool current_gc_should_unl // The increment has to be done before mutators can be resumed // otherwise, mutators might see a stale value Atomic::inc(&mmtk_start_the_world_count); - + + MMTkHeap::heap()->set_is_gc_active(false); log_debug(gc)("Requesting the VM to resume all mutators..."); MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_resumed, true); log_debug(gc)("Mutators resumed. Now notify any mutators waiting for GC to finish..."); From 27a8307ac8fa24e5e4c886593a033bf141ce301e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 24 Mar 2023 12:17:40 -0700 Subject: [PATCH 295/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 61010554..490a2a44 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b3377359852489b2c021670c3c4ca04bf68740d6#b3377359852489b2c021670c3c4ca04bf68740d6" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f7e4450e2566a6d64a5ca5112a908f84fbf7da0c#f7e4450e2566a6d64a5ca5112a908f84fbf7da0c" dependencies = [ "atomic", "atomic-traits", @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b3377359852489b2c021670c3c4ca04bf68740d6#b3377359852489b2c021670c3c4ca04bf68740d6" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f7e4450e2566a6d64a5ca5112a908f84fbf7da0c#f7e4450e2566a6d64a5ca5112a908f84fbf7da0c" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 47cde05f..cf321366 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "b3377359852489b2c021670c3c4ca04bf68740d6" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f7e4450e2566a6d64a5ca5112a908f84fbf7da0c" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 956fb128c6a72529d2aeface7e1d3b56ae9c61e2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 27 Mar 2023 03:41:44 +0000 Subject: [PATCH 296/494] Temporarily increase rust thread stack size --- openjdk/mmtkHeap.cpp | 2 +- openjdk/mmtkHeap.hpp | 2 ++ openjdk/mmtkUpcalls.cpp | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 806b4156..121a6c74 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -131,7 +131,7 @@ jint MMTkHeap::initialize() { BarrierSet::set_barrier_set(barrier_set); _companion_thread = new MMTkVMCompanionThread(); - if (!os::create_thread(_companion_thread, os::pgc_thread)) { + if (!os::create_thread(_companion_thread, os::pgc_thread, WORKER_STACK_SIZE)) { fprintf(stderr, "Failed to create thread"); guarantee(false, "panic"); } diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index fe6d6ebb..8590a6d3 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -42,6 +42,8 @@ #include "utilities/growableArray.hpp" #include "utilities/ostream.hpp" +#define WORKER_STACK_SIZE (64 * 1024 * 1024) + class GCMemoryManager; class MemoryPool; //class mmtkGCTaskManager; diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 1e1a5eda..c5789767 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -234,7 +234,7 @@ static void mmtk_spawn_gc_thread(void* tls, int kind, void* ctx) { switch (kind) { case GC_THREAD_KIND_CONTROLLER: { MMTkContextThread* t = new MMTkContextThread(ctx); - if (!os::create_thread(t, os::pgc_thread)) { + if (!os::create_thread(t, os::pgc_thread, WORKER_STACK_SIZE)) { printf("Failed to create thread"); guarantee(false, "panic"); } @@ -244,7 +244,7 @@ static void mmtk_spawn_gc_thread(void* tls, int kind, void* ctx) { case GC_THREAD_KIND_WORKER: { MMTkHeap::heap()->new_collector_thread(); MMTkCollectorThread* t = new MMTkCollectorThread(ctx); - if (!os::create_thread(t, os::pgc_thread)) { + if (!os::create_thread(t, os::pgc_thread, WORKER_STACK_SIZE)) { printf("Failed to create thread"); guarantee(false, "panic"); } From cd28e449fbc7954a0d188e2247e58f40dbde4685 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 27 Mar 2023 07:33:52 +0000 Subject: [PATCH 297/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index e5555ce0..218cbb0b 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=740ad64ed297e91586c4eb739f0a21a9a60506ec#740ad64ed297e91586c4eb739f0a21a9a60506ec" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=81cf9f1fc2f33e4936b9b45cdc9b588da9ddfbc8#81cf9f1fc2f33e4936b9b45cdc9b588da9ddfbc8" dependencies = [ "atomic", "atomic-traits", @@ -479,7 +479,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=740ad64ed297e91586c4eb739f0a21a9a60506ec#740ad64ed297e91586c4eb739f0a21a9a60506ec" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=81cf9f1fc2f33e4936b9b45cdc9b588da9ddfbc8#81cf9f1fc2f33e4936b9b45cdc9b588da9ddfbc8" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 350d12b5..718cd660 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "740ad64ed297e91586c4eb739f0a21a9a60506ec" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "81cf9f1fc2f33e4936b9b45cdc9b588da9ddfbc8" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 81222aee38c6da811e0ee5338741875ff0a7c674 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 28 Mar 2023 12:25:24 -0700 Subject: [PATCH 298/494] Add JNI GC Locker check --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/api.rs | 4 ++-- openjdk/mmtk.h | 2 +- openjdk/mmtkHeap.cpp | 9 +++------ openjdk/mmtkVMOperation.cpp | 12 ++++++++++++ 6 files changed, 21 insertions(+), 12 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 490a2a44..31e8f38b 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f7e4450e2566a6d64a5ca5112a908f84fbf7da0c#f7e4450e2566a6d64a5ca5112a908f84fbf7da0c" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=c3c84a2a9a465a81ca9b269a27602e878ab91b6a#c3c84a2a9a465a81ca9b269a27602e878ab91b6a" dependencies = [ "atomic", "atomic-traits", @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f7e4450e2566a6d64a5ca5112a908f84fbf7da0c#f7e4450e2566a6d64a5ca5112a908f84fbf7da0c" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=c3c84a2a9a465a81ca9b269a27602e878ab91b6a#c3c84a2a9a465a81ca9b269a27602e878ab91b6a" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index cf321366..5f76a62a 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f7e4450e2566a6d64a5ca5112a908f84fbf7da0c" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "c3c84a2a9a465a81ca9b269a27602e878ab91b6a" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index e8842e5b..42958db7 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -208,8 +208,8 @@ pub extern "C" fn total_bytes() -> usize { } #[no_mangle] -pub extern "C" fn handle_user_collection_request(tls: VMMutatorThread) { - memory_manager::handle_user_collection_request::(&SINGLETON, tls); +pub extern "C" fn handle_user_collection_request(tls: VMMutatorThread, force: bool) { + memory_manager::handle_user_collection_request::(&SINGLETON, tls, force); } #[no_mangle] diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index be7d4855..5e15cbcc 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -116,7 +116,7 @@ extern bool will_never_move(void* object); extern bool process(char* name, char* value); extern bool process_bulk(char* options); extern void scan_region(); -extern void handle_user_collection_request(void *tls); +extern void handle_user_collection_request(void *tls, bool force); extern void start_control_collector(void *tls, void *context); extern void start_worker(void *tls, void* worker); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 121a6c74..511acd98 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -267,20 +267,17 @@ bool MMTkHeap::card_mark_must_follow_store() const { //OK } void MMTkHeap::collect(GCCause::Cause cause) {//later when gc is implemented in rust - if (DisableExplicitGC) return; - handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, cause != GCCause::_java_lang_system_gc); } // Perform a full collection void MMTkHeap::do_full_collection(bool clear_all_soft_refs) {//later when gc is implemented in rust - if (DisableExplicitGC) return; - handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, true); } void MMTkHeap::collect_as_vm_thread(GCCause::Cause cause) { - if (DisableExplicitGC) return; MMTkHeap::heap()->companion_thread()->vm_thread_requires_gc_pause(); - handle_user_collection_request(NULL); + handle_user_collection_request(NULL, true); MMTkHeap::heap()->companion_thread()->block_vm_thread(); } diff --git a/openjdk/mmtkVMOperation.cpp b/openjdk/mmtkVMOperation.cpp index 596df2f9..29d5bd60 100644 --- a/openjdk/mmtkVMOperation.cpp +++ b/openjdk/mmtkVMOperation.cpp @@ -40,6 +40,18 @@ bool VM_MMTkSTWOperation::doit_prologue() { } void VM_MMTkSTWOperation::doit() { + if (GCLocker::check_active_before_gc()) { + // If some threads is in JNI critical region, we don't do a GC for now, + // and end this VM operation earlier. Under such case, `GCLocker::check_active_before_gc` + // will remember there is a pending GC. After the thread exits the critical region, + // if a pending GC needs to be triggered, the java thread will + // call `MMTkHeap::collect(GCCause::_gc_locker)`. + // Since we've already have a unfinished GC request inside mmtk, + // mmtk will not trigger another GC, but simply blocking this thread. + // After all threads are successfully blocked, the previously + // triggered pending GC will proceed. + return; + } log_trace(vmthread)("Entered VM_MMTkSTWOperation::doit()."); _companion_thread->reach_suspended_and_wait_for_resume(); log_trace(vmthread)("Leaving VM_MMTkSTWOperation::doit()"); From 8432f53686a5163ad199dcf6dda94cde5bd5a730 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 28 Mar 2023 14:11:58 -0700 Subject: [PATCH 299/494] Fix MMTkVMCompanionThread busy looping when waiting for JNI critical region --- openjdk/mmtkHeap.cpp | 5 +++++ openjdk/mmtkVMCompanionThread.cpp | 27 ++++++++++++++++++++++++--- openjdk/mmtkVMCompanionThread.hpp | 3 +++ openjdk/mmtkVMOperation.cpp | 2 ++ 4 files changed, 34 insertions(+), 3 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 511acd98..9dd5ca01 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -267,6 +267,11 @@ bool MMTkHeap::card_mark_must_follow_store() const { //OK } void MMTkHeap::collect(GCCause::Cause cause) {//later when gc is implemented in rust + if (cause == GCCause::_gc_locker) { + MutexLockerEx locker(JNICritical_lock, Mutex::_no_safepoint_check_flag); + // Notify the VMCompanionThread to trigger another VM_MMTkSTWOperation. + JNICritical_lock->notify_all(); + } handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, cause != GCCause::_java_lang_system_gc); } diff --git a/openjdk/mmtkVMCompanionThread.cpp b/openjdk/mmtkVMCompanionThread.cpp index 101bdd67..8312f072 100644 --- a/openjdk/mmtkVMCompanionThread.cpp +++ b/openjdk/mmtkVMCompanionThread.cpp @@ -27,6 +27,7 @@ #include "mmtkVMCompanionThread.hpp" #include "runtime/mutex.hpp" #include "logging/log.hpp" +#include "gc/shared/gcLocker.hpp" MMTkVMCompanionThread::MMTkVMCompanionThread(): NamedThread(), @@ -49,7 +50,7 @@ void MMTkVMCompanionThread::run() { for (;;) { // Wait for suspend request log_trace(gc)("MMTkVMCompanionThread: Waiting for suspend request..."); - { + if (!_wait_for_gc_locker) { MutexLockerEx locker(_lock, Mutex::_no_safepoint_check_flag); assert(_reached_state == _threads_resumed, "Threads should be running at this moment."); while (_desired_state != _threads_suspended) { @@ -61,6 +62,7 @@ void MMTkVMCompanionThread::run() { // Let the VM thread stop the world. log_trace(gc)("MMTkVMCompanionThread: Letting VMThread execute VM op..."); if (_vm_thread_requires_gc_pause) { + guarantee(!_wait_for_gc_locker, "VM thread is triggering a GC when the MMTkVMCompanionThread is waiting for GC locker"); MutexLockerEx locker(_lock, Mutex::_no_safepoint_check_flag); _vm_thread_requires_gc_pause = false; _vm_thread_suspend_for_gc = true; @@ -69,6 +71,25 @@ void MMTkVMCompanionThread::run() { _lock->wait(Mutex::_no_safepoint_check_flag); } } else { + if (_wait_for_gc_locker) { + // When VM_MMTkSTWOperation early exits due to a jni critical region, + // _wait_for_gc_locker will be set to true before exsting safepoint. + // This main loop will continue one more iteration and reach here. + // We wait until GC locker is inactive, to avoid busy looping. + #ifndef PRODUCT + auto safepoint_check_required = JNICritical_lock->_safepoint_check_required; + JNICritical_lock->_safepoint_check_required = Monitor::_safepoint_check_sometimes; + #endif + MutexLockerEx locker(JNICritical_lock, Mutex::_no_safepoint_check_flag); + while (GCLocker::is_active_and_needs_gc()) { + JNICritical_lock->wait(true); + } + #ifndef PRODUCT + JNICritical_lock->_safepoint_check_required = safepoint_check_required; + #endif + // clear the flag + _wait_for_gc_locker = false; + } VM_MMTkSTWOperation op(this); // VMThread::execute() is blocking. The companion thread will be blocked // here waiting for the VM thread to execute op, and the VM thread will @@ -79,7 +100,7 @@ void MMTkVMCompanionThread::run() { // Tell the waiter thread that the world has resumed. log_trace(gc)("MMTkVMCompanionThread: Notifying threads resumption..."); - { + if (!_wait_for_gc_locker) { MutexLockerEx locker(_lock, Mutex::_no_safepoint_check_flag); assert(_desired_state == _threads_resumed, "start-the-world should be requested."); assert(_reached_state == _threads_suspended, "Threads should still be suspended at this moment."); @@ -158,7 +179,7 @@ void MMTkVMCompanionThread::wait_for_reached(stw_state desired_state) { // This method will block until the GC requests start-the-world. void MMTkVMCompanionThread::reach_suspended_and_wait_for_resume() { assert(Thread::current()->is_VM_thread(), "reach_suspended_and_wait_for_resume can only be executed by the VM thread"); - + MutexLockerEx locker(_lock, Mutex::_no_safepoint_check_flag); // Tell the waiter thread that the world has stopped. diff --git a/openjdk/mmtkVMCompanionThread.hpp b/openjdk/mmtkVMCompanionThread.hpp index c62aa622..af70fa60 100644 --- a/openjdk/mmtkVMCompanionThread.hpp +++ b/openjdk/mmtkVMCompanionThread.hpp @@ -52,6 +52,9 @@ class MMTkVMCompanionThread: public NamedThread { stw_state _reached_state; bool _vm_thread_requires_gc_pause = false; bool _vm_thread_suspend_for_gc = false; + bool _wait_for_gc_locker = false; + + friend class VM_MMTkSTWOperation; public: // Constructor diff --git a/openjdk/mmtkVMOperation.cpp b/openjdk/mmtkVMOperation.cpp index 29d5bd60..00060fae 100644 --- a/openjdk/mmtkVMOperation.cpp +++ b/openjdk/mmtkVMOperation.cpp @@ -29,6 +29,7 @@ #include "interpreter/oopMapCache.hpp" #include "logging/log.hpp" #include "interpreter/oopMapCache.hpp" +#include "gc/shared/gcLocker.hpp" VM_MMTkSTWOperation::VM_MMTkSTWOperation(MMTkVMCompanionThread *companion_thread): _companion_thread(companion_thread) { @@ -50,6 +51,7 @@ void VM_MMTkSTWOperation::doit() { // mmtk will not trigger another GC, but simply blocking this thread. // After all threads are successfully blocked, the previously // triggered pending GC will proceed. + _companion_thread->_wait_for_gc_locker = true; return; } log_trace(vmthread)("Entered VM_MMTkSTWOperation::doit()."); From 23a727ad097e51d512df1c5c5a18e481fbfed35d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 30 Mar 2023 10:49:20 -0700 Subject: [PATCH 300/494] Fix potential memory leak --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/api.rs | 5 +++++ openjdk/mmtk.h | 1 + openjdk/mmtkMutator.cpp | 10 +++++++++- openjdk/mmtkMutator.hpp | 1 + 6 files changed, 19 insertions(+), 4 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 31e8f38b..5c6880f5 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=c3c84a2a9a465a81ca9b269a27602e878ab91b6a#c3c84a2a9a465a81ca9b269a27602e878ab91b6a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6a22bfc134532d785d1bd256b72dcab74fd584d0#6a22bfc134532d785d1bd256b72dcab74fd584d0" dependencies = [ "atomic", "atomic-traits", @@ -477,7 +477,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.16.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=c3c84a2a9a465a81ca9b269a27602e878ab91b6a#c3c84a2a9a465a81ca9b269a27602e878ab91b6a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6a22bfc134532d785d1bd256b72dcab74fd584d0#6a22bfc134532d785d1bd256b72dcab74fd584d0" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 5f76a62a..f6a712ca 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "c3c84a2a9a465a81ca9b269a27602e878ab91b6a" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "6a22bfc134532d785d1bd256b72dcab74fd584d0" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 42958db7..28f791e8 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -125,6 +125,11 @@ pub extern "C" fn flush_mutator(mutator: *mut Mutator) { memory_manager::flush_mutator(unsafe { &mut *mutator }) } +#[no_mangle] +pub unsafe extern "C" fn release_mutator(mutator: *mut Mutator) { + let _boxed = Box::from_raw(mutator); +} + #[no_mangle] // We trust the mutator pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 5e15cbcc..f44accf9 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -42,6 +42,7 @@ extern const char* get_mmtk_version(); extern MMTk_Mutator bind_mutator(void *tls); extern void destroy_mutator(MMTk_Mutator mutator); extern void flush_mutator(MMTk_Mutator mutator); +extern void release_mutator(MMTk_Mutator mutator); extern void mmtk_report_gc_start(); diff --git a/openjdk/mmtkMutator.cpp b/openjdk/mmtkMutator.cpp index c78c35df..6e941ddf 100644 --- a/openjdk/mmtkMutator.cpp +++ b/openjdk/mmtkMutator.cpp @@ -15,7 +15,10 @@ MMTkMutatorContext MMTkMutatorContext::bind(::Thread* current) { printf("ERROR: Unmatched free list allocator size: rs=%zu cpp=%zu\n", FREE_LIST_ALLOCATOR_SIZE, sizeof(FreeListAllocator)); guarantee(false, "ERROR"); } - return *((MMTkMutatorContext*) ::bind_mutator((void*) current)); + auto original_rust_mutator_pointer = (MMTkMutatorContext*) ::bind_mutator((void*) current); + MMTkMutatorContext context = *original_rust_mutator_pointer; + context.original_rust_mutator_pointer = original_rust_mutator_pointer; + return context; } bool MMTkMutatorContext::is_ready_to_bind() { @@ -57,4 +60,9 @@ void MMTkMutatorContext::flush() { void MMTkMutatorContext::destroy() { ::destroy_mutator((MMTk_Mutator) this); + if (original_rust_mutator_pointer != NULL) { + *original_rust_mutator_pointer = *this; + release_mutator(original_rust_mutator_pointer); + original_rust_mutator_pointer = NULL; + } } diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp index 6248a958..c3070e03 100644 --- a/openjdk/mmtkMutator.hpp +++ b/openjdk/mmtkMutator.hpp @@ -114,6 +114,7 @@ struct MMTkMutatorContext { void* mutator_tls; RustDynPtr plan; MutatorConfig config; + MMTkMutatorContext* original_rust_mutator_pointer; HeapWord* alloc(size_t bytes, Allocator allocator = AllocatorDefault); From d8fd43313b9b8f9504da9cf4965583037ba209bc Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 31 Mar 2023 11:22:54 -0700 Subject: [PATCH 301/494] Temporarily remove memory leak fix --- openjdk/mmtkMutator.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/openjdk/mmtkMutator.cpp b/openjdk/mmtkMutator.cpp index 6e941ddf..06a118b8 100644 --- a/openjdk/mmtkMutator.cpp +++ b/openjdk/mmtkMutator.cpp @@ -60,9 +60,9 @@ void MMTkMutatorContext::flush() { void MMTkMutatorContext::destroy() { ::destroy_mutator((MMTk_Mutator) this); - if (original_rust_mutator_pointer != NULL) { - *original_rust_mutator_pointer = *this; - release_mutator(original_rust_mutator_pointer); - original_rust_mutator_pointer = NULL; - } + // if (original_rust_mutator_pointer != NULL) { + // *original_rust_mutator_pointer = *this; + // release_mutator(original_rust_mutator_pointer); + // original_rust_mutator_pointer = NULL; + // } } From a2d43d5cc4d44a4227ae8dec449850a52732a1c9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 3 Apr 2023 01:51:05 +0000 Subject: [PATCH 302/494] Refactor and cleanup compressed pointer related code --- mmtk/src/abi.rs | 49 +++--- mmtk/src/active_plan.rs | 15 +- mmtk/src/api.rs | 259 +++++++++++++++++++----------- mmtk/src/collection.rs | 55 +++---- mmtk/src/gc_work.rs | 92 +++++++---- mmtk/src/lib.rs | 131 +++++++++------ mmtk/src/object_model.rs | 26 +-- mmtk/src/object_scanning.rs | 131 ++++++++------- mmtk/src/reference_glue.rs | 196 +++++++++------------- mmtk/src/scanning.rs | 55 ++++--- mmtk/src/vm_metadata/constants.rs | 4 +- 11 files changed, 545 insertions(+), 468 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 3f39e9e1..26f8db3a 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -1,6 +1,5 @@ -use crate::OpenJDKEdge; - use super::UPCALLS; +use crate::Edge; use atomic::Ordering; use mmtk::util::constants::*; use mmtk::util::conversions; @@ -273,11 +272,11 @@ impl InstanceRefKlass { } *DISCOVERED_OFFSET } - pub fn referent_address(oop: Oop) -> OpenJDKEdge { - OpenJDKEdge(oop.get_field_address(Self::referent_offset())) + pub fn referent_address(oop: Oop) -> E { + E::from_address(oop.get_field_address(Self::referent_offset())) } - pub fn discovered_address(oop: Oop) -> OpenJDKEdge { - OpenJDKEdge(oop.get_field_address(Self::discovered_offset())) + pub fn discovered_address(oop: Oop) -> E { + E::from_address(oop.get_field_address(Self::discovered_offset())) } } @@ -293,19 +292,17 @@ pub struct OopDesc { klass: KlassField, } +lazy_static! { + static ref COMPRESSED_KLASS_BASE: Address = unsafe { ((*UPCALLS).compressed_klass_base)() }; + static ref COMPRESSED_KLASS_SHIFT: usize = unsafe { ((*UPCALLS).compressed_klass_shift)() }; +} + impl OopDesc { pub fn start(&self) -> Address { unsafe { mem::transmute(self) } } pub(crate) fn klass_ptr(&self) -> Address { - // self.klass if COMPRESSED { - lazy_static! { - static ref COMPRESSED_KLASS_BASE: Address = - unsafe { ((*UPCALLS).compressed_klass_base)() }; - static ref COMPRESSED_KLASS_SHIFT: usize = - unsafe { ((*UPCALLS).compressed_klass_shift)() }; - } let compressed = unsafe { self.klass.narrow_klass }; let addr = *COMPRESSED_KLASS_BASE + ((compressed as usize) << *COMPRESSED_KLASS_SHIFT); addr @@ -314,14 +311,7 @@ impl OopDesc { } } pub fn klass(&self) -> &'static Klass { - // self.klass if COMPRESSED { - lazy_static! { - static ref COMPRESSED_KLASS_BASE: Address = - unsafe { ((*UPCALLS).compressed_klass_base)() }; - static ref COMPRESSED_KLASS_SHIFT: usize = - unsafe { ((*UPCALLS).compressed_klass_shift)() }; - } let compressed = unsafe { self.klass.narrow_klass }; let addr = *COMPRESSED_KLASS_BASE + ((compressed as usize) << *COMPRESSED_KLASS_SHIFT); unsafe { &*addr.to_ptr::() } @@ -456,7 +446,10 @@ impl ArrayOopDesc { self.length::() as _, ) } - pub unsafe fn slice(&self, ty: BasicType) -> crate::OpenJDKEdgeRange { + pub unsafe fn slice( + &self, + ty: BasicType, + ) -> crate::OpenJDKEdgeRange { let base = self.base::(ty); let start = crate::OpenJDKEdge(base); let end = crate::OpenJDKEdge( @@ -504,7 +497,7 @@ struct ChunkedHandleList { } impl ChunkedHandleList { - unsafe fn oops_do_chunk, const COMPRESSED: bool>( + unsafe fn oops_do_chunk, const COMPRESSED: bool>( &self, chunk: &'static Chunk, size: u32, @@ -517,22 +510,22 @@ impl ChunkedHandleList { if COMPRESSED { word = word | (1usize << 63); } - closure.visit_edge(OpenJDKEdge(Address::from_usize(word))) + closure.visit_edge(E::from_address(Address::from_usize(word))) } } } - fn oops_do, const COMPRESSED: bool>(&self, closure: &mut V) { + fn oops_do, const COMPRESSED: bool>(&self, closure: &mut V) { let head = self.head.load(Ordering::Acquire); if !head.is_null() { let head = unsafe { &*head }; let size = head.size.load(Ordering::Acquire); - unsafe { self.oops_do_chunk::<_, COMPRESSED>(head, size, closure) }; + unsafe { self.oops_do_chunk::<_, _, COMPRESSED>(head, size, closure) }; let mut c = head.next; while !c.is_null() { let chunk = unsafe { &*c }; let size = chunk.size.load(Ordering::Relaxed); - unsafe { self.oops_do_chunk::<_, COMPRESSED>(chunk, size, closure) }; + unsafe { self.oops_do_chunk::<_, _, COMPRESSED>(chunk, size, closure) }; c = chunk.next; } } @@ -567,10 +560,10 @@ impl ClassLoaderData { .is_ok() } - pub fn oops_do, const COMPRESSED: bool>(&self, closure: &mut V) { + pub fn oops_do, const COMPRESSED: bool>(&self, closure: &mut V) { if closure.should_claim_clds() && !self.claim() { return; } - self.handles.oops_do::<_, COMPRESSED>(closure); + self.handles.oops_do::<_, _, COMPRESSED>(closure); } } diff --git a/mmtk/src/active_plan.rs b/mmtk/src/active_plan.rs index 88d12958..a702ac53 100644 --- a/mmtk/src/active_plan.rs +++ b/mmtk/src/active_plan.rs @@ -1,5 +1,4 @@ use crate::OpenJDK; -use crate::SINGLETON; use crate::UPCALLS; use mmtk::util::opaque_pointer::*; use mmtk::vm::ActivePlan; @@ -9,19 +8,19 @@ use std::sync::Mutex; pub struct VMActivePlan {} -impl ActivePlan for VMActivePlan { - fn global() -> &'static dyn Plan { - SINGLETON.get_plan() +impl ActivePlan> for VMActivePlan { + fn global() -> &'static dyn Plan> { + crate::singleton::().get_plan() } fn is_mutator(tls: VMThread) -> bool { unsafe { ((*UPCALLS).is_mutator)(tls) } } - fn mutator(tls: VMMutatorThread) -> &'static mut Mutator { + fn mutator(tls: VMMutatorThread) -> &'static mut Mutator> { unsafe { let m = ((*UPCALLS).get_mmtk_mutator)(tls); - &mut *m + &mut *(m as *mut Mutator>) } } @@ -31,14 +30,14 @@ impl ActivePlan for VMActivePlan { } } - fn get_next_mutator() -> Option<&'static mut Mutator> { + fn get_next_mutator() -> Option<&'static mut Mutator>> { let _guard = MUTATOR_ITERATOR_LOCK.lock().unwrap(); unsafe { let m = ((*UPCALLS).get_next_mutator)(); if m.is_null() { None } else { - Some(&mut *m) + Some(&mut *(m as *mut Mutator>)) } } } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 28f791e8..1cb74cfb 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -3,7 +3,6 @@ use crate::OpenJDKEdge; use crate::OpenJDKEdgeRange; use crate::OpenJDK_Upcalls; use crate::BUILDER; -use crate::SINGLETON; use crate::UPCALLS; use libc::c_char; use mmtk::memory_manager; @@ -21,6 +20,18 @@ use std::cell::RefCell; use std::ffi::{CStr, CString}; use std::sync::atomic::Ordering; +macro_rules! with_mutator { + (|$x: ident| $($expr:tt)*) => { + if crate::use_compressed_oops() { + let $x = unsafe { &mut *($x as *mut Mutator>) }; + $($expr)* + } else { + let $x = unsafe { &mut *($x as *mut Mutator>) }; + $($expr)* + } + }; +} + // Supported barriers: static NO_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("NoBarrier").unwrap()); static OBJECT_BARRIER: sync::Lazy = @@ -35,19 +46,21 @@ pub extern "C" fn get_mmtk_version() -> *const c_char { #[no_mangle] pub extern "C" fn mmtk_active_barrier() -> *const c_char { - match SINGLETON.get_plan().constraints().barrier { - BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), - BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), - BarrierSelector::FieldBarrier => FIELD_LOGGING_BARRIER.as_ptr(), - // In case we have more barriers in mmtk-core. - #[allow(unreachable_patterns)] - _ => unimplemented!(), - } + with_singleton!(|singleton| { + match singleton.get_plan().constraints().barrier { + BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), + BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), + BarrierSelector::FieldBarrier => FIELD_LOGGING_BARRIER.as_ptr(), + // In case we have more barriers in mmtk-core. + #[allow(unreachable_patterns)] + _ => unimplemented!(), + } + }) } #[no_mangle] pub extern "C" fn mmtk_report_gc_start() { - mmtk::memory_manager::report_gc_start::(&SINGLETON) + with_singleton!(|singleton| mmtk::memory_manager::report_gc_start(&singleton)); } /// # Safety @@ -92,7 +105,11 @@ pub extern "C" fn openjdk_gc_init(calls: *const OpenJDK_Upcalls) { // Make sure that we haven't initialized MMTk (by accident) yet assert!(!crate::MMTK_INITIALIZED.load(Ordering::SeqCst)); // Make sure we initialize MMTk here - lazy_static::initialize(&SINGLETON); + if crate::use_compressed_oops() { + lazy_static::initialize(&crate::SINGLETON_COMPRESSED); + } else { + lazy_static::initialize(&crate::SINGLETON_UNCOMPRESSED); + } } #[no_mangle] @@ -107,65 +124,75 @@ pub extern "C" fn mmtk_set_heap_size(size: usize) -> bool { } #[no_mangle] -pub extern "C" fn bind_mutator(tls: VMMutatorThread) -> *mut Mutator { - Box::into_raw(memory_manager::bind_mutator(&SINGLETON, tls)) +pub extern "C" fn bind_mutator(tls: VMMutatorThread) -> *mut libc::c_void { + with_singleton!(|singleton| { + Box::into_raw(memory_manager::bind_mutator(&singleton, tls)) as *mut libc::c_void + }) } #[no_mangle] // It is fine we turn the pointer back to box, as we turned a boxed value to the raw pointer in bind_mutator() #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn destroy_mutator(mutator: *mut Mutator) { - memory_manager::destroy_mutator(unsafe { &mut *mutator }) +pub extern "C" fn destroy_mutator(mutator: *mut libc::c_void) { + with_mutator!(|mutator| memory_manager::destroy_mutator(mutator)) } #[no_mangle] // We trust the mutator pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn flush_mutator(mutator: *mut Mutator) { - memory_manager::flush_mutator(unsafe { &mut *mutator }) +pub extern "C" fn flush_mutator(mutator: *mut libc::c_void) { + with_mutator!(|mutator| memory_manager::flush_mutator(mutator)) } #[no_mangle] -pub unsafe extern "C" fn release_mutator(mutator: *mut Mutator) { - let _boxed = Box::from_raw(mutator); +pub unsafe extern "C" fn release_mutator(mutator: *mut libc::c_void) { + if crate::use_compressed_oops() { + let _boxed: Box>> = + Box::from_raw(mutator as *mut Mutator>); + } else { + let _boxed: Box>> = + Box::from_raw(mutator as *mut Mutator>); + } } #[no_mangle] // We trust the mutator pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] pub extern "C" fn alloc( - mutator: *mut Mutator, + mutator: *mut libc::c_void, size: usize, align: usize, offset: isize, allocator: AllocationSemantics, ) -> Address { - memory_manager::alloc::(unsafe { &mut *mutator }, size, align, offset, allocator) + with_mutator!(|mutator| memory_manager::alloc(mutator, size, align, offset, allocator)) } #[no_mangle] pub extern "C" fn get_allocator_mapping(allocator: AllocationSemantics) -> AllocatorSelector { - memory_manager::get_allocator_mapping(&SINGLETON, allocator) + with_singleton!(|singleton| memory_manager::get_allocator_mapping(&singleton, allocator)) } #[no_mangle] pub extern "C" fn get_max_non_los_default_alloc_bytes() -> usize { - SINGLETON - .get_plan() - .constraints() - .max_non_los_default_alloc_bytes + with_singleton!(|singleton| { + singleton + .get_plan() + .constraints() + .max_non_los_default_alloc_bytes + }) } #[no_mangle] // We trust the mutator pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] pub extern "C" fn post_alloc( - mutator: *mut Mutator, + mutator: *mut libc::c_void, refer: ObjectReference, bytes: usize, allocator: AllocationSemantics, ) { - memory_manager::post_alloc::(unsafe { &mut *mutator }, refer, bytes, allocator) + with_mutator!(|mutator| memory_manager::post_alloc(mutator, refer, bytes, allocator)) } #[no_mangle] @@ -176,45 +203,68 @@ pub extern "C" fn will_never_move(object: ObjectReference) -> bool { #[no_mangle] // We trust the gc_collector pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn start_control_collector( - tls: VMWorkerThread, - gc_controller: *mut GCController, -) { - let mut gc_controller = unsafe { Box::from_raw(gc_controller) }; - memory_manager::start_control_collector(&SINGLETON, tls, &mut gc_controller); +pub extern "C" fn start_control_collector(tls: VMWorkerThread, gc_controller: *mut libc::c_void) { + if crate::use_compressed_oops() { + let mut gc_controller = + unsafe { Box::from_raw(gc_controller as *mut GCController>) }; + memory_manager::start_control_collector( + &crate::singleton::(), + tls, + &mut gc_controller, + ); + } else { + let mut gc_controller = + unsafe { Box::from_raw(gc_controller as *mut GCController>) }; + memory_manager::start_control_collector( + &crate::singleton::(), + tls, + &mut gc_controller, + ); + } } #[no_mangle] // We trust the worker pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn start_worker(tls: VMWorkerThread, worker: *mut GCWorker) { - let mut worker = unsafe { Box::from_raw(worker) }; - memory_manager::start_worker::(&SINGLETON, tls, &mut worker) +pub extern "C" fn start_worker(tls: VMWorkerThread, worker: *mut libc::c_void) { + if crate::use_compressed_oops() { + let mut worker = unsafe { Box::from_raw(worker as *mut GCWorker>) }; + memory_manager::start_worker::>(&crate::singleton::(), tls, &mut worker) + } else { + let mut worker = unsafe { Box::from_raw(worker as *mut GCWorker>) }; + memory_manager::start_worker::>( + &crate::singleton::(), + tls, + &mut worker, + ) + } } #[no_mangle] pub extern "C" fn initialize_collection(tls: VMThread) { - memory_manager::initialize_collection(&SINGLETON, tls) + with_singleton!(|singleton| memory_manager::initialize_collection(&singleton, tls)) } #[no_mangle] pub extern "C" fn used_bytes() -> usize { - memory_manager::used_bytes(&SINGLETON) + with_singleton!(|singleton| memory_manager::used_bytes(&singleton)) } #[no_mangle] pub extern "C" fn free_bytes() -> usize { - memory_manager::free_bytes(&SINGLETON) + with_singleton!(|singleton| memory_manager::free_bytes(&singleton)) } #[no_mangle] pub extern "C" fn total_bytes() -> usize { - memory_manager::total_bytes(&SINGLETON) + with_singleton!(|singleton| memory_manager::total_bytes(&singleton)) } #[no_mangle] pub extern "C" fn handle_user_collection_request(tls: VMMutatorThread, force: bool) { - memory_manager::handle_user_collection_request::(&SINGLETON, tls, force); + with_singleton!(|singleton| { + memory_manager::handle_user_collection_request(&singleton, tls, force); + }) } #[no_mangle] @@ -224,7 +274,11 @@ pub extern "C" fn mmtk_use_compressed_ptrs() { #[no_mangle] pub extern "C" fn is_in_mmtk_spaces(object: ObjectReference) -> bool { - memory_manager::is_in_mmtk_spaces::(object) + if crate::use_compressed_oops() { + memory_manager::is_in_mmtk_spaces::>(object) + } else { + memory_manager::is_in_mmtk_spaces::>(object) + } } #[no_mangle] @@ -234,22 +288,22 @@ pub extern "C" fn is_mapped_address(addr: Address) -> bool { #[no_mangle] pub extern "C" fn modify_check(object: ObjectReference) { - memory_manager::modify_check(&SINGLETON, object) + with_singleton!(|singleton| memory_manager::modify_check(&singleton, object)) } #[no_mangle] pub extern "C" fn add_weak_candidate(reff: ObjectReference) { - memory_manager::add_weak_candidate(&SINGLETON, reff) + with_singleton!(|singleton| memory_manager::add_weak_candidate(&singleton, reff)) } #[no_mangle] pub extern "C" fn add_soft_candidate(reff: ObjectReference) { - memory_manager::add_soft_candidate(&SINGLETON, reff) + with_singleton!(|singleton| memory_manager::add_soft_candidate(&singleton, reff)) } #[no_mangle] pub extern "C" fn add_phantom_candidate(reff: ObjectReference) { - memory_manager::add_phantom_candidate(&SINGLETON, reff) + with_singleton!(|singleton| memory_manager::add_phantom_candidate(&singleton, reff)) } // The harness_begin()/end() functions are different than other API functions in terms of the thread state. @@ -266,7 +320,9 @@ pub extern "C" fn harness_begin(_id: usize) { #[no_mangle] pub extern "C" fn mmtk_harness_begin_impl() { // Pass null as tls, OpenJDK binding does not rely on the tls value to block the current thread and do a GC - memory_manager::harness_begin(&SINGLETON, VMMutatorThread(VMThread::UNINITIALIZED)); + with_singleton!(|singleton| { + memory_manager::harness_begin(&singleton, VMMutatorThread(VMThread::UNINITIALIZED)); + }) } #[no_mangle] @@ -276,7 +332,7 @@ pub extern "C" fn harness_end(_id: usize) { #[no_mangle] pub extern "C" fn mmtk_harness_end_impl() { - memory_manager::harness_end(&SINGLETON); + with_singleton!(|singleton| memory_manager::harness_end(&singleton)) } #[no_mangle] @@ -324,7 +380,7 @@ pub extern "C" fn last_heap_address() -> Address { #[no_mangle] pub extern "C" fn openjdk_max_capacity() -> usize { - memory_manager::total_bytes(&SINGLETON) + with_singleton!(|singleton| memory_manager::total_bytes(&singleton)) } #[no_mangle] @@ -333,97 +389,107 @@ pub extern "C" fn executable() -> bool { } #[no_mangle] -pub extern "C" fn mmtk_load_reference(mutator: &'static mut Mutator, o: ObjectReference) { - mutator.barrier().load_reference(o); +pub extern "C" fn mmtk_load_reference(mutator: *mut libc::c_void, o: ObjectReference) { + with_mutator!(|mutator| mutator.barrier().load_reference(o)) } #[no_mangle] pub extern "C" fn mmtk_object_reference_clone_pre( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, obj: ObjectReference, ) { - mutator.barrier().object_reference_clone_pre(obj); + with_mutator!(|mutator| mutator.barrier().object_reference_clone_pre(obj)) } /// Full pre barrier #[no_mangle] pub extern "C" fn mmtk_object_reference_write_pre( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: ObjectReference, ) { - mutator - .barrier() - .object_reference_write_pre(src, OpenJDKEdge(slot), target); + with_mutator!(|mutator| { + mutator + .barrier() + .object_reference_write_pre(src, OpenJDKEdge(slot), target); + }) } /// Full post barrier #[no_mangle] pub extern "C" fn mmtk_object_reference_write_post( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: ObjectReference, ) { - mutator - .barrier() - .object_reference_write_post(src, OpenJDKEdge(slot), target); + with_mutator!(|mutator| { + mutator + .barrier() + .object_reference_write_post(src, OpenJDKEdge(slot), target); + }) } /// Barrier slow-path call #[no_mangle] pub extern "C" fn mmtk_object_reference_write_slow( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: ObjectReference, ) { - mutator - .barrier() - .object_reference_write_slow(src, OpenJDKEdge(slot), target); + with_mutator!(|mutator| { + mutator + .barrier() + .object_reference_write_slow(src, OpenJDKEdge(slot), target); + }) } /// Array-copy pre-barrier #[no_mangle] pub extern "C" fn mmtk_array_copy_pre( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: Address, dst: Address, count: usize, ) { let bytes = count << crate::log_bytes_in_field(); - mutator.barrier().memory_region_copy_pre( - OpenJDKEdgeRange { - start: OpenJDKEdge(src), - end: OpenJDKEdge(src + bytes), - }, - OpenJDKEdgeRange { - start: OpenJDKEdge(dst), - end: OpenJDKEdge(dst + bytes), - }, - ); + with_mutator!(|mutator| { + mutator.barrier().memory_region_copy_pre( + OpenJDKEdgeRange { + start: OpenJDKEdge(src), + end: OpenJDKEdge(src + bytes), + }, + OpenJDKEdgeRange { + start: OpenJDKEdge(dst), + end: OpenJDKEdge(dst + bytes), + }, + ); + }) } /// Array-copy post-barrier #[no_mangle] pub extern "C" fn mmtk_array_copy_post( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: Address, dst: Address, count: usize, ) { - let bytes = count << crate::log_bytes_in_field(); - mutator.barrier().memory_region_copy_post( - OpenJDKEdgeRange { - start: OpenJDKEdge(src), - end: OpenJDKEdge(src + bytes), - }, - OpenJDKEdgeRange { - start: OpenJDKEdge(dst), - end: OpenJDKEdge(dst + bytes), - }, - ); + with_mutator!(|mutator| { + let bytes = count << crate::log_bytes_in_field(); + mutator.barrier().memory_region_copy_post( + OpenJDKEdgeRange { + start: OpenJDKEdge(src), + end: OpenJDKEdge(src + bytes), + }, + OpenJDKEdgeRange { + start: OpenJDKEdge(dst), + end: OpenJDKEdge(dst + bytes), + }, + ); + }) } // finalization @@ -434,10 +500,12 @@ pub extern "C" fn add_finalizer(_object: ObjectReference) { #[no_mangle] pub extern "C" fn get_finalized_object() -> ObjectReference { - match memory_manager::get_finalized_object(&SINGLETON) { - Some(obj) => obj, - None => ObjectReference::NULL, - } + with_singleton!(|singleton| { + match memory_manager::get_finalized_object(&singleton) { + Some(obj) => obj, + None => ObjectReference::NULL, + } + }) } /// Test if an object is live at the end of a GC. @@ -447,8 +515,7 @@ pub extern "C" fn mmtk_is_live(object: ObjectReference) -> usize { if object.is_null() { return 0; } - debug_assert!(object.to_address::().is_mapped()); - debug_assert!(object.class_is_valid::()); + debug_assert!(object.to_raw_address().is_mapped()); object.is_live() as _ } @@ -511,5 +578,5 @@ pub extern "C" fn mmtk_unregister_nmethod(nm: Address) { #[no_mangle] pub extern "C" fn mmtk_verbose() -> usize { - *crate::SINGLETON.options.verbose + with_singleton!(|singleton| *singleton.options.verbose) } diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index fbbd2dae..fe121865 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -10,22 +10,23 @@ use crate::{MutatorClosure, OpenJDK}; pub struct VMCollection {} -extern "C" fn report_mutator_stop( - mutator: *mut Mutator, +extern "C" fn report_mutator_stop( + mutator: *mut libc::c_void, callback_ptr: *mut libc::c_void, ) where - F: FnMut(&'static mut Mutator), + F: FnMut(&'static mut Mutator>), { + let mutator = mutator as *mut Mutator>; let callback: &mut F = unsafe { &mut *(callback_ptr as *mut F) }; callback(unsafe { &mut *mutator }); } -fn to_mutator_closure(callback: &mut F) -> MutatorClosure +fn to_mutator_closure(callback: &mut F) -> MutatorClosure where - F: FnMut(&'static mut Mutator), + F: FnMut(&'static mut Mutator>), { MutatorClosure { - func: report_mutator_stop::, + func: report_mutator_stop::, data: callback as *mut F as *mut libc::c_void, } } @@ -33,7 +34,7 @@ where const GC_THREAD_KIND_CONTROLLER: libc::c_int = 0; const GC_THREAD_KIND_WORKER: libc::c_int = 1; -impl Collection for VMCollection { +impl Collection> for VMCollection { /// With the presence of the "VM companion thread", /// the OpenJDK binding allows any MMTk GC thread to stop/start the world. const COORDINATOR_ONLY_STW: bool = false; @@ -43,16 +44,16 @@ impl Collection for VMCollection { mut mutator_visitor: F, current_gc_should_unload_classes: bool, ) where - F: FnMut(&'static mut Mutator), + F: FnMut(&'static mut Mutator>), { let scan_mutators_in_safepoint = - ::VMScanning::SCAN_MUTATORS_IN_SAFEPOINT; + < as VMBinding>::VMScanning as Scanning>>::SCAN_MUTATORS_IN_SAFEPOINT; unsafe { ((*UPCALLS).stop_all_mutators)( tls, scan_mutators_in_safepoint, - to_mutator_closure(&mut mutator_visitor), + to_mutator_closure::<_, COMPRESSED>(&mut mutator_visitor), current_gc_should_unload_classes, ); } @@ -71,7 +72,7 @@ impl Collection for VMCollection { } } - fn spawn_gc_thread(tls: VMThread, ctx: GCThreadContext) { + fn spawn_gc_thread(tls: VMThread, ctx: GCThreadContext>) { let (ctx_ptr, kind) = match ctx { GCThreadContext::Controller(c) => ( Box::into_raw(c) as *mut libc::c_void, @@ -86,7 +87,7 @@ impl Collection for VMCollection { } } - fn prepare_mutator>( + fn prepare_mutator>>( _tls_w: VMWorkerThread, _tls_m: VMMutatorThread, _m: &T, @@ -104,28 +105,22 @@ impl Collection for VMCollection { unreachable!() } - fn process_weak_refs>(worker: &mut GCWorker) { - if crate::use_compressed_oops() { - DISCOVERED_LISTS.process_soft_weak_final_refs::(worker) - } else { - DISCOVERED_LISTS.process_soft_weak_final_refs::(worker) - } + fn process_weak_refs>>( + worker: &mut GCWorker>, + ) { + DISCOVERED_LISTS.process_soft_weak_final_refs::(worker) } - fn process_final_refs>(worker: &mut GCWorker) { - if crate::use_compressed_oops() { - DISCOVERED_LISTS.resurrect_final_refs::(worker) - } else { - DISCOVERED_LISTS.resurrect_final_refs::(worker) - } + fn process_final_refs>>( + worker: &mut GCWorker>, + ) { + DISCOVERED_LISTS.resurrect_final_refs::(worker) } - fn process_phantom_refs>(worker: &mut GCWorker) { - if crate::use_compressed_oops() { - DISCOVERED_LISTS.process_phantom_refs::(worker) - } else { - DISCOVERED_LISTS.process_phantom_refs::(worker) - } + fn process_phantom_refs>>( + worker: &mut GCWorker>, + ) { + DISCOVERED_LISTS.process_phantom_refs::(worker) } fn update_weak_processor(lxr: bool) { diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 283d1629..11984b63 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -1,27 +1,34 @@ use crate::scanning::to_edges_closure; use crate::Address; +use crate::Edge; use crate::EdgesClosure; use crate::NewBuffer; -use crate::{OpenJDK, OpenJDKEdge, UPCALLS}; +use crate::UPCALLS; use mmtk::scheduler::*; use mmtk::util::ObjectReference; use mmtk::vm::RootsWorkFactory; +use mmtk::vm::*; use mmtk::MMTK; +use std::marker::PhantomData; macro_rules! scan_roots_work { ($struct_name: ident, $func_name: ident) => { - pub struct $struct_name> { + pub struct $struct_name> { factory: F, + _p: std::marker::PhantomData, } - impl> $struct_name { + impl> $struct_name { pub fn new(factory: F) -> Self { - Self { factory } + Self { + factory, + _p: std::marker::PhantomData, + } } } - impl> GCWork for $struct_name { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { + impl> GCWork for $struct_name { + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { unsafe { ((*UPCALLS).$func_name)(to_edges_closure(&mut self.factory)); } @@ -40,18 +47,23 @@ scan_roots_work!(ScanSystemDictionaryRoots, scan_system_dictionary_roots); scan_roots_work!(ScanStringTableRoots, scan_string_table_roots); scan_roots_work!(ScanVMThreadRoots, scan_vm_thread_roots); -pub struct ScanClassLoaderDataGraphRoots> { +pub struct ScanClassLoaderDataGraphRoots> { factory: F, + _p: PhantomData, } -impl> ScanClassLoaderDataGraphRoots { +impl> ScanClassLoaderDataGraphRoots { pub fn new(factory: F) -> Self { - Self { factory } + Self { + factory, + _p: PhantomData, + } } } extern "C" fn report_edges_and_renew_buffer_cld< - F: RootsWorkFactory, + E: Edge, + F: RootsWorkFactory, const WEAK: bool, >( ptr: *mut Address, @@ -60,8 +72,8 @@ extern "C" fn report_edges_and_renew_buffer_cld< factory_ptr: *mut libc::c_void, ) -> NewBuffer { if !ptr.is_null() { - let ptr = ptr as *mut OpenJDKEdge; - let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; + let ptr = ptr as *mut E; + let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; factory.create_process_edge_roots_work_for_cld_roots(buf, WEAK); } @@ -75,20 +87,22 @@ extern "C" fn report_edges_and_renew_buffer_cld< NewBuffer { ptr, capacity } } -fn to_edges_closure_cld, const WEAK: bool>( +fn to_edges_closure_cld, const WEAK: bool>( factory: &mut F, ) -> EdgesClosure { EdgesClosure { - func: report_edges_and_renew_buffer_cld::, + func: report_edges_and_renew_buffer_cld::, data: factory as *mut F as *mut libc::c_void, } } -impl> GCWork for ScanClassLoaderDataGraphRoots { - fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { +impl> GCWork + for ScanClassLoaderDataGraphRoots +{ + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { unsafe { ((*UPCALLS).scan_class_loader_data_graph_roots)( - to_edges_closure_cld::(&mut self.factory), - to_edges_closure_cld::(&mut self.factory), + to_edges_closure_cld::(&mut self.factory), + to_edges_closure_cld::(&mut self.factory), mmtk.get_plan() .current_gc_should_scan_all_classloader_strong_roots(), ); @@ -96,23 +110,29 @@ impl> GCWork for ScanClassLoaderDataGr } } -pub struct ScanCodeCacheRoots> { +pub struct ScanCodeCacheRoots> { factory: F, + _p: PhantomData, } -impl> ScanCodeCacheRoots { +impl> ScanCodeCacheRoots { pub fn new(factory: F) -> Self { - Self { factory } + Self { + factory, + _p: PhantomData, + } } } -impl> GCWork for ScanCodeCacheRoots { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { +impl> GCWork + for ScanCodeCacheRoots +{ + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { // Collect all the cached roots let mut edges = Vec::with_capacity(F::BUFFER_SIZE); for roots in (*crate::CODE_CACHE_ROOTS.lock().unwrap()).values() { for r in roots { - edges.push(OpenJDKEdge(*r)); + edges.push(VM::VMEdge::from_address(*r)); if edges.len() >= F::BUFFER_SIZE { self.factory .create_process_edge_roots_work(std::mem::take(&mut edges)); @@ -131,7 +151,7 @@ impl> GCWork for ScanCodeCacheRoots } } -extern "C" fn report_edges_and_renew_buffer_weakref>( +extern "C" fn report_edges_and_renew_buffer_weakref>( ptr: *mut Address, length: usize, capacity: usize, @@ -155,27 +175,33 @@ extern "C" fn report_edges_and_renew_buffer_weakref>(factory: &mut F) -> EdgesClosure { +fn to_edges_closure_weakref>(factory: &mut F) -> EdgesClosure { EdgesClosure { - func: report_edges_and_renew_buffer_weakref::, + func: report_edges_and_renew_buffer_weakref::, data: factory as *mut F as *mut libc::c_void, } } -pub struct ScaWeakProcessorRoots> { +pub struct ScaWeakProcessorRoots> { factory: F, + _p: PhantomData, } -impl> ScaWeakProcessorRoots { +impl> ScaWeakProcessorRoots { pub fn new(factory: F) -> Self { - Self { factory } + Self { + factory, + _p: PhantomData, + } } } -impl> GCWork for ScaWeakProcessorRoots { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { +impl> GCWork + for ScaWeakProcessorRoots +{ + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { unsafe { - ((*UPCALLS).scan_weak_processor_roots)(to_edges_closure_weakref::<_>( + ((*UPCALLS).scan_weak_processor_roots)(to_edges_closure_weakref::<_, _>( &mut self.factory, )); } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 88340ebf..779bca3e 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -10,7 +10,6 @@ use std::sync::atomic::{AtomicU32, AtomicUsize}; use std::sync::Mutex; use libc::{c_char, c_void, uintptr_t}; -use mmtk::scheduler::GCWorker; use mmtk::util::alloc::AllocationError; use mmtk::util::constants::{ BYTES_IN_ADDRESS, BYTES_IN_INT, LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_INT, @@ -20,7 +19,19 @@ use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::edge_shape::{Edge, MemorySlice}; use mmtk::vm::VMBinding; -use mmtk::{MMTKBuilder, Mutator, MMTK}; +use mmtk::{MMTKBuilder, MMTK}; + +macro_rules! with_singleton { + (|$x: ident| $($expr:tt)*) => { + if crate::use_compressed_oops() { + let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_COMPRESSED; + $($expr)* + } else { + let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_UNCOMPRESSED; + $($expr)* + } + }; +} mod abi; pub mod active_plan; @@ -43,7 +54,7 @@ pub struct NewBuffer { /// A closure for reporting mutators. The C++ code should pass `data` back as the last argument. #[repr(C)] pub struct MutatorClosure { - pub func: extern "C" fn(mutator: *mut Mutator, data: *mut libc::c_void), + pub func: extern "C" fn(mutator: *mut libc::c_void, data: *mut libc::c_void), pub data: *mut libc::c_void, } @@ -72,7 +83,7 @@ pub struct OpenJDK_Upcalls { pub spawn_gc_thread: extern "C" fn(tls: VMThread, kind: libc::c_int, ctx: *mut libc::c_void), pub block_for_gc: extern "C" fn(), pub out_of_memory: extern "C" fn(tls: VMThread, err_kind: AllocationError), - pub get_next_mutator: extern "C" fn() -> *mut Mutator, + pub get_next_mutator: extern "C" fn() -> *mut libc::c_void, pub reset_mutator_iterator: extern "C" fn(), pub scan_object: extern "C" fn( trace: *mut c_void, @@ -83,7 +94,7 @@ pub struct OpenJDK_Upcalls { ), pub dump_object: extern "C" fn(object: ObjectReference), pub get_object_size: extern "C" fn(object: ObjectReference) -> usize, - pub get_mmtk_mutator: extern "C" fn(tls: VMMutatorThread) -> *mut Mutator, + pub get_mmtk_mutator: extern "C" fn(tls: VMMutatorThread) -> *mut libc::c_void, pub is_mutator: extern "C" fn(tls: VMThread) -> bool, pub harness_begin: extern "C" fn(), pub harness_end: extern "C" fn(), @@ -129,10 +140,6 @@ lazy_static! { unsafe { ((*UPCALLS).java_lang_classloader_loader_data_offset)() }; } -pub fn current_worker() -> &'static mut GCWorker { - GCWorker::::current() -} - pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); #[no_mangle] @@ -141,11 +148,7 @@ pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS: uintptr_t = #[no_mangle] pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS_COMPRESSED: uintptr_t = - crate::vm_metadata::LOGGING_SIDE_METADATA_SPEC - .as_spec() - .extract_side_spec() - .upper_bound_address_for_contiguous() - .as_usize(); + mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize(); #[no_mangle] pub static RC_TABLE_BASE_ADDRESS: uintptr_t = @@ -157,14 +160,14 @@ pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t = #[no_mangle] pub static FREE_LIST_ALLOCATOR_SIZE: uintptr_t = - std::mem::size_of::>(); + std::mem::size_of::>>(); #[no_mangle] pub static DISABLE_ALLOCATION_FAST_PATH: i32 = cfg!(feature = "no_fast_alloc") as _; #[no_mangle] pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = - std::mem::size_of::>(); + std::mem::size_of::>>(); #[no_mangle] pub static mut CONCURRENT_MARKING_ACTIVE: u8 = 0; @@ -206,7 +209,7 @@ fn compress(o: ObjectReference) -> u32 { if o.is_null() { 0u32 } else { - unsafe { ((o.to_address::() - BASE) >> SHIFT) as u32 } + unsafe { ((o.to_raw_address() - BASE) >> SHIFT) as u32 } } } @@ -214,7 +217,7 @@ fn decompress(v: u32) -> ObjectReference { if v == 0 { ObjectReference::NULL } else { - unsafe { (BASE + ((v as usize) << SHIFT)).to_object_reference::() } + unsafe { (BASE + ((v as usize) << SHIFT)).to_object_reference::>() } } } @@ -239,14 +242,14 @@ fn initialize_compressed_oops() { } #[derive(Default)] -pub struct OpenJDK; +pub struct OpenJDK; /// The type of edges in OpenJDK. #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] #[repr(transparent)] -pub struct OpenJDKEdge(pub Address); +pub struct OpenJDKEdge(pub Address); -impl OpenJDKEdge { +impl OpenJDKEdge { const MASK: usize = 1usize << 63; const fn is_compressed(&self) -> bool { @@ -258,9 +261,9 @@ impl OpenJDKEdge { } } -impl Edge for OpenJDKEdge { +impl Edge for OpenJDKEdge { /// Load object reference from the edge. - fn load(&self) -> ObjectReference { + fn load(&self) -> ObjectReference { if COMPRESSED { let slot = self.untagged_address(); if self.is_compressed() { @@ -274,7 +277,7 @@ impl Edge for OpenJDKEdge { } /// Store the object reference `object` into the edge. - fn store(&self, object: ObjectReference) { + fn store(&self, object: ObjectReference) { if COMPRESSED { let slot = self.untagged_address(); if self.is_compressed() { @@ -287,7 +290,7 @@ impl Edge for OpenJDKEdge { } } - fn compare_exchange( + fn compare_exchange( &self, old_object: ObjectReference, new_object: ObjectReference, @@ -310,8 +313,8 @@ impl Edge for OpenJDKEdge { let slot = self.untagged_address(); unsafe { match slot.compare_exchange::( - old_object.to_address::().as_usize(), - new_object.to_address::().as_usize(), + old_object.to_raw_address().as_usize(), + new_object.to_raw_address().as_usize(), success, failure, ) { @@ -323,8 +326,8 @@ impl Edge for OpenJDKEdge { } else { unsafe { match self.0.compare_exchange::( - old_object.to_address::().as_usize(), - new_object.to_address::().as_usize(), + old_object.to_raw_address().as_usize(), + new_object.to_raw_address().as_usize(), success, failure, ) { @@ -349,20 +352,20 @@ impl Edge for OpenJDKEdge { } #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] -pub struct OpenJDKEdgeRange { - pub start: OpenJDKEdge, - pub end: OpenJDKEdge, +pub struct OpenJDKEdgeRange { + pub start: OpenJDKEdge, + pub end: OpenJDKEdge, } /// Iterate edges within `Range
`. -pub struct AddressRangeIterator { +pub struct AddressRangeIterator { cursor: Address, limit: Address, width: usize, } -impl Iterator for AddressRangeIterator { - type Item = OpenJDKEdge; +impl Iterator for AddressRangeIterator { + type Item = OpenJDKEdge; fn next(&mut self) -> Option { if self.cursor >= self.limit { @@ -375,14 +378,14 @@ impl Iterator for AddressRangeIterator { } } -pub struct ChunkIterator { +pub struct ChunkIterator { cursor: Address, limit: Address, step: usize, } -impl Iterator for ChunkIterator { - type Item = OpenJDKEdgeRange; +impl Iterator for ChunkIterator { + type Item = OpenJDKEdgeRange; fn next(&mut self) -> Option { if self.cursor >= self.limit { @@ -402,10 +405,10 @@ impl Iterator for ChunkIterator { } } -impl MemorySlice for OpenJDKEdgeRange { - type Edge = OpenJDKEdge; - type EdgeIterator = AddressRangeIterator; - type ChunkIterator = ChunkIterator; +impl MemorySlice for OpenJDKEdgeRange { + type Edge = OpenJDKEdge; + type EdgeIterator = AddressRangeIterator; + type ChunkIterator = ChunkIterator; fn iter_edges(&self) -> Self::EdgeIterator { AddressRangeIterator { @@ -461,15 +464,15 @@ impl MemorySlice for OpenJDKEdgeRange { } } -impl VMBinding for OpenJDK { +impl VMBinding for OpenJDK { type VMObjectModel = object_model::VMObjectModel; type VMScanning = scanning::VMScanning; type VMCollection = collection::VMCollection; type VMActivePlan = active_plan::VMActivePlan; type VMReferenceGlue = reference_glue::VMReferenceGlue; - type VMEdge = OpenJDKEdge; - type VMMemorySlice = OpenJDKEdgeRange; + type VMEdge = OpenJDKEdge; + type VMMemorySlice = OpenJDKEdgeRange; const MIN_ALIGNMENT: usize = 8; const MAX_ALIGNMENT: usize = 8; @@ -483,18 +486,26 @@ pub static MMTK_INITIALIZED: AtomicBool = AtomicBool::new(false); lazy_static! { pub static ref BUILDER: Mutex = Mutex::new(MMTKBuilder::new()); - pub static ref SINGLETON: MMTK = { + pub static ref SINGLETON_COMPRESSED: MMTK> = { let mut builder = BUILDER.lock().unwrap(); - if use_compressed_oops() { - builder.set_option("use_35bit_address_space", "true"); - builder.set_option("use_35bit_address_space", "true"); - } + assert!(use_compressed_oops()); + builder.set_option("use_35bit_address_space", "true"); + builder.set_option("use_35bit_address_space", "true"); assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); - if use_compressed_oops() { - initialize_compressed_oops(); + initialize_compressed_oops(); + unsafe { + HEAP_START = VM_LAYOUT_CONSTANTS.heap_start; + HEAP_END = VM_LAYOUT_CONSTANTS.heap_end; } + *ret + }; + pub static ref SINGLETON_UNCOMPRESSED: MMTK> = { + let builder = BUILDER.lock().unwrap(); + assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); + let ret = mmtk::memory_manager::mmtk_init(&builder); + MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); unsafe { HEAP_START = VM_LAYOUT_CONSTANTS.heap_start; HEAP_END = VM_LAYOUT_CONSTANTS.heap_end; @@ -503,9 +514,23 @@ lazy_static! { }; } +fn singleton() -> &'static MMTK> { + if COMPRESSED { + unsafe { + &*(&*SINGLETON_COMPRESSED as *const MMTK> + as *const MMTK>) + } + } else { + unsafe { + &*(&*SINGLETON_UNCOMPRESSED as *const MMTK> + as *const MMTK>) + } + } +} + #[no_mangle] pub static MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES: usize = - mmtk::util::alloc::MarkCompactAllocator::::HEADER_RESERVED_IN_BYTES; + mmtk::util::alloc::MarkCompactAllocator::>::HEADER_RESERVED_IN_BYTES; lazy_static! { /// A global storage for all the cached CodeCache root pointers diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 6f336a92..3e43347d 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -8,10 +8,12 @@ use mmtk::vm::*; pub struct VMObjectModel {} -impl ObjectModel for VMObjectModel { - const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = vm_metadata::LOGGING_SIDE_METADATA_SPEC; - const GLOBAL_LOG_BIT_SPEC_COMPRESSED: VMGlobalLogBitSpecCompressed = - vm_metadata::LOGGING_SIDE_METADATA_SPEC_COMPRESSED; +impl ObjectModel> for VMObjectModel { + const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = if COMPRESSED { + vm_metadata::LOGGING_SIDE_METADATA_SPEC_COMPRESSED + } else { + vm_metadata::LOGGING_SIDE_METADATA_SPEC + }; const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = vm_metadata::FORWARDING_POINTER_METADATA_SPEC; @@ -23,10 +25,12 @@ impl ObjectModel for VMObjectModel { const UNIFIED_OBJECT_REFERENCE_ADDRESS: bool = true; const OBJECT_REF_OFFSET_LOWER_BOUND: isize = 0; + const COMPRESSED_PTR_ENABLED: bool = COMPRESSED; + fn copy( from: ObjectReference, copy: CopySemantics, - copy_context: &mut GCWorkerCopyContext, + copy_context: &mut GCWorkerCopyContext>, ) -> ObjectReference { let bytes = if crate::use_compressed_oops() { unsafe { Oop::from(from).size::() } @@ -54,9 +58,9 @@ impl ObjectModel for VMObjectModel { unsafe { (dst + i).store((src + i).load::()) }; } } - let start = Self::ref_to_object_start(to); + let start = >>::ref_to_object_start(to); if region != Address::ZERO { - fill_alignment_gap::(region, start); + fill_alignment_gap::>(region, start); } start + bytes } @@ -66,7 +70,7 @@ impl ObjectModel for VMObjectModel { } fn get_current_size(object: ObjectReference) -> usize { - if crate::use_compressed_oops() { + if COMPRESSED { unsafe { Oop::from(object).size::() } } else { unsafe { Oop::from(object).size::() } @@ -74,7 +78,7 @@ impl ObjectModel for VMObjectModel { } fn get_size_when_copied(object: ObjectReference) -> usize { - Self::get_current_size(object) + >>::get_current_size(object) } fn get_align_when_copied(_object: ObjectReference) -> usize { @@ -122,10 +126,6 @@ impl ObjectModel for VMObjectModel { s.to_string() } - fn compressed_pointers_enabled() -> bool { - crate::use_compressed_oops() - } - fn get_class_pointer(object: ObjectReference) -> Address { let oop: Oop = unsafe { std::mem::transmute(object) }; if crate::use_compressed_oops() { diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 269b51a3..064b644f 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -1,7 +1,6 @@ use super::abi::*; -use super::{OpenJDKEdge, UPCALLS}; +use super::UPCALLS; use crate::reference_glue::DISCOVERED_LISTS; -use crate::SINGLETON; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::edge_shape::Edge; @@ -10,7 +9,7 @@ use std::cell::UnsafeCell; use std::{mem, slice}; trait OopIterate: Sized { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate, const COMPRESSED: bool>( &self, oop: Oop, closure: &mut V, @@ -18,7 +17,7 @@ trait OopIterate: Sized { } impl OopIterate for OopMapBlock { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate, const COMPRESSED: bool>( &self, oop: Oop, closure: &mut V, @@ -26,36 +25,36 @@ impl OopIterate for OopMapBlock { let log_bytes_in_oop = if COMPRESSED { 2 } else { 3 }; let start = oop.get_field_address(self.offset); for i in 0..self.count as usize { - let edge = OpenJDKEdge(start + (i << log_bytes_in_oop)); + let edge = E::from_address(start + (i << log_bytes_in_oop)); closure.visit_edge(edge); } } } impl OopIterate for InstanceKlass { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate, const COMPRESSED: bool>( &self, oop: Oop, closure: &mut V, ) { if closure.should_follow_clds() { - do_klass::<_, COMPRESSED>(oop.klass::(), closure); + do_klass::<_, _, COMPRESSED>(oop.klass::(), closure); } let oop_maps = self.nonstatic_oop_maps(); for map in oop_maps { - map.oop_iterate::(oop, closure) + map.oop_iterate::(oop, closure) } } } impl OopIterate for InstanceMirrorKlass { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate, const COMPRESSED: bool>( &self, oop: Oop, closure: &mut V, ) { self.instance_klass - .oop_iterate::(oop, closure); + .oop_iterate::<_, _, COMPRESSED>(oop, closure); if closure.should_follow_clds() { let klass = unsafe { (oop.start() + *crate::JAVA_LANG_CLASS_KLASS_OFFSET_IN_BYTES).load::<*mut Klass>() @@ -65,9 +64,9 @@ impl OopIterate for InstanceMirrorKlass { if klass.is_instance_klass() && (unsafe { klass.cast::().is_anonymous() }) { - do_cld::<_, COMPRESSED>(&klass.class_loader_data, closure) + do_cld::<_, _, COMPRESSED>(&klass.class_loader_data, closure) } else { - do_klass::<_, COMPRESSED>(klass, closure) + do_klass::<_, _, COMPRESSED>(klass, closure) } } } @@ -79,62 +78,62 @@ impl OopIterate for InstanceMirrorKlass { let start: *const NarrowOop = start.to_ptr::(); let slice = unsafe { slice::from_raw_parts(start, len as _) }; for narrow_oop in slice { - closure.visit_edge(OpenJDKEdge(narrow_oop.slot())); + closure.visit_edge(E::from_address(narrow_oop.slot())); } } else { let start: *const Oop = start.to_ptr::(); let slice = unsafe { slice::from_raw_parts(start, len as _) }; for oop in slice { - closure.visit_edge(OpenJDKEdge(Address::from_ref(oop as &Oop))); + closure.visit_edge(E::from_address(Address::from_ref(oop as &Oop))); } } } } impl OopIterate for InstanceClassLoaderKlass { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate, const COMPRESSED: bool>( &self, oop: Oop, - closure: &mut C, + closure: &mut V, ) { self.instance_klass - .oop_iterate::(oop, closure); + .oop_iterate::<_, _, COMPRESSED>(oop, closure); if closure.should_follow_clds() { let cld = unsafe { (oop.start() + *crate::JAVA_LANG_CLASSLOADER_LOADER_DATA_OFFSET) .load::<*mut ClassLoaderData>() }; if !cld.is_null() { - do_cld::<_, COMPRESSED>(unsafe { &*cld }, closure); + do_cld::<_, _, COMPRESSED>(unsafe { &*cld }, closure); } } } } impl OopIterate for ObjArrayKlass { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate, const COMPRESSED: bool>( &self, oop: Oop, closure: &mut V, ) { if closure.should_follow_clds() { - do_klass::<_, COMPRESSED>(oop.klass::(), closure); + do_klass::<_, _, COMPRESSED>(oop.klass::(), closure); } let array = unsafe { oop.as_array_oop() }; if COMPRESSED { for narrow_oop in unsafe { array.data::(BasicType::T_OBJECT) } { - closure.visit_edge(OpenJDKEdge(narrow_oop.slot())); + closure.visit_edge(E::from_address(narrow_oop.slot())); } } else { for oop in unsafe { array.data::(BasicType::T_OBJECT) } { - closure.visit_edge(OpenJDKEdge(Address::from_ref(oop as &Oop))); + closure.visit_edge(E::from_address(Address::from_ref(oop as &Oop))); } } } } impl OopIterate for TypeArrayKlass { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate, const COMPRESSED: bool>( &self, _oop: Oop, _closure: &mut V, @@ -145,24 +144,27 @@ impl OopIterate for TypeArrayKlass { } impl OopIterate for InstanceRefKlass { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate, const COMPRESSED: bool>( &self, oop: Oop, closure: &mut V, ) { use crate::abi::*; self.instance_klass - .oop_iterate::(oop, closure); + .oop_iterate::<_, _, COMPRESSED>(oop, closure); let disable_discovery = !closure.should_discover_references(); - if Self::should_discover_refs(self.instance_klass.reference_type, disable_discovery) { + if Self::should_discover_refs::( + self.instance_klass.reference_type, + disable_discovery, + ) { match self.instance_klass.reference_type { ReferenceType::None => { panic!("oop_iterate on InstanceRefKlass with reference_type as None") } rt => { - if !Self::discover_reference::(oop, rt) { + if !Self::discover_reference::(oop, rt) { Self::process_ref_as_strong(oop, closure) } } @@ -174,64 +176,73 @@ impl OopIterate for InstanceRefKlass { } impl InstanceRefKlass { - fn should_discover_refs(mut rt: ReferenceType, disable_discovery: bool) -> bool { + fn should_discover_refs( + mut rt: ReferenceType, + disable_discovery: bool, + ) -> bool { if rt == ReferenceType::Other { rt = ReferenceType::Weak; } if disable_discovery { return false; } - if *SINGLETON.get_options().no_finalizer && rt == ReferenceType::Final { + if *crate::singleton::().get_options().no_finalizer + && rt == ReferenceType::Final + { return false; } - if *SINGLETON.get_options().no_reference_types && rt != ReferenceType::Final { + if *crate::singleton::() + .get_options() + .no_reference_types + && rt != ReferenceType::Final + { return false; } true } - fn process_ref_as_strong(oop: Oop, closure: &mut impl EdgeVisitor) { - let referent_addr = Self::referent_address(oop); + fn process_ref_as_strong>(oop: Oop, closure: &mut V) { + let referent_addr = Self::referent_address::(oop); closure.visit_edge(referent_addr); - let discovered_addr = Self::discovered_address(oop); + let discovered_addr = Self::discovered_address::(oop); closure.visit_edge(discovered_addr); } - fn discover_reference(oop: Oop, rt: ReferenceType) -> bool { + fn discover_reference(oop: Oop, rt: ReferenceType) -> bool { // Do not discover new refs during reference processing. if !DISCOVERED_LISTS.allow_discover() { return false; } // Do not discover if the referent is live. - let addr = InstanceRefKlass::referent_address(oop); - let referent: ObjectReference = addr.load::(); + let addr = InstanceRefKlass::referent_address::(oop); + let referent: ObjectReference = addr.load(); // Skip live or null referents if referent.is_reachable() || referent.is_null() { return false; } // Skip young referents let reference: ObjectReference = oop.into(); - if !SINGLETON + if !crate::singleton::() .get_plan() .should_process_reference(reference, referent) { return false; } - if rt == ReferenceType::Final && DISCOVERED_LISTS.is_discovered::(reference) { + if rt == ReferenceType::Final && DISCOVERED_LISTS.is_discovered::(reference) { return false; } // Add to reference list DISCOVERED_LISTS .get(rt) - .add::(reference, referent); + .add::(reference, referent); true } } #[allow(unused)] -fn oop_iterate_slow>(oop: Oop, closure: &mut V, tls: OpaquePointer) { +fn oop_iterate_slow>(oop: Oop, closure: &mut V, tls: OpaquePointer) { unsafe { CLOSURE.with(|x| *x.get() = closure as *mut V as *mut u8); ((*UPCALLS).scan_object)( - mem::transmute(scan_object_fn:: as *const unsafe extern "C" fn(edge: Address)), + mem::transmute(scan_object_fn:: as *const unsafe extern "C" fn(edge: Address)), mem::transmute(oop), tls, closure.should_follow_clds(), @@ -240,7 +251,7 @@ fn oop_iterate_slow>(oop: Oop, closure: &mut V, tls: } } -fn oop_iterate, const COMPRESSED: bool>( +fn oop_iterate, const COMPRESSED: bool>( oop: Oop, closure: &mut V, klass: Option
, @@ -260,19 +271,19 @@ fn oop_iterate, const COMPRESSED: bool>( match klass_id { KlassID::Instance => { let instance_klass = unsafe { klass.cast::() }; - instance_klass.oop_iterate::(oop, closure); + instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceClassLoader => { let instance_klass = unsafe { klass.cast::() }; - instance_klass.oop_iterate::(oop, closure); + instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceMirror => { let instance_klass = unsafe { klass.cast::() }; - instance_klass.oop_iterate::(oop, closure); + instance_klass.oop_iterate::(oop, closure); } KlassID::ObjArray => { let array_klass = unsafe { klass.cast::() }; - array_klass.oop_iterate::(oop, closure); + array_klass.oop_iterate::(oop, closure); } KlassID::TypeArray => { // let array_klass = unsafe { oop.klass::().cast::() }; @@ -280,28 +291,28 @@ fn oop_iterate, const COMPRESSED: bool>( } KlassID::InstanceRef => { let instance_klass = unsafe { klass.cast::() }; - instance_klass.oop_iterate::(oop, closure); + instance_klass.oop_iterate::(oop, closure); } #[allow(unreachable_patterns)] _ => unreachable!(), // _ => oop_iterate_slow(oop, closure, OpaquePointer::UNINITIALIZED), } } -fn do_cld, const COMPRESSED: bool>( +fn do_cld, const COMPRESSED: bool>( cld: &ClassLoaderData, closure: &mut V, ) { if !closure.should_follow_clds() { return; } - cld.oops_do::<_, COMPRESSED>(closure) + cld.oops_do::<_, _, COMPRESSED>(closure) } -fn do_klass, const COMPRESSED: bool>(klass: &Klass, closure: &mut V) { +fn do_klass, const COMPRESSED: bool>(klass: &Klass, closure: &mut V) { if !closure.should_follow_clds() { return; } - do_cld::<_, COMPRESSED>(&klass.class_loader_data, closure) + do_cld::<_, _, COMPRESSED>(&klass.class_loader_data, closure) } pub fn is_obj_array(oop: Oop) -> bool { @@ -312,7 +323,7 @@ pub fn is_val_array(oop: Oop) -> bool { oop.klass::().id == KlassID::TypeArray } -pub fn obj_array_data(oop: Oop) -> crate::OpenJDKEdgeRange { +pub fn obj_array_data(oop: Oop) -> crate::OpenJDKEdgeRange { unsafe { let array = oop.as_array_oop(); array.slice::(BasicType::T_OBJECT) @@ -323,25 +334,25 @@ thread_local! { static CLOSURE: UnsafeCell<*mut u8> = UnsafeCell::new(std::ptr::null_mut()); } -pub unsafe extern "C" fn scan_object_fn>(edge: Address) { +pub unsafe extern "C" fn scan_object_fn>(edge: Address) { let ptr: *mut u8 = CLOSURE.with(|x| *x.get()); let closure = &mut *(ptr as *mut V); - closure.visit_edge(OpenJDKEdge(edge)); + closure.visit_edge(E::from_address(edge)); } -pub fn scan_object, const COMPRESSED: bool>( +pub fn scan_object, const COMPRESSED: bool>( object: ObjectReference, - closure: &mut C, + closure: &mut V, _tls: VMWorkerThread, ) { - unsafe { oop_iterate::(mem::transmute(object), closure, None) } + unsafe { oop_iterate::(mem::transmute(object), closure, None) } } -pub fn scan_object_with_klass, const COMPRESSED: bool>( +pub fn scan_object_with_klass, const COMPRESSED: bool>( object: ObjectReference, - closure: &mut C, + closure: &mut V, _tls: VMWorkerThread, klass: Address, ) { - unsafe { oop_iterate::(mem::transmute(object), closure, Some(klass)) } + unsafe { oop_iterate::(mem::transmute(object), closure, Some(klass)) } } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 922c74e7..848fc620 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -3,45 +3,45 @@ use std::sync::atomic::AtomicBool; use std::sync::Mutex; use crate::abi::{InstanceRefKlass, Oop, ReferenceType}; -use crate::{OpenJDK, SINGLETON}; +use crate::OpenJDK; use atomic::{Atomic, Ordering}; use mmtk::scheduler::{GCWork, GCWorker, ProcessEdgesWork, WorkBucketStage}; use mmtk::util::opaque_pointer::VMWorkerThread; use mmtk::util::ObjectReference; use mmtk::vm::edge_shape::Edge; -use mmtk::vm::ReferenceGlue; +use mmtk::vm::{ReferenceGlue, VMBinding}; use mmtk::MMTK; -pub fn set_referent(reff: ObjectReference, referent: ObjectReference) { +fn set_referent(reff: ObjectReference, referent: ObjectReference) { let oop = Oop::from(reff); - let slot = InstanceRefKlass::referent_address(oop); + let slot = InstanceRefKlass::referent_address::(oop); mmtk::plan::lxr::record_edge_for_validation(slot, referent); - slot.store::(referent) + slot.store(referent) } -fn get_referent(object: ObjectReference) -> ObjectReference { +fn get_referent(object: ObjectReference) -> ObjectReference { let oop = Oop::from(object); - InstanceRefKlass::referent_address(oop).load::() + InstanceRefKlass::referent_address::(oop).load() } -fn get_next_reference_slot(object: ObjectReference) -> crate::OpenJDKEdge { +fn get_next_reference_slot(object: ObjectReference) -> E { let oop = Oop::from(object); - InstanceRefKlass::discovered_address(oop) + InstanceRefKlass::discovered_address::(oop) } -fn get_next_reference(object: ObjectReference) -> ObjectReference { - get_next_reference_slot(object).load::() +fn get_next_reference(object: ObjectReference) -> ObjectReference { + get_next_reference_slot::(object).load() } -fn set_next_reference(object: ObjectReference, next: ObjectReference) { - let slot = get_next_reference_slot(object); +fn set_next_reference(object: ObjectReference, next: ObjectReference) { + let slot = get_next_reference_slot::(object); mmtk::plan::lxr::record_edge_for_validation(slot, next); - slot.store::(next) + slot.store(next) } pub struct VMReferenceGlue {} -impl ReferenceGlue for VMReferenceGlue { +impl ReferenceGlue> for VMReferenceGlue { type FinalizableType = ObjectReference; fn set_referent(_reff: ObjectReference, _referent: ObjectReference) { @@ -66,59 +66,34 @@ impl DiscoveredList { } } - pub fn add( + pub fn add( &self, reference: ObjectReference, referent: ObjectReference, ) { // Keep reference and referent alive during SATB - SINGLETON.get_plan().discover_reference(reference, referent); + crate::singleton::() + .get_plan() + .discover_reference(reference, referent); // Add to the corresponding list // Note that the list is a singly-linked list, and the tail object should point to itself. let oop = Oop::from(reference); let head = self.head.load(Ordering::Relaxed); - let addr = InstanceRefKlass::discovered_address(oop); - if COMPRESSED { - let discovered_addr = unsafe { addr.to_address().as_ref::>() }; - let next_discovered = if head.is_null() { reference } else { head }; - debug_assert!(!next_discovered.is_null()); - if discovered_addr - .compare_exchange( - 0, - crate::compress(next_discovered), - Ordering::SeqCst, - Ordering::SeqCst, - ) - .is_ok() - { - self.head.store(reference, Ordering::Relaxed); - } - debug_assert!(!get_next_reference::(reference).is_null()); - } else { - let discovered_addr = unsafe { addr.to_address().as_ref::>() }; - let next_discovered = if head.is_null() { reference } else { head }; - debug_assert!(!next_discovered.is_null()); - if discovered_addr - .compare_exchange( - ObjectReference::NULL, - next_discovered, - Ordering::SeqCst, - Ordering::SeqCst, - ) - .is_ok() - { - self.head.store(reference, Ordering::SeqCst); - } - debug_assert!( - !get_next_reference::(reference).is_null(), - "discover ref {:?}->{:?} next={:?} real-next={:?} bin={:?}", - reference, - referent, + let addr = InstanceRefKlass::discovered_address::(oop); + let next_discovered = if head.is_null() { reference } else { head }; + debug_assert!(!next_discovered.is_null()); + if addr + .compare_exchange( + ObjectReference::NULL, next_discovered, - get_next_reference::(reference), - mmtk::scheduler::worker::current_worker_ordinal().unwrap() - ); + Ordering::SeqCst, + Ordering::SeqCst, + ) + .is_ok() + { + self.head.store(reference, Ordering::Relaxed); } + debug_assert!(!get_next_reference::(reference).is_null()); } } @@ -132,7 +107,7 @@ pub struct DiscoveredLists { impl DiscoveredLists { pub fn new() -> Self { - let workers = crate::SINGLETON.scheduler.num_workers(); + let workers = with_singleton!(|singleton| singleton.scheduler.num_workers()); let build_lists = |rt| (0..workers).map(|_| DiscoveredList::new(rt)).collect(); Self { soft: build_lists(ReferenceType::Soft), @@ -151,8 +126,8 @@ impl DiscoveredLists { self.allow_discover.load(Ordering::SeqCst) } - pub fn is_discovered(&self, reference: ObjectReference) -> bool { - !get_next_reference::(reference).is_null() + pub fn is_discovered(&self, reference: ObjectReference) -> bool { + !get_next_reference::(reference).is_null() } pub fn get_by_rt_and_index(&self, rt: ReferenceType, index: usize) -> &DiscoveredList { @@ -170,9 +145,9 @@ impl DiscoveredLists { self.get_by_rt_and_index(rt, id) } - pub fn process_lists, const COMPRESSED: bool>( + pub fn process_lists( &self, - worker: &mut GCWorker, + worker: &mut GCWorker, rt: ReferenceType, lists: &[DiscoveredList], clear: bool, @@ -186,45 +161,32 @@ impl DiscoveredLists { if head.is_null() { continue; } - let w = ProcessDiscoveredList::<_, COMPRESSED> { + let w = ProcessDiscoveredList { list_index: i, head, rt, _p: PhantomData::, }; - packets.push(Box::new(w) as Box>); + packets.push(Box::new(w) as Box>); } worker.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets); } - pub fn reconsider_soft_refs>( - &self, - _worker: &mut GCWorker, - ) { - } + pub fn reconsider_soft_refs(&self, _worker: &mut GCWorker) {} - pub fn process_soft_weak_final_refs< - E: ProcessEdgesWork, - const COMPRESSED: bool, - >( - &self, - worker: &mut GCWorker, - ) { + pub fn process_soft_weak_final_refs(&self, worker: &mut GCWorker) { self.allow_discover.store(false, Ordering::SeqCst); - if !*SINGLETON.get_options().no_reference_types { - self.process_lists::(worker, ReferenceType::Soft, &self.soft, true); - self.process_lists::(worker, ReferenceType::Weak, &self.weak, true); + if !*worker.mmtk.get_options().no_reference_types { + self.process_lists::(worker, ReferenceType::Soft, &self.soft, true); + self.process_lists::(worker, ReferenceType::Weak, &self.weak, true); } - if !*SINGLETON.get_options().no_finalizer { - self.process_lists::(worker, ReferenceType::Final, &self.r#final, false); + if !*worker.mmtk.get_options().no_finalizer { + self.process_lists::(worker, ReferenceType::Final, &self.r#final, false); } } - pub fn resurrect_final_refs, const COMPRESSED: bool>( - &self, - worker: &mut GCWorker, - ) { - assert!(!*SINGLETON.get_options().no_finalizer); + pub fn resurrect_final_refs(&self, worker: &mut GCWorker) { + assert!(!*worker.mmtk.get_options().no_finalizer); let lists = &self.r#final; let mut packets = vec![]; for i in 0..lists.len() { @@ -233,22 +195,19 @@ impl DiscoveredLists { if head.is_null() { continue; } - let w = ResurrectFinalizables::<_, COMPRESSED> { + let w = ResurrectFinalizables { list_index: i, head, _p: PhantomData::, }; - packets.push(Box::new(w) as Box>); + packets.push(Box::new(w) as Box>); } worker.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets); } - pub fn process_phantom_refs, const COMPRESSED: bool>( - &self, - worker: &mut GCWorker, - ) { - assert!(!*SINGLETON.get_options().no_reference_types); - self.process_lists::(worker, ReferenceType::Phantom, &self.phantom, true); + pub fn process_phantom_refs(&self, worker: &mut GCWorker) { + assert!(!*worker.mmtk.get_options().no_reference_types); + self.process_lists::(worker, ReferenceType::Phantom, &self.phantom, true); } } @@ -257,30 +216,28 @@ lazy_static! { pub static ref DISCOVERED_LISTS: DiscoveredLists = DiscoveredLists::new(); } -pub struct ProcessDiscoveredList, const COMPRESSED: bool> { +pub struct ProcessDiscoveredList { list_index: usize, head: ObjectReference, rt: ReferenceType, _p: PhantomData, } -impl, const COMPRESSED: bool> GCWork - for ProcessDiscoveredList -{ - fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { +impl GCWork for ProcessDiscoveredList { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { let mut trace = E::new(vec![], false, mmtk); trace.set_worker(worker); let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); - let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { + let new_list = iterate_list::<_, ::VMEdge>(self.head, |reference| { debug_assert!( - get_next_reference::(reference).is_null(), + get_next_reference::<::VMEdge>(reference).is_null(), "next must be null. ref={:?} {:?} bin={}", reference, self.rt, self.list_index ); let reference = trace.trace_object(reference); - let referent = get_referent::(reference); + let referent = get_referent::<::VMEdge>(reference); if referent.is_null() { // Remove from the discovered list return DiscoveredListIterationResult::Remove; @@ -289,13 +246,13 @@ impl, const COMPRESSED: bool> GCWork let forwarded = trace.trace_object(referent); debug_assert!(forwarded.get_forwarded_object().is_none()); debug_assert!(reference.get_forwarded_object().is_none()); - set_referent::(reference, forwarded); + set_referent::<::VMEdge>(reference, forwarded); // Remove from the discovered list return DiscoveredListIterationResult::Remove; } else { if self.rt != ReferenceType::Final { // Clear this referent - set_referent::(reference, ObjectReference::NULL); + set_referent::<::VMEdge>(reference, ObjectReference::NULL); // set_referent(reference, ObjectReference::NULL); } // Keep the reference @@ -312,7 +269,7 @@ impl, const COMPRESSED: bool> GCWork .store(head, Ordering::SeqCst); } else { let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; - set_next_reference::(tail, old_head); + set_next_reference::<::VMEdge>(tail, old_head); } } else { if self.rt == ReferenceType::Final { @@ -325,23 +282,21 @@ impl, const COMPRESSED: bool> GCWork } } -pub struct ResurrectFinalizables, const COMPRESSED: bool> { +pub struct ResurrectFinalizables { list_index: usize, head: ObjectReference, _p: PhantomData, } -impl, const COMPRESSED: bool> GCWork - for ResurrectFinalizables -{ - fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { +impl GCWork for ResurrectFinalizables { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { let mut trace = E::new(vec![], false, mmtk); trace.set_worker(worker); - let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { + let new_list = iterate_list::<_, ::VMEdge>(self.head, |reference| { let reference = trace.trace_object(reference); - let referent = get_referent::(reference); + let referent = get_referent::<::VMEdge>(reference); let forwarded = trace.trace_object(referent); - set_referent::(reference, forwarded); + set_referent::<::VMEdge>(reference, forwarded); debug_assert!(forwarded.get_forwarded_object().is_none()); return DiscoveredListIterationResult::Enqueue(reference); }); @@ -353,7 +308,7 @@ impl, const COMPRESSED: bool> GCWork .head .store(ObjectReference::NULL, Ordering::SeqCst); let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; - set_next_reference::(tail, old_head); + set_next_reference::<::VMEdge>(tail, old_head); } trace.flush(); } @@ -364,10 +319,7 @@ enum DiscoveredListIterationResult { Enqueue(ObjectReference), } -fn iterate_list< - F: FnMut(ObjectReference) -> DiscoveredListIterationResult, - const COMPRESSED: bool, ->( +fn iterate_list DiscoveredListIterationResult, E: Edge>( head: ObjectReference, mut visitor: F, ) -> Option<(ObjectReference, ObjectReference)> { @@ -384,15 +336,15 @@ fn iterate_list< debug_assert!(reference.get_forwarded_object().is_none()); debug_assert!(reference.is_reachable()); // Update next_ref forwarding pointer - let next_ref = get_next_reference::(reference); + let next_ref = get_next_reference::(reference); let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); debug_assert!(next_ref.get_forwarded_object().is_none()); // Reaches the end of the list? let end_of_list = next_ref == reference || next_ref.is_null(); // Remove `reference` from current list - set_next_reference::(reference, ObjectReference::NULL); + set_next_reference::(reference, ObjectReference::NULL); if let Some(forwarded_ref) = reference.get_forwarded_object() { - set_next_reference::(forwarded_ref, ObjectReference::NULL); + set_next_reference::(forwarded_ref, ObjectReference::NULL); } // Process reference let result = visitor(reference); @@ -401,7 +353,7 @@ fn iterate_list< DiscoveredListIterationResult::Enqueue(reference) => { // Add to new list if let Some(new_head) = new_head { - set_next_reference::(reference, new_head); + set_next_reference::(reference, new_head); } else { new_tail = Some(reference); } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 3e55abc2..e817d69f 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -1,6 +1,7 @@ use crate::gc_work::*; +use crate::Edge; use crate::{EdgesClosure, OpenJDK}; -use crate::{NewBuffer, OpenJDKEdge, SINGLETON, UPCALLS}; +use crate::{NewBuffer, OpenJDKEdge, UPCALLS}; use mmtk::memory_manager; use mmtk::scheduler::WorkBucketStage; use mmtk::util::opaque_pointer::*; @@ -11,15 +12,15 @@ use mmtk::MutatorContext; pub struct VMScanning {} -extern "C" fn report_edges_and_renew_buffer>( +extern "C" fn report_edges_and_renew_buffer>( ptr: *mut Address, length: usize, capacity: usize, factory_ptr: *mut libc::c_void, ) -> NewBuffer { if !ptr.is_null() { - let ptr = ptr as *mut OpenJDKEdge; - let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; + let ptr = ptr as *mut E; + let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; factory.create_process_edge_roots_work(buf); } @@ -33,32 +34,32 @@ extern "C" fn report_edges_and_renew_buffer>( NewBuffer { ptr, capacity } } -pub(crate) fn to_edges_closure>(factory: &mut F) -> EdgesClosure { +pub(crate) fn to_edges_closure>(factory: &mut F) -> EdgesClosure { EdgesClosure { - func: report_edges_and_renew_buffer::, + func: report_edges_and_renew_buffer::, data: factory as *mut F as *mut libc::c_void, } } -impl Scanning for VMScanning { +impl Scanning> for VMScanning { const SCAN_MUTATORS_IN_SAFEPOINT: bool = false; const SINGLE_THREAD_MUTATOR_SCANNING: bool = false; - fn scan_object, const COMPRESSED: bool>( + fn scan_object( tls: VMWorkerThread, object: ObjectReference, - edge_visitor: &mut EV, + edge_visitor: &mut impl EdgeVisitor>, ) { - crate::object_scanning::scan_object::(object, edge_visitor, tls); + crate::object_scanning::scan_object::<_, _, COMPRESSED>(object, edge_visitor, tls); } - fn scan_object_with_klass, const COMPRESSED: bool>( + fn scan_object_with_klass( tls: VMWorkerThread, object: ObjectReference, - edge_visitor: &mut EV, + edge_visitor: &mut impl EdgeVisitor>, klass: Address, ) { - crate::object_scanning::scan_object_with_klass::( + crate::object_scanning::scan_object_with_klass::<_, _, COMPRESSED>( object, edge_visitor, tls, @@ -66,15 +67,15 @@ impl Scanning for VMScanning { ); } - fn obj_array_data(o: ObjectReference) -> crate::OpenJDKEdgeRange { + fn obj_array_data(o: ObjectReference) -> crate::OpenJDKEdgeRange { crate::object_scanning::obj_array_data::(unsafe { std::mem::transmute(o) }) } - fn is_obj_array(o: ObjectReference) -> bool { + fn is_obj_array(o: ObjectReference) -> bool { crate::object_scanning::is_obj_array::(unsafe { std::mem::transmute(o) }) } - fn is_val_array(o: ObjectReference) -> bool { + fn is_val_array(o: ObjectReference) -> bool { crate::object_scanning::is_val_array::(unsafe { std::mem::transmute(o) }) } @@ -83,7 +84,10 @@ impl Scanning for VMScanning { // TODO } - fn scan_thread_roots(_tls: VMWorkerThread, mut factory: impl RootsWorkFactory) { + fn scan_thread_roots( + _tls: VMWorkerThread, + mut factory: impl RootsWorkFactory>, + ) { unsafe { ((*UPCALLS).scan_all_thread_roots)(to_edges_closure(&mut factory)); } @@ -91,8 +95,8 @@ impl Scanning for VMScanning { fn scan_thread_root( _tls: VMWorkerThread, - mutator: &'static mut Mutator, - mut factory: impl RootsWorkFactory, + mutator: &'static mut Mutator>, + mut factory: impl RootsWorkFactory>, ) { let tls = mutator.get_tls(); unsafe { @@ -100,9 +104,12 @@ impl Scanning for VMScanning { } } - fn scan_vm_specific_roots(_tls: VMWorkerThread, factory: impl RootsWorkFactory) { + fn scan_vm_specific_roots( + _tls: VMWorkerThread, + factory: impl RootsWorkFactory>, + ) { memory_manager::add_work_packets( - &SINGLETON, + &crate::singleton::(), WorkBucketStage::RCProcessIncs, vec![ Box::new(ScanUniverseRoots::new(factory.clone())) as _, @@ -118,9 +125,11 @@ impl Scanning for VMScanning { Box::new(ScaWeakProcessorRoots::new(factory.clone())) as _, ], ); - if !(Self::SCAN_MUTATORS_IN_SAFEPOINT && Self::SINGLE_THREAD_MUTATOR_SCANNING) { + if !(>>::SCAN_MUTATORS_IN_SAFEPOINT + && >>::SINGLE_THREAD_MUTATOR_SCANNING) + { memory_manager::add_work_packet( - &SINGLETON, + &crate::singleton::(), WorkBucketStage::RCProcessIncs, ScanVMThreadRoots::new(factory), ); diff --git a/mmtk/src/vm_metadata/constants.rs b/mmtk/src/vm_metadata/constants.rs index 642c7ce3..5d9e9043 100644 --- a/mmtk/src/vm_metadata/constants.rs +++ b/mmtk/src/vm_metadata/constants.rs @@ -13,8 +13,8 @@ pub(crate) const FORWARDING_POINTER_OFFSET: isize = 0; /// 1 bit per object pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); -pub(crate) const LOGGING_SIDE_METADATA_SPEC_COMPRESSED: VMGlobalLogBitSpecCompressed = - VMGlobalLogBitSpecCompressed::side_after(LOGGING_SIDE_METADATA_SPEC.as_spec()); +pub(crate) const LOGGING_SIDE_METADATA_SPEC_COMPRESSED: VMGlobalLogBitSpec = + VMGlobalLogBitSpec::side_first_compressed(); // Global MetadataSpecs - End From bf675ed0c8d02531695c8d150f21d47322bbd7c3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 3 Apr 2023 21:49:59 +0000 Subject: [PATCH 303/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 218cbb0b..f49d863e 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=81cf9f1fc2f33e4936b9b45cdc9b588da9ddfbc8#81cf9f1fc2f33e4936b9b45cdc9b588da9ddfbc8" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f0f1ebe06a4f25c23b3eb1548e3778fbf9ad3925#f0f1ebe06a4f25c23b3eb1548e3778fbf9ad3925" dependencies = [ "atomic", "atomic-traits", @@ -479,7 +479,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=81cf9f1fc2f33e4936b9b45cdc9b588da9ddfbc8#81cf9f1fc2f33e4936b9b45cdc9b588da9ddfbc8" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f0f1ebe06a4f25c23b3eb1548e3778fbf9ad3925#f0f1ebe06a4f25c23b3eb1548e3778fbf9ad3925" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 718cd660..72e20660 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "81cf9f1fc2f33e4936b9b45cdc9b588da9ddfbc8" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f0f1ebe06a4f25c23b3eb1548e3778fbf9ad3925" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 9f196d5e65549ed86d5d69ae4a2fc470d663e036 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 3 Apr 2023 16:01:04 -0700 Subject: [PATCH 304/494] Add missing header file --- openjdk/mmtkHeap.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index cc097c14..f4c63578 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -34,6 +34,7 @@ #include "gc/shared/oopStorageParState.hpp" #include "gc/shared/strongRootsScope.hpp" #include "gc/shared/workgroup.hpp" +#include "gc/shared/softRefPolicy.hpp" #include "memory/iterator.hpp" #include "memory/metaspace.hpp" #include "mmtkCollectorPolicy.hpp" From 1d5dfc9c5df03460e559ad1de08e1dbb9884b006 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 4 Apr 2023 09:17:34 -0700 Subject: [PATCH 305/494] Opportunstic copying based on virtual space constraint --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/object_model.rs | 22 ++++++++++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index f49d863e..d7c8cda8 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f0f1ebe06a4f25c23b3eb1548e3778fbf9ad3925#f0f1ebe06a4f25c23b3eb1548e3778fbf9ad3925" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6989b5d965a8aa49d58e0fc479636a6ca2a9cfc5#6989b5d965a8aa49d58e0fc479636a6ca2a9cfc5" dependencies = [ "atomic", "atomic-traits", @@ -479,7 +479,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=f0f1ebe06a4f25c23b3eb1548e3778fbf9ad3925#f0f1ebe06a4f25c23b3eb1548e3778fbf9ad3925" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6989b5d965a8aa49d58e0fc479636a6ca2a9cfc5#6989b5d965a8aa49d58e0fc479636a6ca2a9cfc5" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 72e20660..20e66c9b 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "f0f1ebe06a4f25c23b3eb1548e3778fbf9ad3925" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "6989b5d965a8aa49d58e0fc479636a6ca2a9cfc5" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 3e43347d..5ddc8bb2 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -46,6 +46,28 @@ impl ObjectModel> for VMObjectModel to_obj } + fn try_copy( + from: ObjectReference, + copy: CopySemantics, + copy_context: &mut GCWorkerCopyContext>, + ) -> Option { + let bytes = if crate::use_compressed_oops() { + unsafe { Oop::from(from).size::() } + } else { + unsafe { Oop::from(from).size::() } + }; + let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::(), 0, copy); + if dst.is_zero() { + return None; + } + // Copy + let src = from.to_raw_address(); + unsafe { std::ptr::copy_nonoverlapping::(src.to_ptr(), dst.to_mut_ptr(), bytes) } + let to_obj = ObjectReference::from_raw_address(dst); + copy_context.post_copy(to_obj, bytes, copy); + Some(to_obj) + } + fn copy_to(from: ObjectReference, to: ObjectReference, region: Address) -> Address { let need_copy = from != to; let bytes = unsafe { ((*UPCALLS).get_object_size)(from) }; From b109883d77689c61f1ffec2d67de4734c60c27a3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 10 Apr 2023 14:46:39 -0700 Subject: [PATCH 306/494] object size distribution --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 3 ++- mmtk/src/api.rs | 3 +++ mmtk/src/collection.rs | 3 +++ mmtk/src/lib.rs | 20 +++++++++++++++++++- openjdk/mmtkBarrierSet.cpp | 6 +++++- 6 files changed, 34 insertions(+), 5 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index d7c8cda8..d2d0224e 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6989b5d965a8aa49d58e0fc479636a6ca2a9cfc5#6989b5d965a8aa49d58e0fc479636a6ca2a9cfc5" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=1570c684b7b2c97319588cbdcad84017472afb9a#1570c684b7b2c97319588cbdcad84017472afb9a" dependencies = [ "atomic", "atomic-traits", @@ -479,7 +479,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6989b5d965a8aa49d58e0fc479636a6ca2a9cfc5#6989b5d965a8aa49d58e0fc479636a6ca2a9cfc5" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=1570c684b7b2c97319588cbdcad84017472afb9a#1570c684b7b2c97319588cbdcad84017472afb9a" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 20e66c9b..29458b96 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "6989b5d965a8aa49d58e0fc479636a6ca2a9cfc5" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "1570c684b7b2c97319588cbdcad84017472afb9a" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } @@ -83,3 +83,4 @@ instrumentation = ["mmtk/instrumentation", "no_fast_alloc"] log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] log_work_packets = ["mmtk/log_work_packets"] +object_size_distribution = ["mmtk/object_size_distribution"] \ No newline at end of file diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 0a6fbdce..e717d223 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -171,6 +171,9 @@ pub extern "C" fn alloc( offset: isize, allocator: AllocationSemantics, ) -> Address { + if cfg!(feature = "object_size_distribution") { + crate::record_alloc(size); + } with_mutator!(|mutator| memory_manager::alloc(mutator, size, align, offset, allocator)) } diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index fe121865..67c3332a 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -60,6 +60,9 @@ impl Collection> for VMCollection { } fn resume_mutators(tls: VMWorkerThread, lxr: bool, current_gc_should_unload_classes: bool) { + if cfg!(feature = "object_size_distribution") { + crate::dump_and_reset_obj_dist(); + } DISCOVERED_LISTS.enable_discover(); unsafe { ((*UPCALLS).resume_mutators)(tls, lxr, current_gc_should_unload_classes); diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 779bca3e..bdc2c1f9 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -163,7 +163,7 @@ pub static FREE_LIST_ALLOCATOR_SIZE: uintptr_t = std::mem::size_of::>>(); #[no_mangle] -pub static DISABLE_ALLOCATION_FAST_PATH: i32 = cfg!(feature = "no_fast_alloc") as _; +pub static DISABLE_ALLOCATION_FAST_PATH: i32 = (cfg!(feature = "no_fast_alloc") || cfg!(feature = "object_size_distribution")) as _; #[no_mangle] pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = @@ -539,3 +539,21 @@ lazy_static! { /// A counter tracking the total size of the `CODE_CACHE_ROOTS`. static CODE_CACHE_ROOTS_SIZE: AtomicUsize = AtomicUsize::new(0); + +lazy_static! { + static ref OBJ_COUNT: Mutex> = Mutex::new(HashMap::new()); +} + +fn record_alloc(size: usize) { + assert!(cfg!(feature = "object_size_distribution")); + let mut counts = OBJ_COUNT.lock().unwrap(); + counts.entry(size.next_power_of_two()).and_modify(|x| { + x.0 += 1; + x.1 += size; + }).or_insert((1, size)); +} + +extern fn dump_and_reset_obj_dist() { + assert!(cfg!(feature = "object_size_distribution")); + mmtk::dump_and_reset_obj_dist("Dynamic", &mut OBJ_COUNT.lock().unwrap()); +} \ No newline at end of file diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index 1df9024e..fc6d9596 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -92,7 +92,11 @@ MMTkBarrierSet::MMTkBarrierSet(MemRegion whole_heap): get_selected_barrier()->create_c2(), BarrierSet::FakeRtti(BarrierSet::ThirdPartyHeapBarrierSet)), _whole_heap(whole_heap), - _runtime(get_selected_barrier()->create_runtime()) {} + _runtime(get_selected_barrier()->create_runtime()) { + if (!MMTK_ENABLE_ALLOCATION_FASTPATH || disable_fast_alloc()) { + fprintf(stderr, "Allocation fast-path disabled\n"); + } + } void MMTkBarrierSet::write_ref_array_work(MemRegion mr) { guarantee(false, "NoBarrier::write_ref_arrey_work not supported"); From 86a5cd246644f795e41988867b07186ccae15233 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 11 Apr 2023 11:11:35 -0700 Subject: [PATCH 307/494] Make MMTk respect to JVM `ParallelGCThreads` flag --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/api.rs | 22 ++++++---------------- openjdk/mmtk.h | 3 +-- openjdk/mmtkHeap.cpp | 7 +++---- openjdk/thirdPartyHeapArguments.cpp | 9 +++++++++ 6 files changed, 22 insertions(+), 25 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index d2d0224e..0fe6757d 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=1570c684b7b2c97319588cbdcad84017472afb9a#1570c684b7b2c97319588cbdcad84017472afb9a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eca01be5ee0e592b1838db31fea720460e04bade#eca01be5ee0e592b1838db31fea720460e04bade" dependencies = [ "atomic", "atomic-traits", @@ -479,7 +479,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=1570c684b7b2c97319588cbdcad84017472afb9a#1570c684b7b2c97319588cbdcad84017472afb9a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eca01be5ee0e592b1838db31fea720460e04bade#eca01be5ee0e592b1838db31fea720460e04bade" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 29458b96..291c360b 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "1570c684b7b2c97319588cbdcad84017472afb9a" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "eca01be5ee0e592b1838db31fea720460e04bade" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index e717d223..54930278 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -347,24 +347,14 @@ pub extern "C" fn mmtk_harness_end_impl() { #[no_mangle] // We trust the name/value pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn process(name: *const c_char, value: *const c_char) -> bool { - let name_str: &CStr = unsafe { CStr::from_ptr(name) }; - let value_str: &CStr = unsafe { CStr::from_ptr(value) }; - let mut builder = BUILDER.lock().unwrap(); - memory_manager::process( - &mut builder, - name_str.to_str().unwrap(), - value_str.to_str().unwrap(), - ) -} - -#[no_mangle] -// We trust the name/value pointer is valid. -#[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn process_bulk(options: *const c_char) -> bool { +pub extern "C" fn process_bulk(options: *const c_char, parallel_gc_threads: usize) -> bool { let options_str: &CStr = unsafe { CStr::from_ptr(options) }; let mut builder = BUILDER.lock().unwrap(); - memory_manager::process_bulk(&mut builder, options_str.to_str().unwrap()) + let mut result = memory_manager::process_bulk(&mut builder, options_str.to_str().unwrap()); + if builder.options.threads.is_default { + result |= builder.set_option("threads", &format!("{}", parallel_gc_threads)); + } + result } #[no_mangle] diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index dd5c7dc1..0cf8879c 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -114,8 +114,7 @@ extern char* mmtk_active_barrier(); extern void initialize_collection(void *tls); extern void gc_init(size_t heap_size); extern bool will_never_move(void* object); -extern bool process(char* name, char* value); -extern bool process_bulk(char* options); +extern bool process_bulk(char* options, size_t threads); extern void scan_region(); extern void handle_user_collection_request(void *tls, bool force); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index a95ad185..c26c4a09 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -92,10 +92,9 @@ jint MMTkHeap::initialize() { // printf("policy max heap size %zu, min heap size %zu\n", heap_size, collector_policy()->min_heap_byte_size()); // Set options - if (ThirdPartyHeapOptions != NULL) { - bool set_options = process_bulk(strdup(ThirdPartyHeapOptions)); - guarantee(set_options, "Failed to set MMTk options. Please check if the options are valid: %s\n", ThirdPartyHeapOptions); - } + bool set_options = process_bulk(strdup(ThirdPartyHeapOptions), ParallelGCThreads); + guarantee(set_options, "Failed to set MMTk options. Please check if the options are valid: %s\n", ThirdPartyHeapOptions); + // Set heap size bool set_heap_size = mmtk_set_heap_size(min_heap_size, max_heap_size); guarantee(set_heap_size, "Failed to set MMTk heap size. Please check if the heap size is valid: min = %ld, max = %ld\n", min_heap_size, max_heap_size); diff --git a/openjdk/thirdPartyHeapArguments.cpp b/openjdk/thirdPartyHeapArguments.cpp index f2530ded..e459366d 100644 --- a/openjdk/thirdPartyHeapArguments.cpp +++ b/openjdk/thirdPartyHeapArguments.cpp @@ -34,6 +34,7 @@ #include "runtime/vm_version.hpp" #include "thirdPartyHeapArguments.hpp" #include "utilities/defaultStream.hpp" +#include "runtime/abstract_vm_version.hpp" size_t ThirdPartyHeapArguments::conservative_max_heap_alignment() { return CollectorPolicy::compute_heap_alignment(); @@ -47,6 +48,14 @@ void ThirdPartyHeapArguments::initialize() { // auto compressed_oops = mmtk_use_compressed_ptrs(); // FLAG_SET_DEFAULT(UseCompressedOops, compressed_oops); FLAG_SET_DEFAULT(UseCompressedClassPointers, UseCompressedOops); + + FLAG_SET_DEFAULT(ParallelGCThreads, Abstract_VM_Version::parallel_worker_threads()); + if (ParallelGCThreads == 0) { + assert(!FLAG_IS_DEFAULT(ParallelGCThreads), "The default value for ParallelGCThreads should not be 0."); + vm_exit_during_initialization("The flag -XX:+UseG1GC can not be combined with -XX:ParallelGCThreads=0", NULL); + } + + if (ThirdPartyHeapOptions == NULL) ThirdPartyHeapOptions = ""; } CollectedHeap* ThirdPartyHeapArguments::create_heap() { From 5c23a1857bff11cc44c128782e7be116e5118ff3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 14 Apr 2023 01:39:56 +0000 Subject: [PATCH 308/494] Fix different RC bits support --- openjdk/mmtk.h | 19 ++++++++++++++++++- openjdk/mmtkUpcalls.cpp | 5 +---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 0cf8879c..1a2c73f3 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -244,13 +244,30 @@ extern void add_phantom_candidate(void* ref, void* referent); extern void mmtk_harness_begin_impl(); extern void mmtk_harness_end_impl(); -inline uint8_t mmtk_get_rc(void* o) { +inline uint8_t mmtk_get_rc_2bits(void* o) { const uintptr_t index = uintptr_t((void*) o) >> 4; const uint8_t byte = *((uint8_t*) (RC_TABLE_BASE_ADDRESS + (index >> 2))); auto v = byte >> ((index & 0b11) << 1); return v & 0b11; } +inline uint8_t mmtk_get_rc_4bits(void* o) { + const uintptr_t index = uintptr_t((void*) o) >> 4; + const uint8_t byte = *((uint8_t*) (RC_TABLE_BASE_ADDRESS + (index >> 1))); + auto v = byte >> ((index & 0b1) << 2); + return v & 0b1111; +} + +inline uint8_t mmtk_get_rc_8bits(void* o) { + const uintptr_t index = uintptr_t((void*) o) >> 4; + const uint8_t byte = *((uint8_t*) (RC_TABLE_BASE_ADDRESS + index)); + return byte; +} + +inline uint8_t mmtk_get_rc(void* o) { + return mmtk_get_rc_2bits(o); +} + #ifdef __cplusplus } #endif diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index c5789767..126dd04e 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -97,10 +97,7 @@ class MMTkForwardClosure : public OopClosure { class MMTkLXRFastIsAliveClosure : public BoolObjectClosure { public: static inline bool rc_live(oop o) { - const uintptr_t index = uintptr_t((void*) o) >> 4; - const uint8_t byte = *((uint8_t*) (RC_TABLE_BASE_ADDRESS + (index >> 2))); - const uint8_t byte_mask = 0b11 << ((index & 0b11) << 1); - return (byte & byte_mask) != 0; + return mmtk_get_rc((void*) o) != 0; } static inline bool is_forwarded(oop o) { From f363e40b9c1e763c4ebc640cbcfac6b8e4411159 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 14 Apr 2023 11:59:11 -0700 Subject: [PATCH 309/494] Update openjdk revision --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 291c360b..6c086ea0 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -26,7 +26,7 @@ codegen-units = 1 [package.metadata.openjdk] # Our CI matches the following line and extract mmtk/openjdk. If this line is updated, please check ci yaml files and make sure it works. openjdk_repo = "https://github.com/mmtk/openjdk.git" -openjdk_version = "c558e3eeea9b16fa9f481d1fdd18fead2f462e19" +openjdk_version = "91259ca9d6062020b7c4bc28006c1c6944495173" [dependencies] libc = "0.2.138" From 5445dfe55aa99f0c938279a48cb7b6e55276f678 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 14 Apr 2023 11:59:44 -0700 Subject: [PATCH 310/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 0fe6757d..26f66634 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eca01be5ee0e592b1838db31fea720460e04bade#eca01be5ee0e592b1838db31fea720460e04bade" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=dd0acf1871205da734714e4efb55ea85647ac974#dd0acf1871205da734714e4efb55ea85647ac974" dependencies = [ "atomic", "atomic-traits", @@ -479,7 +479,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eca01be5ee0e592b1838db31fea720460e04bade#eca01be5ee0e592b1838db31fea720460e04bade" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=dd0acf1871205da734714e4efb55ea85647ac974#dd0acf1871205da734714e4efb55ea85647ac974" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 6c086ea0..ff35e8d2 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "eca01be5ee0e592b1838db31fea720460e04bade" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "dd0acf1871205da734714e4efb55ea85647ac974" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 75a6d0471f24bdb72bb918287ac5c17cd9e45fc8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 14 Apr 2023 12:02:39 -0700 Subject: [PATCH 311/494] Update cargo version --- mmtk/rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/rust-toolchain b/mmtk/rust-toolchain index 65ee0959..ee2f4ca9 100644 --- a/mmtk/rust-toolchain +++ b/mmtk/rust-toolchain @@ -1 +1 @@ -1.67.0 +1.68.0 From 530aa92cbae07a81f8914e45a8fe3ce5b7db84fa Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 14 Apr 2023 15:02:12 -0700 Subject: [PATCH 312/494] WIP: Rename field unlog bits side metadata --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/lib.rs | 24 ++++++++++++++++-------- mmtk/src/object_model.rs | 7 ++++--- mmtk/src/vm_metadata/constants.rs | 7 +++++-- openjdk/barriers/mmtkFieldBarrier.cpp | 6 +++--- openjdk/mmtkBarrierSet.hpp | 4 +++- 7 files changed, 34 insertions(+), 20 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 26f66634..f6f0cb7e 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=dd0acf1871205da734714e4efb55ea85647ac974#dd0acf1871205da734714e4efb55ea85647ac974" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=922e649a6f6851c78d0e421a266fd7fd100a541d#922e649a6f6851c78d0e421a266fd7fd100a541d" dependencies = [ "atomic", "atomic-traits", @@ -479,7 +479,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=dd0acf1871205da734714e4efb55ea85647ac974#dd0acf1871205da734714e4efb55ea85647ac974" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=922e649a6f6851c78d0e421a266fd7fd100a541d#922e649a6f6851c78d0e421a266fd7fd100a541d" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index ff35e8d2..eeb1a4c5 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "dd0acf1871205da734714e4efb55ea85647ac974" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "922e649a6f6851c78d0e421a266fd7fd100a541d" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index bdc2c1f9..b8b44c4e 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -147,7 +147,11 @@ pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS: uintptr_t = mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize(); #[no_mangle] -pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS_COMPRESSED: uintptr_t = +pub static FIELD_UNLOG_BITS_BASE_ADDRESS: uintptr_t = + mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize(); + +#[no_mangle] +pub static FIELD_UNLOG_BITS_BASE_ADDRESS_COMPRESSED: uintptr_t = mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize(); #[no_mangle] @@ -163,7 +167,8 @@ pub static FREE_LIST_ALLOCATOR_SIZE: uintptr_t = std::mem::size_of::>>(); #[no_mangle] -pub static DISABLE_ALLOCATION_FAST_PATH: i32 = (cfg!(feature = "no_fast_alloc") || cfg!(feature = "object_size_distribution")) as _; +pub static DISABLE_ALLOCATION_FAST_PATH: i32 = + (cfg!(feature = "no_fast_alloc") || cfg!(feature = "object_size_distribution")) as _; #[no_mangle] pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = @@ -547,13 +552,16 @@ lazy_static! { fn record_alloc(size: usize) { assert!(cfg!(feature = "object_size_distribution")); let mut counts = OBJ_COUNT.lock().unwrap(); - counts.entry(size.next_power_of_two()).and_modify(|x| { - x.0 += 1; - x.1 += size; - }).or_insert((1, size)); + counts + .entry(size.next_power_of_two()) + .and_modify(|x| { + x.0 += 1; + x.1 += size; + }) + .or_insert((1, size)); } -extern fn dump_and_reset_obj_dist() { +extern "C" fn dump_and_reset_obj_dist() { assert!(cfg!(feature = "object_size_distribution")); mmtk::dump_and_reset_obj_dist("Dynamic", &mut OBJ_COUNT.lock().unwrap()); -} \ No newline at end of file +} diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 5ddc8bb2..661add6d 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -9,10 +9,11 @@ use mmtk::vm::*; pub struct VMObjectModel {} impl ObjectModel> for VMObjectModel { - const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = if COMPRESSED { - vm_metadata::LOGGING_SIDE_METADATA_SPEC_COMPRESSED + const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = vm_metadata::LOGGING_SIDE_METADATA_SPEC; + const GLOBAL_FIELD_UNLOG_BIT_SPEC: VMGlobalFieldUnlogBitSpec = if COMPRESSED { + vm_metadata::FIELD_LOGGING_SIDE_METADATA_SPEC_COMPRESSED } else { - vm_metadata::LOGGING_SIDE_METADATA_SPEC + vm_metadata::FIELD_LOGGING_SIDE_METADATA_SPEC }; const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = diff --git a/mmtk/src/vm_metadata/constants.rs b/mmtk/src/vm_metadata/constants.rs index 5d9e9043..9af15398 100644 --- a/mmtk/src/vm_metadata/constants.rs +++ b/mmtk/src/vm_metadata/constants.rs @@ -13,8 +13,11 @@ pub(crate) const FORWARDING_POINTER_OFFSET: isize = 0; /// 1 bit per object pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); -pub(crate) const LOGGING_SIDE_METADATA_SPEC_COMPRESSED: VMGlobalLogBitSpec = - VMGlobalLogBitSpec::side_first_compressed(); +pub(crate) const FIELD_LOGGING_SIDE_METADATA_SPEC: VMGlobalFieldUnlogBitSpec = + VMGlobalFieldUnlogBitSpec::side_first(); + +pub(crate) const FIELD_LOGGING_SIDE_METADATA_SPEC_COMPRESSED: VMGlobalFieldUnlogBitSpec = + VMGlobalFieldUnlogBitSpec::side_first_compressed(); // Global MetadataSpecs - End diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index fb7aad6b..69c12dc9 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -6,7 +6,7 @@ constexpr int kUnloggedValue = 1; static inline intptr_t side_metadata_base_address() { - return UseCompressedOops ? SIDE_METADATA_BASE_ADDRESS_COMPRESSED : SIDE_METADATA_BASE_ADDRESS; + return UseCompressedOops ? FIELD_UNLOG_BITS_BASE_ADDRESS_COMPRESSED : FIELD_UNLOG_BITS_BASE_ADDRESS; } @@ -78,7 +78,7 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma Register tmp4 = rscratch2; Register tmp5 = tmp1 == dst.base() || tmp1 == dst.index() ? tmp2 : tmp1; - // tmp5 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); + // tmp5 = load-byte (side_metadata_base_address() + (obj >> 6)); __ lea(tmp3, dst); __ shrptr(tmp3, UseCompressedOops ? 5 : 6); __ movptr(tmp5, side_metadata_base_address()); @@ -246,7 +246,7 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op __ jump(slow); } else { LIR_Opr addr = slot; - // uint8_t* meta_addr = (uint8_t*) (SIDE_METADATA_BASE_ADDRESS + (addr >> 6)); + // uint8_t* meta_addr = (uint8_t*) (side_metadata_base_address() + (addr >> 6)); LIR_Opr offset = gen->new_pointer_register(); __ move(addr, offset); __ unsigned_shift_right(offset, UseCompressedOops ? 5 : 6, offset); diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 8fd37c72..492b6722 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -42,7 +42,9 @@ const intptr_t SIDE_METADATA_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; -const intptr_t SIDE_METADATA_BASE_ADDRESS_COMPRESSED = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS_COMPRESSED; +const intptr_t FIELD_UNLOG_BITS_BASE_ADDRESS = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; + +const intptr_t FIELD_UNLOG_BITS_BASE_ADDRESS_COMPRESSED = (intptr_t) GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; const intptr_t ALLOC_BIT_BASE_ADDRESS = GLOBAL_ALLOC_BIT_ADDRESS; From dc21642daf28b0d716a10380c244783f0e92375b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 15 Apr 2023 00:15:48 +0000 Subject: [PATCH 313/494] Fix barrier --- openjdk/barriers/mmtkObjectBarrier.cpp | 6 +++++- openjdk/barriers/mmtkObjectBarrier.hpp | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index bcc977b9..dea1824f 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -17,6 +17,10 @@ void MMTkObjectBarrierSetRuntime::object_reference_write_post(oop src, oop* slot #endif } +void MMTkObjectBarrierSetRuntime::on_slowpath_allocation_exit(oop new_obj) const { + ::mmtk_object_reference_clone_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) new_obj); +} + #define __ masm-> void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const { @@ -103,7 +107,7 @@ void MMTkObjectBarrierSetC1::object_reference_write_post(LIRAccess& access, LIR_ if (!address->index()->is_valid() && address->disp() == 0) { __ move(address->base(), ptr); } else { - assert(address->disp() != max_jint, "lea doesn't support patched addresses!"); + // assert(address->disp() != max_jint, "lea doesn't support patched addresses!"); __ leal(slot, ptr); } slot = ptr; diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index 475fe7ad..9eec8713 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -23,6 +23,7 @@ class MMTkObjectBarrierSetRuntime: public MMTkBarrierSetRuntime { virtual void object_reference_array_copy_post(oop* src, oop* dst, size_t count) const override { object_reference_array_copy_post_call((void*) src, (void*) dst, count); } + virtual void on_slowpath_allocation_exit(oop new_obj) const override; }; class MMTkObjectBarrierSetAssembler: public MMTkBarrierSetAssembler { @@ -44,6 +45,10 @@ class MMTkObjectBarrierSetC1: public MMTkBarrierSetC1 { class MMTkObjectBarrierSetC2: public MMTkBarrierSetC2 { protected: virtual void object_reference_write_post(GraphKit* kit, Node* src, Node* slot, Node* val) const override; +public: + virtual bool array_copy_requires_gc_barriers(BasicType type) const override { + return false; + } }; struct MMTkObjectBarrier: MMTkBarrierImpl< From 1b7bb697920490779f78ca6793294844c04204d1 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 15 Apr 2023 04:59:57 +0000 Subject: [PATCH 314/494] Compressed pointer support --- mmtk/src/abi.rs | 108 ++++++++-- mmtk/src/active_plan.rs | 15 +- mmtk/src/api.rs | 249 +++++++++++++++------- mmtk/src/collection.rs | 25 +-- mmtk/src/gc_work.rs | 41 ++-- mmtk/src/lib.rs | 317 ++++++++++++++++++++++++++-- mmtk/src/object_model.rs | 29 ++- mmtk/src/object_scanning.rs | 218 +++++++++++-------- mmtk/src/reference_glue.rs | 8 +- mmtk/src/scanning.rs | 40 ++-- openjdk/mmtk.h | 8 + openjdk/mmtkHeap.cpp | 6 +- openjdk/mmtkRootsClosure.hpp | 81 ++----- openjdk/mmtkUpcalls.cpp | 12 +- openjdk/thirdPartyHeapArguments.cpp | 4 +- 15 files changed, 828 insertions(+), 333 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 0bd27a20..70df7110 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -1,4 +1,5 @@ -use crate::UPCALLS; +use super::UPCALLS; +use crate::Edge; use mmtk::util::constants::*; use mmtk::util::conversions; use mmtk::util::ObjectReference; @@ -80,7 +81,7 @@ impl Klass { pub const LH_HEADER_SIZE_SHIFT: i32 = BITS_IN_BYTE as i32 * 2; pub const LH_HEADER_SIZE_MASK: i32 = (1 << BITS_IN_BYTE) - 1; pub unsafe fn cast<'a, T>(&self) -> &'a T { - &*(self as *const _ as usize as *const T) + &*(self as *const Self as *const T) } /// Force slow-path for instance size calculation? const fn layout_helper_needs_slow_path(lh: i32) -> bool { @@ -94,6 +95,10 @@ impl Klass { const fn layout_helper_header_size(lh: i32) -> i32 { (lh >> Self::LH_HEADER_SIZE_SHIFT) & Self::LH_HEADER_SIZE_MASK } + + pub const fn is_instance_klass(&self) -> bool { + self.layout_helper > Self::LH_NEUTRAL_VALUE + } } #[repr(C)] @@ -152,7 +157,7 @@ pub struct InstanceKlass { } #[repr(u8)] -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] #[allow(dead_code)] pub enum ReferenceType { None, // Regular class @@ -168,7 +173,7 @@ impl InstanceKlass { const VTABLE_START_OFFSET: usize = Self::HEADER_SIZE * BYTES_IN_WORD; fn start_of_vtable(&self) -> *const usize { - unsafe { (self as *const _ as *const u8).add(Self::VTABLE_START_OFFSET) as _ } + unsafe { (self as *const Self as *const u8).add(Self::VTABLE_START_OFFSET) as _ } } fn start_of_itable(&self) -> *const usize { @@ -263,24 +268,45 @@ impl InstanceRefKlass { } *DISCOVERED_OFFSET } - pub fn referent_address(oop: Oop) -> Address { - oop.get_field_address(Self::referent_offset()) + pub fn referent_address(oop: Oop) -> E { + E::from_address(oop.get_field_address(Self::referent_offset())) } - pub fn discovered_address(oop: Oop) -> Address { - oop.get_field_address(Self::discovered_offset()) + pub fn discovered_address(oop: Oop) -> E { + E::from_address(oop.get_field_address(Self::discovered_offset())) } } +#[repr(C)] +union KlassField { + klass: &'static Klass, + narrow_klass: u32, +} + #[repr(C)] pub struct OopDesc { pub mark: usize, - pub klass: &'static Klass, + klass: KlassField, +} + +lazy_static! { + static ref COMPRESSED_KLASS_BASE: Address = unsafe { ((*UPCALLS).compressed_klass_base)() }; + static ref COMPRESSED_KLASS_SHIFT: usize = unsafe { ((*UPCALLS).compressed_klass_shift)() }; } impl OopDesc { pub fn start(&self) -> Address { unsafe { mem::transmute(self) } } + + pub fn klass(&self) -> &'static Klass { + if COMPRESSED { + let compressed = unsafe { self.klass.narrow_klass }; + let addr = *COMPRESSED_KLASS_BASE + ((compressed as usize) << *COMPRESSED_KLASS_SHIFT); + unsafe { &*addr.to_ptr::() } + } else { + unsafe { self.klass.klass } + } + } } impl fmt::Debug for OopDesc { @@ -294,6 +320,16 @@ impl fmt::Debug for OopDesc { pub type Oop = &'static OopDesc; +#[repr(transparent)] +#[derive(Clone, Copy)] +pub struct NarrowOop(u32); + +impl NarrowOop { + pub fn slot(&self) -> Address { + Address::from_ref(self) + } +} + /// Convert ObjectReference to Oop impl From for &OopDesc { fn from(o: ObjectReference) -> Self { @@ -323,8 +359,8 @@ impl OopDesc { } /// Calculate object instance size - pub unsafe fn size(&self) -> usize { - let klass = self.klass; + pub unsafe fn size(&self) -> usize { + let klass = self.klass::(); let lh = klass.layout_helper; // The (scalar) instance size is pre-recorded in the TIB? if lh > Klass::LH_NEUTRAL_VALUE { @@ -336,7 +372,7 @@ impl OopDesc { } else if lh <= Klass::LH_NEUTRAL_VALUE { if lh < Klass::LH_NEUTRAL_VALUE { // Calculate array size - let array_length = self.as_array_oop().length(); + let array_length = self.as_array_oop().length::(); let mut size_in_bytes: usize = (array_length as usize) << Klass::layout_helper_log2_element_size(lh); size_in_bytes += Klass::layout_helper_header_size(lh) as usize; @@ -356,34 +392,60 @@ pub struct ArrayOopDesc(OopDesc); pub type ArrayOop = &'static ArrayOopDesc; impl ArrayOopDesc { - const LENGTH_OFFSET: usize = mem::size_of::(); + fn length_offset() -> usize { + if COMPRESSED { + mem::size_of::() + mem::size_of::() + } else { + mem::size_of::() + } + } fn element_type_should_be_aligned(ty: BasicType) -> bool { ty == BasicType::T_DOUBLE || ty == BasicType::T_LONG } - fn header_size(ty: BasicType) -> usize { - let typesize_in_bytes = - conversions::raw_align_up(Self::LENGTH_OFFSET + BYTES_IN_INT, BYTES_IN_LONG); + fn header_size(ty: BasicType) -> usize { + let typesize_in_bytes = conversions::raw_align_up( + Self::length_offset::() + BYTES_IN_INT, + BYTES_IN_LONG, + ); if Self::element_type_should_be_aligned(ty) { conversions::raw_align_up(typesize_in_bytes / BYTES_IN_WORD, BYTES_IN_LONG) } else { typesize_in_bytes / BYTES_IN_WORD } } - fn length(&self) -> i32 { - unsafe { *((self as *const _ as *const u8).add(Self::LENGTH_OFFSET) as *const i32) } + fn length(&self) -> i32 { + unsafe { + *((self as *const Self as *const u8).add(Self::length_offset::()) + as *const i32) + } } - fn base(&self, ty: BasicType) -> Address { - let base_offset_in_bytes = Self::header_size(ty) * BYTES_IN_WORD; - Address::from_ptr(unsafe { (self as *const _ as *const u8).add(base_offset_in_bytes) }) + fn base(&self, ty: BasicType) -> Address { + let base_offset_in_bytes = Self::header_size::(ty) * BYTES_IN_WORD; + Address::from_ptr(unsafe { (self as *const Self as *const u8).add(base_offset_in_bytes) }) } // This provides an easy way to access the array data in Rust. However, the array data // is Java types, so we have to map Java types to Rust types. The caller needs to guarantee: // 1. matches the actual Java type // 2. matches the argument, BasicType `ty` - pub unsafe fn data(&self, ty: BasicType) -> &[T] { - slice::from_raw_parts(self.base(ty).to_ptr(), self.length() as _) + pub unsafe fn data(&self, ty: BasicType) -> &[T] { + slice::from_raw_parts( + self.base::(ty).to_ptr(), + self.length::() as _, + ) + } + + pub unsafe fn slice( + &self, + ty: BasicType, + ) -> crate::OpenJDKEdgeRange { + let base = self.base::(ty); + let start = crate::OpenJDKEdge(base); + let end = crate::OpenJDKEdge( + base + ((self.length::() as usize) << if COMPRESSED { 2 } else { 3 }), + ); + crate::OpenJDKEdgeRange { start, end } } } diff --git a/mmtk/src/active_plan.rs b/mmtk/src/active_plan.rs index 88d12958..a702ac53 100644 --- a/mmtk/src/active_plan.rs +++ b/mmtk/src/active_plan.rs @@ -1,5 +1,4 @@ use crate::OpenJDK; -use crate::SINGLETON; use crate::UPCALLS; use mmtk::util::opaque_pointer::*; use mmtk::vm::ActivePlan; @@ -9,19 +8,19 @@ use std::sync::Mutex; pub struct VMActivePlan {} -impl ActivePlan for VMActivePlan { - fn global() -> &'static dyn Plan { - SINGLETON.get_plan() +impl ActivePlan> for VMActivePlan { + fn global() -> &'static dyn Plan> { + crate::singleton::().get_plan() } fn is_mutator(tls: VMThread) -> bool { unsafe { ((*UPCALLS).is_mutator)(tls) } } - fn mutator(tls: VMMutatorThread) -> &'static mut Mutator { + fn mutator(tls: VMMutatorThread) -> &'static mut Mutator> { unsafe { let m = ((*UPCALLS).get_mmtk_mutator)(tls); - &mut *m + &mut *(m as *mut Mutator>) } } @@ -31,14 +30,14 @@ impl ActivePlan for VMActivePlan { } } - fn get_next_mutator() -> Option<&'static mut Mutator> { + fn get_next_mutator() -> Option<&'static mut Mutator>> { let _guard = MUTATOR_ITERATOR_LOCK.lock().unwrap(); unsafe { let m = ((*UPCALLS).get_next_mutator)(); if m.is_null() { None } else { - Some(&mut *m) + Some(&mut *(m as *mut Mutator>)) } } } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 5bd6b820..9325abc7 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -1,7 +1,8 @@ use crate::OpenJDK; +use crate::OpenJDKEdge; +use crate::OpenJDKEdgeRange; use crate::OpenJDK_Upcalls; use crate::BUILDER; -use crate::SINGLETON; use crate::UPCALLS; use libc::c_char; use mmtk::memory_manager; @@ -9,7 +10,6 @@ use mmtk::plan::BarrierSelector; use mmtk::scheduler::GCController; use mmtk::scheduler::GCWorker; use mmtk::util::alloc::AllocatorSelector; -use mmtk::util::constants::LOG_BYTES_IN_ADDRESS; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::AllocationSemantics; @@ -20,6 +20,18 @@ use std::cell::RefCell; use std::ffi::{CStr, CString}; use std::sync::atomic::Ordering; +macro_rules! with_mutator { + (|$x: ident| $($expr:tt)*) => { + if crate::use_compressed_oops() { + let $x = unsafe { &mut *($x as *mut Mutator>) }; + $($expr)* + } else { + let $x = unsafe { &mut *($x as *mut Mutator>) }; + $($expr)* + } + }; +} + // Supported barriers: static NO_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("NoBarrier").unwrap()); static OBJECT_BARRIER: sync::Lazy = @@ -32,13 +44,15 @@ pub extern "C" fn get_mmtk_version() -> *const c_char { #[no_mangle] pub extern "C" fn mmtk_active_barrier() -> *const c_char { - match SINGLETON.get_plan().constraints().barrier { - BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), - BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), - // In case we have more barriers in mmtk-core. - #[allow(unreachable_patterns)] - _ => unimplemented!(), - } + with_singleton!(|singleton| { + match singleton.get_plan().constraints().barrier { + BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), + BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), + // In case we have more barriers in mmtk-core. + #[allow(unreachable_patterns)] + _ => unimplemented!(), + } + }) } /// # Safety @@ -83,7 +97,11 @@ pub extern "C" fn openjdk_gc_init(calls: *const OpenJDK_Upcalls) { // Make sure that we haven't initialized MMTk (by accident) yet assert!(!crate::MMTK_INITIALIZED.load(Ordering::SeqCst)); // Make sure we initialize MMTk here - lazy_static::initialize(&SINGLETON); + if crate::use_compressed_oops() { + lazy_static::initialize(&crate::SINGLETON_COMPRESSED); + } else { + lazy_static::initialize(&crate::SINGLETON_UNCOMPRESSED); + } } #[no_mangle] @@ -104,60 +122,64 @@ pub extern "C" fn mmtk_set_heap_size(min: usize, max: usize) -> bool { } #[no_mangle] -pub extern "C" fn bind_mutator(tls: VMMutatorThread) -> *mut Mutator { - Box::into_raw(memory_manager::bind_mutator(&SINGLETON, tls)) +pub extern "C" fn bind_mutator(tls: VMMutatorThread) -> *mut libc::c_void { + with_singleton!(|singleton| { + Box::into_raw(memory_manager::bind_mutator(&singleton, tls)) as *mut libc::c_void + }) } #[no_mangle] // It is fine we turn the pointer back to box, as we turned a boxed value to the raw pointer in bind_mutator() #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn destroy_mutator(mutator: *mut Mutator) { - memory_manager::destroy_mutator(unsafe { &mut *mutator }) +pub extern "C" fn destroy_mutator(mutator: *mut libc::c_void) { + with_mutator!(|mutator| memory_manager::destroy_mutator(mutator)) } #[no_mangle] // We trust the mutator pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn flush_mutator(mutator: *mut Mutator) { - memory_manager::flush_mutator(unsafe { &mut *mutator }) +pub extern "C" fn flush_mutator(mutator: *mut libc::c_void) { + with_mutator!(|mutator| memory_manager::flush_mutator(mutator)) } #[no_mangle] // We trust the mutator pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] pub extern "C" fn alloc( - mutator: *mut Mutator, + mutator: *mut libc::c_void, size: usize, align: usize, offset: isize, allocator: AllocationSemantics, ) -> Address { - memory_manager::alloc::(unsafe { &mut *mutator }, size, align, offset, allocator) + with_mutator!(|mutator| memory_manager::alloc(mutator, size, align, offset, allocator)) } #[no_mangle] pub extern "C" fn get_allocator_mapping(allocator: AllocationSemantics) -> AllocatorSelector { - memory_manager::get_allocator_mapping(&SINGLETON, allocator) + with_singleton!(|singleton| memory_manager::get_allocator_mapping(&singleton, allocator)) } #[no_mangle] pub extern "C" fn get_max_non_los_default_alloc_bytes() -> usize { - SINGLETON - .get_plan() - .constraints() - .max_non_los_default_alloc_bytes + with_singleton!(|singleton| { + singleton + .get_plan() + .constraints() + .max_non_los_default_alloc_bytes + }) } #[no_mangle] // We trust the mutator pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] pub extern "C" fn post_alloc( - mutator: *mut Mutator, + mutator: *mut libc::c_void, refer: ObjectReference, bytes: usize, allocator: AllocationSemantics, ) { - memory_manager::post_alloc::(unsafe { &mut *mutator }, refer, bytes, allocator) + with_mutator!(|mutator| memory_manager::post_alloc(mutator, refer, bytes, allocator)) } #[no_mangle] @@ -168,56 +190,88 @@ pub extern "C" fn will_never_move(object: ObjectReference) -> bool { #[no_mangle] // We trust the gc_collector pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn start_control_collector( - tls: VMWorkerThread, - gc_controller: *mut GCController, -) { - let mut gc_controller = unsafe { Box::from_raw(gc_controller) }; - memory_manager::start_control_collector(&SINGLETON, tls, &mut gc_controller); +pub extern "C" fn start_control_collector(tls: VMWorkerThread, gc_controller: *mut libc::c_void) { + if crate::use_compressed_oops() { + let mut gc_controller = + unsafe { Box::from_raw(gc_controller as *mut GCController>) }; + memory_manager::start_control_collector( + &crate::singleton::(), + tls, + &mut gc_controller, + ); + } else { + let mut gc_controller = + unsafe { Box::from_raw(gc_controller as *mut GCController>) }; + memory_manager::start_control_collector( + &crate::singleton::(), + tls, + &mut gc_controller, + ); + } } #[no_mangle] // We trust the worker pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] -pub extern "C" fn start_worker(tls: VMWorkerThread, worker: *mut GCWorker) { - let mut worker = unsafe { Box::from_raw(worker) }; - memory_manager::start_worker::(&SINGLETON, tls, &mut worker) +pub extern "C" fn start_worker(tls: VMWorkerThread, worker: *mut libc::c_void) { + if crate::use_compressed_oops() { + let mut worker = unsafe { Box::from_raw(worker as *mut GCWorker>) }; + memory_manager::start_worker::>(&crate::singleton::(), tls, &mut worker) + } else { + let mut worker = unsafe { Box::from_raw(worker as *mut GCWorker>) }; + memory_manager::start_worker::>( + &crate::singleton::(), + tls, + &mut worker, + ) + } } #[no_mangle] pub extern "C" fn initialize_collection(tls: VMThread) { - memory_manager::initialize_collection(&SINGLETON, tls) + with_singleton!(|singleton| memory_manager::initialize_collection(&singleton, tls)) } #[no_mangle] pub extern "C" fn used_bytes() -> usize { - memory_manager::used_bytes(&SINGLETON) + with_singleton!(|singleton| memory_manager::used_bytes(&singleton)) } #[no_mangle] pub extern "C" fn free_bytes() -> usize { - memory_manager::free_bytes(&SINGLETON) + with_singleton!(|singleton| memory_manager::free_bytes(&singleton)) } #[no_mangle] pub extern "C" fn total_bytes() -> usize { - memory_manager::total_bytes(&SINGLETON) + with_singleton!(|singleton| memory_manager::total_bytes(&singleton)) } #[no_mangle] #[cfg(feature = "sanity")] pub extern "C" fn scan_region() { - memory_manager::scan_region(&SINGLETON) + with_singleton!(|singleton| memory_manager::scan_region(&singleton)) } #[no_mangle] pub extern "C" fn handle_user_collection_request(tls: VMMutatorThread) { - memory_manager::handle_user_collection_request::(&SINGLETON, tls); + with_singleton!(|singleton| { + memory_manager::handle_user_collection_request(&singleton, tls); + }) +} + +#[no_mangle] +pub extern "C" fn mmtk_use_compressed_ptrs() { + crate::init_compressed_oop_constants() } #[no_mangle] pub extern "C" fn is_in_mmtk_spaces(object: ObjectReference) -> bool { - memory_manager::is_in_mmtk_spaces::(object) + if crate::use_compressed_oops() { + memory_manager::is_in_mmtk_spaces::>(object) + } else { + memory_manager::is_in_mmtk_spaces::>(object) + } } #[no_mangle] @@ -227,22 +281,22 @@ pub extern "C" fn is_mapped_address(addr: Address) -> bool { #[no_mangle] pub extern "C" fn modify_check(object: ObjectReference) { - memory_manager::modify_check(&SINGLETON, object) + with_singleton!(|singleton| memory_manager::modify_check(&singleton, object)) } #[no_mangle] pub extern "C" fn add_weak_candidate(reff: ObjectReference) { - memory_manager::add_weak_candidate(&SINGLETON, reff) + with_singleton!(|singleton| memory_manager::add_weak_candidate(&singleton, reff)) } #[no_mangle] pub extern "C" fn add_soft_candidate(reff: ObjectReference) { - memory_manager::add_soft_candidate(&SINGLETON, reff) + with_singleton!(|singleton| memory_manager::add_soft_candidate(&singleton, reff)) } #[no_mangle] pub extern "C" fn add_phantom_candidate(reff: ObjectReference) { - memory_manager::add_phantom_candidate(&SINGLETON, reff) + with_singleton!(|singleton| memory_manager::add_phantom_candidate(&singleton, reff)) } // The harness_begin()/end() functions are different than other API functions in terms of the thread state. @@ -259,7 +313,9 @@ pub extern "C" fn harness_begin(_id: usize) { #[no_mangle] pub extern "C" fn mmtk_harness_begin_impl() { // Pass null as tls, OpenJDK binding does not rely on the tls value to block the current thread and do a GC - memory_manager::harness_begin(&SINGLETON, VMMutatorThread(VMThread::UNINITIALIZED)); + with_singleton!(|singleton| { + memory_manager::harness_begin(&singleton, VMMutatorThread(VMThread::UNINITIALIZED)); + }) } #[no_mangle] @@ -269,7 +325,7 @@ pub extern "C" fn harness_end(_id: usize) { #[no_mangle] pub extern "C" fn mmtk_harness_end_impl() { - memory_manager::harness_end(&SINGLETON); + with_singleton!(|singleton| memory_manager::harness_end(&singleton)) } #[no_mangle] @@ -295,6 +351,16 @@ pub extern "C" fn process_bulk(options: *const c_char) -> bool { memory_manager::process_bulk(&mut builder, options_str.to_str().unwrap()) } +#[no_mangle] +pub extern "C" fn mmtk_narrow_oop_base() -> Address { + unsafe { crate::BASE } +} + +#[no_mangle] +pub extern "C" fn mmtk_narrow_oop_shift() -> usize { + unsafe { crate::SHIFT } +} + #[no_mangle] pub extern "C" fn starting_heap_address() -> Address { memory_manager::starting_heap_address() @@ -307,7 +373,7 @@ pub extern "C" fn last_heap_address() -> Address { #[no_mangle] pub extern "C" fn openjdk_max_capacity() -> usize { - memory_manager::total_bytes(&SINGLETON) + with_singleton!(|singleton| memory_manager::total_bytes(&singleton)) } #[no_mangle] @@ -318,91 +384,114 @@ pub extern "C" fn executable() -> bool { /// Full pre barrier #[no_mangle] pub extern "C" fn mmtk_object_reference_write_pre( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: ObjectReference, ) { - mutator - .barrier() - .object_reference_write_pre(src, slot, target); + with_mutator!(|mutator| { + mutator + .barrier() + .object_reference_write_pre(src, OpenJDKEdge(slot), target); + }) } /// Full post barrier #[no_mangle] pub extern "C" fn mmtk_object_reference_write_post( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: ObjectReference, ) { - mutator - .barrier() - .object_reference_write_post(src, slot, target); + with_mutator!(|mutator| { + mutator + .barrier() + .object_reference_write_post(src, OpenJDKEdge(slot), target); + }) } /// Barrier slow-path call #[no_mangle] pub extern "C" fn mmtk_object_reference_write_slow( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: ObjectReference, ) { - mutator - .barrier() - .object_reference_write_slow(src, slot, target); + with_mutator!(|mutator| { + mutator + .barrier() + .object_reference_write_slow(src, OpenJDKEdge(slot), target); + }) } /// Array-copy pre-barrier #[no_mangle] pub extern "C" fn mmtk_array_copy_pre( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: Address, dst: Address, count: usize, ) { - let bytes = count << LOG_BYTES_IN_ADDRESS; - mutator - .barrier() - .memory_region_copy_pre(src..src + bytes, dst..dst + bytes); + let bytes = count << crate::log_bytes_in_field(); + with_mutator!(|mutator| { + mutator.barrier().memory_region_copy_pre( + OpenJDKEdgeRange { + start: OpenJDKEdge(src), + end: OpenJDKEdge(src + bytes), + }, + OpenJDKEdgeRange { + start: OpenJDKEdge(dst), + end: OpenJDKEdge(dst + bytes), + }, + ); + }) } /// Array-copy post-barrier #[no_mangle] pub extern "C" fn mmtk_array_copy_post( - mutator: &'static mut Mutator, + mutator: *mut libc::c_void, src: Address, dst: Address, count: usize, ) { - let bytes = count << LOG_BYTES_IN_ADDRESS; - mutator - .barrier() - .memory_region_copy_post(src..src + bytes, dst..dst + bytes); + with_mutator!(|mutator| { + let bytes = count << crate::log_bytes_in_field(); + mutator.barrier().memory_region_copy_post( + OpenJDKEdgeRange { + start: OpenJDKEdge(src), + end: OpenJDKEdge(src + bytes), + }, + OpenJDKEdgeRange { + start: OpenJDKEdge(dst), + end: OpenJDKEdge(dst + bytes), + }, + ); + }) } /// C2 Slowpath allocation barrier #[no_mangle] -pub extern "C" fn mmtk_object_probable_write( - mutator: &'static mut Mutator, - obj: ObjectReference, -) { - mutator.barrier().object_probable_write(obj); +pub extern "C" fn mmtk_object_probable_write(mutator: *mut libc::c_void, obj: ObjectReference) { + with_mutator!(|mutator| mutator.barrier().object_probable_write(obj)); } // finalization #[no_mangle] pub extern "C" fn add_finalizer(object: ObjectReference) { - memory_manager::add_finalizer(&SINGLETON, object); + with_singleton!(|singleton| memory_manager::add_finalizer(&singleton, object)); } #[no_mangle] pub extern "C" fn get_finalized_object() -> ObjectReference { - match memory_manager::get_finalized_object(&SINGLETON) { - Some(obj) => obj, - None => ObjectReference::NULL, - } + with_singleton!(|singleton| { + match memory_manager::get_finalized_object(&singleton) { + Some(obj) => obj, + None => ObjectReference::NULL, + } + }) } thread_local! { diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index f02e3f56..66d8fe48 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -8,22 +8,23 @@ use crate::{MutatorClosure, OpenJDK}; pub struct VMCollection {} -extern "C" fn report_mutator_stop( - mutator: *mut Mutator, +extern "C" fn report_mutator_stop( + mutator: *mut libc::c_void, callback_ptr: *mut libc::c_void, ) where - F: FnMut(&'static mut Mutator), + F: FnMut(&'static mut Mutator>), { + let mutator = mutator as *mut Mutator>; let callback: &mut F = unsafe { &mut *(callback_ptr as *mut F) }; callback(unsafe { &mut *mutator }); } -fn to_mutator_closure(callback: &mut F) -> MutatorClosure +fn to_mutator_closure(callback: &mut F) -> MutatorClosure where - F: FnMut(&'static mut Mutator), + F: FnMut(&'static mut Mutator>), { MutatorClosure { - func: report_mutator_stop::, + func: report_mutator_stop::, data: callback as *mut F as *mut libc::c_void, } } @@ -31,23 +32,23 @@ where const GC_THREAD_KIND_CONTROLLER: libc::c_int = 0; const GC_THREAD_KIND_WORKER: libc::c_int = 1; -impl Collection for VMCollection { +impl Collection> for VMCollection { /// With the presence of the "VM companion thread", /// the OpenJDK binding allows any MMTk GC thread to stop/start the world. const COORDINATOR_ONLY_STW: bool = false; fn stop_all_mutators(tls: VMWorkerThread, mut mutator_visitor: F) where - F: FnMut(&'static mut Mutator), + F: FnMut(&'static mut Mutator>), { let scan_mutators_in_safepoint = - ::VMScanning::SCAN_MUTATORS_IN_SAFEPOINT; + < as VMBinding>::VMScanning as Scanning>>::SCAN_MUTATORS_IN_SAFEPOINT; unsafe { ((*UPCALLS).stop_all_mutators)( tls, scan_mutators_in_safepoint, - to_mutator_closure(&mut mutator_visitor), + to_mutator_closure::<_, COMPRESSED>(&mut mutator_visitor), ); } } @@ -64,7 +65,7 @@ impl Collection for VMCollection { } } - fn spawn_gc_thread(tls: VMThread, ctx: GCThreadContext) { + fn spawn_gc_thread(tls: VMThread, ctx: GCThreadContext>) { let (ctx_ptr, kind) = match ctx { GCThreadContext::Controller(c) => ( Box::into_raw(c) as *mut libc::c_void, @@ -79,7 +80,7 @@ impl Collection for VMCollection { } } - fn prepare_mutator>( + fn prepare_mutator>>( _tls_w: VMWorkerThread, _tls_m: VMMutatorThread, _m: &T, diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 4273f9f7..d6ca64e1 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -1,25 +1,32 @@ use std::sync::atomic::Ordering; use crate::scanning::to_edges_closure; -use crate::{OpenJDK, OpenJDKEdge, UPCALLS}; +use crate::Edge; +use crate::UPCALLS; use mmtk::scheduler::*; use mmtk::vm::RootsWorkFactory; +use mmtk::vm::*; use mmtk::MMTK; +use std::marker::PhantomData; macro_rules! scan_roots_work { ($struct_name: ident, $func_name: ident) => { - pub struct $struct_name> { + pub struct $struct_name> { factory: F, + _p: std::marker::PhantomData, } - impl> $struct_name { + impl> $struct_name { pub fn new(factory: F) -> Self { - Self { factory } + Self { + factory, + _p: std::marker::PhantomData, + } } } - impl> GCWork for $struct_name { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { + impl> GCWork for $struct_name { + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { unsafe { ((*UPCALLS).$func_name)(to_edges_closure(&mut self.factory)); } @@ -43,27 +50,35 @@ scan_roots_work!( scan_roots_work!(ScanWeakProcessorRoots, scan_weak_processor_roots); scan_roots_work!(ScanVMThreadRoots, scan_vm_thread_roots); -pub struct ScanCodeCacheRoots> { +pub struct ScanCodeCacheRoots> { factory: F, + _p: PhantomData, } -impl> ScanCodeCacheRoots { +impl> ScanCodeCacheRoots { pub fn new(factory: F) -> Self { - Self { factory } + Self { + factory, + _p: PhantomData, + } } } -impl> GCWork for ScanCodeCacheRoots { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { +impl> GCWork + for ScanCodeCacheRoots +{ + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { // Collect all the cached roots let mut edges = Vec::with_capacity(crate::CODE_CACHE_ROOTS_SIZE.load(Ordering::Relaxed)); for roots in (*crate::CODE_CACHE_ROOTS.lock().unwrap()).values() { for r in roots { - edges.push(*r) + edges.push(VM::VMEdge::from_address(*r)) } } // Create work packet - self.factory.create_process_edge_roots_work(edges); + if !edges.is_empty() { + self.factory.create_process_edge_roots_work(edges); + } // Use the following code to scan CodeCache directly, instead of scanning the "remembered set". // unsafe { // ((*UPCALLS).scan_code_cache_roots)(create_process_edges_work:: as _); diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 45820e47..8cd7eee3 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -2,17 +2,33 @@ extern crate lazy_static; use std::collections::HashMap; -use std::ops::Range; use std::ptr::null_mut; use std::sync::atomic::AtomicUsize; use std::sync::Mutex; use libc::{c_char, c_void, uintptr_t}; use mmtk::util::alloc::AllocationError; +use mmtk::util::constants::{ + BYTES_IN_ADDRESS, BYTES_IN_INT, LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_INT, +}; +use mmtk::util::heap::layout::vm_layout_constants::VM_LAYOUT_CONSTANTS; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; +use mmtk::vm::edge_shape::{Edge, MemorySlice}; use mmtk::vm::VMBinding; -use mmtk::{MMTKBuilder, Mutator, MMTK}; +use mmtk::{MMTKBuilder, MMTK}; + +macro_rules! with_singleton { + (|$x: ident| $($expr:tt)*) => { + if crate::use_compressed_oops() { + let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_COMPRESSED; + $($expr)* + } else { + let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_UNCOMPRESSED; + $($expr)* + } + }; +} mod abi; pub mod active_plan; @@ -35,7 +51,7 @@ pub struct NewBuffer { /// A closure for reporting mutators. The C++ code should pass `data` back as the last argument. #[repr(C)] pub struct MutatorClosure { - pub func: extern "C" fn(mutator: *mut Mutator, data: *mut libc::c_void), + pub func: extern "C" fn(mutator: *mut libc::c_void, data: *mut libc::c_void), pub data: *mut libc::c_void, } @@ -62,12 +78,12 @@ pub struct OpenJDK_Upcalls { pub spawn_gc_thread: extern "C" fn(tls: VMThread, kind: libc::c_int, ctx: *mut libc::c_void), pub block_for_gc: extern "C" fn(), pub out_of_memory: extern "C" fn(tls: VMThread, err_kind: AllocationError), - pub get_next_mutator: extern "C" fn() -> *mut Mutator, + pub get_next_mutator: extern "C" fn() -> *mut libc::c_void, pub reset_mutator_iterator: extern "C" fn(), pub scan_object: extern "C" fn(trace: *mut c_void, object: ObjectReference, tls: OpaquePointer), pub dump_object: extern "C" fn(object: ObjectReference), pub get_object_size: extern "C" fn(object: ObjectReference) -> usize, - pub get_mmtk_mutator: extern "C" fn(tls: VMMutatorThread) -> *mut Mutator, + pub get_mmtk_mutator: extern "C" fn(tls: VMMutatorThread) -> *mut libc::c_void, pub is_mutator: extern "C" fn(tls: VMThread) -> bool, pub harness_begin: extern "C" fn(), pub harness_end: extern "C" fn(), @@ -95,6 +111,8 @@ pub struct OpenJDK_Upcalls { pub schedule_finalizer: extern "C" fn(), pub prepare_for_roots_re_scanning: extern "C" fn(), pub enqueue_references: extern "C" fn(objects: *const ObjectReference, len: usize), + pub compressed_klass_base: extern "C" fn() -> Address, + pub compressed_klass_shift: extern "C" fn() -> usize, } pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); @@ -113,27 +131,259 @@ pub static GLOBAL_ALLOC_BIT_ADDRESS: uintptr_t = #[no_mangle] pub static FREE_LIST_ALLOCATOR_SIZE: uintptr_t = - std::mem::size_of::>(); + std::mem::size_of::>>(); + +#[no_mangle] +pub static DISABLE_ALLOCATION_FAST_PATH: i32 = + (cfg!(feature = "no_fast_alloc") || cfg!(feature = "object_size_distribution")) as _; + +#[no_mangle] +pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = + std::mem::size_of::>>(); + +#[no_mangle] +pub static mut CONCURRENT_MARKING_ACTIVE: u8 = 0; + +#[no_mangle] +pub static mut HEAP_START: Address = Address::ZERO; + +#[no_mangle] +pub static mut HEAP_END: Address = Address::ZERO; + +static mut USE_COMPRESSED_OOPS: bool = false; +static mut LOG_BYTES_IN_FIELD: usize = LOG_BYTES_IN_ADDRESS as _; +static mut BYTES_IN_FIELD: usize = BYTES_IN_ADDRESS as _; + +fn init_compressed_oop_constants() { + unsafe { + USE_COMPRESSED_OOPS = true; + LOG_BYTES_IN_FIELD = LOG_BYTES_IN_INT as _; + BYTES_IN_FIELD = BYTES_IN_INT as _; + } +} + +fn use_compressed_oops() -> bool { + unsafe { USE_COMPRESSED_OOPS } +} + +fn log_bytes_in_field() -> usize { + unsafe { LOG_BYTES_IN_FIELD } +} + +fn bytes_in_field() -> usize { + unsafe { BYTES_IN_FIELD } +} + +static mut BASE: Address = Address::ZERO; +static mut SHIFT: usize = 0; + +fn compress(o: ObjectReference) -> u32 { + if o.is_null() { + 0u32 + } else { + unsafe { ((o.to_raw_address() - BASE) >> SHIFT) as u32 } + } +} + +fn decompress(v: u32) -> ObjectReference { + if v == 0 { + ObjectReference::NULL + } else { + unsafe { ObjectReference::from_raw_address(BASE + ((v as usize) << SHIFT)) } + } +} + +fn initialize_compressed_oops() { + let heap_end = VM_LAYOUT_CONSTANTS.heap_end.as_usize(); + if heap_end <= (4usize << 30) { + unsafe { + BASE = Address::ZERO; + SHIFT = 0; + } + } else if heap_end <= (32usize << 30) { + unsafe { + BASE = Address::ZERO; + SHIFT = 3; + } + } else { + unsafe { + BASE = VM_LAYOUT_CONSTANTS.heap_start - 4096; + SHIFT = 3; + } + } +} #[derive(Default)] -pub struct OpenJDK; +pub struct OpenJDK; /// The type of edges in OpenJDK. -/// -/// TODO: We currently make it an alias to Address to make the change minimal. -/// If we support CompressedOOPs, we should define an enum type to support both -/// compressed and uncompressed OOPs. -pub type OpenJDKEdge = Address; +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +#[repr(transparent)] +pub struct OpenJDKEdge(pub Address); + +impl OpenJDKEdge { + const MASK: usize = 1usize << 63; + + const fn is_compressed(&self) -> bool { + self.0.as_usize() & Self::MASK == 0 + } + + const fn untagged_address(&self) -> Address { + unsafe { Address::from_usize(self.0.as_usize() << 1 >> 1) } + } +} + +impl Edge for OpenJDKEdge { + /// Load object reference from the edge. + fn load(&self) -> ObjectReference { + if COMPRESSED { + let slot = self.untagged_address(); + if self.is_compressed() { + decompress(unsafe { slot.load::() }) + } else { + unsafe { slot.load::() } + } + } else { + unsafe { self.0.load::() } + } + } + + /// Store the object reference `object` into the edge. + fn store(&self, object: ObjectReference) { + if COMPRESSED { + let slot = self.untagged_address(); + if self.is_compressed() { + unsafe { slot.store(compress(object)) } + } else { + unsafe { slot.store(object) } + } + } else { + unsafe { self.0.store(object) } + } + } + + fn to_address(&self) -> Address { + self.untagged_address() + } + + // fn raw_address(&self) -> Address { + // self.0 + // } + + fn from_address(a: Address) -> Self { + Self(a) + } +} + +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +pub struct OpenJDKEdgeRange { + pub start: OpenJDKEdge, + pub end: OpenJDKEdge, +} + +/// Iterate edges within `Range
`. +pub struct AddressRangeIterator { + cursor: Address, + limit: Address, + width: usize, +} + +impl Iterator for AddressRangeIterator { + type Item = OpenJDKEdge; + + fn next(&mut self) -> Option { + if self.cursor >= self.limit { + None + } else { + let edge = self.cursor; + self.cursor += self.width; + Some(OpenJDKEdge(edge)) + } + } +} + +pub struct ChunkIterator { + cursor: Address, + limit: Address, + step: usize, +} -impl VMBinding for OpenJDK { +impl Iterator for ChunkIterator { + type Item = OpenJDKEdgeRange; + + fn next(&mut self) -> Option { + if self.cursor >= self.limit { + None + } else { + let start = self.cursor; + let mut end = start + self.step; + if end > self.limit { + end = self.limit; + } + self.cursor = end; + Some(OpenJDKEdgeRange { + start: OpenJDKEdge(start), + end: OpenJDKEdge(end), + }) + } + } +} + +impl MemorySlice for OpenJDKEdgeRange { + type Edge = OpenJDKEdge; + type EdgeIterator = AddressRangeIterator; + + fn iter_edges(&self) -> Self::EdgeIterator { + AddressRangeIterator { + cursor: self.start.0, + limit: self.end.0, + width: crate::bytes_in_field(), + } + } + + fn start(&self) -> Address { + self.start.0 + } + + fn bytes(&self) -> usize { + self.end.0 - self.start.0 + } + + fn copy(src: &Self, tgt: &Self) { + debug_assert_eq!(src.bytes(), tgt.bytes()); + debug_assert_eq!( + src.bytes() & ((1 << LOG_BYTES_IN_ADDRESS) - 1), + 0, + "bytes are not a multiple of words" + ); + // Raw memory copy + if crate::use_compressed_oops() { + unsafe { + let words = tgt.bytes() >> LOG_BYTES_IN_INT; + let src = src.start().to_ptr::(); + let tgt = tgt.start().to_mut_ptr::(); + std::ptr::copy(src, tgt, words) + } + } else { + unsafe { + let words = tgt.bytes() >> LOG_BYTES_IN_ADDRESS; + let src = src.start().to_ptr::(); + let tgt = tgt.start().to_mut_ptr::(); + std::ptr::copy(src, tgt, words) + } + } + } +} + +impl VMBinding for OpenJDK { type VMObjectModel = object_model::VMObjectModel; type VMScanning = scanning::VMScanning; type VMCollection = collection::VMCollection; type VMActivePlan = active_plan::VMActivePlan; type VMReferenceGlue = reference_glue::VMReferenceGlue; - type VMEdge = OpenJDKEdge; - type VMMemorySlice = Range
; + type VMEdge = OpenJDKEdge; + type VMMemorySlice = OpenJDKEdgeRange; const MIN_ALIGNMENT: usize = 8; const MAX_ALIGNMENT: usize = 8; @@ -147,18 +397,51 @@ pub static MMTK_INITIALIZED: AtomicBool = AtomicBool::new(false); lazy_static! { pub static ref BUILDER: Mutex = Mutex::new(MMTKBuilder::new()); - pub static ref SINGLETON: MMTK = { + pub static ref SINGLETON_COMPRESSED: MMTK> = { + let mut builder = BUILDER.lock().unwrap(); + assert!(use_compressed_oops()); + builder.set_option("use_35bit_address_space", "true"); + builder.set_option("use_35bit_address_space", "true"); + assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); + let ret = mmtk::memory_manager::mmtk_init(&builder); + MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); + initialize_compressed_oops(); + unsafe { + HEAP_START = VM_LAYOUT_CONSTANTS.heap_start; + HEAP_END = VM_LAYOUT_CONSTANTS.heap_end; + } + *ret + }; + pub static ref SINGLETON_UNCOMPRESSED: MMTK> = { let builder = BUILDER.lock().unwrap(); assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); + unsafe { + HEAP_START = VM_LAYOUT_CONSTANTS.heap_start; + HEAP_END = VM_LAYOUT_CONSTANTS.heap_end; + } *ret }; } +fn singleton() -> &'static MMTK> { + if COMPRESSED { + unsafe { + &*(&*SINGLETON_COMPRESSED as *const MMTK> + as *const MMTK>) + } + } else { + unsafe { + &*(&*SINGLETON_UNCOMPRESSED as *const MMTK> + as *const MMTK>) + } + } +} + #[no_mangle] pub static MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES: usize = - mmtk::util::alloc::MarkCompactAllocator::::HEADER_RESERVED_IN_BYTES; + mmtk::util::alloc::MarkCompactAllocator::>::HEADER_RESERVED_IN_BYTES; lazy_static! { /// A global storage for all the cached CodeCache root pointers diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 7ee39948..774e74a5 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -8,7 +8,7 @@ use mmtk::vm::*; pub struct VMObjectModel {} -impl ObjectModel for VMObjectModel { +impl ObjectModel> for VMObjectModel { const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = vm_metadata::LOGGING_SIDE_METADATA_SPEC; const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = @@ -24,9 +24,13 @@ impl ObjectModel for VMObjectModel { fn copy( from: ObjectReference, copy: CopySemantics, - copy_context: &mut GCWorkerCopyContext, + copy_context: &mut GCWorkerCopyContext>, ) -> ObjectReference { - let bytes = unsafe { Oop::from(from).size() }; + let bytes = if crate::use_compressed_oops() { + unsafe { Oop::from(from).size::() } + } else { + unsafe { Oop::from(from).size::() } + }; let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::(), 0, copy); // Copy let src = from.to_raw_address(); @@ -48,9 +52,9 @@ impl ObjectModel for VMObjectModel { unsafe { (dst + i).store((src + i).load::()) }; } } - let start = Self::ref_to_object_start(to); + let start = >>::ref_to_object_start(to); if region != Address::ZERO { - fill_alignment_gap::(region, start); + fill_alignment_gap::>(region, start); } start + bytes } @@ -60,11 +64,15 @@ impl ObjectModel for VMObjectModel { } fn get_current_size(object: ObjectReference) -> usize { - unsafe { Oop::from(object).size() } + if COMPRESSED { + unsafe { Oop::from(object).size::() } + } else { + unsafe { Oop::from(object).size::() } + } } fn get_size_when_copied(object: ObjectReference) -> usize { - Self::get_current_size(object) + >>::get_current_size(object) } fn get_align_when_copied(_object: ObjectReference) -> usize { @@ -106,6 +114,11 @@ impl ObjectModel for VMObjectModel { let oop = Oop::from(object); // It is only valid if klass.id is between 0 and 5 (see KlassID in openjdk/src/hotspot/share/oops/klass.hpp) // If oop.klass is not a valid pointer, we may segfault here. - oop.klass.id as i32 >= 0 && (oop.klass.id as i32) < 6 + let klass_id = if crate::use_compressed_oops() { + oop.klass::().id + } else { + oop.klass::().id + } as i32; + klass_id >= 0 && klass_id < 6 } } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 6ca17e7f..a265f353 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -1,112 +1,129 @@ -use crate::SINGLETON; - -use crate::abi::*; -use crate::{OpenJDKEdge, UPCALLS}; -use mmtk::util::constants::*; +use super::abi::*; +use super::UPCALLS; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; +use mmtk::vm::edge_shape::Edge; use mmtk::vm::EdgeVisitor; +use std::cell::UnsafeCell; use std::{mem, slice}; trait OopIterate: Sized { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor); + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut V, + ); } impl OopIterate for OopMapBlock { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut V, + ) { + let log_bytes_in_oop = if COMPRESSED { 2 } else { 3 }; let start = oop.get_field_address(self.offset); for i in 0..self.count as usize { - let edge = start + (i << LOG_BYTES_IN_ADDRESS); + let edge = E::from_address(start + (i << log_bytes_in_oop)); closure.visit_edge(edge); } } } impl OopIterate for InstanceKlass { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut V, + ) { let oop_maps = self.nonstatic_oop_maps(); for map in oop_maps { - map.oop_iterate(oop, closure) + map.oop_iterate::(oop, closure) } } } impl OopIterate for InstanceMirrorKlass { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { - self.instance_klass.oop_iterate(oop, closure); - // if (Devirtualizer::do_metadata(closure)) { - // Klass* klass = java_lang_Class::as_Klass(obj); - // // We'll get NULL for primitive mirrors. - // if (klass != NULL) { - // if (klass->is_instance_klass() && InstanceKlass::cast(klass)->is_anonymous()) { - // // An anonymous class doesn't have its own class loader, so when handling - // // the java mirror for an anonymous class we need to make sure its class - // // loader data is claimed, this is done by calling do_cld explicitly. - // // For non-anonymous classes the call to do_cld is made when the class - // // loader itself is handled. - // Devirtualizer::do_cld(closure, klass->class_loader_data()); - // } else { - // Devirtualizer::do_klass(closure, klass); - // } - // } else { - // // We would like to assert here (as below) that if klass has been NULL, then - // // this has been a mirror for a primitive type that we do not need to follow - // // as they are always strong roots. - // // However, we might get across a klass that just changed during CMS concurrent - // // marking if allocation occurred in the old generation. - // // This is benign here, as we keep alive all CLDs that were loaded during the - // // CMS concurrent phase in the class loading, i.e. they will be iterated over - // // and kept alive during remark. - // // assert(java_lang_Class::is_primitive(obj), "Sanity check"); - // } - // } + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut V, + ) { + self.instance_klass + .oop_iterate::<_, _, COMPRESSED>(oop, closure); // static fields - let start: *const Oop = Self::start_of_static_fields(oop).to_ptr::(); + let start = Self::start_of_static_fields(oop); let len = Self::static_oop_field_count(oop); - let slice = unsafe { slice::from_raw_parts(start, len as _) }; - for oop in slice { - closure.visit_edge(Address::from_ref(oop as &Oop)); + if COMPRESSED { + let start: *const NarrowOop = start.to_ptr::(); + let slice = unsafe { slice::from_raw_parts(start, len as _) }; + for narrow_oop in slice { + closure.visit_edge(E::from_address(narrow_oop.slot())); + } + } else { + let start: *const Oop = start.to_ptr::(); + let slice = unsafe { slice::from_raw_parts(start, len as _) }; + for oop in slice { + closure.visit_edge(E::from_address(Address::from_ref(oop as &Oop))); + } } } } impl OopIterate for InstanceClassLoaderKlass { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { - self.instance_klass.oop_iterate(oop, closure); - // if (Devirtualizer::do_metadata(closure)) { - // ClassLoaderData* cld = java_lang_ClassLoader::loader_data(obj); - // // cld can be null if we have a non-registered class loader. - // if (cld != NULL) { - // Devirtualizer::do_cld(closure, cld); - // } - // } + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut V, + ) { + self.instance_klass + .oop_iterate::<_, _, COMPRESSED>(oop, closure); } } impl OopIterate for ObjArrayKlass { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut V, + ) { let array = unsafe { oop.as_array_oop() }; - for oop in unsafe { array.data::(BasicType::T_OBJECT) } { - closure.visit_edge(Address::from_ref(oop as &Oop)); + if COMPRESSED { + for narrow_oop in unsafe { array.data::(BasicType::T_OBJECT) } { + closure.visit_edge(E::from_address(narrow_oop.slot())); + } + } else { + for oop in unsafe { array.data::(BasicType::T_OBJECT) } { + closure.visit_edge(E::from_address(Address::from_ref(oop as &Oop))); + } } } } impl OopIterate for TypeArrayKlass { - fn oop_iterate(&self, _oop: Oop, _closure: &mut impl EdgeVisitor) { + fn oop_iterate, const COMPRESSED: bool>( + &self, + _oop: Oop, + _closure: &mut V, + ) { // Performance tweak: We skip processing the klass pointer since all // TypeArrayKlasses are guaranteed processed via the null class loader. } } impl OopIterate for InstanceRefKlass { - fn oop_iterate(&self, oop: Oop, closure: &mut impl EdgeVisitor) { + fn oop_iterate, const COMPRESSED: bool>( + &self, + oop: Oop, + closure: &mut V, + ) { use crate::abi::*; use crate::api::{add_phantom_candidate, add_soft_candidate, add_weak_candidate}; - self.instance_klass.oop_iterate(oop, closure); + self.instance_klass + .oop_iterate::<_, _, COMPRESSED>(oop, closure); - if Self::should_scan_weak_refs() { + if Self::should_scan_weak_refs::() { let reference = ObjectReference::from(oop); match self.instance_klass.reference_type { ReferenceType::None => { @@ -128,27 +145,43 @@ impl OopIterate for InstanceRefKlass { } impl InstanceRefKlass { - fn should_scan_weak_refs() -> bool { - !*SINGLETON.get_options().no_reference_types + fn should_scan_weak_refs() -> bool { + !*crate::singleton::() + .get_options() + .no_reference_types } - fn process_ref_as_strong(oop: Oop, closure: &mut impl EdgeVisitor) { - let referent_addr = Self::referent_address(oop); + fn process_ref_as_strong>(oop: Oop, closure: &mut V) { + let referent_addr = Self::referent_address::(oop); closure.visit_edge(referent_addr); - let discovered_addr = Self::discovered_address(oop); + let discovered_addr = Self::discovered_address::(oop); closure.visit_edge(discovered_addr); } } #[allow(unused)] -fn oop_iterate_slow(oop: Oop, closure: &mut impl EdgeVisitor, tls: OpaquePointer) { +fn oop_iterate_slow>(oop: Oop, closure: &mut V, tls: OpaquePointer) { unsafe { - ((*UPCALLS).scan_object)(closure as *mut _ as _, mem::transmute(oop), tls); + CLOSURE.with(|x| *x.get() = closure as *mut V as *mut u8); + ((*UPCALLS).scan_object)( + mem::transmute(scan_object_fn:: as *const unsafe extern "C" fn(edge: Address)), + mem::transmute(oop), + tls, + ); } } -fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { - let klass_id = oop.klass.id; - debug_assert!( +fn oop_iterate, const COMPRESSED: bool>( + oop: Oop, + closure: &mut V, + klass: Option
, +) { + let klass = if let Some(klass) = klass { + unsafe { &*(klass.as_usize() as *const Klass) } + } else { + oop.klass::() + }; + let klass_id = klass.id; + assert!( klass_id as i32 >= 0 && (klass_id as i32) < 6, "Invalid klass-id: {:x} for oop: {:x}", klass_id as i32, @@ -156,41 +189,48 @@ fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor) { ); match klass_id { KlassID::Instance => { - let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.oop_iterate(oop, closure); + let instance_klass = unsafe { klass.cast::() }; + instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceClassLoader => { - let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.oop_iterate(oop, closure); + let instance_klass = unsafe { klass.cast::() }; + instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceMirror => { - let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.oop_iterate(oop, closure); + let instance_klass = unsafe { klass.cast::() }; + instance_klass.oop_iterate::(oop, closure); } KlassID::ObjArray => { - let array_klass = unsafe { oop.klass.cast::() }; - array_klass.oop_iterate(oop, closure); + let array_klass = unsafe { klass.cast::() }; + array_klass.oop_iterate::(oop, closure); } KlassID::TypeArray => { - let array_klass = unsafe { oop.klass.cast::() }; - array_klass.oop_iterate(oop, closure); + // let array_klass = unsafe { oop.klass::().cast::() }; + // array_klass.oop_iterate::(oop, closure); } KlassID::InstanceRef => { - let instance_klass = unsafe { oop.klass.cast::() }; - instance_klass.oop_iterate(oop, closure); - } // _ => oop_iterate_slow(oop, closure, tls), + let instance_klass = unsafe { klass.cast::() }; + instance_klass.oop_iterate::(oop, closure); + } + #[allow(unreachable_patterns)] + _ => unreachable!(), // _ => oop_iterate_slow(oop, closure, OpaquePointer::UNINITIALIZED), } } -pub fn scan_object( +thread_local! { + static CLOSURE: UnsafeCell<*mut u8> = UnsafeCell::new(std::ptr::null_mut()); +} + +pub unsafe extern "C" fn scan_object_fn>(edge: Address) { + let ptr: *mut u8 = CLOSURE.with(|x| *x.get()); + let closure = &mut *(ptr as *mut V); + closure.visit_edge(E::from_address(edge)); +} + +pub fn scan_object, const COMPRESSED: bool>( object: ObjectReference, - closure: &mut impl EdgeVisitor, + closure: &mut V, _tls: VMWorkerThread, ) { - // println!("*****scan_object(0x{:x}) -> \n 0x{:x}, 0x{:x} \n", - // object, - // unsafe { *(object.value() as *const usize) }, - // unsafe { *((object.value() + 8) as *const usize) } - // ); - unsafe { oop_iterate(mem::transmute(object), closure) } + unsafe { oop_iterate::(mem::transmute(object), closure, None) } } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 966f4e92..d85c1dd6 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -1,22 +1,24 @@ use crate::abi::{InstanceRefKlass, Oop}; use crate::OpenJDK; +use crate::OpenJDKEdge; use crate::UPCALLS; use mmtk::util::opaque_pointer::VMWorkerThread; use mmtk::util::ObjectReference; +use mmtk::vm::edge_shape::Edge; use mmtk::vm::ReferenceGlue; pub struct VMReferenceGlue {} -impl ReferenceGlue for VMReferenceGlue { +impl ReferenceGlue> for VMReferenceGlue { type FinalizableType = ObjectReference; fn set_referent(reff: ObjectReference, referent: ObjectReference) { let oop = Oop::from(reff); - unsafe { InstanceRefKlass::referent_address(oop).store(referent) }; + InstanceRefKlass::referent_address::>(oop).store(referent); } fn get_referent(object: ObjectReference) -> ObjectReference { let oop = Oop::from(object); - unsafe { InstanceRefKlass::referent_address(oop).load::() } + InstanceRefKlass::referent_address::>(oop).load() } fn enqueue_references(references: &[ObjectReference], _tls: VMWorkerThread) { unsafe { diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 17192e9c..5efa2401 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -1,6 +1,7 @@ use crate::gc_work::*; +use crate::Edge; use crate::{EdgesClosure, OpenJDK}; -use crate::{NewBuffer, OpenJDKEdge, SINGLETON, UPCALLS}; +use crate::{NewBuffer, OpenJDKEdge, UPCALLS}; use mmtk::memory_manager; use mmtk::scheduler::WorkBucketStage; use mmtk::util::opaque_pointer::*; @@ -13,14 +14,15 @@ pub struct VMScanning {} const WORK_PACKET_CAPACITY: usize = 4096; -extern "C" fn report_edges_and_renew_buffer>( +extern "C" fn report_edges_and_renew_buffer>( ptr: *mut Address, length: usize, capacity: usize, factory_ptr: *mut libc::c_void, ) -> NewBuffer { if !ptr.is_null() { - let buf = unsafe { Vec::
::from_raw_parts(ptr, length, capacity) }; + let ptr = ptr as *mut E; + let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; factory.create_process_edge_roots_work(buf); } @@ -34,23 +36,23 @@ extern "C" fn report_edges_and_renew_buffer>( NewBuffer { ptr, capacity } } -pub(crate) fn to_edges_closure>(factory: &mut F) -> EdgesClosure { +pub(crate) fn to_edges_closure>(factory: &mut F) -> EdgesClosure { EdgesClosure { - func: report_edges_and_renew_buffer::, + func: report_edges_and_renew_buffer::, data: factory as *mut F as *mut libc::c_void, } } -impl Scanning for VMScanning { +impl Scanning> for VMScanning { const SCAN_MUTATORS_IN_SAFEPOINT: bool = false; const SINGLE_THREAD_MUTATOR_SCANNING: bool = false; - fn scan_object>( + fn scan_object>>( tls: VMWorkerThread, object: ObjectReference, edge_visitor: &mut EV, ) { - crate::object_scanning::scan_object(object, edge_visitor, tls) + crate::object_scanning::scan_object::<_, _, COMPRESSED>(object, edge_visitor, tls); } fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { @@ -58,7 +60,10 @@ impl Scanning for VMScanning { // TODO } - fn scan_thread_roots(_tls: VMWorkerThread, mut factory: impl RootsWorkFactory) { + fn scan_thread_roots( + _tls: VMWorkerThread, + mut factory: impl RootsWorkFactory>, + ) { unsafe { ((*UPCALLS).scan_all_thread_roots)(to_edges_closure(&mut factory)); } @@ -66,8 +71,8 @@ impl Scanning for VMScanning { fn scan_thread_root( _tls: VMWorkerThread, - mutator: &'static mut Mutator, - mut factory: impl RootsWorkFactory, + mutator: &'static mut Mutator>, + mut factory: impl RootsWorkFactory>, ) { let tls = mutator.get_tls(); unsafe { @@ -75,9 +80,12 @@ impl Scanning for VMScanning { } } - fn scan_vm_specific_roots(_tls: VMWorkerThread, factory: impl RootsWorkFactory) { + fn scan_vm_specific_roots( + _tls: VMWorkerThread, + factory: impl RootsWorkFactory>, + ) { memory_manager::add_work_packets( - &SINGLETON, + &crate::singleton::(), WorkBucketStage::Prepare, vec![ Box::new(ScanUniverseRoots::new(factory.clone())) as _, @@ -93,9 +101,11 @@ impl Scanning for VMScanning { Box::new(ScanWeakProcessorRoots::new(factory.clone())) as _, ], ); - if !(Self::SCAN_MUTATORS_IN_SAFEPOINT && Self::SINGLE_THREAD_MUTATOR_SCANNING) { + if !(>>::SCAN_MUTATORS_IN_SAFEPOINT + && >>::SINGLE_THREAD_MUTATOR_SCANNING) + { memory_manager::add_work_packet( - &SINGLETON, + &crate::singleton::(), WorkBucketStage::Prepare, ScanVMThreadRoots::new(factory), ); diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index b1da9454..8e291179 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -23,6 +23,8 @@ extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; extern const uintptr_t GLOBAL_ALLOC_BIT_ADDRESS; extern const size_t MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; extern const uintptr_t FREE_LIST_ALLOCATOR_SIZE; +extern uintptr_t HEAP_START; +extern uintptr_t HEAP_END; extern const char* get_mmtk_version(); @@ -182,6 +184,8 @@ typedef struct { void (*schedule_finalizer)(); void (*prepare_for_roots_re_scanning)(); void (*enqueue_references)(void** objects, size_t len); + void* (*compressed_klass_base)(); + size_t (*compressed_klass_shift)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); @@ -189,6 +193,10 @@ extern bool openjdk_is_gc_initialized(); extern bool mmtk_set_heap_size(size_t min, size_t max); +extern bool mmtk_use_compressed_ptrs(); +extern void* mmtk_narrow_oop_base(); +extern size_t mmtk_narrow_oop_shift(); + extern size_t used_bytes(); extern void* starting_heap_address(); extern void* last_heap_address(); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 6a573f74..a71134a3 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -77,8 +77,6 @@ MMTkHeap::MMTkHeap(MMTkCollectorPolicy* policy) : jint MMTkHeap::initialize() { assert(!UseTLAB , "should disable UseTLAB"); - assert(!UseCompressedOops , "should disable CompressedOops"); - assert(!UseCompressedClassPointers , "should disable UseCompressedClassPointers"); const size_t min_heap_size = collector_policy()->min_heap_byte_size(); const size_t max_heap_size = collector_policy()->max_heap_byte_size(); // printf("policy max heap size %zu, min heap size %zu\n", heap_size, collector_policy()->min_heap_byte_size()); @@ -113,6 +111,10 @@ jint MMTkHeap::initialize() { _start = (HeapWord*) starting_heap_address(); _end = (HeapWord*) last_heap_address(); // printf("start: %p, end: %p\n", _start, _end); + if (UseCompressedOops) { + Universe::set_narrow_oop_base((address) mmtk_narrow_oop_base()); + Universe::set_narrow_oop_shift(mmtk_narrow_oop_shift()); + } initialize_reserved_region(_start, _end); diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 990f6d5e..841d83de 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -7,46 +7,6 @@ #include "oops/oop.inline.hpp" #include "utilities/globalDefinitions.hpp" -#define ROOTS_BUFFER_SIZE 4096 - -class MMTkRootsClosure : public OopClosure { - void* _trace; - void* _buffer[ROOTS_BUFFER_SIZE]; - size_t _cursor; - - template - void do_oop_work(T* p) { - // T heap_oop = RawAccess<>::oop_load(p); - // if (!CompressedOops::is_null(heap_oop)) { - // oop obj = CompressedOops::decode_not_null(heap_oop); - // oop fwd = (oop) trace_root_object(_trace, obj); - // RawAccess<>::oop_store(p, fwd); - // } - _buffer[_cursor++] = (void*) p; - if (_cursor >= ROOTS_BUFFER_SIZE) { - flush(); - } - } - - NOINLINE void flush() { - // bulk_report_delayed_root_edge(_trace, _buffer, _cursor); - _cursor = 0; - } - -public: - MMTkRootsClosure(void* trace): _trace(trace), _cursor(0) {} - - ~MMTkRootsClosure() { - if (_cursor > 0) flush(); - } - - virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { - // printf("narrowoop root %p -> %d %p %p\n", (void*) p, *p, *((void**) p), (void*) oopDesc::load_decode_heap_oop(p)); - do_oop_work(p); - } -}; - class MMTkRootsClosure2 : public OopClosure { EdgesClosure _edges_closure; void** _buffer; @@ -54,16 +14,17 @@ class MMTkRootsClosure2 : public OopClosure { size_t _cursor; template - void do_oop_work(T* p) { - // T heap_oop = RawAccess<>::oop_load(p); - // if (!CompressedOops::is_null(heap_oop)) { - // oop obj = CompressedOops::decode_not_null(heap_oop); - // oop fwd = (oop) trace_root_object(_trace, obj); - // RawAccess<>::oop_store(p, fwd); - // } - _buffer[_cursor++] = (void*) p; - if (_cursor >= _cap) { - flush(); + void do_oop_work(T* p, bool narrow) { + T heap_oop = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(heap_oop)) { + if (UseCompressedOops && !narrow) { + guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); + p = (T*) (uintptr_t(p) | (1ull << 63)); + } + _buffer[_cursor++] = (void*) p; + if (_cursor >= _cap) { + flush(); + } } } @@ -90,8 +51,8 @@ class MMTkRootsClosure2 : public OopClosure { } } - virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop(oop* p) { do_oop_work(p, false); } + virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } }; class MMTkScanObjectClosure : public BasicOopIterateClosure { @@ -99,19 +60,19 @@ class MMTkScanObjectClosure : public BasicOopIterateClosure { CLDToOopClosure follow_cld_closure; template - void do_oop_work(T* p) { - // oop ref = (void*) oopDesc::decode_heap_oop(oopDesc::load_heap_oop(p)); - // process_edge(_trace, (void*) p); + void do_oop_work(T* p, bool narrow) { + if (UseCompressedOops && !narrow) { + guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); + p = (T*) (uintptr_t(p) | (1ull << 63)); + } + // _trace((void*) p); } public: MMTkScanObjectClosure(void* trace): _trace(trace), follow_cld_closure(this, false) {} - virtual void do_oop(oop* p) { do_oop_work(p); } - virtual void do_oop(narrowOop* p) { - // printf("narrowoop edge %p -> %d %p %p\n", (void*) p, *p, *((void**) p), (void*) oopDesc::load_decode_heap_oop(p)); - do_oop_work(p); - } + virtual void do_oop(oop* p) { do_oop_work(p, false); } + virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } virtual bool do_metadata() { return true; diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 3b983bcb..c3cb9058 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -341,6 +341,14 @@ static void mmtk_enqueue_references(void** objects, size_t len) { assert(Universe::has_reference_pending_list(), "Reference pending list is empty after swap"); } +static void* mmtk_compressed_klass_base() { + return (void*) Universe::narrow_klass_base(); +} + +static size_t mmtk_compressed_klass_shift() { + return (size_t) Universe::narrow_klass_shift(); +} + OpenJDK_Upcalls mmtk_upcalls = { mmtk_stop_all_mutators, mmtk_resume_mutators, @@ -379,5 +387,7 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_number_of_mutators, mmtk_schedule_finalizer, mmtk_prepare_for_roots_re_scanning, - mmtk_enqueue_references + mmtk_enqueue_references, + mmtk_compressed_klass_base, + mmtk_compressed_klass_shift, }; diff --git a/openjdk/thirdPartyHeapArguments.cpp b/openjdk/thirdPartyHeapArguments.cpp index 745250df..acb67367 100644 --- a/openjdk/thirdPartyHeapArguments.cpp +++ b/openjdk/thirdPartyHeapArguments.cpp @@ -43,8 +43,8 @@ void ThirdPartyHeapArguments::initialize() { GCArguments::initialize(); assert(UseThirdPartyHeap , "Error, should UseThirdPartyHeap"); FLAG_SET_DEFAULT(UseTLAB, false); - FLAG_SET_DEFAULT(UseCompressedOops, false); - FLAG_SET_DEFAULT(UseCompressedClassPointers, false); + if (UseCompressedOops) mmtk_use_compressed_ptrs(); + FLAG_SET_DEFAULT(UseCompressedClassPointers, UseCompressedOops); } CollectedHeap* ThirdPartyHeapArguments::create_heap() { From d27da840f3dd5dd83d59861f0539f27cb35bffb7 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 15 Apr 2023 05:14:37 +0000 Subject: [PATCH 315/494] Fix import --- mmtk/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 8cd7eee3..3647656b 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -11,8 +11,8 @@ use mmtk::util::alloc::AllocationError; use mmtk::util::constants::{ BYTES_IN_ADDRESS, BYTES_IN_INT, LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_INT, }; -use mmtk::util::heap::layout::vm_layout_constants::VM_LAYOUT_CONSTANTS; use mmtk::util::opaque_pointer::*; +use mmtk::util::VM_LAYOUT_CONSTANTS; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::edge_shape::{Edge, MemorySlice}; use mmtk::vm::VMBinding; From c77edee4df17bd2936b8f1634352fa5d55c8b0ea Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 19 Apr 2023 06:52:45 +0000 Subject: [PATCH 316/494] Fix Generational GCs --- openjdk/barriers/mmtkObjectBarrier.cpp | 32 +++++++++++++++++++++++++- openjdk/barriers/mmtkObjectBarrier.hpp | 1 + 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/openjdk/barriers/mmtkObjectBarrier.cpp b/openjdk/barriers/mmtkObjectBarrier.cpp index dea1824f..4ef3d68f 100644 --- a/openjdk/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/barriers/mmtkObjectBarrier.cpp @@ -67,9 +67,39 @@ void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* #endif } +void MMTkObjectBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { + // See https://github.com/openjdk/jdk/blob/jdk-11%2B19/src/hotspot/cpu/x86/gc/shared/modRefBarrierSetAssembler_x86.cpp#L37-L50 + bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; + bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; + bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); + if (type == T_OBJECT || type == T_ARRAY) { + if (!checkcast) { + if (!obj_int) { + // Save count for barrier + __ movptr(r11, count); + } else if (disjoint) { + // Save dst in r11 in the disjoint case + __ movq(r11, dst); + } + } + } +} + void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { + bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; + bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; + bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); const bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; - if ((type == T_OBJECT || type == T_ARRAY) && !dest_uninitialized) { + if ((type == T_OBJECT || type == T_ARRAY) ) { + if (!checkcast) { + if (!obj_int) { + // Save count for barrier + count = r11; + } else if (disjoint) { + // Use the saved dst in the disjoint case + dst = r11; + } + } __ pusha(); __ movptr(c_rarg0, src); __ movptr(c_rarg1, dst); diff --git a/openjdk/barriers/mmtkObjectBarrier.hpp b/openjdk/barriers/mmtkObjectBarrier.hpp index 9eec8713..26b59ab6 100644 --- a/openjdk/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/barriers/mmtkObjectBarrier.hpp @@ -30,6 +30,7 @@ class MMTkObjectBarrierSetAssembler: public MMTkBarrierSetAssembler { protected: virtual void object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const override; public: + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; }; From c6bd5d4980121d010081c585351afee82632d904 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 19 Apr 2023 07:43:57 +0000 Subject: [PATCH 317/494] Fix assertion failures --- openjdk/mmtkBarrierSet.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index fc6d9596..6abd031c 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -139,30 +139,30 @@ bool MMTkBarrierSet::is_slow_path_call(address call) { return runtime()->is_slow_path_call(call); } -JRT_LEAF(void, MMTkBarrierSetRuntime::object_reference_write_pre_call(void* src, void* slot, void* target)) +void MMTkBarrierSetRuntime::object_reference_write_pre_call(void* src, void* slot, void* target) { ::mmtk_object_reference_write_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, target); -JRT_END +} -JRT_LEAF(void, MMTkBarrierSetRuntime::object_reference_write_post_call(void* src, void* slot, void* target)) +void MMTkBarrierSetRuntime::object_reference_write_post_call(void* src, void* slot, void* target) { ::mmtk_object_reference_write_post((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, target); -JRT_END +} -JRT_LEAF(void, MMTkBarrierSetRuntime::object_reference_write_slow_call(void* src, void* slot, void* target)) +void MMTkBarrierSetRuntime::object_reference_write_slow_call(void* src, void* slot, void* target) { ::mmtk_object_reference_write_slow((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, target); -JRT_END +} -JRT_LEAF(void, MMTkBarrierSetRuntime::object_reference_array_copy_pre_call(void* src, void* dst, size_t count)) +void MMTkBarrierSetRuntime::object_reference_array_copy_pre_call(void* src, void* dst, size_t count) { ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); -JRT_END +} -JRT_LEAF(void, MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, void* dst, size_t count)) +void MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, void* dst, size_t count) { ::mmtk_array_copy_post((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); -JRT_END +} -JRT_LEAF(void, MMTkBarrierSetRuntime::load_reference_call(void* ref)) +void MMTkBarrierSetRuntime::load_reference_call(void* ref) { ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); -JRT_END +} -JRT_LEAF(void, MMTkBarrierSetRuntime::object_reference_clone_pre_call(void* ref)) +void MMTkBarrierSetRuntime::object_reference_clone_pre_call(void* ref) { ::mmtk_object_reference_clone_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); -JRT_END \ No newline at end of file +} \ No newline at end of file From c991bc4995384cc3bbac50b3672504545c64e29b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 19 Apr 2023 10:27:32 -0700 Subject: [PATCH 318/494] cargo fmt --- mmtk/src/lib.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index bdc2c1f9..0a477067 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -163,7 +163,8 @@ pub static FREE_LIST_ALLOCATOR_SIZE: uintptr_t = std::mem::size_of::>>(); #[no_mangle] -pub static DISABLE_ALLOCATION_FAST_PATH: i32 = (cfg!(feature = "no_fast_alloc") || cfg!(feature = "object_size_distribution")) as _; +pub static DISABLE_ALLOCATION_FAST_PATH: i32 = + (cfg!(feature = "no_fast_alloc") || cfg!(feature = "object_size_distribution")) as _; #[no_mangle] pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = @@ -547,13 +548,16 @@ lazy_static! { fn record_alloc(size: usize) { assert!(cfg!(feature = "object_size_distribution")); let mut counts = OBJ_COUNT.lock().unwrap(); - counts.entry(size.next_power_of_two()).and_modify(|x| { - x.0 += 1; - x.1 += size; - }).or_insert((1, size)); + counts + .entry(size.next_power_of_two()) + .and_modify(|x| { + x.0 += 1; + x.1 += size; + }) + .or_insert((1, size)); } -extern fn dump_and_reset_obj_dist() { +extern "C" fn dump_and_reset_obj_dist() { assert!(cfg!(feature = "object_size_distribution")); mmtk::dump_and_reset_obj_dist("Dynamic", &mut OBJ_COUNT.lock().unwrap()); -} \ No newline at end of file +} From bde4b0539301df07c81ec267ec23ce7b714f3656 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 19 Apr 2023 21:27:23 +0000 Subject: [PATCH 319/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 20243c3b..213d74f5 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=295e070b09a2f8dbe2f343c8a8f45407bc6121f2#295e070b09a2f8dbe2f343c8a8f45407bc6121f2" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ab5d1a9723fdfd38e7be3f8bc15b245f26e9837f#ab5d1a9723fdfd38e7be3f8bc15b245f26e9837f" dependencies = [ "atomic", "atomic-traits", @@ -479,7 +479,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=295e070b09a2f8dbe2f343c8a8f45407bc6121f2#295e070b09a2f8dbe2f343c8a8f45407bc6121f2" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ab5d1a9723fdfd38e7be3f8bc15b245f26e9837f#ab5d1a9723fdfd38e7be3f8bc15b245f26e9837f" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 12348a83..3be3512a 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "295e070b09a2f8dbe2f343c8a8f45407bc6121f2" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "ab5d1a9723fdfd38e7be3f8bc15b245f26e9837f" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 3be937926f9f50c91ecd8f758d0b940b64c2bee7 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 19 Apr 2023 14:55:42 -0700 Subject: [PATCH 320/494] Remove `WeakProcessor` roots --- mmtk/src/gc_work.rs | 1 + mmtk/src/scanning.rs | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 11984b63..17961880 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -188,6 +188,7 @@ pub struct ScaWeakProcessorRoots> { } impl> ScaWeakProcessorRoots { + #[allow(unused)] pub fn new(factory: F) -> Self { Self { factory, diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index e817d69f..43f77ac9 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -122,7 +122,6 @@ impl Scanning> for VMScanning { Box::new(ScanCodeCacheRoots::new(factory.clone())) as _, Box::new(ScanStringTableRoots::new(factory.clone())) as _, Box::new(ScanClassLoaderDataGraphRoots::new(factory.clone())) as _, - Box::new(ScaWeakProcessorRoots::new(factory.clone())) as _, ], ); if !(>>::SCAN_MUTATORS_IN_SAFEPOINT From e2bbaebd0e3200258a209034f5ef211de7591880 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 19 Apr 2023 16:40:42 -0700 Subject: [PATCH 321/494] Revert "Remove `WeakProcessor` roots" This reverts commit 3be937926f9f50c91ecd8f758d0b940b64c2bee7. --- mmtk/src/gc_work.rs | 1 - mmtk/src/scanning.rs | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 17961880..11984b63 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -188,7 +188,6 @@ pub struct ScaWeakProcessorRoots> { } impl> ScaWeakProcessorRoots { - #[allow(unused)] pub fn new(factory: F) -> Self { Self { factory, diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 43f77ac9..e817d69f 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -122,6 +122,7 @@ impl Scanning> for VMScanning { Box::new(ScanCodeCacheRoots::new(factory.clone())) as _, Box::new(ScanStringTableRoots::new(factory.clone())) as _, Box::new(ScanClassLoaderDataGraphRoots::new(factory.clone())) as _, + Box::new(ScaWeakProcessorRoots::new(factory.clone())) as _, ], ); if !(>>::SCAN_MUTATORS_IN_SAFEPOINT From 3c7ac1b8e00e2c8626b1354a94e13d266a598657 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 21 Apr 2023 21:04:02 +0000 Subject: [PATCH 322/494] Fix time.stw accountng; Improve class unloading performance --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/collection.rs | 10 ++++++++-- mmtk/src/lib.rs | 4 ++-- openjdk/mmtk.h | 3 ++- openjdk/mmtkHeap.cpp | 1 + openjdk/mmtkUpcalls.cpp | 10 +++++++--- 7 files changed, 23 insertions(+), 11 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 26f66634..d07a22b7 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=dd0acf1871205da734714e4efb55ea85647ac974#dd0acf1871205da734714e4efb55ea85647ac974" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=4dc166a29a1768d94765662612bff994871ad5eb#4dc166a29a1768d94765662612bff994871ad5eb" dependencies = [ "atomic", "atomic-traits", @@ -479,7 +479,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=dd0acf1871205da734714e4efb55ea85647ac974#dd0acf1871205da734714e4efb55ea85647ac974" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=4dc166a29a1768d94765662612bff994871ad5eb#4dc166a29a1768d94765662612bff994871ad5eb" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index ff35e8d2..f446ddaa 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "dd0acf1871205da734714e4efb55ea85647ac974" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "4dc166a29a1768d94765662612bff994871ad5eb" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 67c3332a..2db3f63e 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -59,13 +59,13 @@ impl Collection> for VMCollection { } } - fn resume_mutators(tls: VMWorkerThread, lxr: bool, current_gc_should_unload_classes: bool) { + fn resume_mutators(tls: VMWorkerThread) { if cfg!(feature = "object_size_distribution") { crate::dump_and_reset_obj_dist(); } DISCOVERED_LISTS.enable_discover(); unsafe { - ((*UPCALLS).resume_mutators)(tls, lxr, current_gc_should_unload_classes); + ((*UPCALLS).resume_mutators)(tls); } } @@ -141,4 +141,10 @@ impl Collection> for VMCollection { fn set_concurrent_marking_state(active: bool) { unsafe { crate::CONCURRENT_MARKING_ACTIVE = if active { 1 } else { 0 } } } + + fn unload_vm_resources() { + unsafe { + ((*UPCALLS).unload_classes)(); + } + } } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 0a477067..2f199b33 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -78,8 +78,7 @@ pub struct OpenJDK_Upcalls { closure: MutatorClosure, current_gc_should_unload_classes: bool, ), - pub resume_mutators: - extern "C" fn(tls: VMWorkerThread, lxr: bool, current_gc_should_unload_classes: bool), + pub resume_mutators: extern "C" fn(tls: VMWorkerThread), pub spawn_gc_thread: extern "C" fn(tls: VMThread, kind: libc::c_int, ctx: *mut libc::c_void), pub block_for_gc: extern "C" fn(), pub out_of_memory: extern "C" fn(tls: VMThread, err_kind: AllocationError), @@ -131,6 +130,7 @@ pub struct OpenJDK_Upcalls { pub compressed_klass_shift: extern "C" fn() -> usize, pub nmethod_fix_relocation: extern "C" fn(Address), pub clear_claimed_marks: extern "C" fn(), + pub unload_classes: extern "C" fn(), } lazy_static! { diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 1a2c73f3..af236655 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -165,7 +165,7 @@ struct EdgesClosure { */ typedef struct { void (*stop_all_mutators) (void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure, bool current_gc_should_unload_classes); - void (*resume_mutators) (void *tls, bool lxr, bool current_gc_should_unload_classes); + void (*resume_mutators) (void *tls); void (*spawn_gc_thread) (void *tls, int kind, void *ctx); void (*block_for_gc) (); void (*out_of_memory) (void *tls, MMTkAllocationError err_kind); @@ -210,6 +210,7 @@ typedef struct { size_t (*compressed_klass_shift)(); void (*nmethod_fix_relocation)(void* nmethod); void (*clear_claimed_marks)(); + void (*unload_classes)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index c26c4a09..d3c061b2 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -152,6 +152,7 @@ jint MMTkHeap::initialize() { unsigned int ncpus = (unsigned int) os::initial_active_processor_count(); _workers = new WorkGang("GC Thread", ncpus, /* are_GC_task_threads */true, /* are_ConcurrentGC_threads */false); _workers->initialize_workers(); + _workers->update_active_workers(Abstract_VM_Version::parallel_worker_threads()); os::start_thread(_companion_thread); // Set up the GCTaskManager diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 126dd04e..1b736cf3 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -189,9 +189,8 @@ static void mmtk_update_weak_processor(bool lxr) { } } -static void mmtk_resume_mutators(void *tls, bool lxr, bool current_gc_should_unload_classes) { - nmethod::oops_do_marking_epilogue(); - if (ClassUnloading && current_gc_should_unload_classes) { +static void mmtk_unload_classes() { + if (ClassUnloading) { // Unload classes and purge SystemDictionary. auto purged_classes = SystemDictionary::do_unloading(NULL, false /* Defer cleaning */); MMTkIsAliveClosure is_alive; @@ -201,6 +200,10 @@ static void mmtk_resume_mutators(void *tls, bool lxr, bool current_gc_should_unl MetaspaceGC::compute_new_size(); MetaspaceUtils::verify_metrics(); } +} + +static void mmtk_resume_mutators(void *tls) { + nmethod::oops_do_marking_epilogue(); // BiasedLocking::restore_marks(); CodeCache::gc_epilogue(); JvmtiExport::gc_epilogue(); @@ -561,4 +564,5 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_compressed_klass_shift, nmethod_fix_relocation, mmtk_clear_claimed_marks, + mmtk_unload_classes, }; From d04875827043a99e615b0cd2c1017134d967d93c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 21 Apr 2023 21:16:44 +0000 Subject: [PATCH 323/494] Fix time.stw accounting --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/collection.rs | 7 +++++-- mmtk/src/lib.rs | 1 + openjdk/mmtk.h | 1 + openjdk/mmtkUpcalls.cpp | 5 ++++- 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index d07a22b7..f3e9ea72 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=4dc166a29a1768d94765662612bff994871ad5eb#4dc166a29a1768d94765662612bff994871ad5eb" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a0d61e81b10beb93a02a29d1b04538f4d21ba1e9#a0d61e81b10beb93a02a29d1b04538f4d21ba1e9" dependencies = [ "atomic", "atomic-traits", @@ -479,7 +479,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.17.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=4dc166a29a1768d94765662612bff994871ad5eb#4dc166a29a1768d94765662612bff994871ad5eb" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a0d61e81b10beb93a02a29d1b04538f4d21ba1e9#a0d61e81b10beb93a02a29d1b04538f4d21ba1e9" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index f446ddaa..edaecb32 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "4dc166a29a1768d94765662612bff994871ad5eb" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "a0d61e81b10beb93a02a29d1b04538f4d21ba1e9" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 2db3f63e..9bcddeaf 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -142,9 +142,12 @@ impl Collection> for VMCollection { unsafe { crate::CONCURRENT_MARKING_ACTIVE = if active { 1 } else { 0 } } } - fn unload_vm_resources() { + fn vm_release(do_unloading: bool) { unsafe { - ((*UPCALLS).unload_classes)(); + if do_unloading { + ((*UPCALLS).unload_classes)(); + } + ((*UPCALLS).gc_epilogue)(); } } } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 2f199b33..95373e56 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -131,6 +131,7 @@ pub struct OpenJDK_Upcalls { pub nmethod_fix_relocation: extern "C" fn(Address), pub clear_claimed_marks: extern "C" fn(), pub unload_classes: extern "C" fn(), + pub gc_epilogue: extern "C" fn(), } lazy_static! { diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index af236655..bf6143b2 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -211,6 +211,7 @@ typedef struct { void (*nmethod_fix_relocation)(void* nmethod); void (*clear_claimed_marks)(); void (*unload_classes)(); + void (*gc_epilogue)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 1b736cf3..feae599f 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -202,7 +202,7 @@ static void mmtk_unload_classes() { } } -static void mmtk_resume_mutators(void *tls) { +static void mmtk_gc_epilogue() { nmethod::oops_do_marking_epilogue(); // BiasedLocking::restore_marks(); CodeCache::gc_epilogue(); @@ -210,7 +210,9 @@ static void mmtk_resume_mutators(void *tls) { #if COMPILER2_OR_JVMCI DerivedPointerTable::update_pointers(); #endif +} +static void mmtk_resume_mutators(void *tls) { // Note: we don't have to hold gc_lock to increment the counter. // The increment has to be done before mutators can be resumed // otherwise, mutators might see a stale value @@ -565,4 +567,5 @@ OpenJDK_Upcalls mmtk_upcalls = { nmethod_fix_relocation, mmtk_clear_claimed_marks, mmtk_unload_classes, + mmtk_gc_epilogue, }; From cf9cb8a611e34824ab3daebebc73bbd0aea41953 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 9 May 2023 05:25:53 +0000 Subject: [PATCH 324/494] Add jvmti hook --- openjdk/mmtkVMOperation.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/openjdk/mmtkVMOperation.cpp b/openjdk/mmtkVMOperation.cpp index 00060fae..1e141daf 100644 --- a/openjdk/mmtkVMOperation.cpp +++ b/openjdk/mmtkVMOperation.cpp @@ -54,6 +54,7 @@ void VM_MMTkSTWOperation::doit() { _companion_thread->_wait_for_gc_locker = true; return; } + JvmtiGCMarker _jgcm; log_trace(vmthread)("Entered VM_MMTkSTWOperation::doit()."); _companion_thread->reach_suspended_and_wait_for_resume(); log_trace(vmthread)("Leaving VM_MMTkSTWOperation::doit()"); From 26ec22f7dbb974691be5db7f8f522e5352b7d7fa Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 16 May 2023 19:45:08 +0000 Subject: [PATCH 325/494] Optimize update_weak_processor --- openjdk/mmtkUpcalls.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index feae599f..2f843b74 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -182,7 +182,11 @@ static void mmtk_update_weak_processor(bool lxr) { MMTkForwardClosure forward; if (lxr) { MMTkLXRFastIsAliveClosure is_alive; - WeakProcessor::weak_oops_do(&is_alive, &forward); + MMTkLXRFastUpdateClosure fast_update; + JNIHandles::weak_global_handles()->weak_oops_do(&fast_update); + JvmtiExport::weak_oops_do(&is_alive, &forward); + SystemDictionary::vm_weak_oop_storage()->weak_oops_do(&fast_update); + JFR_ONLY(Jfr::weak_oops_do(&is_alive, &forward);) } else { MMTkIsAliveClosure is_alive; WeakProcessor::weak_oops_do(&is_alive, &forward); From 176f2efee092865e6a2714eaf295bc2551027106 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 16 May 2023 22:44:06 +0000 Subject: [PATCH 326/494] Revert weak processor optimization --- openjdk/mmtkUpcalls.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 2f843b74..feae599f 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -182,11 +182,7 @@ static void mmtk_update_weak_processor(bool lxr) { MMTkForwardClosure forward; if (lxr) { MMTkLXRFastIsAliveClosure is_alive; - MMTkLXRFastUpdateClosure fast_update; - JNIHandles::weak_global_handles()->weak_oops_do(&fast_update); - JvmtiExport::weak_oops_do(&is_alive, &forward); - SystemDictionary::vm_weak_oop_storage()->weak_oops_do(&fast_update); - JFR_ONLY(Jfr::weak_oops_do(&is_alive, &forward);) + WeakProcessor::weak_oops_do(&is_alive, &forward); } else { MMTkIsAliveClosure is_alive; WeakProcessor::weak_oops_do(&is_alive, &forward); From 39cb2d671ae400265c545761e3f6f6300cccf783 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 18 May 2023 19:47:24 +0000 Subject: [PATCH 327/494] Optimize code root scanning: Only scan young code roots --- mmtk/Cargo.lock | 5 +-- mmtk/Cargo.toml | 2 +- mmtk/src/api.rs | 19 ++++------ mmtk/src/gc_work.rs | 85 +++++++++++++++++++++++++++++--------------- mmtk/src/lib.rs | 6 ++-- mmtk/src/scanning.rs | 3 +- 6 files changed, 70 insertions(+), 50 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 612f6c74..dabe19ff 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e2383164e377e4d95bf943b6a389e3b7e53e12ff#e2383164e377e4d95bf943b6a389e3b7e53e12ff" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=5051fc88a4785a3ba1f0c6256fad346d49a6f379#5051fc88a4785a3ba1f0c6256fad346d49a6f379" dependencies = [ "atomic", "atomic-traits", @@ -461,6 +461,7 @@ dependencies = [ "downcast-rs", "enum-map", "env_logger", + "humantime", "itertools", "lazy_static", "libc", @@ -479,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e2383164e377e4d95bf943b6a389e3b7e53e12ff#e2383164e377e4d95bf943b6a389e3b7e53e12ff" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=5051fc88a4785a3ba1f0c6256fad346d49a6f379#5051fc88a4785a3ba1f0c6256fad346d49a6f379" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 2d402811..19d6a3e9 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e2383164e377e4d95bf943b6a389e3b7e53e12ff" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "5051fc88a4785a3ba1f0c6256fad346d49a6f379" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index a7346171..7339b8a9 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -561,26 +561,19 @@ pub extern "C" fn mmtk_register_nmethod(nm: Address) { Some(slots) => slots, _ => return, }; - let mut roots = crate::CODE_CACHE_ROOTS.lock().unwrap(); - // Relaxed add instead of `fetch_add`, since we've already acquired the lock. - crate::CODE_CACHE_ROOTS_SIZE.store( - crate::CODE_CACHE_ROOTS_SIZE.load(Ordering::Relaxed) + slots.len(), - Ordering::Relaxed, - ); + let mut roots = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); roots.insert(nm, slots); } /// Unregister a nmethod. #[no_mangle] pub extern "C" fn mmtk_unregister_nmethod(nm: Address) { - let mut roots = crate::CODE_CACHE_ROOTS.lock().unwrap(); - if let Some(slots) = roots.remove(&nm) { - // Relaxed sub instead of `fetch_sub`, since we've already acquired the lock. - crate::CODE_CACHE_ROOTS_SIZE.store( - crate::CODE_CACHE_ROOTS_SIZE.load(Ordering::Relaxed) - slots.len(), - Ordering::Relaxed, - ); + let mut roots = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); + if roots.remove(&nm).is_some() { + return; } + let mut roots = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); + roots.remove(&nm); } #[no_mangle] diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 11984b63..dd102550 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -9,6 +9,7 @@ use mmtk::util::ObjectReference; use mmtk::vm::RootsWorkFactory; use mmtk::vm::*; use mmtk::MMTK; +use std::collections::HashMap; use std::marker::PhantomData; macro_rules! scan_roots_work { @@ -47,20 +48,6 @@ scan_roots_work!(ScanSystemDictionaryRoots, scan_system_dictionary_roots); scan_roots_work!(ScanStringTableRoots, scan_string_table_roots); scan_roots_work!(ScanVMThreadRoots, scan_vm_thread_roots); -pub struct ScanClassLoaderDataGraphRoots> { - factory: F, - _p: PhantomData, -} - -impl> ScanClassLoaderDataGraphRoots { - pub fn new(factory: F) -> Self { - Self { - factory, - _p: PhantomData, - } - } -} - extern "C" fn report_edges_and_renew_buffer_cld< E: Edge, F: RootsWorkFactory, @@ -71,11 +58,16 @@ extern "C" fn report_edges_and_renew_buffer_cld< capacity: usize, factory_ptr: *mut libc::c_void, ) -> NewBuffer { + let root_kind = if WEAK { + RootKind::Weak + } else { + RootKind::Young + }; if !ptr.is_null() { let ptr = ptr as *mut E; let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; - factory.create_process_edge_roots_work_for_cld_roots(buf, WEAK); + factory.create_process_edge_roots_work(buf, root_kind); } let (ptr, _, capacity) = { // TODO: Use Vec::into_raw_parts() when the method is available. @@ -95,6 +87,21 @@ fn to_edges_closure_cld, const WEAK: bool>( data: factory as *mut F as *mut libc::c_void, } } + +pub struct ScanClassLoaderDataGraphRoots> { + factory: F, + _p: PhantomData, +} + +impl> ScanClassLoaderDataGraphRoots { + pub fn new(factory: F) -> Self { + Self { + factory, + _p: PhantomData, + } + } +} + impl> GCWork for ScanClassLoaderDataGraphRoots { @@ -127,27 +134,48 @@ impl> ScanCodeCacheRoots { impl> GCWork for ScanCodeCacheRoots { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { - // Collect all the cached roots + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { let mut edges = Vec::with_capacity(F::BUFFER_SIZE); - for roots in (*crate::CODE_CACHE_ROOTS.lock().unwrap()).values() { - for r in roots { + let scan_all_roots = mmtk + .get_plan() + .current_gc_should_scan_all_classloader_strong_roots() + || mmtk.get_plan().current_gc_should_perform_class_unloading(); + let mut mature = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); + let mut nursery_guard = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); + let nursery = std::mem::take::>>(&mut nursery_guard); + if scan_all_roots { + // Collect all the mature cached roots + for roots in mature.values() { + for r in roots { + edges.push(VM::VMEdge::from_address(*r)); + if edges.len() >= F::BUFFER_SIZE { + self.factory.create_process_edge_roots_work( + std::mem::take(&mut edges), + RootKind::Young, + ); + edges.reserve(F::BUFFER_SIZE); + } + } + } + } + // Young roots + for (key, roots) in nursery { + for r in &roots { edges.push(VM::VMEdge::from_address(*r)); if edges.len() >= F::BUFFER_SIZE { - self.factory - .create_process_edge_roots_work(std::mem::take(&mut edges)); + self.factory.create_process_edge_roots_work( + std::mem::take(&mut edges), + RootKind::Young, + ); edges.reserve(F::BUFFER_SIZE); } } + mature.insert(key, roots); } - // Create work packet if !edges.is_empty() { - self.factory.create_process_edge_roots_work(edges); + self.factory + .create_process_edge_roots_work(edges, RootKind::Young); } - // Use the following code to scan CodeCache directly, instead of scanning the "remembered set". - // unsafe { - // ((*UPCALLS).scan_code_cache_roots)(create_process_edges_work:: as _); - // } } } @@ -162,8 +190,7 @@ extern "C" fn report_edges_and_renew_buffer_weakref::from_raw_parts(ptr as *mut ObjectReference, length, capacity) }; let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; - // println!("weakref nodes {:?}", buf); - factory.create_process_node_roots_work(buf); + factory.create_process_node_roots_work(buf, RootKind::Weak); } let (ptr, _, capacity) = { // TODO: Use Vec::into_raw_parts() when the method is available. diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 7de2c2f4..203ef237 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -540,12 +540,10 @@ pub static MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES: usize = lazy_static! { /// A global storage for all the cached CodeCache root pointers - static ref CODE_CACHE_ROOTS: Mutex>> = Mutex::new(HashMap::new()); + static ref NURSERY_CODE_CACHE_ROOTS: Mutex>> = Mutex::new(HashMap::new()); + static ref MATURE_CODE_CACHE_ROOTS: Mutex>> = Mutex::new(HashMap::new()); } -/// A counter tracking the total size of the `CODE_CACHE_ROOTS`. -static CODE_CACHE_ROOTS_SIZE: AtomicUsize = AtomicUsize::new(0); - lazy_static! { static ref OBJ_COUNT: Mutex> = Mutex::new(HashMap::new()); } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index e817d69f..3269d967 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -3,6 +3,7 @@ use crate::Edge; use crate::{EdgesClosure, OpenJDK}; use crate::{NewBuffer, OpenJDKEdge, UPCALLS}; use mmtk::memory_manager; +use mmtk::scheduler::RootKind; use mmtk::scheduler::WorkBucketStage; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; @@ -22,7 +23,7 @@ extern "C" fn report_edges_and_renew_buffer>( let ptr = ptr as *mut E; let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; - factory.create_process_edge_roots_work(buf); + factory.create_process_edge_roots_work(buf, RootKind::Strong); } let (ptr, _, capacity) = { // TODO: Use Vec::into_raw_parts() when the method is available. From cb69c9df358eede27e21036e813261a302e14373 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 21 May 2023 05:13:07 +0000 Subject: [PATCH 328/494] Fix nmethod unregister --- mmtk/src/api.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 7339b8a9..8e775e4f 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -569,9 +569,7 @@ pub extern "C" fn mmtk_register_nmethod(nm: Address) { #[no_mangle] pub extern "C" fn mmtk_unregister_nmethod(nm: Address) { let mut roots = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); - if roots.remove(&nm).is_some() { - return; - } + roots.remove(&nm); let mut roots = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); roots.remove(&nm); } From 7a043715ee0a73282e293f04c5c3c5744086687d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 25 May 2023 18:58:53 +0000 Subject: [PATCH 329/494] root counters --- mmtk/Cargo.toml | 1 + mmtk/src/gc_work.rs | 49 ++++++++++++++++++++++++++++++++++++++++++++ mmtk/src/scanning.rs | 3 +++ 3 files changed, 53 insertions(+) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 19d6a3e9..e9bc3c09 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -57,6 +57,7 @@ nogc_no_zeroing = ["mmtk/nogc_no_zeroing"] # See README. global_alloc_bit = ["mmtk/global_alloc_bit"] no_fast_alloc = [] +roots_breakdown = [] # This compile time constant places the mark bit in the header of the object instead of on the side. mark_bit_in_header = [] diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index dd102550..52e554b9 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -11,6 +11,26 @@ use mmtk::vm::*; use mmtk::MMTK; use std::collections::HashMap; use std::marker::PhantomData; +use std::sync::atomic::{AtomicUsize, Ordering}; + +thread_local! { + pub static COUNT: AtomicUsize = AtomicUsize::new(0); +} + +pub fn record_roots(len: usize) { + super::gc_work::COUNT.with(|x| { + let c = x.load(Ordering::Relaxed); + x.store(c + len, Ordering::Relaxed); + }); +} + +fn report_roots(name: &str) { + super::gc_work::COUNT.with(|x| { + let c = x.load(Ordering::Relaxed); + eprintln!(" - {} roots count: {}", name, c); + x.store(0, Ordering::Relaxed); + }); +} macro_rules! scan_roots_work { ($struct_name: ident, $func_name: ident) => { @@ -33,6 +53,10 @@ macro_rules! scan_roots_work { unsafe { ((*UPCALLS).$func_name)(to_edges_closure(&mut self.factory)); } + if cfg!(feature = "roots_breakdown") { + let name = stringify!($struct_name); + report_roots(&name[4..name.len() - 5]); + } } } }; @@ -66,6 +90,9 @@ extern "C" fn report_edges_and_renew_buffer_cld< if !ptr.is_null() { let ptr = ptr as *mut E; let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; + if cfg!(feature = "roots_breakdown") { + record_roots(buf.len()); + } let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; factory.create_process_edge_roots_work(buf, root_kind); } @@ -114,6 +141,9 @@ impl> GCWork .current_gc_should_scan_all_classloader_strong_roots(), ); } + if cfg!(feature = "roots_breakdown") { + report_roots("ClassLoaderDataGraph"); + } } } @@ -143,12 +173,16 @@ impl> GCWork let mut mature = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); let mut nursery_guard = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); let nursery = std::mem::take::>>(&mut nursery_guard); + let mut c = 0; if scan_all_roots { // Collect all the mature cached roots for roots in mature.values() { for r in roots { edges.push(VM::VMEdge::from_address(*r)); if edges.len() >= F::BUFFER_SIZE { + if cfg!(feature = "roots_breakdown") { + c += edges.len(); + } self.factory.create_process_edge_roots_work( std::mem::take(&mut edges), RootKind::Young, @@ -163,6 +197,9 @@ impl> GCWork for r in &roots { edges.push(VM::VMEdge::from_address(*r)); if edges.len() >= F::BUFFER_SIZE { + if cfg!(feature = "roots_breakdown") { + c += edges.len(); + } self.factory.create_process_edge_roots_work( std::mem::take(&mut edges), RootKind::Young, @@ -173,9 +210,15 @@ impl> GCWork mature.insert(key, roots); } if !edges.is_empty() { + if cfg!(feature = "roots_breakdown") { + c += edges.len(); + } self.factory .create_process_edge_roots_work(edges, RootKind::Young); } + if cfg!(feature = "roots_breakdown") { + eprintln!(" - CodeCache roots count: {}", c); + } } } @@ -189,6 +232,9 @@ extern "C" fn report_edges_and_renew_buffer_weakref::from_raw_parts(ptr as *mut ObjectReference, length, capacity) }; + if cfg!(feature = "roots_breakdown") { + record_roots(buf.len()); + } let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; factory.create_process_node_roots_work(buf, RootKind::Weak); } @@ -232,5 +278,8 @@ impl> GCWork &mut self.factory, )); } + if cfg!(feature = "roots_breakdown") { + report_roots("WeakProcessor"); + } } } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 3269d967..8db3c5b3 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -22,6 +22,9 @@ extern "C" fn report_edges_and_renew_buffer>( if !ptr.is_null() { let ptr = ptr as *mut E; let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; + if cfg!(feature = "roots_breakdown") { + super::gc_work::record_roots(buf.len()); + } let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; factory.create_process_edge_roots_work(buf, RootKind::Strong); } From 69b78d9356218243a9e131d839c7a583c507ac2f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 31 May 2023 12:39:19 -0700 Subject: [PATCH 330/494] Fix openjdk weak storage unloading --- mmtk/Cargo.lock | 4 +- mmtk/Cargo.toml | 2 +- mmtk/src/gc_work.rs | 118 ++++++++++++++++++++++++++++--- mmtk/src/lib.rs | 5 +- mmtk/src/scanning.rs | 32 +++++---- openjdk/mmtk.h | 6 +- openjdk/mmtkHeap.cpp | 4 +- openjdk/mmtkParallelCleaning.cpp | 3 + openjdk/mmtkRootsClosure.hpp | 20 ++++-- openjdk/mmtkUpcalls.cpp | 52 ++++++++++---- 10 files changed, 195 insertions(+), 51 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index dabe19ff..e73022c3 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=5051fc88a4785a3ba1f0c6256fad346d49a6f379#5051fc88a4785a3ba1f0c6256fad346d49a6f379" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=85334070f1bf9ad48329e6ae7f143e1eda2b4846#85334070f1bf9ad48329e6ae7f143e1eda2b4846" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=5051fc88a4785a3ba1f0c6256fad346d49a6f379#5051fc88a4785a3ba1f0c6256fad346d49a6f379" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=85334070f1bf9ad48329e6ae7f143e1eda2b4846#85334070f1bf9ad48329e6ae7f143e1eda2b4846" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index e9bc3c09..2da50f68 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "5051fc88a4785a3ba1f0c6256fad346d49a6f379" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "85334070f1bf9ad48329e6ae7f143e1eda2b4846" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 52e554b9..b7f00af8 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -24,10 +24,10 @@ pub fn record_roots(len: usize) { }); } -fn report_roots(name: &str) { +fn report_roots(name: &str, ms: f32) { super::gc_work::COUNT.with(|x| { let c = x.load(Ordering::Relaxed); - eprintln!(" - {} roots count: {}", name, c); + eprintln!(" - {} roots count: {} ({:.3}ms)", name, c, ms); x.store(0, Ordering::Relaxed); }); } @@ -50,12 +50,18 @@ macro_rules! scan_roots_work { impl> GCWork for $struct_name { fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { + let t = if cfg!(feature = "roots_breakdown") { + Some(std::time::SystemTime::now()) + } else { + None + }; unsafe { ((*UPCALLS).$func_name)(to_edges_closure(&mut self.factory)); } if cfg!(feature = "roots_breakdown") { let name = stringify!($struct_name); - report_roots(&name[4..name.len() - 5]); + let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; + report_roots(&name[4..name.len() - 5], ms); } } } @@ -69,7 +75,6 @@ scan_roots_work!(ScanManagementRoots, scan_management_roots); scan_roots_work!(ScanJvmtiExportRoots, scan_jvmti_export_roots); scan_roots_work!(ScanAOTLoaderRoots, scan_aot_loader_roots); scan_roots_work!(ScanSystemDictionaryRoots, scan_system_dictionary_roots); -scan_roots_work!(ScanStringTableRoots, scan_string_table_roots); scan_roots_work!(ScanVMThreadRoots, scan_vm_thread_roots); extern "C" fn report_edges_and_renew_buffer_cld< @@ -133,6 +138,11 @@ impl> GCWork for ScanClassLoaderDataGraphRoots { fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let t = if cfg!(feature = "roots_breakdown") { + Some(std::time::SystemTime::now()) + } else { + None + }; unsafe { ((*UPCALLS).scan_class_loader_data_graph_roots)( to_edges_closure_cld::(&mut self.factory), @@ -142,7 +152,78 @@ impl> GCWork ); } if cfg!(feature = "roots_breakdown") { - report_roots("ClassLoaderDataGraph"); + let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; + report_roots("ClassLoaderDataGraph", ms); + } + } +} + +extern "C" fn report_edges_and_renew_buffer_st>( + ptr: *mut Address, + length: usize, + capacity: usize, + factory_ptr: *mut libc::c_void, +) -> NewBuffer { + if !ptr.is_null() { + let ptr = ptr as *mut E; + let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; + if cfg!(feature = "roots_breakdown") { + record_roots(buf.len()); + } + let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; + factory.create_process_edge_roots_work(buf, RootKind::Weak); + } + let (ptr, _, capacity) = { + // TODO: Use Vec::into_raw_parts() when the method is available. + use std::mem::ManuallyDrop; + let new_vec = Vec::with_capacity(F::BUFFER_SIZE); + let mut me = ManuallyDrop::new(new_vec); + (me.as_mut_ptr(), me.len(), me.capacity()) + }; + NewBuffer { ptr, capacity } +} + +fn to_edges_closure_st>(factory: &mut F) -> EdgesClosure { + EdgesClosure { + func: report_edges_and_renew_buffer_st::, + data: factory as *mut F as *mut libc::c_void, + } +} + +pub struct ScanStringTableRoots> { + factory: F, + _p: PhantomData, +} + +impl> ScanStringTableRoots { + pub fn new(factory: F) -> Self { + Self { + factory, + _p: PhantomData, + } + } +} + +impl> GCWork + for ScanStringTableRoots +{ + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let t = if cfg!(feature = "roots_breakdown") { + Some(std::time::SystemTime::now()) + } else { + None + }; + unsafe { + ((*UPCALLS).scan_string_table_roots)( + to_edges_closure_st::(&mut self.factory), + mmtk.get_plan() + .downcast_ref::>() + .is_some(), + ); + } + if cfg!(feature = "roots_breakdown") { + let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; + report_roots("StringTable", ms); } } } @@ -165,6 +246,11 @@ impl> GCWork for ScanCodeCacheRoots { fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let t = if cfg!(feature = "roots_breakdown") { + Some(std::time::SystemTime::now()) + } else { + None + }; let mut edges = Vec::with_capacity(F::BUFFER_SIZE); let scan_all_roots = mmtk .get_plan() @@ -217,7 +303,8 @@ impl> GCWork .create_process_edge_roots_work(edges, RootKind::Young); } if cfg!(feature = "roots_breakdown") { - eprintln!(" - CodeCache roots count: {}", c); + let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; + eprintln!(" - CodeCache roots count: {} ({:.3})", c, ms); } } } @@ -272,14 +359,23 @@ impl> ScaWeakProcessorRoots { impl> GCWork for ScaWeakProcessorRoots { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let t = if cfg!(feature = "roots_breakdown") { + Some(std::time::SystemTime::now()) + } else { + None + }; unsafe { - ((*UPCALLS).scan_weak_processor_roots)(to_edges_closure_weakref::<_, _>( - &mut self.factory, - )); + ((*UPCALLS).scan_weak_processor_roots)( + to_edges_closure_weakref::<_, _>(&mut self.factory), + mmtk.get_plan() + .downcast_ref::>() + .is_some(), + ); } if cfg!(feature = "roots_breakdown") { - report_roots("WeakProcessor"); + let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; + report_roots("WeakProcessor", ms); } } } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 203ef237..4ceac38b 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -113,10 +113,11 @@ pub struct OpenJDK_Upcalls { pub scan_aot_loader_roots: extern "C" fn(closure: EdgesClosure), pub scan_system_dictionary_roots: extern "C" fn(closure: EdgesClosure), pub scan_code_cache_roots: extern "C" fn(closure: EdgesClosure), - pub scan_string_table_roots: extern "C" fn(closure: EdgesClosure), + pub scan_string_table_roots: extern "C" fn(closure: EdgesClosure, rc_non_stuck_objs_only: bool), pub scan_class_loader_data_graph_roots: extern "C" fn(closure: EdgesClosure, weak_closure: EdgesClosure, scan_weak: bool), - pub scan_weak_processor_roots: extern "C" fn(closure: EdgesClosure), + pub scan_weak_processor_roots: + extern "C" fn(closure: EdgesClosure, rc_non_stuck_objs_only: bool), pub scan_vm_thread_roots: extern "C" fn(closure: EdgesClosure), pub number_of_mutators: extern "C" fn() -> usize, pub schedule_finalizer: extern "C" fn(), diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 8db3c5b3..cffa4c84 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -112,22 +112,28 @@ impl Scanning> for VMScanning { _tls: VMWorkerThread, factory: impl RootsWorkFactory>, ) { + let mut w = vec![ + Box::new(ScanUniverseRoots::new(factory.clone())) as _, + Box::new(ScanJNIHandlesRoots::new(factory.clone())) as _, + Box::new(ScanObjectSynchronizerRoots::new(factory.clone())) as _, + Box::new(ScanManagementRoots::new(factory.clone())) as _, + Box::new(ScanJvmtiExportRoots::new(factory.clone())) as _, + Box::new(ScanAOTLoaderRoots::new(factory.clone())) as _, + Box::new(ScanSystemDictionaryRoots::new(factory.clone())) as _, + Box::new(ScanCodeCacheRoots::new(factory.clone())) as _, + Box::new(ScanClassLoaderDataGraphRoots::new(factory.clone())) as _, + ]; + if crate::singleton::() + .get_plan() + .requires_weak_root_scanning() + { + w.push(Box::new(ScanStringTableRoots::new(factory.clone())) as _); + w.push(Box::new(ScaWeakProcessorRoots::new(factory.clone())) as _); + } memory_manager::add_work_packets( &crate::singleton::(), WorkBucketStage::RCProcessIncs, - vec![ - Box::new(ScanUniverseRoots::new(factory.clone())) as _, - Box::new(ScanJNIHandlesRoots::new(factory.clone())) as _, - Box::new(ScanObjectSynchronizerRoots::new(factory.clone())) as _, - Box::new(ScanManagementRoots::new(factory.clone())) as _, - Box::new(ScanJvmtiExportRoots::new(factory.clone())) as _, - Box::new(ScanAOTLoaderRoots::new(factory.clone())) as _, - Box::new(ScanSystemDictionaryRoots::new(factory.clone())) as _, - Box::new(ScanCodeCacheRoots::new(factory.clone())) as _, - Box::new(ScanStringTableRoots::new(factory.clone())) as _, - Box::new(ScanClassLoaderDataGraphRoots::new(factory.clone())) as _, - Box::new(ScaWeakProcessorRoots::new(factory.clone())) as _, - ], + w, ); if !(>>::SCAN_MUTATORS_IN_SAFEPOINT && >>::SINGLE_THREAD_MUTATOR_SCANNING) diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 9efa655b..1043c919 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -197,9 +197,9 @@ typedef struct { void (*scan_aot_loader_roots) (EdgesClosure closure); void (*scan_system_dictionary_roots) (EdgesClosure closure); void (*scan_code_cache_roots) (EdgesClosure closure); - void (*scan_string_table_roots) (EdgesClosure closure); + void (*scan_string_table_roots) (EdgesClosure closure, bool rc_non_stuck_objs_only); void (*scan_class_loader_data_graph_roots) (EdgesClosure closure, EdgesClosure weak_closure, bool scan_weak); - void (*scan_weak_processor_roots) (EdgesClosure closure); + void (*scan_weak_processor_roots) (EdgesClosure closure, bool rc_non_stuck_objs_only); void (*scan_vm_thread_roots) (EdgesClosure closure); size_t (*number_of_mutators)(); void (*schedule_finalizer)(); @@ -277,4 +277,6 @@ inline uint8_t mmtk_get_rc(void* o) { } #endif +namespace JavaClassFile {}; + #endif // MMTK_OPENJDK_MMTK_H diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index d3c061b2..def06e79 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -50,6 +50,9 @@ #include "utilities/vmError.hpp" #include "mmtkParallelCleaning.hpp" #include "mmtkRootsClosure.hpp" + +using namespace JavaClassFile; + /* needed support from rust heap capacity @@ -62,7 +65,6 @@ object iterator??!! //mmtkGCTaskManager* MMTkHeap::_mmtk_gc_task_manager = NULL; - MMTkHeap* MMTkHeap::_heap = NULL; MMTkHeap::MMTkHeap(MMTkCollectorPolicy* policy) : diff --git a/openjdk/mmtkParallelCleaning.cpp b/openjdk/mmtkParallelCleaning.cpp index 3bb2d5df..814112a5 100644 --- a/openjdk/mmtkParallelCleaning.cpp +++ b/openjdk/mmtkParallelCleaning.cpp @@ -33,6 +33,9 @@ #include "gc/shared/gcTraceTime.hpp" #include "gc/shared/gcTraceTime.inline.hpp" #include "mmtkParallelCleaning.hpp" +#include "mmtk.h" + +using namespace JavaClassFile; namespace mmtk { diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 76a8e513..c3a6422b 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -8,7 +8,8 @@ #include "utilities/globalDefinitions.hpp" #include "classfile/classLoaderData.inline.hpp" -class MMTkRootsClosure2 : public OopClosure { +template +class MMTkRootsClosure : public OopClosure { EdgesClosure _edges_closure; void** _buffer; size_t _cap; @@ -18,6 +19,12 @@ class MMTkRootsClosure2 : public OopClosure { void do_oop_work(T* p, bool narrow) { T heap_oop = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(heap_oop)) { + if (UNPINNED_ONLY) { + auto o = CompressedOops::decode(heap_oop); + if (3 == mmtk_get_rc((void*) o)) { + return; + } + } if (UseCompressedOops && !narrow) { guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); p = (T*) (uintptr_t(p) | (1ull << 63)); @@ -39,13 +46,13 @@ class MMTkRootsClosure2 : public OopClosure { } public: - MMTkRootsClosure2(EdgesClosure edges_closure): _edges_closure(edges_closure), _cursor(0) { + MMTkRootsClosure(EdgesClosure edges_closure): _edges_closure(edges_closure), _cursor(0) { NewBuffer buf = edges_closure.invoke(NULL, 0, 0); _buffer = buf.buf; _cap = buf.cap; } - ~MMTkRootsClosure2() { + ~MMTkRootsClosure() { if (_cursor > 0) flush(); if (_buffer != NULL) { release_buffer(_buffer, _cursor, _cap); @@ -56,6 +63,7 @@ class MMTkRootsClosure2 : public OopClosure { virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } }; +template class MMTkCollectRootObjects : public OopClosure { EdgesClosure _edges_closure; void** _buffer; @@ -66,7 +74,11 @@ class MMTkCollectRootObjects : public OopClosure { void do_oop_work(T* p, bool narrow) { T heap_oop = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(heap_oop)) { - _buffer[_cursor++] = (void*) CompressedOops::decode(heap_oop); + auto o = CompressedOops::decode(heap_oop); + if (UNPINNED_ONLY && 3 == mmtk_get_rc((void*) o)) { + return; + } + _buffer[_cursor++] = (void*) o; if (_cursor >= _cap) { flush(); } diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index feae599f..02f5fa06 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -37,6 +37,7 @@ #include "runtime/mutexLocker.hpp" #include "runtime/os.hpp" #include "runtime/safepoint.hpp" +#include "runtime/interfaceSupport.inline.hpp" #include "runtime/thread.hpp" #include "runtime/threadSMR.hpp" #include "runtime/vmThread.hpp" @@ -53,6 +54,8 @@ #include "jfr/jfr.hpp" #endif +using namespace JavaClassFile; + // Note: This counter must be accessed using the Atomic class. static volatile size_t mmtk_start_the_world_count = 0; @@ -186,10 +189,13 @@ static void mmtk_update_weak_processor(bool lxr) { } else { MMTkIsAliveClosure is_alive; WeakProcessor::weak_oops_do(&is_alive, &forward); + StringTable::oops_do(&forward); } } static void mmtk_unload_classes() { + MMTkForwardClosure forward; + StringTable::oops_do(&forward); if (ClassUnloading) { // Unload classes and purge SystemDictionary. auto purged_classes = SystemDictionary::do_unloading(NULL, false /* Defer cleaning */); @@ -351,14 +357,14 @@ static void mmtk_reset_mutator_iterator() { static void mmtk_scan_all_thread_roots(EdgesClosure closure) { - MMTkRootsClosure2 cl(closure); + MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_thread_roots(cl); } static void mmtk_scan_thread_roots(EdgesClosure closure, void* tls) { ResourceMark rm; JavaThread* thread = (JavaThread*) tls; - MMTkRootsClosure2 cl(closure); + MMTkRootsClosure<> cl(closure); thread->oops_do(&cl, NULL); } @@ -442,22 +448,38 @@ static void mmtk_schedule_finalizer() { MMTkHeap::heap()->schedule_finalizer(); } -static void mmtk_scan_universe_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_universe_roots(cl); } -static void mmtk_scan_jni_handle_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_jni_handle_roots(cl); } -static void mmtk_scan_object_synchronizer_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_object_synchronizer_roots(cl); } -static void mmtk_scan_management_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_management_roots(cl); } -static void mmtk_scan_jvmti_export_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_jvmti_export_roots(cl); } -static void mmtk_scan_aot_loader_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_aot_loader_roots(cl); } -static void mmtk_scan_system_dictionary_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_system_dictionary_roots(cl); } -static void mmtk_scan_code_cache_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } -static void mmtk_scan_string_table_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_string_table_roots(cl); } +static void mmtk_scan_universe_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_universe_roots(cl); } +static void mmtk_scan_jni_handle_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_jni_handle_roots(cl); } +static void mmtk_scan_object_synchronizer_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_object_synchronizer_roots(cl); } +static void mmtk_scan_management_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_management_roots(cl); } +static void mmtk_scan_jvmti_export_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_jvmti_export_roots(cl); } +static void mmtk_scan_aot_loader_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_aot_loader_roots(cl); } +static void mmtk_scan_system_dictionary_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_system_dictionary_roots(cl); } +static void mmtk_scan_code_cache_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } +static void mmtk_scan_string_table_roots(EdgesClosure closure, bool rc_non_stuck_objs_only) { + if (rc_non_stuck_objs_only) { + MMTkRootsClosure cl(closure); + MMTkHeap::heap()->scan_string_table_roots(cl); + } else { + MMTkRootsClosure cl(closure); + MMTkHeap::heap()->scan_string_table_roots(cl); + } +} static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, EdgesClosure weak_closure, bool scank_all_strong_roots) { - MMTkRootsClosure2 cl(closure); - MMTkRootsClosure2 weak_cl(weak_closure); + MMTkRootsClosure<> cl(closure); + MMTkRootsClosure<> weak_cl(weak_closure); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl, weak_cl, scank_all_strong_roots); } -static void mmtk_scan_weak_processor_roots(EdgesClosure closure) { MMTkCollectRootObjects cl(closure); MMTkHeap::heap()->scan_weak_processor_roots(cl); } -static void mmtk_scan_vm_thread_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } +static void mmtk_scan_weak_processor_roots(EdgesClosure closure, bool rc_non_stuck_objs_only) { + if (rc_non_stuck_objs_only) { + MMTkCollectRootObjects cl(closure); + MMTkHeap::heap()->scan_weak_processor_roots(cl); + } else { + MMTkCollectRootObjects cl(closure); + MMTkHeap::heap()->scan_weak_processor_roots(cl); + } +} +static void mmtk_scan_vm_thread_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } static size_t mmtk_number_of_mutators() { return Threads::number_of_threads(); From 61c756b8d5f13de9a4bbafacbc1fb889e71e717a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 1 Jun 2023 17:00:03 -0700 Subject: [PATCH 331/494] parallel string table scanning --- mmtk/src/scanning.rs | 10 +++++++++- openjdk/mmtkHeap.cpp | 10 ++++++++-- openjdk/mmtkHeap.hpp | 2 +- openjdk/mmtkUpcalls.cpp | 38 +++++++++++++++++++++++--------------- 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index cffa4c84..589e7adf 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -127,7 +127,15 @@ impl Scanning> for VMScanning { .get_plan() .requires_weak_root_scanning() { - w.push(Box::new(ScanStringTableRoots::new(factory.clone())) as _); + const PARALLEL_STRING_TABLE_SCANNING: bool = true; + let num_string_table_packets: usize = if PARALLEL_STRING_TABLE_SCANNING { + crate::singleton::().scheduler.num_workers() + } else { + 1 + }; + for _ in 0..num_string_table_packets { + w.push(Box::new(ScanStringTableRoots::new(factory.clone())) as _); + } w.push(Box::new(ScaWeakProcessorRoots::new(factory.clone())) as _); } memory_manager::add_work_packets( diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index def06e79..6d3462ea 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -438,9 +438,15 @@ void MMTkHeap::scan_code_cache_roots(OopClosure& cl) { MarkingCodeBlobClosure cb_cl(&cl, false); CodeCache::blobs_do(&cb_cl); } -void MMTkHeap::scan_string_table_roots(OopClosure& cl) { - StringTable::oops_do(&cl); + +void MMTkHeap::scan_string_table_roots(OopClosure& cl, OopStorage::ParState* par_state_string) { + if (par_state_string == NULL) { + StringTable::oops_do(&cl); + } else { + StringTable::possibly_parallel_oops_do(par_state_string, &cl); + } } + void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& weak_cl, bool scan_all_strong_roots) { if (!ClassUnloading) { CLDToOopClosure cld_cl(&cl, false); diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index f4c63578..44580b2f 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -218,7 +218,7 @@ class MMTkHeap : public CollectedHeap { void scan_aot_loader_roots(OopClosure& cl); void scan_system_dictionary_roots(OopClosure& cl); void scan_code_cache_roots(OopClosure& cl); - void scan_string_table_roots(OopClosure& cl); + void scan_string_table_roots(OopClosure& cl, OopStorage::ParState* par_state_string); void scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& weak_cl, bool scan_all_strong_roots); void scan_weak_processor_roots(OopClosure& cl); void scan_vm_thread_roots(OopClosure& cl); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 02f5fa06..eb6115c4 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -151,12 +151,33 @@ class MMTkLXRFastUpdateClosure : public OopClosure { } }; +template +struct MaybeUninit { + MaybeUninit() {} + T* operator->() { + return (T*) &_data; + } + T& operator*() { + return *((T*) &_data); + } + template + void init(Args... args) { + new (&_data) T(args...); + } +private: + char _data[sizeof(T)]; +}; + +static MaybeUninit> par_state_string; + static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure, bool current_gc_should_unload_classes) { log_debug(gc)("Requesting the VM to suspend all mutators..."); MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_suspended, true); log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); MMTkHeap::heap()->set_is_gc_active(true); + par_state_string.init(StringTable::weak_storage()); + mmtk_report_gc_start(); if (ClassUnloading && current_gc_should_unload_classes) { ClassLoaderDataGraph::clear_claimed_marks(); @@ -322,19 +343,6 @@ static bool mmtk_is_mutator(void* tls) { return ((Thread*) tls)->third_party_heap_collector == NULL; } -template -struct MaybeUninit { - MaybeUninit() {} - T* operator->() { - return (T*) &_data; - } - T& operator*() { - return *((T*) &_data); - } -private: - char _data[sizeof(T)]; -}; - static MaybeUninit jtiwh; static bool mutator_iteration_start = true; @@ -459,10 +467,10 @@ static void mmtk_scan_code_cache_roots(EdgesClosure closure) { MMTkRootsClosure< static void mmtk_scan_string_table_roots(EdgesClosure closure, bool rc_non_stuck_objs_only) { if (rc_non_stuck_objs_only) { MMTkRootsClosure cl(closure); - MMTkHeap::heap()->scan_string_table_roots(cl); + MMTkHeap::heap()->scan_string_table_roots(cl, &*par_state_string); } else { MMTkRootsClosure cl(closure); - MMTkHeap::heap()->scan_string_table_roots(cl); + MMTkHeap::heap()->scan_string_table_roots(cl, &*par_state_string); } } static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, EdgesClosure weak_closure, bool scank_all_strong_roots) { From 6c424cd464848edad641ad71f0bde68d40078e3a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 5 Jun 2023 23:05:53 -0700 Subject: [PATCH 332/494] immix field barrier --- mmtk/src/lib.rs | 12 ++++++++++++ openjdk/barriers/mmtkFieldBarrier.cpp | 3 ++- openjdk/mmtk.h | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 4ceac38b..84062939 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -10,6 +10,7 @@ use std::sync::atomic::{AtomicU32, AtomicUsize}; use std::sync::Mutex; use libc::{c_char, c_void, uintptr_t}; +use mmtk::plan::lxr::LXR; use mmtk::util::alloc::AllocationError; use mmtk::util::constants::{ BYTES_IN_ADDRESS, BYTES_IN_INT, LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_INT, @@ -179,6 +180,9 @@ pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = #[no_mangle] pub static mut CONCURRENT_MARKING_ACTIVE: u8 = 0; +#[no_mangle] +pub static mut RC_ENABLED: u8 = 0; + #[no_mangle] pub static mut HEAP_START: Address = Address::ZERO; @@ -505,6 +509,10 @@ lazy_static! { unsafe { HEAP_START = VM_LAYOUT_CONSTANTS.heap_start; HEAP_END = VM_LAYOUT_CONSTANTS.heap_end; + RC_ENABLED = ret + .get_plan() + .downcast_ref::>>() + .is_some() as _; } *ret }; @@ -516,6 +524,10 @@ lazy_static! { unsafe { HEAP_START = VM_LAYOUT_CONSTANTS.heap_start; HEAP_END = VM_LAYOUT_CONSTANTS.heap_end; + RC_ENABLED = ret + .get_plan() + .downcast_ref::>>() + .is_some() as _; } *ret }; diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index fcde6d5a..5dffe9e1 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -1,3 +1,4 @@ +#include "mmtk.h" #include "mmtkFieldBarrier.hpp" #include "runtime/interfaceSupport.inline.hpp" @@ -32,7 +33,7 @@ void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, } void MMTkFieldBarrierSetRuntime::object_probable_write(oop new_obj) const { - if (mmtk_get_rc((void*) new_obj) != 0) { + if (!RC_ENABLED || mmtk_get_rc((void*) new_obj) != 0) { ::mmtk_object_probable_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) new_obj); } } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 1043c919..0773e67b 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -25,6 +25,7 @@ extern const uintptr_t GLOBAL_ALLOC_BIT_ADDRESS; extern const int DISABLE_ALLOCATION_FAST_PATH; extern const uintptr_t IMMIX_ALLOCATOR_SIZE; extern uint8_t CONCURRENT_MARKING_ACTIVE; +extern uint8_t RC_ENABLED; inline bool disable_fast_alloc() { return DISABLE_ALLOCATION_FAST_PATH != 0; From c0623daa95a2457cb4d6f873a96162a9278e1af5 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 7 Jun 2023 12:59:54 -0700 Subject: [PATCH 333/494] Upgrade to rust 1.70 --- mmtk/rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/rust-toolchain b/mmtk/rust-toolchain index ee2f4ca9..832e9afb 100644 --- a/mmtk/rust-toolchain +++ b/mmtk/rust-toolchain @@ -1 +1 @@ -1.68.0 +1.70.0 From 3d96ce638ddc9dc8147b2764beae3122264ce5b6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 7 Jun 2023 14:38:16 -0700 Subject: [PATCH 334/494] Remove the second weak processor scan for RC pauses --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/gc_work.rs | 7 ++----- openjdk/mmtkHeap.cpp | 4 ++-- openjdk/mmtkHeap.hpp | 2 +- openjdk/mmtkParallelCleaning.cpp | 24 +++++++++++++++++++++--- openjdk/mmtkParallelCleaning.hpp | 13 +++++++++++-- openjdk/mmtkUpcalls.cpp | 10 ++++------ 8 files changed, 44 insertions(+), 22 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index e73022c3..28c0ed71 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=85334070f1bf9ad48329e6ae7f143e1eda2b4846#85334070f1bf9ad48329e6ae7f143e1eda2b4846" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e793dbcaa786710918778b1a40d951fecd739f39#e793dbcaa786710918778b1a40d951fecd739f39" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=85334070f1bf9ad48329e6ae7f143e1eda2b4846#85334070f1bf9ad48329e6ae7f143e1eda2b4846" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e793dbcaa786710918778b1a40d951fecd739f39#e793dbcaa786710918778b1a40d951fecd739f39" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 2da50f68..70152ece 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "85334070f1bf9ad48329e6ae7f143e1eda2b4846" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e793dbcaa786710918778b1a40d951fecd739f39" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index b7f00af8..391e5f40 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -5,7 +5,6 @@ use crate::EdgesClosure; use crate::NewBuffer; use crate::UPCALLS; use mmtk::scheduler::*; -use mmtk::util::ObjectReference; use mmtk::vm::RootsWorkFactory; use mmtk::vm::*; use mmtk::MMTK; @@ -316,14 +315,12 @@ extern "C" fn report_edges_and_renew_buffer_weakref NewBuffer { if !ptr.is_null() { - let buf = unsafe { - Vec::::from_raw_parts(ptr as *mut ObjectReference, length, capacity) - }; + let buf = unsafe { Vec::::from_raw_parts(ptr as *mut E, length, capacity) }; if cfg!(feature = "roots_breakdown") { record_roots(buf.len()); } let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; - factory.create_process_node_roots_work(buf, RootKind::Weak); + factory.create_process_edge_roots_work(buf, RootKind::Weak); } let (ptr, _, capacity) = { // TODO: Use Vec::into_raw_parts() when the method is available. diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 6d3462ea..daa01879 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -549,11 +549,11 @@ HeapWord* MMTkHeap::mem_allocate_nonmove(size_t size, bool* gc_overhead_limit_wa return alloc_fast(size << LogHeapWordSize, AllocatorLos); } -void MMTkHeap::complete_cleaning(BoolObjectClosure* is_alive, bool purged_classes) { +void MMTkHeap::complete_cleaning(BoolObjectClosure* is_alive, OopClosure* forward, bool purged_classes) { ResourceMark rm; HandleMark hm; uint num_workers = _workers->active_workers(); - mmtk::ParallelCleaningTask unlink_task(is_alive, num_workers, purged_classes); + mmtk::ParallelCleaningTask unlink_task(is_alive, forward, num_workers, purged_classes); _workers->run_task(&unlink_task); } /* diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index 44580b2f..6352162e 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -223,7 +223,7 @@ class MMTkHeap : public CollectedHeap { void scan_weak_processor_roots(OopClosure& cl); void scan_vm_thread_roots(OopClosure& cl); - void complete_cleaning(BoolObjectClosure* is_alive, bool class_unloading_occurred); + void complete_cleaning(BoolObjectClosure* is_alive, OopClosure* forward, bool class_unloading_occurred); jlong _last_gc_time; }; diff --git a/openjdk/mmtkParallelCleaning.cpp b/openjdk/mmtkParallelCleaning.cpp index 814112a5..eb9b4899 100644 --- a/openjdk/mmtkParallelCleaning.cpp +++ b/openjdk/mmtkParallelCleaning.cpp @@ -39,12 +39,15 @@ using namespace JavaClassFile; namespace mmtk { -StringSymbolTableUnlinkTask::StringSymbolTableUnlinkTask(BoolObjectClosure* is_alive) : +StringSymbolTableUnlinkTask::StringSymbolTableUnlinkTask(uint num_workers, BoolObjectClosure* is_alive, OopClosure* forward) : AbstractGangTask("String/Symbol Unlinking"), _is_alive(is_alive), + _forward(forward), + _par_state_string_fwd(StringTable::weak_storage()), _par_state_string(StringTable::weak_storage()), _strings_processed(0), _strings_removed(0), - _symbols_processed(0), _symbols_removed(0) { + _symbols_processed(0), _symbols_removed(0), + _num_workers(num_workers), _num_entered_barrier(0) { _initial_string_table_size = (int) StringTable::the_table()->table_size(); _initial_symbol_table_size = SymbolTable::the_table()->table_size(); @@ -79,6 +82,8 @@ void StringSymbolTableUnlinkTask::work(uint worker_id) { int symbols_processed = 0; int symbols_removed = 0; // if (_process_strings) { + StringTable::possibly_parallel_oops_do(&_par_state_string_fwd, _forward); + barrier_wait(worker_id); StringTable::possibly_parallel_unlink(&_par_state_string, _is_alive, &strings_processed, &strings_removed); Atomic::add(strings_processed, &_strings_processed); Atomic::add(strings_removed, &_strings_removed); @@ -96,7 +101,19 @@ size_t StringSymbolTableUnlinkTask::strings_removed() const { return (size_t)_ size_t StringSymbolTableUnlinkTask::symbols_processed() const { return (size_t)_symbols_processed; } size_t StringSymbolTableUnlinkTask::symbols_removed() const { return (size_t)_symbols_removed; } +void StringSymbolTableUnlinkTask::barrier_wait(uint worker_id) { + MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag); + _num_entered_barrier++; + if (_num_entered_barrier >= _num_workers) { + ml.notify_all(); + } + while (_num_entered_barrier < _num_workers) { + ml.wait(Mutex::_no_safepoint_check_flag, 0, false); + } +} + +Monitor* StringSymbolTableUnlinkTask::_lock = new Monitor(Mutex::leaf, "String Table Unload lock", false, Monitor::_safepoint_check_never); Monitor* CodeCacheUnloadingTask::_lock = new Monitor(Mutex::leaf, "Code Cache Unload lock", false, Monitor::_safepoint_check_never); CodeCacheUnloadingTask::CodeCacheUnloadingTask(uint num_workers, BoolObjectClosure* is_alive, bool unloading_occurred) : @@ -303,10 +320,11 @@ void ResolvedMethodCleaningTask::work() { } ParallelCleaningTask::ParallelCleaningTask(BoolObjectClosure* is_alive, + OopClosure* forward, uint num_workers, bool unloading_occurred) : AbstractGangTask("Parallel Cleaning"), - _string_symbol_task(is_alive), + _string_symbol_task(num_workers, is_alive, forward), _code_cache_task(num_workers, is_alive, unloading_occurred), _klass_cleaning_task(is_alive), _resolved_method_cleaning_task(is_alive) diff --git a/openjdk/mmtkParallelCleaning.hpp b/openjdk/mmtkParallelCleaning.hpp index c8b680eb..4cdeb6d8 100644 --- a/openjdk/mmtkParallelCleaning.hpp +++ b/openjdk/mmtkParallelCleaning.hpp @@ -32,7 +32,11 @@ namespace mmtk { class StringSymbolTableUnlinkTask : public AbstractGangTask { private: + static Monitor* _lock; + + OopClosure* _forward; BoolObjectClosure* _is_alive; + OopStorage::ParState _par_state_string_fwd; OopStorage::ParState _par_state_string; int _initial_string_table_size; int _initial_symbol_table_size; @@ -45,8 +49,11 @@ class StringSymbolTableUnlinkTask : public AbstractGangTask { volatile int _symbols_processed; volatile int _symbols_removed; + const uint _num_workers; + volatile uint _num_entered_barrier; + public: - StringSymbolTableUnlinkTask(BoolObjectClosure* is_alive); + StringSymbolTableUnlinkTask(uint num_workers, BoolObjectClosure* is_alive, OopClosure* forward); ~StringSymbolTableUnlinkTask(); void work(uint worker_id); @@ -56,6 +63,8 @@ class StringSymbolTableUnlinkTask : public AbstractGangTask { size_t symbols_processed() const; size_t symbols_removed() const; + + void barrier_wait(uint worker_id); }; class CodeCacheUnloadingTask { @@ -147,7 +156,7 @@ class ParallelCleaningTask : public AbstractGangTask { public: // The constructor is run in the VMThread. - ParallelCleaningTask(BoolObjectClosure* is_alive, uint num_workers, bool unloading_occurred); + ParallelCleaningTask(BoolObjectClosure* is_alive, OopClosure* forward, uint num_workers, bool unloading_occurred); // The parallel work done by all worker threads. void work(uint worker_id); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index eb6115c4..3c0a67ac 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -210,18 +210,16 @@ static void mmtk_update_weak_processor(bool lxr) { } else { MMTkIsAliveClosure is_alive; WeakProcessor::weak_oops_do(&is_alive, &forward); - StringTable::oops_do(&forward); } } static void mmtk_unload_classes() { - MMTkForwardClosure forward; - StringTable::oops_do(&forward); if (ClassUnloading) { // Unload classes and purge SystemDictionary. auto purged_classes = SystemDictionary::do_unloading(NULL, false /* Defer cleaning */); MMTkIsAliveClosure is_alive; - MMTkHeap::heap()->complete_cleaning(&is_alive, purged_classes); + MMTkForwardClosure forward; + MMTkHeap::heap()->complete_cleaning(&is_alive, &forward, purged_classes); ClassLoaderDataGraph::purge(); // Resize and verify metaspace MetaspaceGC::compute_new_size(); @@ -480,10 +478,10 @@ static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, EdgesC } static void mmtk_scan_weak_processor_roots(EdgesClosure closure, bool rc_non_stuck_objs_only) { if (rc_non_stuck_objs_only) { - MMTkCollectRootObjects cl(closure); + MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_weak_processor_roots(cl); } else { - MMTkCollectRootObjects cl(closure); + MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_weak_processor_roots(cl); } } From ae9f789bf3a59973136c28fa9755f82245d43b58 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 9 Jun 2023 14:49:13 -0700 Subject: [PATCH 335/494] weak handle barrier support --- mmtk/src/api.rs | 9 ++++ mmtk/src/gc_work.rs | 122 ++++--------------------------------------- mmtk/src/lib.rs | 6 +++ mmtk/src/scanning.rs | 11 +--- openjdk/mmtk.h | 3 ++ openjdk/mmtkHeap.cpp | 7 +++ openjdk/mmtkHeap.hpp | 1 + 7 files changed, 37 insertions(+), 122 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 8e775e4f..0247d69d 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -1,3 +1,4 @@ +use crate::abi::Oop; use crate::OpenJDK; use crate::OpenJDKEdge; use crate::OpenJDKEdgeRange; @@ -578,3 +579,11 @@ pub extern "C" fn mmtk_unregister_nmethod(nm: Address) { pub extern "C" fn mmtk_verbose() -> usize { with_singleton!(|singleton| *singleton.options.verbose) } + +#[no_mangle] +pub unsafe extern "C" fn mmtk_register_new_weak_handle(oop: *const Oop) { + crate::NURSERY_WEAK_HANDLE_ROOTS + .lock() + .unwrap() + .push(Address::from_ptr(oop)); +} diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 391e5f40..222fad58 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -157,44 +157,12 @@ impl> GCWork } } -extern "C" fn report_edges_and_renew_buffer_st>( - ptr: *mut Address, - length: usize, - capacity: usize, - factory_ptr: *mut libc::c_void, -) -> NewBuffer { - if !ptr.is_null() { - let ptr = ptr as *mut E; - let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; - if cfg!(feature = "roots_breakdown") { - record_roots(buf.len()); - } - let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; - factory.create_process_edge_roots_work(buf, RootKind::Weak); - } - let (ptr, _, capacity) = { - // TODO: Use Vec::into_raw_parts() when the method is available. - use std::mem::ManuallyDrop; - let new_vec = Vec::with_capacity(F::BUFFER_SIZE); - let mut me = ManuallyDrop::new(new_vec); - (me.as_mut_ptr(), me.len(), me.capacity()) - }; - NewBuffer { ptr, capacity } -} - -fn to_edges_closure_st>(factory: &mut F) -> EdgesClosure { - EdgesClosure { - func: report_edges_and_renew_buffer_st::, - data: factory as *mut F as *mut libc::c_void, - } -} - -pub struct ScanStringTableRoots> { +pub struct ScanNewWeakHandleRoots> { factory: F, _p: PhantomData, } -impl> ScanStringTableRoots { +impl> ScanNewWeakHandleRoots { pub fn new(factory: F) -> Self { Self { factory, @@ -204,22 +172,21 @@ impl> ScanStringTableRoots { } impl> GCWork - for ScanStringTableRoots + for ScanNewWeakHandleRoots { - fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { let t = if cfg!(feature = "roots_breakdown") { Some(std::time::SystemTime::now()) } else { None }; - unsafe { - ((*UPCALLS).scan_string_table_roots)( - to_edges_closure_st::(&mut self.factory), - mmtk.get_plan() - .downcast_ref::>() - .is_some(), - ); + let mut new_roots = crate::NURSERY_WEAK_HANDLE_ROOTS.lock().unwrap(); + for slice in new_roots.chunks(mmtk::args::BUFFER_SIZE) { + let slice = unsafe { std::mem::transmute::<&[Address], &[VM::VMEdge]>(slice) }; + self.factory + .create_process_edge_roots_work(slice.to_vec(), RootKind::Weak); } + new_roots.clear(); if cfg!(feature = "roots_breakdown") { let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; report_roots("StringTable", ms); @@ -307,72 +274,3 @@ impl> GCWork } } } - -extern "C" fn report_edges_and_renew_buffer_weakref>( - ptr: *mut Address, - length: usize, - capacity: usize, - factory_ptr: *mut libc::c_void, -) -> NewBuffer { - if !ptr.is_null() { - let buf = unsafe { Vec::::from_raw_parts(ptr as *mut E, length, capacity) }; - if cfg!(feature = "roots_breakdown") { - record_roots(buf.len()); - } - let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; - factory.create_process_edge_roots_work(buf, RootKind::Weak); - } - let (ptr, _, capacity) = { - // TODO: Use Vec::into_raw_parts() when the method is available. - use std::mem::ManuallyDrop; - let new_vec = Vec::with_capacity(F::BUFFER_SIZE); - let mut me = ManuallyDrop::new(new_vec); - (me.as_mut_ptr(), me.len(), me.capacity()) - }; - NewBuffer { ptr, capacity } -} - -fn to_edges_closure_weakref>(factory: &mut F) -> EdgesClosure { - EdgesClosure { - func: report_edges_and_renew_buffer_weakref::, - data: factory as *mut F as *mut libc::c_void, - } -} - -pub struct ScaWeakProcessorRoots> { - factory: F, - _p: PhantomData, -} - -impl> ScaWeakProcessorRoots { - pub fn new(factory: F) -> Self { - Self { - factory, - _p: PhantomData, - } - } -} - -impl> GCWork - for ScaWeakProcessorRoots -{ - fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { - let t = if cfg!(feature = "roots_breakdown") { - Some(std::time::SystemTime::now()) - } else { - None - }; - unsafe { - ((*UPCALLS).scan_weak_processor_roots)( - to_edges_closure_weakref::<_, _>(&mut self.factory), - mmtk.get_plan() - .downcast_ref::>() - .is_some(), - ); - } - if cfg!(feature = "roots_breakdown") { - let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; - report_roots("WeakProcessor", ms); - } - } -} diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 84062939..60e25540 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -183,6 +183,9 @@ pub static mut CONCURRENT_MARKING_ACTIVE: u8 = 0; #[no_mangle] pub static mut RC_ENABLED: u8 = 0; +#[no_mangle] +pub static mut REQUIRES_WEAK_HANDLE_BARRIER: u8 = 0; + #[no_mangle] pub static mut HEAP_START: Address = Address::ZERO; @@ -513,6 +516,7 @@ lazy_static! { .get_plan() .downcast_ref::>>() .is_some() as _; + REQUIRES_WEAK_HANDLE_BARRIER = RC_ENABLED; } *ret }; @@ -528,6 +532,7 @@ lazy_static! { .get_plan() .downcast_ref::>>() .is_some() as _; + REQUIRES_WEAK_HANDLE_BARRIER = RC_ENABLED; } *ret }; @@ -555,6 +560,7 @@ lazy_static! { /// A global storage for all the cached CodeCache root pointers static ref NURSERY_CODE_CACHE_ROOTS: Mutex>> = Mutex::new(HashMap::new()); static ref MATURE_CODE_CACHE_ROOTS: Mutex>> = Mutex::new(HashMap::new()); + static ref NURSERY_WEAK_HANDLE_ROOTS: Mutex> = Mutex::new(Vec::new()); } lazy_static! { diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 589e7adf..acb42248 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -127,16 +127,7 @@ impl Scanning> for VMScanning { .get_plan() .requires_weak_root_scanning() { - const PARALLEL_STRING_TABLE_SCANNING: bool = true; - let num_string_table_packets: usize = if PARALLEL_STRING_TABLE_SCANNING { - crate::singleton::().scheduler.num_workers() - } else { - 1 - }; - for _ in 0..num_string_table_packets { - w.push(Box::new(ScanStringTableRoots::new(factory.clone())) as _); - } - w.push(Box::new(ScaWeakProcessorRoots::new(factory.clone())) as _); + w.push(Box::new(ScanNewWeakHandleRoots::new(factory.clone())) as _); } memory_manager::add_work_packets( &crate::singleton::(), diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 0773e67b..a8ae5e15 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -26,6 +26,7 @@ extern const int DISABLE_ALLOCATION_FAST_PATH; extern const uintptr_t IMMIX_ALLOCATOR_SIZE; extern uint8_t CONCURRENT_MARKING_ACTIVE; extern uint8_t RC_ENABLED; +extern uint8_t REQUIRES_WEAK_HANDLE_BARRIER; inline bool disable_fast_alloc() { return DISABLE_ALLOCATION_FAST_PATH != 0; @@ -133,6 +134,8 @@ extern size_t mmtk_add_nmethod_oop(void* object); extern size_t mmtk_register_nmethod(void* nm); extern size_t mmtk_unregister_nmethod(void* nm); +extern size_t mmtk_register_new_weak_handle(void* entry); + extern size_t mmtk_verbose(); /** diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index daa01879..b6550e12 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -556,6 +556,13 @@ void MMTkHeap::complete_cleaning(BoolObjectClosure* is_alive, OopClosure* forwar mmtk::ParallelCleaningTask unlink_task(is_alive, forward, num_workers, purged_classes); _workers->run_task(&unlink_task); } + +void MMTkHeap::register_new_weak_handle(oop* handle) { + if (REQUIRES_WEAK_HANDLE_BARRIER) { + mmtk_register_new_weak_handle((void*) handle); + } +} + /* * files with prints currently: * collectedHeap.inline.hpp, mmtkHeap.cpp, diff --git a/openjdk/mmtkHeap.hpp b/openjdk/mmtkHeap.hpp index 6352162e..e5d1f5df 100644 --- a/openjdk/mmtkHeap.hpp +++ b/openjdk/mmtkHeap.hpp @@ -170,6 +170,7 @@ class MMTkHeap : public CollectedHeap { virtual void prepare_for_verify() override; + virtual void register_new_weak_handle(oop* handle) override; private: From b44ccfc2b39ec9142a0f75c46d50f97b4685b0c0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 10 Jun 2023 13:30:55 -0700 Subject: [PATCH 336/494] minor --- mmtk/src/api.rs | 10 ++++++---- mmtk/src/gc_work.rs | 5 ++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 0247d69d..50ad4c5f 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -582,8 +582,10 @@ pub extern "C" fn mmtk_verbose() -> usize { #[no_mangle] pub unsafe extern "C" fn mmtk_register_new_weak_handle(oop: *const Oop) { - crate::NURSERY_WEAK_HANDLE_ROOTS - .lock() - .unwrap() - .push(Address::from_ptr(oop)); + let addr = if crate::use_compressed_oops() { + Address::from_usize(oop as usize | (1 << 63)) + } else { + Address::from_ptr(oop) + }; + crate::NURSERY_WEAK_HANDLE_ROOTS.lock().unwrap().push(addr); } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 222fad58..acdd2b69 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -181,6 +181,9 @@ impl> GCWork None }; let mut new_roots = crate::NURSERY_WEAK_HANDLE_ROOTS.lock().unwrap(); + if cfg!(feature = "roots_breakdown") { + record_roots(new_roots.len()); + } for slice in new_roots.chunks(mmtk::args::BUFFER_SIZE) { let slice = unsafe { std::mem::transmute::<&[Address], &[VM::VMEdge]>(slice) }; self.factory @@ -189,7 +192,7 @@ impl> GCWork new_roots.clear(); if cfg!(feature = "roots_breakdown") { let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; - report_roots("StringTable", ms); + report_roots("NewWeakHandleRoots", ms); } } } From 9c6f98b7ed6cadd09f251db70aa6706bb1d87607 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 11 Jun 2023 15:13:22 -0700 Subject: [PATCH 337/494] support bulk thread scanning --- mmtk/src/lib.rs | 2 ++ mmtk/src/scanning.rs | 16 ++++++++++++++++ openjdk/mmtk.h | 1 + openjdk/mmtkUpcalls.cpp | 9 +++++++++ 4 files changed, 28 insertions(+) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 60e25540..3d715b42 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -106,6 +106,8 @@ pub struct OpenJDK_Upcalls { pub dump_object_string: extern "C" fn(object: ObjectReference) -> *const c_char, pub scan_all_thread_roots: extern "C" fn(closure: EdgesClosure), pub scan_thread_roots: extern "C" fn(closure: EdgesClosure, tls: VMMutatorThread), + pub scan_multiple_thread_roots: + extern "C" fn(closure: EdgesClosure, ptr: OpaquePointer, len: usize), pub scan_universe_roots: extern "C" fn(closure: EdgesClosure), pub scan_jni_handle_roots: extern "C" fn(closure: EdgesClosure), pub scan_object_synchronizer_roots: extern "C" fn(closure: EdgesClosure), diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index acb42248..c5939756 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -108,6 +108,22 @@ impl Scanning> for VMScanning { } } + fn scan_multiple_thread_root( + _tls: VMWorkerThread, + mutators: Vec, + mut factory: impl RootsWorkFactory< as mmtk::vm::VMBinding>::VMEdge>, + ) { + let len = mutators.len(); + let ptr = mutators.as_ptr(); + unsafe { + ((*UPCALLS).scan_multiple_thread_roots)( + to_edges_closure(&mut factory), + std::mem::transmute(ptr), + len, + ); + } + } + fn scan_vm_specific_roots( _tls: VMWorkerThread, factory: impl RootsWorkFactory>, diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index a8ae5e15..591240a2 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -193,6 +193,7 @@ typedef struct { const char* (*dump_object_string) (void* object); void (*scan_all_thread_roots)(EdgesClosure closure); void (*scan_thread_roots)(EdgesClosure closure, void* tls); + void (*scan_multiple_thread_roots)(EdgesClosure closure, void* ptr, size_t len); void (*scan_universe_roots) (EdgesClosure closure); void (*scan_jni_handle_roots) (EdgesClosure closure); void (*scan_object_synchronizer_roots) (EdgesClosure closure); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 3c0a67ac..de9fb103 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -374,6 +374,14 @@ static void mmtk_scan_thread_roots(EdgesClosure closure, void* tls) { thread->oops_do(&cl, NULL); } +static void mmtk_scan_multiple_thread_roots(EdgesClosure closure, void* ptr, size_t len) { + ResourceMark rm; + auto mutators = (JavaThread**) ptr; + MMTkRootsClosure<> cl(closure); + for (auto i = 0; i < len; i++) + mutators[i]->oops_do(&cl, NULL); +} + static void mmtk_scan_object(void* trace, void* object, void* tls, bool follow_clds, bool claim_clds) { MMTkScanObjectClosure cl(trace, follow_clds, claim_clds); ((oop) object)->oop_iterate(&cl); @@ -570,6 +578,7 @@ OpenJDK_Upcalls mmtk_upcalls = { dump_object_string, mmtk_scan_all_thread_roots, mmtk_scan_thread_roots, + mmtk_scan_multiple_thread_roots, mmtk_scan_universe_roots, mmtk_scan_jni_handle_roots, mmtk_scan_object_synchronizer_roots, From e1449f841dd03a2c0c7cde3285a00e4d6f752ef1 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 11 Jun 2023 23:25:49 -0700 Subject: [PATCH 338/494] Revert bulk thread scan --- mmtk/src/lib.rs | 2 -- mmtk/src/scanning.rs | 16 ---------------- openjdk/mmtk.h | 1 - openjdk/mmtkUpcalls.cpp | 9 --------- 4 files changed, 28 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 3d715b42..60e25540 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -106,8 +106,6 @@ pub struct OpenJDK_Upcalls { pub dump_object_string: extern "C" fn(object: ObjectReference) -> *const c_char, pub scan_all_thread_roots: extern "C" fn(closure: EdgesClosure), pub scan_thread_roots: extern "C" fn(closure: EdgesClosure, tls: VMMutatorThread), - pub scan_multiple_thread_roots: - extern "C" fn(closure: EdgesClosure, ptr: OpaquePointer, len: usize), pub scan_universe_roots: extern "C" fn(closure: EdgesClosure), pub scan_jni_handle_roots: extern "C" fn(closure: EdgesClosure), pub scan_object_synchronizer_roots: extern "C" fn(closure: EdgesClosure), diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index c5939756..acb42248 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -108,22 +108,6 @@ impl Scanning> for VMScanning { } } - fn scan_multiple_thread_root( - _tls: VMWorkerThread, - mutators: Vec, - mut factory: impl RootsWorkFactory< as mmtk::vm::VMBinding>::VMEdge>, - ) { - let len = mutators.len(); - let ptr = mutators.as_ptr(); - unsafe { - ((*UPCALLS).scan_multiple_thread_roots)( - to_edges_closure(&mut factory), - std::mem::transmute(ptr), - len, - ); - } - } - fn scan_vm_specific_roots( _tls: VMWorkerThread, factory: impl RootsWorkFactory>, diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 591240a2..a8ae5e15 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -193,7 +193,6 @@ typedef struct { const char* (*dump_object_string) (void* object); void (*scan_all_thread_roots)(EdgesClosure closure); void (*scan_thread_roots)(EdgesClosure closure, void* tls); - void (*scan_multiple_thread_roots)(EdgesClosure closure, void* ptr, size_t len); void (*scan_universe_roots) (EdgesClosure closure); void (*scan_jni_handle_roots) (EdgesClosure closure); void (*scan_object_synchronizer_roots) (EdgesClosure closure); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index de9fb103..3c0a67ac 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -374,14 +374,6 @@ static void mmtk_scan_thread_roots(EdgesClosure closure, void* tls) { thread->oops_do(&cl, NULL); } -static void mmtk_scan_multiple_thread_roots(EdgesClosure closure, void* ptr, size_t len) { - ResourceMark rm; - auto mutators = (JavaThread**) ptr; - MMTkRootsClosure<> cl(closure); - for (auto i = 0; i < len; i++) - mutators[i]->oops_do(&cl, NULL); -} - static void mmtk_scan_object(void* trace, void* object, void* tls, bool follow_clds, bool claim_clds) { MMTkScanObjectClosure cl(trace, follow_clds, claim_clds); ((oop) object)->oop_iterate(&cl); @@ -578,7 +570,6 @@ OpenJDK_Upcalls mmtk_upcalls = { dump_object_string, mmtk_scan_all_thread_roots, mmtk_scan_thread_roots, - mmtk_scan_multiple_thread_roots, mmtk_scan_universe_roots, mmtk_scan_jni_handle_roots, mmtk_scan_object_synchronizer_roots, From 53f36d18c2c7f481645137b31f2d7b05d8779a50 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 12 Jun 2023 14:30:27 -0700 Subject: [PATCH 339/494] Bulk thread scan --- mmtk/src/lib.rs | 2 ++ mmtk/src/scanning.rs | 16 ++++++++++++++++ openjdk/mmtk.h | 1 + openjdk/mmtkUpcalls.cpp | 9 +++++++++ 4 files changed, 28 insertions(+) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 60e25540..3d715b42 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -106,6 +106,8 @@ pub struct OpenJDK_Upcalls { pub dump_object_string: extern "C" fn(object: ObjectReference) -> *const c_char, pub scan_all_thread_roots: extern "C" fn(closure: EdgesClosure), pub scan_thread_roots: extern "C" fn(closure: EdgesClosure, tls: VMMutatorThread), + pub scan_multiple_thread_roots: + extern "C" fn(closure: EdgesClosure, ptr: OpaquePointer, len: usize), pub scan_universe_roots: extern "C" fn(closure: EdgesClosure), pub scan_jni_handle_roots: extern "C" fn(closure: EdgesClosure), pub scan_object_synchronizer_roots: extern "C" fn(closure: EdgesClosure), diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index acb42248..c5939756 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -108,6 +108,22 @@ impl Scanning> for VMScanning { } } + fn scan_multiple_thread_root( + _tls: VMWorkerThread, + mutators: Vec, + mut factory: impl RootsWorkFactory< as mmtk::vm::VMBinding>::VMEdge>, + ) { + let len = mutators.len(); + let ptr = mutators.as_ptr(); + unsafe { + ((*UPCALLS).scan_multiple_thread_roots)( + to_edges_closure(&mut factory), + std::mem::transmute(ptr), + len, + ); + } + } + fn scan_vm_specific_roots( _tls: VMWorkerThread, factory: impl RootsWorkFactory>, diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index a8ae5e15..591240a2 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -193,6 +193,7 @@ typedef struct { const char* (*dump_object_string) (void* object); void (*scan_all_thread_roots)(EdgesClosure closure); void (*scan_thread_roots)(EdgesClosure closure, void* tls); + void (*scan_multiple_thread_roots)(EdgesClosure closure, void* ptr, size_t len); void (*scan_universe_roots) (EdgesClosure closure); void (*scan_jni_handle_roots) (EdgesClosure closure); void (*scan_object_synchronizer_roots) (EdgesClosure closure); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 3c0a67ac..de9fb103 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -374,6 +374,14 @@ static void mmtk_scan_thread_roots(EdgesClosure closure, void* tls) { thread->oops_do(&cl, NULL); } +static void mmtk_scan_multiple_thread_roots(EdgesClosure closure, void* ptr, size_t len) { + ResourceMark rm; + auto mutators = (JavaThread**) ptr; + MMTkRootsClosure<> cl(closure); + for (auto i = 0; i < len; i++) + mutators[i]->oops_do(&cl, NULL); +} + static void mmtk_scan_object(void* trace, void* object, void* tls, bool follow_clds, bool claim_clds) { MMTkScanObjectClosure cl(trace, follow_clds, claim_clds); ((oop) object)->oop_iterate(&cl); @@ -570,6 +578,7 @@ OpenJDK_Upcalls mmtk_upcalls = { dump_object_string, mmtk_scan_all_thread_roots, mmtk_scan_thread_roots, + mmtk_scan_multiple_thread_roots, mmtk_scan_universe_roots, mmtk_scan_jni_handle_roots, mmtk_scan_object_synchronizer_roots, From eeb5455ab7869802d4f504fb12dce95bc64370c1 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 12 Jun 2023 14:31:11 -0700 Subject: [PATCH 340/494] 8-Byte Mark Bit Coverage --- openjdk/mmtk.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 591240a2..9e107c57 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -254,22 +254,24 @@ extern void add_phantom_candidate(void* ref, void* referent); extern void mmtk_harness_begin_impl(); extern void mmtk_harness_end_impl(); +constexpr size_t log_min_obj_size = 3; + inline uint8_t mmtk_get_rc_2bits(void* o) { - const uintptr_t index = uintptr_t((void*) o) >> 4; + const uintptr_t index = uintptr_t((void*) o) >> log_min_obj_size; const uint8_t byte = *((uint8_t*) (RC_TABLE_BASE_ADDRESS + (index >> 2))); auto v = byte >> ((index & 0b11) << 1); return v & 0b11; } inline uint8_t mmtk_get_rc_4bits(void* o) { - const uintptr_t index = uintptr_t((void*) o) >> 4; + const uintptr_t index = uintptr_t((void*) o) >> log_min_obj_size; const uint8_t byte = *((uint8_t*) (RC_TABLE_BASE_ADDRESS + (index >> 1))); auto v = byte >> ((index & 0b1) << 2); return v & 0b1111; } inline uint8_t mmtk_get_rc_8bits(void* o) { - const uintptr_t index = uintptr_t((void*) o) >> 4; + const uintptr_t index = uintptr_t((void*) o) >> log_min_obj_size; const uint8_t byte = *((uint8_t*) (RC_TABLE_BASE_ADDRESS + index)); return byte; } From b521ade2ebf91e710c8c565bf5b5f5e2d59c7cfc Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 12 Jun 2023 14:31:54 -0700 Subject: [PATCH 341/494] update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 28c0ed71..33ef35bc 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e793dbcaa786710918778b1a40d951fecd739f39#e793dbcaa786710918778b1a40d951fecd739f39" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=4fe32e0742fcef31122d425527685e8441d61f06#4fe32e0742fcef31122d425527685e8441d61f06" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e793dbcaa786710918778b1a40d951fecd739f39#e793dbcaa786710918778b1a40d951fecd739f39" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=4fe32e0742fcef31122d425527685e8441d61f06#4fe32e0742fcef31122d425527685e8441d61f06" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 70152ece..18817c2c 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e793dbcaa786710918778b1a40d951fecd739f39" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "4fe32e0742fcef31122d425527685e8441d61f06" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From abec59ba921199107dadc1829e3896195a9d9e5f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 13 Jun 2023 03:57:53 +0000 Subject: [PATCH 342/494] update openjdk --- mmtk/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 18817c2c..bcc4deef 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -26,7 +26,7 @@ codegen-units = 1 [package.metadata.openjdk] # Our CI matches the following line and extract mmtk/openjdk. If this line is updated, please check ci yaml files and make sure it works. openjdk_repo = "https://github.com/mmtk/openjdk.git" -openjdk_version = "91259ca9d6062020b7c4bc28006c1c6944495173" +openjdk_version = "7caf8f7d19b19a9fd53be6f909db805246790807" [dependencies] libc = "0.2.138" From cd6e40f529533c3ffc212ea06419233ef10bfff8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 15 Jun 2023 00:18:34 -0700 Subject: [PATCH 343/494] stack root scanning timer --- mmtk/src/scanning.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index c5939756..e8f70c3e 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -113,6 +113,11 @@ impl Scanning> for VMScanning { mutators: Vec, mut factory: impl RootsWorkFactory< as mmtk::vm::VMBinding>::VMEdge>, ) { + let t = if cfg!(feature = "roots_breakdown") { + Some(std::time::SystemTime::now()) + } else { + None + }; let len = mutators.len(); let ptr = mutators.as_ptr(); unsafe { @@ -122,6 +127,10 @@ impl Scanning> for VMScanning { len, ); } + if cfg!(feature = "roots_breakdown") { + let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; + eprintln!(" - ScanThreadRoots ({:.3}ms)", ms); + } } fn scan_vm_specific_roots( From 848974d7f23e598b6b0a716ae5198995efeebbf2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 27 Jun 2023 13:57:27 -0700 Subject: [PATCH 344/494] Disable stack root timer --- mmtk/src/scanning.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index e8f70c3e..dda78e8a 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -113,11 +113,11 @@ impl Scanning> for VMScanning { mutators: Vec, mut factory: impl RootsWorkFactory< as mmtk::vm::VMBinding>::VMEdge>, ) { - let t = if cfg!(feature = "roots_breakdown") { - Some(std::time::SystemTime::now()) - } else { - None - }; + // let t = if cfg!(feature = "roots_breakdown") { + // Some(std::time::SystemTime::now()) + // } else { + // None + // }; let len = mutators.len(); let ptr = mutators.as_ptr(); unsafe { @@ -127,10 +127,10 @@ impl Scanning> for VMScanning { len, ); } - if cfg!(feature = "roots_breakdown") { - let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; - eprintln!(" - ScanThreadRoots ({:.3}ms)", ms); - } + // if cfg!(feature = "roots_breakdown") { + // let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; + // eprintln!(" - ScanThreadRoots ({:.3}ms)", ms); + // } } fn scan_vm_specific_roots( From cceda2b6f9816332af533ee6daf18625a99cbc54 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 28 Jun 2023 17:44:40 -0700 Subject: [PATCH 345/494] Report weak CLD roots to remset --- openjdk/mmtkHeap.cpp | 5 +++-- openjdk/mmtkUpcalls.cpp | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index b6550e12..810da35a 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -455,12 +455,13 @@ void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& we } else if (scan_all_strong_roots) { // At the start of full heap trace, we want to scan all the strong CLD roots + all the modified CLDs. MMTkScanCLDClosure cld_cl(&cl); - MMTkScanCLDClosure weak_cld_cl(&cl); + MMTkScanCLDClosure weak_cld_cl(&weak_cl); ClassLoaderDataGraph::roots_cld_do(&cld_cl, &weak_cld_cl); } else { // Normal RC pause: We simply apply increments to modified CLD roots. No decrements will be applied on these roots. MMTkScanCLDClosure cld_cl(&cl); - ClassLoaderDataGraph::cld_do(&cld_cl); + MMTkScanCLDClosure weak_cld_cl(&weak_cl); + ClassLoaderDataGraph::roots_cld_do(&cld_cl, &weak_cld_cl); } } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index de9fb103..4ccdd636 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -378,7 +378,7 @@ static void mmtk_scan_multiple_thread_roots(EdgesClosure closure, void* ptr, siz ResourceMark rm; auto mutators = (JavaThread**) ptr; MMTkRootsClosure<> cl(closure); - for (auto i = 0; i < len; i++) + for (size_t i = 0; i < len; i++) mutators[i]->oops_do(&cl, NULL); } @@ -479,10 +479,10 @@ static void mmtk_scan_string_table_roots(EdgesClosure closure, bool rc_non_stuck MMTkHeap::heap()->scan_string_table_roots(cl, &*par_state_string); } } -static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, EdgesClosure weak_closure, bool scank_all_strong_roots) { - MMTkRootsClosure<> cl(closure); - MMTkRootsClosure<> weak_cl(weak_closure); - MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl, weak_cl, scank_all_strong_roots); +static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, EdgesClosure weak_closure, bool scan_all_strong_roots) { + MMTkRootsClosure cl(closure); + MMTkRootsClosure weak_cl(weak_closure); + MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl, weak_cl, scan_all_strong_roots); } static void mmtk_scan_weak_processor_roots(EdgesClosure closure, bool rc_non_stuck_objs_only) { if (rc_non_stuck_objs_only) { From 10acf7bb974bf848bfc6c4a02a0a59bc153bfaa4 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 28 Jun 2023 18:07:21 -0700 Subject: [PATCH 346/494] minor --- openjdk/mmtkHeap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 810da35a..5a033653 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -455,7 +455,7 @@ void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& we } else if (scan_all_strong_roots) { // At the start of full heap trace, we want to scan all the strong CLD roots + all the modified CLDs. MMTkScanCLDClosure cld_cl(&cl); - MMTkScanCLDClosure weak_cld_cl(&weak_cl); + MMTkScanCLDClosure weak_cld_cl(&weak_cl); ClassLoaderDataGraph::roots_cld_do(&cld_cl, &weak_cld_cl); } else { // Normal RC pause: We simply apply increments to modified CLD roots. No decrements will be applied on these roots. From 5bf81d643da0722fd7a9cca0cd0635165cf231fb Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 29 Jun 2023 10:30:18 -0700 Subject: [PATCH 347/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 33ef35bc..8a81a543 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=4fe32e0742fcef31122d425527685e8441d61f06#4fe32e0742fcef31122d425527685e8441d61f06" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=963b3d755f9ca9650e800b50a07652ea7b54d1b1#963b3d755f9ca9650e800b50a07652ea7b54d1b1" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=4fe32e0742fcef31122d425527685e8441d61f06#4fe32e0742fcef31122d425527685e8441d61f06" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=963b3d755f9ca9650e800b50a07652ea7b54d1b1#963b3d755f9ca9650e800b50a07652ea7b54d1b1" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index bcc4deef..dd7887fb 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "4fe32e0742fcef31122d425527685e8441d61f06" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "963b3d755f9ca9650e800b50a07652ea7b54d1b1" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 98eb6f9fc346aaf7814fa71a9f64cfd26d25c83b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 29 Jun 2023 14:20:38 -0700 Subject: [PATCH 348/494] Weak handle load barrier --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- openjdk/barriers/mmtkFieldBarrier.cpp | 2 +- openjdk/mmtkBarrierSet.hpp | 12 ++++++++++++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 8a81a543..2db34610 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=963b3d755f9ca9650e800b50a07652ea7b54d1b1#963b3d755f9ca9650e800b50a07652ea7b54d1b1" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ca2d0817b07ebd1f4c9c5846c5f4bd2666fef9b0#ca2d0817b07ebd1f4c9c5846c5f4bd2666fef9b0" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=963b3d755f9ca9650e800b50a07652ea7b54d1b1#963b3d755f9ca9650e800b50a07652ea7b54d1b1" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ca2d0817b07ebd1f4c9c5846c5f4bd2666fef9b0#ca2d0817b07ebd1f4c9c5846c5f4bd2666fef9b0" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index dd7887fb..a56f4a1e 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "963b3d755f9ca9650e800b50a07652ea7b54d1b1" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "ca2d0817b07ebd1f4c9c5846c5f4bd2666fef9b0" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 5dffe9e1..ed02b587 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -13,7 +13,7 @@ static inline intptr_t side_metadata_base_address() { void MMTkFieldBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const { #if SOFT_REFERENCE_LOAD_BARRIER - if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) + if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL && mmtk_get_rc((void*) value) != 0) ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); #endif }; diff --git a/openjdk/mmtkBarrierSet.hpp b/openjdk/mmtkBarrierSet.hpp index 58089a58..33e5da3e 100644 --- a/openjdk/mmtkBarrierSet.hpp +++ b/openjdk/mmtkBarrierSet.hpp @@ -199,6 +199,18 @@ class MMTkBarrierSet : public BarrierSet { } return value; } + + template + static oop oop_load_not_in_heap(T* addr) { + oop value = Raw::template oop_load(addr); + const bool on_strong_oop_ref = (decorators & ON_STRONG_OOP_REF) != 0; + const bool peek = (decorators & AS_NO_KEEPALIVE) != 0; + const bool needs_enqueue = (!peek && !on_strong_oop_ref); + if (needs_enqueue && value != NULL) { + runtime()->load_reference(decorators, value); + } + return value; + } // Defensive: will catch weak oops at addresses in heap template From e468630d0f4c8a0ba727ed40c0e80b377ed997b9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 2 Aug 2023 02:26:03 +0000 Subject: [PATCH 349/494] Fix build error --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 6 +++--- mmtk/src/lib.rs | 4 ++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 59be1ce5..37f43c40 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -450,7 +450,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7c0b28bb4da66b307e60382c37a32d82863b931b#7c0b28bb4da66b307e60382c37a32d82863b931b" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=74c557df2809bac28baf514e248f4e967688ebbc#74c557df2809bac28baf514e248f4e967688ebbc" dependencies = [ "atomic", "atomic-traits", @@ -480,7 +480,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7c0b28bb4da66b307e60382c37a32d82863b931b#7c0b28bb4da66b307e60382c37a32d82863b931b" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=74c557df2809bac28baf514e248f4e967688ebbc#74c557df2809bac28baf514e248f4e967688ebbc" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 30105f15..e03a27f9 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -31,7 +31,7 @@ openjdk_version = "7caf8f7d19b19a9fd53be6f909db805246790807" [dependencies] libc = "0.2.138" lazy_static = "1.4.0" -spin = {version = "0.9.4", features = ["std"]} +spin = { version = "0.9.4", features = ["std"] } once_cell = "1.16.0" atomic = "0.5.1" # Be very careful to commit any changes to the following mmtk dependency, as our CI scripts (including mmtk-core CI) @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7c0b28bb4da66b307e60382c37a32d82863b931b" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "74c557df2809bac28baf514e248f4e967688ebbc" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } @@ -84,4 +84,4 @@ instrumentation = ["mmtk/instrumentation", "no_fast_alloc"] log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] log_work_packets = ["mmtk/log_work_packets"] -object_size_distribution = ["mmtk/object_size_distribution"] \ No newline at end of file +object_size_distribution = ["mmtk/object_size_distribution"] diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 00bb4f45..598c9f4b 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -442,6 +442,10 @@ impl MemorySlice for OpenJDKEdgeRange { } } + fn object(&self) -> Option { + None + } + fn start(&self) -> Address { self.start.0 } From 526e043c1c76ed2fca5ddd72ff7db59647b3124e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 2 Aug 2023 04:52:41 +0000 Subject: [PATCH 350/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 6ca2e8e8..5bb422b7 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -448,7 +448,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=9ea00d711d54bf70746d6ef0b20ffd90db2d642a#9ea00d711d54bf70746d6ef0b20ffd90db2d642a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6a12cb8b7b7d776adcd9900b3610aebd40953fcf#6a12cb8b7b7d776adcd9900b3610aebd40953fcf" dependencies = [ "atomic", "atomic-traits", @@ -479,7 +479,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=9ea00d711d54bf70746d6ef0b20ffd90db2d642a#9ea00d711d54bf70746d6ef0b20ffd90db2d642a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6a12cb8b7b7d776adcd9900b3610aebd40953fcf#6a12cb8b7b7d776adcd9900b3610aebd40953fcf" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index ba6250ef..86eb67a4 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "9ea00d711d54bf70746d6ef0b20ffd90db2d642a" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "6a12cb8b7b7d776adcd9900b3610aebd40953fcf" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 9e6214a6a7178d8276011c67f2f6ae64eaee608b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 3 Aug 2023 07:17:57 +0000 Subject: [PATCH 351/494] Update mmtk-core --- mmtk/Cargo.lock | 174 ++++++++++++++++++++++++++++++++++++++++++++++-- mmtk/Cargo.toml | 2 +- 2 files changed, 168 insertions(+), 8 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index a16aa208..7a9d8e9c 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -88,7 +88,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b9c056b9ed43aee5e064b683aa1ec783e19c6acec7559e3ae931b7490472fbe" dependencies = [ "cargo-lock", - "git2", + "git2 0.15.0", +] + +[[package]] +name = "built" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96f9cdd34d6eb553f9ea20e5bf84abb7b13c729f113fc1d8e49dc00ad9fa8738" +dependencies = [ + "cargo-lock", + "git2 0.16.0", ] [[package]] @@ -267,6 +277,27 @@ dependencies = [ "termcolor", ] +[[package]] +name = "errno" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -289,6 +320,19 @@ dependencies = [ "url", ] +[[package]] +name = "git2" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be36bc9e0546df253c0cc41fd0af34f5e92845ad8509462ec76672fac6997f5b" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "url", +] + [[package]] name = "glob" version = "0.3.1" @@ -319,6 +363,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + [[package]] name = "humantime" version = "2.1.0" @@ -335,6 +385,29 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi 0.3.2", + "libc", + "windows-sys", +] + +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +dependencies = [ + "hermit-abi 0.3.2", + "io-lifetimes", + "rustix", + "windows-sys", +] + [[package]] name = "itertools" version = "0.10.5" @@ -405,6 +478,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" + [[package]] name = "lock_api" version = "0.4.9" @@ -448,12 +527,12 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=691d9a86969205c897647447244dad41767c15e0#691d9a86969205c897647447244dad41767c15e0" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=c8bfe4d1eef28e5ed43e022b2182c42beb9c7689#c8bfe4d1eef28e5ed43e022b2182c42beb9c7689" dependencies = [ "atomic", "atomic-traits", "atomic_refcell", - "built", + "built 0.6.0", "cfg-if", "crossbeam", "delegate", @@ -461,6 +540,7 @@ dependencies = [ "enum-map", "env_logger", "humantime", + "is-terminal", "itertools", "lazy_static", "libc", @@ -479,7 +559,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=691d9a86969205c897647447244dad41767c15e0#691d9a86969205c897647447244dad41767c15e0" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=c8bfe4d1eef28e5ed43e022b2182c42beb9c7689#c8bfe4d1eef28e5ed43e022b2182c42beb9c7689" dependencies = [ "proc-macro-error", "proc-macro2", @@ -492,7 +572,7 @@ name = "mmtk_openjdk" version = "0.18.0" dependencies = [ "atomic", - "built", + "built 0.5.2", "lazy_static", "libc", "mmtk", @@ -697,6 +777,20 @@ dependencies = [ "semver 0.11.0", ] +[[package]] +name = "rustix" +version = "0.37.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustversion" version = "1.0.11" @@ -803,9 +897,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.29.7" +version = "0.29.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "165d6d8539689e3d3bc8b98ac59541e1f21c7de7c85d60dc80e43ae0ed2113db" +checksum = "6b949f01f9c23823744b71e0060472ecbde578ef68cc2a9e46d114efd77c3034" dependencies = [ "cfg-if", "core-foundation-sys", @@ -960,3 +1054,69 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 9ab4ae4f..69ac44ea 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "691d9a86969205c897647447244dad41767c15e0" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "c8bfe4d1eef28e5ed43e022b2182c42beb9c7689" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From e461a614844088e8e3bea33be7dd6bf8f68b21a4 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 6 Aug 2023 23:31:57 +0000 Subject: [PATCH 352/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 7a9d8e9c..6d84a229 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -527,7 +527,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=c8bfe4d1eef28e5ed43e022b2182c42beb9c7689#c8bfe4d1eef28e5ed43e022b2182c42beb9c7689" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8228e5e5ae263ac53abc8ca2a78899a0367f281d#8228e5e5ae263ac53abc8ca2a78899a0367f281d" dependencies = [ "atomic", "atomic-traits", @@ -559,7 +559,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=c8bfe4d1eef28e5ed43e022b2182c42beb9c7689#c8bfe4d1eef28e5ed43e022b2182c42beb9c7689" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8228e5e5ae263ac53abc8ca2a78899a0367f281d#8228e5e5ae263ac53abc8ca2a78899a0367f281d" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 69ac44ea..d285c6c3 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "c8bfe4d1eef28e5ed43e022b2182c42beb9c7689" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8228e5e5ae263ac53abc8ca2a78899a0367f281d" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From f1b16c9ff9eea3cc5d321400e068d16e316ad7a8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 8 Aug 2023 17:01:50 +1000 Subject: [PATCH 353/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 6d84a229..a401c21c 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -527,7 +527,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8228e5e5ae263ac53abc8ca2a78899a0367f281d#8228e5e5ae263ac53abc8ca2a78899a0367f281d" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6686566bb8924048d7f415442fc50bf870680bb7#6686566bb8924048d7f415442fc50bf870680bb7" dependencies = [ "atomic", "atomic-traits", @@ -559,7 +559,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8228e5e5ae263ac53abc8ca2a78899a0367f281d#8228e5e5ae263ac53abc8ca2a78899a0367f281d" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6686566bb8924048d7f415442fc50bf870680bb7#6686566bb8924048d7f415442fc50bf870680bb7" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index d285c6c3..6b2e8e81 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8228e5e5ae263ac53abc8ca2a78899a0367f281d" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "6686566bb8924048d7f415442fc50bf870680bb7" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 4a3dd51b2cfc07638e169a63c862cc90a10f3def Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 12 Aug 2023 20:24:16 +1000 Subject: [PATCH 354/494] Fix build errors --- mmtk/src/collection.rs | 16 ++-------------- mmtk/src/lib.rs | 23 +++++++++++------------ mmtk/src/scanning.rs | 22 +--------------------- openjdk/mmtk.h | 2 +- openjdk/mmtkUpcalls.cpp | 11 +++++------ 5 files changed, 20 insertions(+), 54 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 1ba17191..81e5bcc8 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -1,7 +1,7 @@ use mmtk::util::alloc::AllocationError; use mmtk::util::opaque_pointer::*; -use mmtk::vm::{Collection, GCThreadContext, Scanning, VMBinding}; -use mmtk::{Mutator, MutatorContext}; +use mmtk::vm::{Collection, GCThreadContext}; +use mmtk::Mutator; use crate::UPCALLS; use crate::{MutatorClosure, OpenJDK}; @@ -16,13 +16,9 @@ impl Collection> for VMCollection { where F: FnMut(&'static mut Mutator>), { - let scan_mutators_in_safepoint = - < as VMBinding>::VMScanning as Scanning>>::SCAN_MUTATORS_IN_SAFEPOINT; - unsafe { ((*UPCALLS).stop_all_mutators)( tls, - scan_mutators_in_safepoint, MutatorClosure::from_rust_closure::<_, COMPRESSED>(&mut mutator_visitor), ); } @@ -55,14 +51,6 @@ impl Collection> for VMCollection { } } - fn prepare_mutator>>( - _tls_w: VMWorkerThread, - _tls_m: VMMutatorThread, - _m: &T, - ) { - // unimplemented!() - } - fn out_of_memory(tls: VMThread, err_kind: AllocationError) { unsafe { ((*UPCALLS).out_of_memory)(tls, err_kind); diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 59da0214..d305faa4 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -12,7 +12,6 @@ use mmtk::util::constants::{ BYTES_IN_ADDRESS, BYTES_IN_INT, LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_INT, }; use mmtk::util::opaque_pointer::*; -use mmtk::util::VM_LAYOUT_CONSTANTS; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::edge_shape::{Edge, MemorySlice}; use mmtk::vm::VMBinding; @@ -92,11 +91,7 @@ pub struct EdgesClosure { #[repr(C)] pub struct OpenJDK_Upcalls { - pub stop_all_mutators: extern "C" fn( - tls: VMWorkerThread, - scan_mutators_in_safepoint: bool, - closure: MutatorClosure, - ), + pub stop_all_mutators: extern "C" fn(tls: VMWorkerThread, closure: MutatorClosure), pub resume_mutators: extern "C" fn(tls: VMWorkerThread), pub spawn_gc_thread: extern "C" fn(tls: VMThread, kind: libc::c_int, ctx: *mut libc::c_void), pub block_for_gc: extern "C" fn(), @@ -216,7 +211,7 @@ fn decompress(v: u32) -> ObjectReference { } fn initialize_compressed_oops() { - let heap_end = VM_LAYOUT_CONSTANTS.heap_end.as_usize(); + let heap_end = mmtk::memory_manager::last_heap_address().as_usize(); if heap_end <= (4usize << 30) { unsafe { BASE = Address::ZERO; @@ -229,7 +224,7 @@ fn initialize_compressed_oops() { } } else { unsafe { - BASE = VM_LAYOUT_CONSTANTS.heap_start - 4096; + BASE = mmtk::memory_manager::starting_heap_address() - 4096; SHIFT = 3; } } @@ -395,6 +390,10 @@ impl MemorySlice for OpenJDKEdgeRange { } } } + + fn object(&self) -> Option { + None + } } impl VMBinding for OpenJDK { @@ -429,8 +428,8 @@ lazy_static! { MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); initialize_compressed_oops(); unsafe { - HEAP_START = VM_LAYOUT_CONSTANTS.heap_start; - HEAP_END = VM_LAYOUT_CONSTANTS.heap_end; + HEAP_START = mmtk::memory_manager::starting_heap_address(); + HEAP_END = mmtk::memory_manager::last_heap_address(); } *ret }; @@ -440,8 +439,8 @@ lazy_static! { let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); unsafe { - HEAP_START = VM_LAYOUT_CONSTANTS.heap_start; - HEAP_END = VM_LAYOUT_CONSTANTS.heap_end; + HEAP_START = mmtk::memory_manager::starting_heap_address(); + HEAP_END = mmtk::memory_manager::last_heap_address(); } *ret }; diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index d5089866..50a59763 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -44,9 +44,6 @@ pub(crate) fn to_edges_closure>(factory: &mut F) } impl Scanning> for VMScanning { - const SCAN_MUTATORS_IN_SAFEPOINT: bool = false; - const SINGLE_THREAD_MUTATOR_SCANNING: bool = false; - fn scan_object>>( tls: VMWorkerThread, object: ObjectReference, @@ -60,15 +57,6 @@ impl Scanning> for VMScanning { // TODO } - fn scan_roots_in_all_mutator_threads( - _tls: VMWorkerThread, - mut factory: impl RootsWorkFactory>, - ) { - unsafe { - ((*UPCALLS).scan_roots_in_all_mutator_threads)(to_edges_closure(&mut factory)); - } - } - fn scan_roots_in_mutator_thread( _tls: VMWorkerThread, mutator: &'static mut Mutator>, @@ -99,17 +87,9 @@ impl Scanning> for VMScanning { Box::new(ScanStringTableRoots::new(factory.clone())) as _, Box::new(ScanClassLoaderDataGraphRoots::new(factory.clone())) as _, Box::new(ScanWeakProcessorRoots::new(factory.clone())) as _, + Box::new(ScanVMThreadRoots::new(factory)) as _, ], ); - if !(>>::SCAN_MUTATORS_IN_SAFEPOINT - && >>::SINGLE_THREAD_MUTATOR_SCANNING) - { - memory_manager::add_work_packet( - &crate::singleton::(), - WorkBucketStage::Prepare, - ScanVMThreadRoots::new(factory), - ); - } } fn supports_return_barrier() -> bool { diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 89540765..d358eb9d 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -146,7 +146,7 @@ struct EdgesClosure { * OpenJDK-specific */ typedef struct { - void (*stop_all_mutators) (void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure); + void (*stop_all_mutators) (void *tls, MutatorClosure closure); void (*resume_mutators) (void *tls); void (*spawn_gc_thread) (void *tls, int kind, void *ctx); void (*block_for_gc) (); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index c3c3f03a..4a4f85cc 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -45,7 +45,7 @@ // Note: This counter must be accessed using the Atomic class. static volatile size_t mmtk_start_the_world_count = 0; -static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, MutatorClosure closure) { +static void mmtk_stop_all_mutators(void *tls, MutatorClosure closure) { ClassLoaderDataGraph::clear_claimed_marks(); CodeCache::gc_prologue(); #if COMPILER2_OR_JVMCI @@ -56,12 +56,11 @@ static void mmtk_stop_all_mutators(void *tls, bool scan_mutators_in_safepoint, M MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_suspended, true); log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); - if (!scan_mutators_in_safepoint) { - JavaThreadIteratorWithHandle jtiwh; - while (JavaThread *cur = jtiwh.next()) { - closure.invoke((void*)&cur->third_party_heap_mutator); - } + JavaThreadIteratorWithHandle jtiwh; + while (JavaThread *cur = jtiwh.next()) { + closure.invoke((void*)&cur->third_party_heap_mutator); } + log_debug(gc)("Finished enumerating threads."); nmethod::oops_do_marking_prologue(); } From 4612446d447f1523546d5e5b4ef4145295aa750f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 16 Aug 2023 10:53:45 +1000 Subject: [PATCH 355/494] Use custom heap args --- mmtk/src/lib.rs | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index d305faa4..951dc4a8 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -9,9 +9,10 @@ use std::sync::Mutex; use libc::{c_char, c_void, uintptr_t}; use mmtk::util::alloc::AllocationError; use mmtk::util::constants::{ - BYTES_IN_ADDRESS, BYTES_IN_INT, LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_INT, + BYTES_IN_ADDRESS, BYTES_IN_INT, LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_GBYTE, LOG_BYTES_IN_INT, }; -use mmtk::util::opaque_pointer::*; +use mmtk::util::heap::vm_layout_constants::{VMLayoutConstants, LOG_BYTES_IN_CHUNK}; +use mmtk::util::{conversions, opaque_pointer::*}; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::edge_shape::{Edge, MemorySlice}; use mmtk::vm::VMBinding; @@ -424,6 +425,7 @@ lazy_static! { builder.set_option("use_35bit_address_space", "true"); builder.set_option("use_35bit_address_space", "true"); assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); + set_custom_vm_layout_constants(builder.options.gc_trigger.max_heap_size()); let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); initialize_compressed_oops(); @@ -471,3 +473,30 @@ lazy_static! { /// A counter tracking the total size of the `CODE_CACHE_ROOTS`. static CODE_CACHE_ROOTS_SIZE: AtomicUsize = AtomicUsize::new(0); + +fn set_custom_vm_layout_constants(max_heap_size: usize) { + assert!( + max_heap_size <= (32usize << LOG_BYTES_IN_GBYTE), + "Heap size is larger than 32 GB" + ); + let start = 0x4000_0000; + let end = match start + max_heap_size { + end if end <= (4usize << 30) => 4usize << 30, + end if end <= (32usize << 30) => 32usize << 30, + _ => 0x4000_0000 + (32usize << 30), + }; + let constants = VMLayoutConstants { + log_address_space: 35, + heap_start: conversions::chunk_align_down(unsafe { Address::from_usize(start) }), + heap_end: conversions::chunk_align_up(unsafe { Address::from_usize(end) }), + vm_space_size: conversions::chunk_align_up(unsafe { Address::from_usize(0x800_0000) }) + .as_usize(), + log_max_chunks: VMLayoutConstants::LOG_ARCH_ADDRESS_SPACE - LOG_BYTES_IN_CHUNK, + log_space_extent: 31, + space_shift_64: 0, + space_mask_64: 0, + space_size_64: 0, + force_use_contiguous_spaces: false, + }; + VMLayoutConstants::set_custom_vm_layout_constants(constants); +} From e9c357c3b0b35d94e4777f59e188c64cabb14988 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 16 Aug 2023 11:56:14 +1000 Subject: [PATCH 356/494] Update inferfaces --- mmtk/src/abi.rs | 10 ++-- mmtk/src/gc_work.rs | 29 +++++----- mmtk/src/lib.rs | 20 +++---- mmtk/src/object_scanning.rs | 110 +++++++++++++++++++++--------------- mmtk/src/reference_glue.rs | 5 +- mmtk/src/scanning.rs | 2 +- 6 files changed, 96 insertions(+), 80 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 70df7110..59614df5 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -1,5 +1,5 @@ use super::UPCALLS; -use crate::Edge; +use crate::OpenJDKEdge; use mmtk::util::constants::*; use mmtk::util::conversions; use mmtk::util::ObjectReference; @@ -268,11 +268,11 @@ impl InstanceRefKlass { } *DISCOVERED_OFFSET } - pub fn referent_address(oop: Oop) -> E { - E::from_address(oop.get_field_address(Self::referent_offset())) + pub fn referent_address(oop: Oop) -> OpenJDKEdge { + OpenJDKEdge::::from_address(oop.get_field_address(Self::referent_offset())) } - pub fn discovered_address(oop: Oop) -> E { - E::from_address(oop.get_field_address(Self::discovered_offset())) + pub fn discovered_address(oop: Oop) -> OpenJDKEdge { + OpenJDKEdge::::from_address(oop.get_field_address(Self::discovered_offset())) } } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index d6ca64e1..9ec420a4 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -1,13 +1,13 @@ use std::sync::atomic::Ordering; use crate::scanning::to_edges_closure; -use crate::Edge; +use crate::OpenJDK; +use crate::OpenJDKEdge; use crate::UPCALLS; use mmtk::scheduler::*; use mmtk::vm::RootsWorkFactory; use mmtk::vm::*; use mmtk::MMTK; -use std::marker::PhantomData; macro_rules! scan_roots_work { ($struct_name: ident, $func_name: ident) => { @@ -50,29 +50,32 @@ scan_roots_work!( scan_roots_work!(ScanWeakProcessorRoots, scan_weak_processor_roots); scan_roots_work!(ScanVMThreadRoots, scan_vm_thread_roots); -pub struct ScanCodeCacheRoots> { +pub struct ScanCodeCacheRoots>> +{ factory: F, - _p: PhantomData, } -impl> ScanCodeCacheRoots { +impl>> + ScanCodeCacheRoots +{ pub fn new(factory: F) -> Self { - Self { - factory, - _p: PhantomData, - } + Self { factory } } } -impl> GCWork - for ScanCodeCacheRoots +impl>> + GCWork> for ScanCodeCacheRoots { - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { + fn do_work( + &mut self, + _worker: &mut GCWorker>, + _mmtk: &'static MMTK>, + ) { // Collect all the cached roots let mut edges = Vec::with_capacity(crate::CODE_CACHE_ROOTS_SIZE.load(Ordering::Relaxed)); for roots in (*crate::CODE_CACHE_ROOTS.lock().unwrap()).values() { for r in roots { - edges.push(VM::VMEdge::from_address(*r)) + edges.push(OpenJDKEdge::::from_address(*r)) } } // Create work packet diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 951dc4a8..3e9b8a71 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -249,6 +249,14 @@ impl OpenJDKEdge { const fn untagged_address(&self) -> Address { unsafe { Address::from_usize(self.0.as_usize() << 1 >> 1) } } + + pub fn to_address(&self) -> Address { + self.untagged_address() + } + + pub fn from_address(a: Address) -> Self { + Self(a) + } } impl Edge for OpenJDKEdge { @@ -279,18 +287,6 @@ impl Edge for OpenJDKEdge { unsafe { self.0.store(object) } } } - - fn to_address(&self) -> Address { - self.untagged_address() - } - - // fn raw_address(&self) -> Address { - // self.0 - // } - - fn from_address(a: Address) -> Self { - Self(a) - } } #[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index a265f353..d9d37d49 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -1,56 +1,58 @@ +use crate::OpenJDKEdge; + use super::abi::*; use super::UPCALLS; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; -use mmtk::vm::edge_shape::Edge; use mmtk::vm::EdgeVisitor; use std::cell::UnsafeCell; use std::{mem, slice}; +type E = OpenJDKEdge; + trait OopIterate: Sized { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate( &self, oop: Oop, - closure: &mut V, + closure: &mut impl EdgeVisitor>, ); } impl OopIterate for OopMapBlock { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate( &self, oop: Oop, - closure: &mut V, + closure: &mut impl EdgeVisitor>, ) { let log_bytes_in_oop = if COMPRESSED { 2 } else { 3 }; let start = oop.get_field_address(self.offset); for i in 0..self.count as usize { - let edge = E::from_address(start + (i << log_bytes_in_oop)); + let edge = OpenJDKEdge::::from_address(start + (i << log_bytes_in_oop)); closure.visit_edge(edge); } } } impl OopIterate for InstanceKlass { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate( &self, oop: Oop, - closure: &mut V, + closure: &mut impl EdgeVisitor>, ) { let oop_maps = self.nonstatic_oop_maps(); for map in oop_maps { - map.oop_iterate::(oop, closure) + map.oop_iterate::(oop, closure) } } } impl OopIterate for InstanceMirrorKlass { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate( &self, oop: Oop, - closure: &mut V, + closure: &mut impl EdgeVisitor>, ) { - self.instance_klass - .oop_iterate::<_, _, COMPRESSED>(oop, closure); + self.instance_klass.oop_iterate::(oop, closure); // static fields let start = Self::start_of_static_fields(oop); @@ -59,53 +61,56 @@ impl OopIterate for InstanceMirrorKlass { let start: *const NarrowOop = start.to_ptr::(); let slice = unsafe { slice::from_raw_parts(start, len as _) }; for narrow_oop in slice { - closure.visit_edge(E::from_address(narrow_oop.slot())); + closure.visit_edge(OpenJDKEdge::::from_address(narrow_oop.slot())); } } else { let start: *const Oop = start.to_ptr::(); let slice = unsafe { slice::from_raw_parts(start, len as _) }; for oop in slice { - closure.visit_edge(E::from_address(Address::from_ref(oop as &Oop))); + closure.visit_edge(OpenJDKEdge::::from_address(Address::from_ref( + oop as &Oop, + ))); } } } } impl OopIterate for InstanceClassLoaderKlass { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate( &self, oop: Oop, - closure: &mut V, + closure: &mut impl EdgeVisitor>, ) { - self.instance_klass - .oop_iterate::<_, _, COMPRESSED>(oop, closure); + self.instance_klass.oop_iterate::(oop, closure); } } impl OopIterate for ObjArrayKlass { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate( &self, oop: Oop, - closure: &mut V, + closure: &mut impl EdgeVisitor>, ) { let array = unsafe { oop.as_array_oop() }; if COMPRESSED { for narrow_oop in unsafe { array.data::(BasicType::T_OBJECT) } { - closure.visit_edge(E::from_address(narrow_oop.slot())); + closure.visit_edge(OpenJDKEdge::::from_address(narrow_oop.slot())); } } else { for oop in unsafe { array.data::(BasicType::T_OBJECT) } { - closure.visit_edge(E::from_address(Address::from_ref(oop as &Oop))); + closure.visit_edge(OpenJDKEdge::::from_address(Address::from_ref( + oop as &Oop, + ))); } } } } impl OopIterate for TypeArrayKlass { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate( &self, _oop: Oop, - _closure: &mut V, + _closure: &mut impl EdgeVisitor>, ) { // Performance tweak: We skip processing the klass pointer since all // TypeArrayKlasses are guaranteed processed via the null class loader. @@ -113,15 +118,14 @@ impl OopIterate for TypeArrayKlass { } impl OopIterate for InstanceRefKlass { - fn oop_iterate, const COMPRESSED: bool>( + fn oop_iterate( &self, oop: Oop, - closure: &mut V, + closure: &mut impl EdgeVisitor>, ) { use crate::abi::*; use crate::api::{add_phantom_candidate, add_soft_candidate, add_weak_candidate}; - self.instance_klass - .oop_iterate::<_, _, COMPRESSED>(oop, closure); + self.instance_klass.oop_iterate::(oop, closure); if Self::should_scan_weak_refs::() { let reference = ObjectReference::from(oop); @@ -150,29 +154,38 @@ impl InstanceRefKlass { .get_options() .no_reference_types } - fn process_ref_as_strong>(oop: Oop, closure: &mut V) { - let referent_addr = Self::referent_address::(oop); + fn process_ref_as_strong( + oop: Oop, + closure: &mut impl EdgeVisitor>, + ) { + let referent_addr = Self::referent_address::(oop); closure.visit_edge(referent_addr); - let discovered_addr = Self::discovered_address::(oop); + let discovered_addr = Self::discovered_address::(oop); closure.visit_edge(discovered_addr); } } #[allow(unused)] -fn oop_iterate_slow>(oop: Oop, closure: &mut V, tls: OpaquePointer) { +fn oop_iterate_slow>>( + oop: Oop, + closure: &mut V, + tls: OpaquePointer, +) { unsafe { CLOSURE.with(|x| *x.get() = closure as *mut V as *mut u8); ((*UPCALLS).scan_object)( - mem::transmute(scan_object_fn:: as *const unsafe extern "C" fn(edge: Address)), + mem::transmute( + scan_object_fn:: as *const unsafe extern "C" fn(edge: Address), + ), mem::transmute(oop), tls, ); } } -fn oop_iterate, const COMPRESSED: bool>( +fn oop_iterate( oop: Oop, - closure: &mut V, + closure: &mut impl EdgeVisitor>, klass: Option
, ) { let klass = if let Some(klass) = klass { @@ -190,19 +203,19 @@ fn oop_iterate, const COMPRESSED: bool>( match klass_id { KlassID::Instance => { let instance_klass = unsafe { klass.cast::() }; - instance_klass.oop_iterate::(oop, closure); + instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceClassLoader => { let instance_klass = unsafe { klass.cast::() }; - instance_klass.oop_iterate::(oop, closure); + instance_klass.oop_iterate::(oop, closure); } KlassID::InstanceMirror => { let instance_klass = unsafe { klass.cast::() }; - instance_klass.oop_iterate::(oop, closure); + instance_klass.oop_iterate::(oop, closure); } KlassID::ObjArray => { let array_klass = unsafe { klass.cast::() }; - array_klass.oop_iterate::(oop, closure); + array_klass.oop_iterate::(oop, closure); } KlassID::TypeArray => { // let array_klass = unsafe { oop.klass::().cast::() }; @@ -210,7 +223,7 @@ fn oop_iterate, const COMPRESSED: bool>( } KlassID::InstanceRef => { let instance_klass = unsafe { klass.cast::() }; - instance_klass.oop_iterate::(oop, closure); + instance_klass.oop_iterate::(oop, closure); } #[allow(unreachable_patterns)] _ => unreachable!(), // _ => oop_iterate_slow(oop, closure, OpaquePointer::UNINITIALIZED), @@ -221,16 +234,21 @@ thread_local! { static CLOSURE: UnsafeCell<*mut u8> = UnsafeCell::new(std::ptr::null_mut()); } -pub unsafe extern "C" fn scan_object_fn>(edge: Address) { +pub unsafe extern "C" fn scan_object_fn< + const COMPRESSED: bool, + V: EdgeVisitor>, +>( + edge: Address, +) { let ptr: *mut u8 = CLOSURE.with(|x| *x.get()); let closure = &mut *(ptr as *mut V); - closure.visit_edge(E::from_address(edge)); + closure.visit_edge(OpenJDKEdge::::from_address(edge)); } -pub fn scan_object, const COMPRESSED: bool>( +pub fn scan_object( object: ObjectReference, - closure: &mut V, + closure: &mut impl EdgeVisitor>, _tls: VMWorkerThread, ) { - unsafe { oop_iterate::(mem::transmute(object), closure, None) } + unsafe { oop_iterate::(mem::transmute(object), closure, None) } } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index d85c1dd6..3ecd5f1a 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -1,6 +1,5 @@ use crate::abi::{InstanceRefKlass, Oop}; use crate::OpenJDK; -use crate::OpenJDKEdge; use crate::UPCALLS; use mmtk::util::opaque_pointer::VMWorkerThread; use mmtk::util::ObjectReference; @@ -14,11 +13,11 @@ impl ReferenceGlue> for VMReferenceG fn set_referent(reff: ObjectReference, referent: ObjectReference) { let oop = Oop::from(reff); - InstanceRefKlass::referent_address::>(oop).store(referent); + InstanceRefKlass::referent_address::(oop).store(referent); } fn get_referent(object: ObjectReference) -> ObjectReference { let oop = Oop::from(object); - InstanceRefKlass::referent_address::>(oop).load() + InstanceRefKlass::referent_address::(oop).load() } fn enqueue_references(references: &[ObjectReference], _tls: VMWorkerThread) { unsafe { diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 50a59763..11f90de8 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -49,7 +49,7 @@ impl Scanning> for VMScanning { object: ObjectReference, edge_visitor: &mut EV, ) { - crate::object_scanning::scan_object::<_, _, COMPRESSED>(object, edge_visitor, tls); + crate::object_scanning::scan_object::(object, edge_visitor, tls); } fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { From cd73c37a69cce4697d3b689e98d455e60609aa3d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 16 Aug 2023 14:09:30 +1000 Subject: [PATCH 357/494] remove redundant --- mmtk/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 3e9b8a71..f0d88380 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -419,7 +419,6 @@ lazy_static! { let mut builder = BUILDER.lock().unwrap(); assert!(use_compressed_oops()); builder.set_option("use_35bit_address_space", "true"); - builder.set_option("use_35bit_address_space", "true"); assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); set_custom_vm_layout_constants(builder.options.gc_trigger.max_heap_size()); let ret = mmtk::memory_manager::mmtk_init(&builder); From ec820077188d32748d549abffc0d34878152800d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 17 Aug 2023 12:24:48 +1000 Subject: [PATCH 358/494] cleanup --- mmtk/src/abi.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 59614df5..813e65c5 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -95,10 +95,6 @@ impl Klass { const fn layout_helper_header_size(lh: i32) -> i32 { (lh >> Self::LH_HEADER_SIZE_SHIFT) & Self::LH_HEADER_SIZE_MASK } - - pub const fn is_instance_klass(&self) -> bool { - self.layout_helper > Self::LH_NEUTRAL_VALUE - } } #[repr(C)] @@ -157,7 +153,7 @@ pub struct InstanceKlass { } #[repr(u8)] -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, Debug)] #[allow(dead_code)] pub enum ReferenceType { None, // Regular class From 465c4d57828666107bf2c705fe253c206f37ea5e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 17 Aug 2023 12:59:33 +1000 Subject: [PATCH 359/494] Delete derivable constants --- mmtk/src/lib.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index f0d88380..90b0076b 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -486,11 +486,7 @@ fn set_custom_vm_layout_constants(max_heap_size: usize) { heap_end: conversions::chunk_align_up(unsafe { Address::from_usize(end) }), vm_space_size: conversions::chunk_align_up(unsafe { Address::from_usize(0x800_0000) }) .as_usize(), - log_max_chunks: VMLayoutConstants::LOG_ARCH_ADDRESS_SPACE - LOG_BYTES_IN_CHUNK, log_space_extent: 31, - space_shift_64: 0, - space_mask_64: 0, - space_size_64: 0, force_use_contiguous_spaces: false, }; VMLayoutConstants::set_custom_vm_layout_constants(constants); From c1b7b6fc83f6553aa48873fa28d33aeedb1316fe Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 17 Aug 2023 13:06:05 +1000 Subject: [PATCH 360/494] Remove `vm_space_size` --- mmtk/src/lib.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 90b0076b..03641be0 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -11,7 +11,7 @@ use mmtk::util::alloc::AllocationError; use mmtk::util::constants::{ BYTES_IN_ADDRESS, BYTES_IN_INT, LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_GBYTE, LOG_BYTES_IN_INT, }; -use mmtk::util::heap::vm_layout_constants::{VMLayoutConstants, LOG_BYTES_IN_CHUNK}; +use mmtk::util::heap::vm_layout_constants::VMLayoutConstants; use mmtk::util::{conversions, opaque_pointer::*}; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::edge_shape::{Edge, MemorySlice}; @@ -484,8 +484,6 @@ fn set_custom_vm_layout_constants(max_heap_size: usize) { log_address_space: 35, heap_start: conversions::chunk_align_down(unsafe { Address::from_usize(start) }), heap_end: conversions::chunk_align_up(unsafe { Address::from_usize(end) }), - vm_space_size: conversions::chunk_align_up(unsafe { Address::from_usize(0x800_0000) }) - .as_usize(), log_space_extent: 31, force_use_contiguous_spaces: false, }; From de66ed387de784e6b5669a2896e013c9f75d171d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 17 Aug 2023 13:34:30 +1000 Subject: [PATCH 361/494] Update --- mmtk/src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 03641be0..fa7d2381 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -11,7 +11,7 @@ use mmtk::util::alloc::AllocationError; use mmtk::util::constants::{ BYTES_IN_ADDRESS, BYTES_IN_INT, LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_GBYTE, LOG_BYTES_IN_INT, }; -use mmtk::util::heap::vm_layout_constants::VMLayoutConstants; +use mmtk::util::heap::vm_layout::VMLayout; use mmtk::util::{conversions, opaque_pointer::*}; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::edge_shape::{Edge, MemorySlice}; @@ -420,7 +420,7 @@ lazy_static! { assert!(use_compressed_oops()); builder.set_option("use_35bit_address_space", "true"); assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); - set_custom_vm_layout_constants(builder.options.gc_trigger.max_heap_size()); + set_custom_vm_layout(builder.options.gc_trigger.max_heap_size()); let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); initialize_compressed_oops(); @@ -469,7 +469,7 @@ lazy_static! { /// A counter tracking the total size of the `CODE_CACHE_ROOTS`. static CODE_CACHE_ROOTS_SIZE: AtomicUsize = AtomicUsize::new(0); -fn set_custom_vm_layout_constants(max_heap_size: usize) { +fn set_custom_vm_layout(max_heap_size: usize) { assert!( max_heap_size <= (32usize << LOG_BYTES_IN_GBYTE), "Heap size is larger than 32 GB" @@ -480,12 +480,12 @@ fn set_custom_vm_layout_constants(max_heap_size: usize) { end if end <= (32usize << 30) => 32usize << 30, _ => 0x4000_0000 + (32usize << 30), }; - let constants = VMLayoutConstants { + let constants = VMLayout { log_address_space: 35, heap_start: conversions::chunk_align_down(unsafe { Address::from_usize(start) }), heap_end: conversions::chunk_align_up(unsafe { Address::from_usize(end) }), log_space_extent: 31, force_use_contiguous_spaces: false, }; - VMLayoutConstants::set_custom_vm_layout_constants(constants); + VMLayout::set_custom_vm_layout(constants); } From bf901a6744877695d415a507f544614ec5cef006 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 22 Aug 2023 14:25:35 +1000 Subject: [PATCH 362/494] Update api --- mmtk/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index fa7d2381..9e3632c6 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -418,9 +418,8 @@ lazy_static! { pub static ref SINGLETON_COMPRESSED: MMTK> = { let mut builder = BUILDER.lock().unwrap(); assert!(use_compressed_oops()); - builder.set_option("use_35bit_address_space", "true"); assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); - set_custom_vm_layout(builder.options.gc_trigger.max_heap_size()); + set_compressed_pointer_vm_layout(&mut builder); let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); initialize_compressed_oops(); @@ -469,7 +468,8 @@ lazy_static! { /// A counter tracking the total size of the `CODE_CACHE_ROOTS`. static CODE_CACHE_ROOTS_SIZE: AtomicUsize = AtomicUsize::new(0); -fn set_custom_vm_layout(max_heap_size: usize) { +fn set_compressed_pointer_vm_layout(builder: &mut MMTKBuilder) { + let max_heap_size = builder.options.gc_trigger.max_heap_size(); assert!( max_heap_size <= (32usize << LOG_BYTES_IN_GBYTE), "Heap size is larger than 32 GB" @@ -487,5 +487,5 @@ fn set_custom_vm_layout(max_heap_size: usize) { log_space_extent: 31, force_use_contiguous_spaces: false, }; - VMLayout::set_custom_vm_layout(constants); + builder.set_vm_layout(constants); } From c615a5a604cf066f5cc524c8648858729c5b2738 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 24 Aug 2023 10:47:44 +1000 Subject: [PATCH 363/494] Update Cargo.lock --- mmtk/Cargo.lock | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index f04fc916..2a7a35b8 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -145,7 +145,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", + "memoffset 0.7.1", "scopeguard", ] @@ -433,6 +433,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "mmtk" version = "0.18.0" @@ -452,6 +461,7 @@ dependencies = [ "lazy_static", "libc", "log", + "memoffset 0.9.0", "mmtk-macros", "num-traits", "num_cpus", From 64169873fe150c7e0ec7e661dc74ea5d4ceb0f5b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 24 Aug 2023 12:02:34 +1000 Subject: [PATCH 364/494] cleanup --- mmtk/src/api.rs | 12 ++++++++++++ mmtk/src/edges.rs | 11 +++++++++++ mmtk/src/lib.rs | 12 ------------ mmtk/src/object_model.rs | 12 ++---------- mmtk/src/object_scanning.rs | 16 +++------------- openjdk/mmtkRootsClosure.hpp | 7 +++---- openjdk/mmtkUpcalls.cpp | 28 ++++++++++++++-------------- 7 files changed, 45 insertions(+), 53 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index b56c0b44..a95792a2 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -19,6 +19,18 @@ use std::cell::RefCell; use std::ffi::{CStr, CString}; use std::sync::atomic::Ordering; +macro_rules! with_singleton { + (|$x: ident| $($expr:tt)*) => { + if crate::use_compressed_oops() { + let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_COMPRESSED; + $($expr)* + } else { + let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_UNCOMPRESSED; + $($expr)* + } + }; +} + macro_rules! with_mutator { (|$x: ident| $($expr:tt)*) => { if crate::use_compressed_oops() { diff --git a/mmtk/src/edges.rs b/mmtk/src/edges.rs index f421cf07..8a496ba2 100644 --- a/mmtk/src/edges.rs +++ b/mmtk/src/edges.rs @@ -17,6 +17,15 @@ pub static BASE: Atomic
= Atomic::new(Address::ZERO); pub static SHIFT: AtomicUsize = AtomicUsize::new(0); pub fn enable_compressed_oops() { + static COMPRESSED_OOPS_INITIALIZED: AtomicBool = AtomicBool::new(false); + assert!( + !COMPRESSED_OOPS_INITIALIZED.fetch_or(true, Ordering::Relaxed), + "cannot enable compressed pointers twice." + ); + if cfg!(not(target_arch = "x86_64")) { + panic!("Compressed pointer is only enable on x86_64 platforms.\ + For other RISC architectures, we need to find a way to process compressed embeded pointers in code objects first."); + } USE_COMPRESSED_OOPS.store(true, Ordering::Relaxed) } @@ -102,6 +111,7 @@ impl OpenJDKEdge { } } + /// encode an object pointer to its u32 compressed form fn compress(o: ObjectReference) -> u32 { if o.is_null() { 0u32 @@ -111,6 +121,7 @@ impl OpenJDKEdge { } } + /// decode an object pointer from its u32 compressed form fn decompress(v: u32) -> ObjectReference { if v == 0 { ObjectReference::NULL diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index f6c191f7..2cdbc484 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -18,18 +18,6 @@ use mmtk::vm::edge_shape::Edge; use mmtk::vm::VMBinding; use mmtk::{MMTKBuilder, Mutator, MMTK}; -macro_rules! with_singleton { - (|$x: ident| $($expr:tt)*) => { - if crate::use_compressed_oops() { - let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_COMPRESSED; - $($expr)* - } else { - let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_UNCOMPRESSED; - $($expr)* - } - }; -} - mod abi; pub mod active_plan; pub mod api; diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 17770bc5..b7449a2e 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -26,11 +26,7 @@ impl ObjectModel> for VMObjectModel copy: CopySemantics, copy_context: &mut GCWorkerCopyContext>, ) -> ObjectReference { - let bytes = if crate::use_compressed_oops() { - unsafe { Oop::from(from).size::() } - } else { - unsafe { Oop::from(from).size::() } - }; + let bytes = unsafe { Oop::from(from).size::() }; let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::(), 0, copy); // Copy let src = from.to_raw_address(); @@ -114,11 +110,7 @@ impl ObjectModel> for VMObjectModel let oop = Oop::from(object); // It is only valid if klass.id is between 0 and 5 (see KlassID in openjdk/src/hotspot/share/oops/klass.hpp) // If oop.klass is not a valid pointer, we may segfault here. - let klass_id = if crate::use_compressed_oops() { - oop.klass::().id - } else { - oop.klass::().id - } as i32; + let klass_id = oop.klass::().id as i32; klass_id >= 0 && klass_id < 6 } } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 8de380fd..3868ff67 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -179,16 +179,8 @@ fn oop_iterate_slow>>( } } -fn oop_iterate( - oop: Oop, - closure: &mut impl EdgeVisitor>, - klass: Option
, -) { - let klass = if let Some(klass) = klass { - unsafe { &*(klass.as_usize() as *const Klass) } - } else { - oop.klass::() - }; +fn oop_iterate(oop: Oop, closure: &mut impl EdgeVisitor>) { + let klass = oop.klass::(); let klass_id = klass.id; assert!( klass_id as i32 >= 0 && (klass_id as i32) < 6, @@ -221,8 +213,6 @@ fn oop_iterate( let instance_klass = unsafe { klass.cast::() }; instance_klass.oop_iterate::(oop, closure); } - #[allow(unreachable_patterns)] - _ => unreachable!(), // _ => oop_iterate_slow(oop, closure, OpaquePointer::UNINITIALIZED), } } @@ -246,5 +236,5 @@ pub fn scan_object( closure: &mut impl EdgeVisitor>, _tls: VMWorkerThread, ) { - unsafe { oop_iterate::(mem::transmute(object), closure, None) } + unsafe { oop_iterate::(mem::transmute(object), closure) } } diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 841d83de..ee92b223 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -7,7 +7,7 @@ #include "oops/oop.inline.hpp" #include "utilities/globalDefinitions.hpp" -class MMTkRootsClosure2 : public OopClosure { +class MMTkRootsClosure : public OopClosure { EdgesClosure _edges_closure; void** _buffer; size_t _cap; @@ -38,13 +38,13 @@ class MMTkRootsClosure2 : public OopClosure { } public: - MMTkRootsClosure2(EdgesClosure edges_closure): _edges_closure(edges_closure), _cursor(0) { + MMTkRootsClosure(EdgesClosure edges_closure): _edges_closure(edges_closure), _cursor(0) { NewBuffer buf = edges_closure.invoke(NULL, 0, 0); _buffer = buf.buf; _cap = buf.cap; } - ~MMTkRootsClosure2() { + ~MMTkRootsClosure() { if (_cursor > 0) flush(); if (_buffer != NULL) { release_buffer(_buffer, _cursor, _cap); @@ -65,7 +65,6 @@ class MMTkScanObjectClosure : public BasicOopIterateClosure { guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); p = (T*) (uintptr_t(p) | (1ull << 63)); } - // _trace((void*) p); } public: diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 4a4f85cc..8da5fd84 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -184,14 +184,14 @@ static void mmtk_get_mutators(MutatorClosure closure) { } static void mmtk_scan_roots_in_all_mutator_threads(EdgesClosure closure) { - MMTkRootsClosure2 cl(closure); + MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_roots_in_all_mutator_threads(cl); } static void mmtk_scan_roots_in_mutator_thread(EdgesClosure closure, void* tls) { ResourceMark rm; JavaThread* thread = (JavaThread*) tls; - MMTkRootsClosure2 cl(closure); + MMTkRootsClosure cl(closure); thread->oops_do(&cl, NULL); } @@ -268,18 +268,18 @@ static void mmtk_schedule_finalizer() { MMTkHeap::heap()->schedule_finalizer(); } -static void mmtk_scan_universe_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_universe_roots(cl); } -static void mmtk_scan_jni_handle_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_jni_handle_roots(cl); } -static void mmtk_scan_object_synchronizer_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_object_synchronizer_roots(cl); } -static void mmtk_scan_management_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_management_roots(cl); } -static void mmtk_scan_jvmti_export_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_jvmti_export_roots(cl); } -static void mmtk_scan_aot_loader_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_aot_loader_roots(cl); } -static void mmtk_scan_system_dictionary_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_system_dictionary_roots(cl); } -static void mmtk_scan_code_cache_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } -static void mmtk_scan_string_table_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_string_table_roots(cl); } -static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl); } -static void mmtk_scan_weak_processor_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_weak_processor_roots(cl); } -static void mmtk_scan_vm_thread_roots(EdgesClosure closure) { MMTkRootsClosure2 cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } +static void mmtk_scan_universe_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_universe_roots(cl); } +static void mmtk_scan_jni_handle_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_jni_handle_roots(cl); } +static void mmtk_scan_object_synchronizer_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_object_synchronizer_roots(cl); } +static void mmtk_scan_management_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_management_roots(cl); } +static void mmtk_scan_jvmti_export_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_jvmti_export_roots(cl); } +static void mmtk_scan_aot_loader_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_aot_loader_roots(cl); } +static void mmtk_scan_system_dictionary_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_system_dictionary_roots(cl); } +static void mmtk_scan_code_cache_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } +static void mmtk_scan_string_table_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_string_table_roots(cl); } +static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl); } +static void mmtk_scan_weak_processor_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_weak_processor_roots(cl); } +static void mmtk_scan_vm_thread_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } static size_t mmtk_number_of_mutators() { return Threads::number_of_threads(); From 89c109d7e78cfa901eabd8f076e50c1b29827d7e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 24 Aug 2023 13:52:43 +1000 Subject: [PATCH 365/494] cleanup --- mmtk/src/abi.rs | 23 +++++++++++++++++------ mmtk/src/api.rs | 7 +++++++ mmtk/src/lib.rs | 27 --------------------------- openjdk/mmtk.h | 5 +---- openjdk/mmtkHeap.cpp | 3 +++ openjdk/mmtkUpcalls.cpp | 12 +----------- 6 files changed, 29 insertions(+), 48 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index f8b7f31d..f83a0c63 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -1,11 +1,14 @@ use super::UPCALLS; use crate::OpenJDKEdge; +use atomic::Atomic; +use atomic::Ordering; use mmtk::util::constants::*; use mmtk::util::conversions; use mmtk::util::ObjectReference; use mmtk::util::{Address, OpaquePointer}; use std::ffi::CStr; use std::fmt; +use std::sync::atomic::AtomicUsize; use std::{mem, slice}; #[repr(i32)] @@ -273,20 +276,27 @@ impl InstanceRefKlass { } #[repr(C)] -union KlassField { +union KlassPointer { + /// uncompressed Klass pointer klass: &'static Klass, + /// compressed Klass pointer narrow_klass: u32, } #[repr(C)] pub struct OopDesc { pub mark: usize, - klass: KlassField, + klass: KlassPointer, } -lazy_static! { - static ref COMPRESSED_KLASS_BASE: Address = unsafe { ((*UPCALLS).compressed_klass_base)() }; - static ref COMPRESSED_KLASS_SHIFT: usize = unsafe { ((*UPCALLS).compressed_klass_shift)() }; +static COMPRESSED_KLASS_BASE: Atomic
= Atomic::new(Address::ZERO); +static COMPRESSED_KLASS_SHIFT: AtomicUsize = AtomicUsize::new(0); + +/// When enabling compressed pointers, the class pointers are also compressed. +/// The c++ part of the binding should pass the compressed klass base and shift to rust binding, as object scanning will need it. +pub fn set_compressed_klass_base_and_shift(base: Address, shift: usize) { + COMPRESSED_KLASS_BASE.store(base, Ordering::Relaxed); + COMPRESSED_KLASS_SHIFT.store(shift, Ordering::Relaxed); } impl OopDesc { @@ -297,7 +307,8 @@ impl OopDesc { pub fn klass(&self) -> &'static Klass { if COMPRESSED { let compressed = unsafe { self.klass.narrow_klass }; - let addr = *COMPRESSED_KLASS_BASE + ((compressed as usize) << *COMPRESSED_KLASS_SHIFT); + let addr = COMPRESSED_KLASS_BASE.load(Ordering::Relaxed) + + ((compressed as usize) << COMPRESSED_KLASS_SHIFT.load(Ordering::Relaxed)); unsafe { &*addr.to_ptr::() } } else { unsafe { self.klass.klass } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index a95792a2..6798da23 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -280,6 +280,11 @@ pub extern "C" fn mmtk_use_compressed_ptrs() { crate::edges::enable_compressed_oops() } +#[no_mangle] +pub extern "C" fn mmtk_set_compressed_klass_base_and_shift(base: Address, shift: usize) { + crate::abi::set_compressed_klass_base_and_shift(base, shift) +} + #[no_mangle] pub extern "C" fn is_in_mmtk_spaces(object: ObjectReference) -> bool { if crate::use_compressed_oops() { @@ -382,11 +387,13 @@ pub extern "C" fn process_bulk(options: *const c_char) -> bool { #[no_mangle] pub extern "C" fn mmtk_narrow_oop_base() -> Address { + debug_assert!(crate::use_compressed_oops()); crate::edges::BASE.load(Ordering::Relaxed) } #[no_mangle] pub extern "C" fn mmtk_narrow_oop_shift() -> usize { + debug_assert!(crate::use_compressed_oops()); crate::edges::SHIFT.load(Ordering::Relaxed) } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 2cdbc484..7e3e48b6 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -118,8 +118,6 @@ pub struct OpenJDK_Upcalls { pub schedule_finalizer: extern "C" fn(), pub prepare_for_roots_re_scanning: extern "C" fn(), pub enqueue_references: extern "C" fn(objects: *const ObjectReference, len: usize), - pub compressed_klass_base: extern "C" fn() -> Address, - pub compressed_klass_shift: extern "C" fn() -> usize, } pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); @@ -140,23 +138,6 @@ pub static VO_BIT_ADDRESS: uintptr_t = pub static FREE_LIST_ALLOCATOR_SIZE: uintptr_t = std::mem::size_of::>>(); -#[no_mangle] -pub static DISABLE_ALLOCATION_FAST_PATH: i32 = - (cfg!(feature = "no_fast_alloc") || cfg!(feature = "object_size_distribution")) as _; - -#[no_mangle] -pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = - std::mem::size_of::>>(); - -#[no_mangle] -pub static mut CONCURRENT_MARKING_ACTIVE: u8 = 0; - -#[no_mangle] -pub static mut HEAP_START: Address = Address::ZERO; - -#[no_mangle] -pub static mut HEAP_END: Address = Address::ZERO; - #[derive(Default)] pub struct OpenJDK; @@ -190,10 +171,6 @@ lazy_static! { let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); edges::initialize_compressed_oops_base_and_shift(); - unsafe { - HEAP_START = mmtk::memory_manager::starting_heap_address(); - HEAP_END = mmtk::memory_manager::last_heap_address(); - } *ret }; pub static ref SINGLETON_UNCOMPRESSED: MMTK> = { @@ -202,10 +179,6 @@ lazy_static! { assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); - unsafe { - HEAP_START = mmtk::memory_manager::starting_heap_address(); - HEAP_END = mmtk::memory_manager::last_heap_address(); - } *ret }; } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 750bb363..3bdb2a2a 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -23,8 +23,6 @@ extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; extern const uintptr_t VO_BIT_ADDRESS; extern const size_t MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; extern const uintptr_t FREE_LIST_ALLOCATOR_SIZE; -extern uintptr_t HEAP_START; -extern uintptr_t HEAP_END; extern const char* get_mmtk_version(); @@ -183,8 +181,6 @@ typedef struct { void (*schedule_finalizer)(); void (*prepare_for_roots_re_scanning)(); void (*enqueue_references)(void** objects, size_t len); - void* (*compressed_klass_base)(); - size_t (*compressed_klass_shift)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); @@ -195,6 +191,7 @@ extern bool mmtk_set_heap_size(size_t min, size_t max); extern bool mmtk_use_compressed_ptrs(); extern void* mmtk_narrow_oop_base(); extern size_t mmtk_narrow_oop_shift(); +extern size_t mmtk_set_compressed_klass_base_and_shift(void* base, size_t shift); extern size_t used_bytes(); extern void* starting_heap_address(); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index a9a97ea2..ecdd822e 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -148,6 +148,9 @@ void MMTkHeap::schedule_finalizer() { void MMTkHeap::post_initialize() { CollectedHeap::post_initialize(); + if (UseCompressedOops) { + mmtk_set_compressed_klass_base_and_shift((void*) Universe::narrow_klass_base(), (size_t) Universe::narrow_klass_shift()); + } } void MMTkHeap::enable_collection() { diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 8da5fd84..af3033ae 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -313,14 +313,6 @@ static void mmtk_enqueue_references(void** objects, size_t len) { assert(Universe::has_reference_pending_list(), "Reference pending list is empty after swap"); } -static void* mmtk_compressed_klass_base() { - return (void*) Universe::narrow_klass_base(); -} - -static size_t mmtk_compressed_klass_shift() { - return (size_t) Universe::narrow_klass_shift(); -} - OpenJDK_Upcalls mmtk_upcalls = { mmtk_stop_all_mutators, mmtk_resume_mutators, @@ -358,7 +350,5 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_number_of_mutators, mmtk_schedule_finalizer, mmtk_prepare_for_roots_re_scanning, - mmtk_enqueue_references, - mmtk_compressed_klass_base, - mmtk_compressed_klass_shift, + mmtk_enqueue_references }; From 39da263e5c1fb144bff0abc0e879a171c4a866df Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 25 Aug 2023 09:16:08 +1000 Subject: [PATCH 366/494] Update Cargo.lock --- mmtk/Cargo.lock | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 5213b63e..1c8ff157 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -145,7 +145,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset 0.7.1", + "memoffset", "scopeguard", ] @@ -433,15 +433,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] - [[package]] name = "mmtk" version = "0.18.0" @@ -461,7 +452,6 @@ dependencies = [ "lazy_static", "libc", "log", - "memoffset 0.9.0", "mmtk-macros", "num-traits", "num_cpus", From 83035359b9e0aca59813f64f6e23614470f08786 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 25 Aug 2023 09:17:47 +1000 Subject: [PATCH 367/494] Update Cargo.toml --- mmtk/Cargo.lock | 22 +++++++++++++++++----- mmtk/Cargo.toml | 4 ++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 1c8ff157..e0af6798 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -145,7 +145,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", + "memoffset 0.7.1", "scopeguard", ] @@ -433,9 +433,19 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "mmtk" -version = "0.18.0" +version = "0.19.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=039055ab9a94ac48919a8f3fac090248e406373d#039055ab9a94ac48919a8f3fac090248e406373d" dependencies = [ "atomic", "atomic-traits", @@ -452,6 +462,7 @@ dependencies = [ "lazy_static", "libc", "log", + "memoffset 0.9.0", "mmtk-macros", "num-traits", "num_cpus", @@ -466,7 +477,8 @@ dependencies = [ [[package]] name = "mmtk-macros" -version = "0.18.0" +version = "0.19.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=039055ab9a94ac48919a8f3fac090248e406373d#039055ab9a94ac48919a8f3fac090248e406373d" dependencies = [ "proc-macro-error", "proc-macro2", @@ -544,9 +556,9 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "probe" -version = "0.3.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216e81fcf486280f0b8b18ca43ceafd32739eb0eb703eb024a8d00814eeba556" +checksum = "d8e2d2444b730c8f027344c60f9e1f1554d7a3342df9bdd425142ed119a6e5a3" [[package]] name = "proc-macro-error" diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 7900b045..f647f9af 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -31,9 +31,9 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -# mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "f1a0bb7fbec97dd84e35a40e8be01cf5018f2f9e" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "039055ab9a94ac48919a8f3fac090248e406373d" } # Uncomment the following to build locally -mmtk = { path = "../../mmtk-core" } +# mmtk = { path = "../repos/mmtk-core" } [build-dependencies] built = { version = "0.5.1", features = ["git2"] } From 59b3cbcbe83fb961ea89340b929501952b058643 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 25 Aug 2023 12:08:41 +1000 Subject: [PATCH 368/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/collection.rs | 17 +++-------------- mmtk/src/scanning.rs | 26 +++++--------------------- 4 files changed, 11 insertions(+), 38 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index a401c21c..5b04da98 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -527,7 +527,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6686566bb8924048d7f415442fc50bf870680bb7#6686566bb8924048d7f415442fc50bf870680bb7" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=746be259a8348ed895d1d4ce8790d1c8f6942b39#746be259a8348ed895d1d4ce8790d1c8f6942b39" dependencies = [ "atomic", "atomic-traits", @@ -559,7 +559,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.18.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=6686566bb8924048d7f415442fc50bf870680bb7#6686566bb8924048d7f415442fc50bf870680bb7" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=746be259a8348ed895d1d4ce8790d1c8f6942b39#746be259a8348ed895d1d4ce8790d1c8f6942b39" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 6b2e8e81..c3c64a59 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "6686566bb8924048d7f415442fc50bf870680bb7" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "746be259a8348ed895d1d4ce8790d1c8f6942b39" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index fac5b140..9091e44c 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -1,8 +1,8 @@ use mmtk::scheduler::{GCWorker, ProcessEdgesWork}; use mmtk::util::alloc::AllocationError; use mmtk::util::opaque_pointer::*; -use mmtk::vm::{Collection, GCThreadContext, Scanning, VMBinding}; -use mmtk::{Mutator, MutatorContext}; +use mmtk::vm::{Collection, GCThreadContext}; +use mmtk::Mutator; use crate::reference_glue::DISCOVERED_LISTS; use crate::UPCALLS; @@ -21,13 +21,10 @@ impl Collection> for VMCollection { ) where F: FnMut(&'static mut Mutator>), { - let scan_mutators_in_safepoint = - < as VMBinding>::VMScanning as Scanning>>::SCAN_MUTATORS_IN_SAFEPOINT; - unsafe { ((*UPCALLS).stop_all_mutators)( tls, - scan_mutators_in_safepoint, + false, MutatorClosure::from_rust_closure::<_, COMPRESSED>(&mut mutator_visitor), current_gc_should_unload_classes, ); @@ -65,14 +62,6 @@ impl Collection> for VMCollection { } } - fn prepare_mutator>>( - _tls_w: VMWorkerThread, - _tls_m: VMMutatorThread, - _m: &T, - ) { - // unimplemented!() - } - fn out_of_memory(tls: VMThread, err_kind: AllocationError) { unsafe { ((*UPCALLS).out_of_memory)(tls, err_kind); diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 44da1eb7..59bb03a3 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -46,9 +46,6 @@ pub(crate) fn to_edges_closure>(factory: &mut F) } impl Scanning> for VMScanning { - const SCAN_MUTATORS_IN_SAFEPOINT: bool = false; - const SINGLE_THREAD_MUTATOR_SCANNING: bool = false; - fn scan_object( tls: VMWorkerThread, object: ObjectReference, @@ -88,15 +85,6 @@ impl Scanning> for VMScanning { // TODO } - fn scan_roots_in_all_mutator_threads( - _tls: VMWorkerThread, - mut factory: impl RootsWorkFactory>, - ) { - unsafe { - ((*UPCALLS).scan_roots_in_all_mutator_threads)(to_edges_closure(&mut factory)); - } - } - fn scan_roots_in_mutator_thread( _tls: VMWorkerThread, mutator: &'static mut Mutator>, @@ -159,15 +147,11 @@ impl Scanning> for VMScanning { WorkBucketStage::RCProcessIncs, w, ); - if !(>>::SCAN_MUTATORS_IN_SAFEPOINT - && >>::SINGLE_THREAD_MUTATOR_SCANNING) - { - memory_manager::add_work_packet( - &crate::singleton::(), - WorkBucketStage::RCProcessIncs, - ScanVMThreadRoots::new(factory), - ); - } + memory_manager::add_work_packet( + &crate::singleton::(), + WorkBucketStage::RCProcessIncs, + ScanVMThreadRoots::new(factory), + ); } fn supports_return_barrier() -> bool { From d382526c041b44aa18c5fd2f1a8205ccce0fac7a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 28 Aug 2023 10:26:01 +1000 Subject: [PATCH 369/494] Update mmtk-core --- mmtk/Cargo.lock | 4 +-- mmtk/Cargo.toml | 2 +- mmtk/src/api.rs | 1 - mmtk/src/edges.rs | 38 +-------------------- mmtk/src/lib.rs | 86 ++++++++++++++++++++++++++++++----------------- 5 files changed, 59 insertions(+), 72 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index ca08cc15..5675ca5b 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -507,7 +507,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.19.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=acbb579b21f124934c65ada8646eb32a4a5b22b5#acbb579b21f124934c65ada8646eb32a4a5b22b5" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=72aad377bc2aace56dbce9d0623d801383d64e43#72aad377bc2aace56dbce9d0623d801383d64e43" dependencies = [ "atomic", "atomic-traits", @@ -540,7 +540,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.19.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=acbb579b21f124934c65ada8646eb32a4a5b22b5#acbb579b21f124934c65ada8646eb32a4a5b22b5" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=72aad377bc2aace56dbce9d0623d801383d64e43#72aad377bc2aace56dbce9d0623d801383d64e43" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index b97df870..cea9dba1 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "acbb579b21f124934c65ada8646eb32a4a5b22b5" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "72aad377bc2aace56dbce9d0623d801383d64e43" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index a7e33e8c..9d96b06b 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -1,7 +1,6 @@ use crate::abi::Oop; use crate::OpenJDK; use crate::OpenJDKEdge; -use crate::OpenJDKEdgeRange; use crate::OpenJDK_Upcalls; use crate::BUILDER; use crate::UPCALLS; diff --git a/mmtk/src/edges.rs b/mmtk/src/edges.rs index eef071a4..6ea5ee83 100644 --- a/mmtk/src/edges.rs +++ b/mmtk/src/edges.rs @@ -1,6 +1,6 @@ use std::{ ops::Range, - sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}, + sync::atomic::{AtomicU32, AtomicUsize, Ordering}, }; use atomic::Atomic; @@ -12,45 +12,9 @@ use mmtk::{ vm::edge_shape::{Edge, MemorySlice}, }; -static USE_COMPRESSED_OOPS: AtomicBool = AtomicBool::new(false); pub static BASE: Atomic
= Atomic::new(Address::ZERO); pub static SHIFT: AtomicUsize = AtomicUsize::new(0); -pub fn enable_compressed_oops() { - static COMPRESSED_OOPS_INITIALIZED: AtomicBool = AtomicBool::new(false); - assert!( - !COMPRESSED_OOPS_INITIALIZED.fetch_or(true, Ordering::Relaxed), - "cannot enable compressed pointers twice." - ); - if cfg!(not(target_arch = "x86_64")) { - panic!("Compressed pointer is only enable on x86_64 platforms.\ - For other RISC architectures, we need to find a way to process compressed embeded pointers in code objects first."); - } - USE_COMPRESSED_OOPS.store(true, Ordering::Relaxed) -} - -pub fn use_compressed_oops() -> bool { - USE_COMPRESSED_OOPS.load(Ordering::Relaxed) -} - -pub fn initialize_compressed_oops_base_and_shift() { - let heap_end = mmtk::memory_manager::last_heap_address().as_usize(); - if heap_end <= (4usize << 30) { - BASE.store(Address::ZERO, Ordering::Relaxed); - SHIFT.store(0, Ordering::Relaxed); - } else if heap_end <= (32usize << 30) { - BASE.store(Address::ZERO, Ordering::Relaxed); - SHIFT.store(3, Ordering::Relaxed); - } else { - // set heap base as HEAP_START - 4096, to make sure null pointer value is not conflict with HEAP_START - BASE.store( - mmtk::memory_manager::starting_heap_address() - 4096, - Ordering::Relaxed, - ); - SHIFT.store(3, Ordering::Relaxed); - } -} - /// The type of edges in OpenJDK. /// Currently it has the same layout as `Address`, but we override its load and store methods. #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 5f30a1bc..53b99cf8 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -6,7 +6,6 @@ extern crate spin; use std::collections::HashMap; use std::ptr::null_mut; -use std::sync::atomic::{AtomicU32, AtomicUsize}; use std::sync::Mutex; use edges::{OpenJDKEdge, OpenJDKEdgeRange}; @@ -14,12 +13,12 @@ use libc::{c_char, c_void, uintptr_t}; use mmtk::plan::lxr::LXR; use mmtk::util::alloc::AllocationError; use mmtk::util::constants::{ - BYTES_IN_ADDRESS, BYTES_IN_INT, LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_INT, + BYTES_IN_ADDRESS, BYTES_IN_INT, LOG_BYTES_IN_ADDRESS, LOG_BYTES_IN_GBYTE, LOG_BYTES_IN_INT, }; -use mmtk::util::heap::layout::vm_layout_constants::VM_LAYOUT_CONSTANTS; -use mmtk::util::opaque_pointer::*; +use mmtk::util::heap::vm_layout::{vm_layout, VMLayout, BYTES_IN_CHUNK}; +use mmtk::util::{conversions, opaque_pointer::*}; use mmtk::util::{Address, ObjectReference}; -use mmtk::vm::edge_shape::{Edge, MemorySlice}; +use mmtk::vm::edge_shape::Edge; use mmtk::vm::VMBinding; use mmtk::{MMTKBuilder, Mutator, MMTK}; @@ -237,31 +236,11 @@ fn log_bytes_in_field() -> usize { unsafe { LOG_BYTES_IN_FIELD } } -fn bytes_in_field() -> usize { - unsafe { BYTES_IN_FIELD } -} - static mut BASE: Address = Address::ZERO; static mut SHIFT: usize = 0; -fn compress(o: ObjectReference) -> u32 { - if o.is_null() { - 0u32 - } else { - unsafe { ((o.to_raw_address() - BASE) >> SHIFT) as u32 } - } -} - -fn decompress(v: u32) -> ObjectReference { - if v == 0 { - ObjectReference::NULL - } else { - unsafe { (BASE + ((v as usize) << SHIFT)).to_object_reference::>() } - } -} - fn initialize_compressed_oops() { - let heap_end = VM_LAYOUT_CONSTANTS.heap_end.as_usize(); + let heap_end = vm_layout().heap_end.as_usize(); if heap_end <= (4usize << 30) { unsafe { BASE = Address::ZERO; @@ -274,7 +253,7 @@ fn initialize_compressed_oops() { } } else { unsafe { - BASE = VM_LAYOUT_CONSTANTS.heap_start - 4096; + BASE = vm_layout().heap_start - 4096; SHIFT = 3; } } @@ -311,12 +290,13 @@ lazy_static! { builder.set_option("use_35bit_address_space", "true"); builder.set_option("use_35bit_address_space", "true"); assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); + set_compressed_pointer_vm_layout(&mut builder); let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); initialize_compressed_oops(); unsafe { - HEAP_START = VM_LAYOUT_CONSTANTS.heap_start; - HEAP_END = VM_LAYOUT_CONSTANTS.heap_end; + HEAP_START = vm_layout().heap_start; + HEAP_END = vm_layout().heap_end; RC_ENABLED = ret .get_plan() .downcast_ref::>>() @@ -331,8 +311,8 @@ lazy_static! { let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); unsafe { - HEAP_START = VM_LAYOUT_CONSTANTS.heap_start; - HEAP_END = VM_LAYOUT_CONSTANTS.heap_end; + HEAP_START = vm_layout().heap_start; + HEAP_END = vm_layout().heap_end; RC_ENABLED = ret .get_plan() .downcast_ref::>>() @@ -388,3 +368,47 @@ extern "C" fn dump_and_reset_obj_dist() { assert!(cfg!(feature = "object_size_distribution")); mmtk::dump_and_reset_obj_dist("Dynamic", &mut OBJ_COUNT.lock().unwrap()); } + +fn set_compressed_pointer_vm_layout(builder: &mut MMTKBuilder) { + let max_heap_size = builder.options.gc_trigger.max_heap_size(); + assert!( + max_heap_size <= (32usize << LOG_BYTES_IN_GBYTE), + "Heap size is larger than 32 GB" + ); + let rounded_heap_size = (max_heap_size + (BYTES_IN_CHUNK - 1)) & !(BYTES_IN_CHUNK - 1); + let mut start: usize = 0x4000_0000; // block lowest 1G + let (end, mut small_chunk_space_size) = match rounded_heap_size { + // heap <= 2G; virtual = 3G; max-small-space=2G; min-small-space=1.5G + heap if heap <= 2 << 30 => { + let end = 4usize << 30; + let small_space = 2 << 30; + (end, small_space) + } + // heap <= 29G; virtual = 31G; max-small-space=29G; + heap if heap <= 29 << 30 => { + let end = 32usize << 30; + let small_space = usize::min(heap * 3 / 2, 29 << 30); + (end, small_space) + } + // heap > 29G; virtual = 32G - 1chunk; max-small-space=30G; start=0x200_0000_0000 + heap => { + // A workaround to avoid address conflict with the OpenJDK + // MetaSpace, which may start from 0x8_0000_0000 + start = 0x200_0000_0000; + let end = start + 0x8_0000_0000 - BYTES_IN_CHUNK; + let small_space = usize::min(heap * 3 / 2, 30 << 30); + (end, small_space) + } + }; + small_chunk_space_size = + (small_chunk_space_size + (BYTES_IN_CHUNK - 1)) & !(BYTES_IN_CHUNK - 1); + let constants = VMLayout { + log_address_space: 35, + heap_start: conversions::chunk_align_down(unsafe { Address::from_usize(start) }), + heap_end: conversions::chunk_align_up(unsafe { Address::from_usize(end) }), + log_space_extent: 31, + force_use_contiguous_spaces: false, + small_chunk_space_size: Some(small_chunk_space_size), + }; + builder.set_vm_layout(constants); +} From 8a84dd4205d24d42bee54cf1e2efd8262401e798 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 28 Aug 2023 11:51:57 +1000 Subject: [PATCH 370/494] cleanup --- mmtk/src/abi.rs | 9 +-- mmtk/src/collection.rs | 6 +- mmtk/src/lib.rs | 14 ----- mmtk/src/object_scanning.rs | 19 +++--- mmtk/src/reference_glue.rs | 120 ++++++++++++++++++++++-------------- openjdk/mmtk.h | 2 - openjdk/mmtkUpcalls.cpp | 10 --- 7 files changed, 94 insertions(+), 86 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 9117d857..99f9e689 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -1,5 +1,6 @@ use super::UPCALLS; use crate::Edge; +use crate::OpenJDKEdge; use atomic::Atomic; use atomic::Ordering; use mmtk::util::constants::*; @@ -274,11 +275,11 @@ impl InstanceRefKlass { } *DISCOVERED_OFFSET } - pub fn referent_address(oop: Oop) -> E { - E::from_address(oop.get_field_address(Self::referent_offset())) + pub fn referent_address(oop: Oop) -> OpenJDKEdge { + oop.get_field_address(Self::referent_offset()).into() } - pub fn discovered_address(oop: Oop) -> E { - E::from_address(oop.get_field_address(Self::discovered_offset())) + pub fn discovered_address(oop: Oop) -> OpenJDKEdge { + oop.get_field_address(Self::discovered_offset()).into() } } diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index e823e52b..3b163c4c 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -74,19 +74,19 @@ impl Collection> for VMCollection { fn process_weak_refs>>( worker: &mut GCWorker>, ) { - DISCOVERED_LISTS.process_soft_weak_final_refs::(worker) + DISCOVERED_LISTS.process_soft_weak_final_refs::(worker) } fn process_final_refs>>( worker: &mut GCWorker>, ) { - DISCOVERED_LISTS.resurrect_final_refs::(worker) + DISCOVERED_LISTS.resurrect_final_refs::(worker) } fn process_phantom_refs>>( worker: &mut GCWorker>, ) { - DISCOVERED_LISTS.process_phantom_refs::(worker) + DISCOVERED_LISTS.process_phantom_refs::(worker) } fn update_weak_processor(lxr: bool) { diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 5d0f3ec7..be47a084 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -21,18 +21,6 @@ use mmtk::vm::edge_shape::Edge; use mmtk::vm::VMBinding; use mmtk::{MMTKBuilder, Mutator, MMTK}; -macro_rules! with_singleton { - (|$x: ident| $($expr:tt)*) => { - if crate::use_compressed_oops() { - let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_COMPRESSED; - $($expr)* - } else { - let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_UNCOMPRESSED; - $($expr)* - } - }; -} - mod abi; pub mod active_plan; pub mod api; @@ -151,8 +139,6 @@ pub struct OpenJDK_Upcalls { pub swap_reference_pending_list: extern "C" fn(objects: ObjectReference) -> ObjectReference, pub java_lang_class_klass_offset_in_bytes: extern "C" fn() -> usize, pub java_lang_classloader_loader_data_offset: extern "C" fn() -> usize, - pub compressed_klass_base: extern "C" fn() -> Address, - pub compressed_klass_shift: extern "C" fn() -> usize, pub nmethod_fix_relocation: extern "C" fn(Address), pub clear_claimed_marks: extern "C" fn(), pub unload_classes: extern "C" fn(), diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 5ef47166..fa241b77 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -164,7 +164,7 @@ impl OopIterate for InstanceRefKlass { panic!("oop_iterate on InstanceRefKlass with reference_type as None") } rt => { - if !Self::discover_reference::, COMPRESSED>(oop, rt) { + if !Self::discover_reference::(oop, rt) { Self::process_ref_as_strong(oop, closure) } } @@ -200,19 +200,22 @@ impl InstanceRefKlass { } true } - fn process_ref_as_strong>(oop: Oop, closure: &mut V) { - let referent_addr = Self::referent_address::(oop); + fn process_ref_as_strong( + oop: Oop, + closure: &mut impl EdgeVisitor>, + ) { + let referent_addr = Self::referent_address::(oop); closure.visit_edge(referent_addr); - let discovered_addr = Self::discovered_address::(oop); + let discovered_addr = Self::discovered_address::(oop); closure.visit_edge(discovered_addr); } - fn discover_reference(oop: Oop, rt: ReferenceType) -> bool { + fn discover_reference(oop: Oop, rt: ReferenceType) -> bool { // Do not discover new refs during reference processing. if !DISCOVERED_LISTS.allow_discover() { return false; } // Do not discover if the referent is live. - let addr = InstanceRefKlass::referent_address::(oop); + let addr = InstanceRefKlass::referent_address::(oop); let referent: ObjectReference = addr.load(); // Skip live or null referents if referent.is_reachable() || referent.is_null() { @@ -226,13 +229,13 @@ impl InstanceRefKlass { { return false; } - if rt == ReferenceType::Final && DISCOVERED_LISTS.is_discovered::(reference) { + if rt == ReferenceType::Final && DISCOVERED_LISTS.is_discovered::(reference) { return false; } // Add to reference list DISCOVERED_LISTS .get(rt) - .add::(reference, referent); + .add::(reference, referent); true } } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 848fc620..ea952943 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -3,38 +3,41 @@ use std::sync::atomic::AtomicBool; use std::sync::Mutex; use crate::abi::{InstanceRefKlass, Oop, ReferenceType}; +use crate::edges::OpenJDKEdge; use crate::OpenJDK; use atomic::{Atomic, Ordering}; use mmtk::scheduler::{GCWork, GCWorker, ProcessEdgesWork, WorkBucketStage}; use mmtk::util::opaque_pointer::VMWorkerThread; use mmtk::util::ObjectReference; use mmtk::vm::edge_shape::Edge; -use mmtk::vm::{ReferenceGlue, VMBinding}; +use mmtk::vm::ReferenceGlue; use mmtk::MMTK; -fn set_referent(reff: ObjectReference, referent: ObjectReference) { +fn set_referent(reff: ObjectReference, referent: ObjectReference) { let oop = Oop::from(reff); - let slot = InstanceRefKlass::referent_address::(oop); + let slot = InstanceRefKlass::referent_address::(oop); mmtk::plan::lxr::record_edge_for_validation(slot, referent); slot.store(referent) } -fn get_referent(object: ObjectReference) -> ObjectReference { +fn get_referent(object: ObjectReference) -> ObjectReference { let oop = Oop::from(object); - InstanceRefKlass::referent_address::(oop).load() + InstanceRefKlass::referent_address::(oop).load() } -fn get_next_reference_slot(object: ObjectReference) -> E { +fn get_next_reference_slot( + object: ObjectReference, +) -> OpenJDKEdge { let oop = Oop::from(object); - InstanceRefKlass::discovered_address::(oop) + InstanceRefKlass::discovered_address::(oop) } -fn get_next_reference(object: ObjectReference) -> ObjectReference { - get_next_reference_slot::(object).load() +fn get_next_reference(object: ObjectReference) -> ObjectReference { + get_next_reference_slot::(object).load() } -fn set_next_reference(object: ObjectReference, next: ObjectReference) { - let slot = get_next_reference_slot::(object); +fn set_next_reference(object: ObjectReference, next: ObjectReference) { + let slot = get_next_reference_slot::(object); mmtk::plan::lxr::record_edge_for_validation(slot, next); slot.store(next) } @@ -66,7 +69,7 @@ impl DiscoveredList { } } - pub fn add( + pub fn add( &self, reference: ObjectReference, referent: ObjectReference, @@ -79,7 +82,7 @@ impl DiscoveredList { // Note that the list is a singly-linked list, and the tail object should point to itself. let oop = Oop::from(reference); let head = self.head.load(Ordering::Relaxed); - let addr = InstanceRefKlass::discovered_address::(oop); + let addr = InstanceRefKlass::discovered_address::(oop); let next_discovered = if head.is_null() { reference } else { head }; debug_assert!(!next_discovered.is_null()); if addr @@ -93,7 +96,7 @@ impl DiscoveredList { { self.head.store(reference, Ordering::Relaxed); } - debug_assert!(!get_next_reference::(reference).is_null()); + debug_assert!(!get_next_reference::(reference).is_null()); } } @@ -107,6 +110,17 @@ pub struct DiscoveredLists { impl DiscoveredLists { pub fn new() -> Self { + macro_rules! with_singleton { + (|$x: ident| $($expr:tt)*) => { + if crate::use_compressed_oops() { + let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_COMPRESSED; + $($expr)* + } else { + let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_UNCOMPRESSED; + $($expr)* + } + }; + } let workers = with_singleton!(|singleton| singleton.scheduler.num_workers()); let build_lists = |rt| (0..workers).map(|_| DiscoveredList::new(rt)).collect(); Self { @@ -126,8 +140,8 @@ impl DiscoveredLists { self.allow_discover.load(Ordering::SeqCst) } - pub fn is_discovered(&self, reference: ObjectReference) -> bool { - !get_next_reference::(reference).is_null() + pub fn is_discovered(&self, reference: ObjectReference) -> bool { + !get_next_reference::(reference).is_null() } pub fn get_by_rt_and_index(&self, rt: ReferenceType, index: usize) -> &DiscoveredList { @@ -145,7 +159,7 @@ impl DiscoveredLists { self.get_by_rt_and_index(rt, id) } - pub fn process_lists( + fn process_lists( &self, worker: &mut GCWorker, rt: ReferenceType, @@ -161,7 +175,7 @@ impl DiscoveredLists { if head.is_null() { continue; } - let w = ProcessDiscoveredList { + let w = ProcessDiscoveredList::<_, COMPRESSED> { list_index: i, head, rt, @@ -174,18 +188,24 @@ impl DiscoveredLists { pub fn reconsider_soft_refs(&self, _worker: &mut GCWorker) {} - pub fn process_soft_weak_final_refs(&self, worker: &mut GCWorker) { + pub fn process_soft_weak_final_refs( + &self, + worker: &mut GCWorker, + ) { self.allow_discover.store(false, Ordering::SeqCst); if !*worker.mmtk.get_options().no_reference_types { - self.process_lists::(worker, ReferenceType::Soft, &self.soft, true); - self.process_lists::(worker, ReferenceType::Weak, &self.weak, true); + self.process_lists::(worker, ReferenceType::Soft, &self.soft, true); + self.process_lists::(worker, ReferenceType::Weak, &self.weak, true); } if !*worker.mmtk.get_options().no_finalizer { - self.process_lists::(worker, ReferenceType::Final, &self.r#final, false); + self.process_lists::(worker, ReferenceType::Final, &self.r#final, false); } } - pub fn resurrect_final_refs(&self, worker: &mut GCWorker) { + pub fn resurrect_final_refs( + &self, + worker: &mut GCWorker, + ) { assert!(!*worker.mmtk.get_options().no_finalizer); let lists = &self.r#final; let mut packets = vec![]; @@ -195,7 +215,7 @@ impl DiscoveredLists { if head.is_null() { continue; } - let w = ResurrectFinalizables { + let w = ResurrectFinalizables::<_, COMPRESSED> { list_index: i, head, _p: PhantomData::, @@ -205,9 +225,12 @@ impl DiscoveredLists { worker.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets); } - pub fn process_phantom_refs(&self, worker: &mut GCWorker) { + pub fn process_phantom_refs( + &self, + worker: &mut GCWorker, + ) { assert!(!*worker.mmtk.get_options().no_reference_types); - self.process_lists::(worker, ReferenceType::Phantom, &self.phantom, true); + self.process_lists::(worker, ReferenceType::Phantom, &self.phantom, true); } } @@ -216,28 +239,30 @@ lazy_static! { pub static ref DISCOVERED_LISTS: DiscoveredLists = DiscoveredLists::new(); } -pub struct ProcessDiscoveredList { +pub struct ProcessDiscoveredList { list_index: usize, head: ObjectReference, rt: ReferenceType, _p: PhantomData, } -impl GCWork for ProcessDiscoveredList { +impl GCWork + for ProcessDiscoveredList +{ fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { let mut trace = E::new(vec![], false, mmtk); trace.set_worker(worker); let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); - let new_list = iterate_list::<_, ::VMEdge>(self.head, |reference| { + let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { debug_assert!( - get_next_reference::<::VMEdge>(reference).is_null(), + get_next_reference::(reference).is_null(), "next must be null. ref={:?} {:?} bin={}", reference, self.rt, self.list_index ); let reference = trace.trace_object(reference); - let referent = get_referent::<::VMEdge>(reference); + let referent = get_referent::(reference); if referent.is_null() { // Remove from the discovered list return DiscoveredListIterationResult::Remove; @@ -246,13 +271,13 @@ impl GCWork for ProcessDiscoveredList { let forwarded = trace.trace_object(referent); debug_assert!(forwarded.get_forwarded_object().is_none()); debug_assert!(reference.get_forwarded_object().is_none()); - set_referent::<::VMEdge>(reference, forwarded); + set_referent::(reference, forwarded); // Remove from the discovered list return DiscoveredListIterationResult::Remove; } else { if self.rt != ReferenceType::Final { // Clear this referent - set_referent::<::VMEdge>(reference, ObjectReference::NULL); + set_referent::(reference, ObjectReference::NULL); // set_referent(reference, ObjectReference::NULL); } // Keep the reference @@ -269,7 +294,7 @@ impl GCWork for ProcessDiscoveredList { .store(head, Ordering::SeqCst); } else { let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; - set_next_reference::<::VMEdge>(tail, old_head); + set_next_reference::(tail, old_head); } } else { if self.rt == ReferenceType::Final { @@ -282,21 +307,23 @@ impl GCWork for ProcessDiscoveredList { } } -pub struct ResurrectFinalizables { +pub struct ResurrectFinalizables { list_index: usize, head: ObjectReference, _p: PhantomData, } -impl GCWork for ResurrectFinalizables { +impl GCWork + for ResurrectFinalizables +{ fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { let mut trace = E::new(vec![], false, mmtk); trace.set_worker(worker); - let new_list = iterate_list::<_, ::VMEdge>(self.head, |reference| { + let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { let reference = trace.trace_object(reference); - let referent = get_referent::<::VMEdge>(reference); + let referent = get_referent::(reference); let forwarded = trace.trace_object(referent); - set_referent::<::VMEdge>(reference, forwarded); + set_referent::(reference, forwarded); debug_assert!(forwarded.get_forwarded_object().is_none()); return DiscoveredListIterationResult::Enqueue(reference); }); @@ -308,7 +335,7 @@ impl GCWork for ResurrectFinalizables { .head .store(ObjectReference::NULL, Ordering::SeqCst); let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; - set_next_reference::<::VMEdge>(tail, old_head); + set_next_reference::(tail, old_head); } trace.flush(); } @@ -319,7 +346,10 @@ enum DiscoveredListIterationResult { Enqueue(ObjectReference), } -fn iterate_list DiscoveredListIterationResult, E: Edge>( +fn iterate_list< + F: FnMut(ObjectReference) -> DiscoveredListIterationResult, + const COMPRESSED: bool, +>( head: ObjectReference, mut visitor: F, ) -> Option<(ObjectReference, ObjectReference)> { @@ -336,15 +366,15 @@ fn iterate_list DiscoveredListIterationResult, E: E debug_assert!(reference.get_forwarded_object().is_none()); debug_assert!(reference.is_reachable()); // Update next_ref forwarding pointer - let next_ref = get_next_reference::(reference); + let next_ref = get_next_reference::(reference); let next_ref = next_ref.get_forwarded_object().unwrap_or(next_ref); debug_assert!(next_ref.get_forwarded_object().is_none()); // Reaches the end of the list? let end_of_list = next_ref == reference || next_ref.is_null(); // Remove `reference` from current list - set_next_reference::(reference, ObjectReference::NULL); + set_next_reference::(reference, ObjectReference::NULL); if let Some(forwarded_ref) = reference.get_forwarded_object() { - set_next_reference::(forwarded_ref, ObjectReference::NULL); + set_next_reference::(forwarded_ref, ObjectReference::NULL); } // Process reference let result = visitor(reference); @@ -353,7 +383,7 @@ fn iterate_list DiscoveredListIterationResult, E: E DiscoveredListIterationResult::Enqueue(reference) => { // Add to new list if let Some(new_head) = new_head { - set_next_reference::(reference, new_head); + set_next_reference::(reference, new_head); } else { new_tail = Some(reference); } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index b53e850e..cc72ca61 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -213,8 +213,6 @@ typedef struct { void* (*swap_reference_pending_list)(void* objects); size_t (*java_lang_class_klass_offset_in_bytes)(); size_t (*java_lang_classloader_loader_data_offset)(); - void* (*compressed_klass_base)(); - size_t (*compressed_klass_shift)(); void (*nmethod_fix_relocation)(void* nmethod); void (*clear_claimed_marks)(); void (*unload_classes)(); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index a0ec3ca6..f3c6e249 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -528,14 +528,6 @@ static size_t mmtk_java_lang_classloader_loader_data_offset() { return v; } -static void* mmtk_compressed_klass_base() { - return (void*) Universe::narrow_klass_base(); -} - -static size_t mmtk_compressed_klass_shift() { - return (size_t) Universe::narrow_klass_shift(); -} - static void nmethod_fix_relocation(void* nm) { ((nmethod*) nm)->fix_oop_relocations(); } @@ -583,8 +575,6 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_swap_reference_pending_list, mmtk_java_lang_class_klass_offset_in_bytes, mmtk_java_lang_classloader_loader_data_offset, - mmtk_compressed_klass_base, - mmtk_compressed_klass_shift, nmethod_fix_relocation, mmtk_clear_claimed_marks, mmtk_unload_classes, From 1141a30e279974462b25d5b101755bcbfefec5cb Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 30 Aug 2023 14:09:08 +1000 Subject: [PATCH 371/494] Update mmtk-core and rust toolchain --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/rust-toolchain | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 5675ca5b..17a0681f 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -507,7 +507,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.19.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=72aad377bc2aace56dbce9d0623d801383d64e43#72aad377bc2aace56dbce9d0623d801383d64e43" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=c4aaeae4e81aa1f1f82aa1b681acfff2a1557421#c4aaeae4e81aa1f1f82aa1b681acfff2a1557421" dependencies = [ "atomic", "atomic-traits", @@ -540,7 +540,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.19.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=72aad377bc2aace56dbce9d0623d801383d64e43#72aad377bc2aace56dbce9d0623d801383d64e43" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=c4aaeae4e81aa1f1f82aa1b681acfff2a1557421#c4aaeae4e81aa1f1f82aa1b681acfff2a1557421" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index cea9dba1..724d8fc9 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "72aad377bc2aace56dbce9d0623d801383d64e43" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "c4aaeae4e81aa1f1f82aa1b681acfff2a1557421" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/rust-toolchain b/mmtk/rust-toolchain index 68bc7ff2..0834888f 100644 --- a/mmtk/rust-toolchain +++ b/mmtk/rust-toolchain @@ -1 +1 @@ -1.71.1 +1.72.0 From 08218fcce9aa89a609b9a1bfd3225f9d6ee5641f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 22 Sep 2023 09:48:59 +0000 Subject: [PATCH 372/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 17a0681f..f48ff4ac 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -507,7 +507,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.19.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=c4aaeae4e81aa1f1f82aa1b681acfff2a1557421#c4aaeae4e81aa1f1f82aa1b681acfff2a1557421" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=50fbb1a9a0ec4cccdb2c8579d1be3ba833ebadf9#50fbb1a9a0ec4cccdb2c8579d1be3ba833ebadf9" dependencies = [ "atomic", "atomic-traits", @@ -540,7 +540,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.19.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=c4aaeae4e81aa1f1f82aa1b681acfff2a1557421#c4aaeae4e81aa1f1f82aa1b681acfff2a1557421" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=50fbb1a9a0ec4cccdb2c8579d1be3ba833ebadf9#50fbb1a9a0ec4cccdb2c8579d1be3ba833ebadf9" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 724d8fc9..3116fdeb 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "c4aaeae4e81aa1f1f82aa1b681acfff2a1557421" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "50fbb1a9a0ec4cccdb2c8579d1be3ba833ebadf9" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 297a9d1de1a85c4129aac0d9fde3cb56f76c23ff Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 10 Oct 2023 06:33:00 +0000 Subject: [PATCH 373/494] minor --- openjdk/mmtkHeap.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 77e03483..6cfeff9c 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -392,9 +392,11 @@ class MMTkRegisterNMethodOopClosure: public OopClosure { template void do_oop_work(T* p, bool narrow) { if (UseCompressedOops && !narrow) { guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); - p = (T*) (uintptr_t(p) | (1ull << 63)); + auto tagged_p = (T*) (uintptr_t(p) | (1ull << 63)); + mmtk_add_nmethod_oop((void*) tagged_p); + } else { + mmtk_add_nmethod_oop((void*) p); } - mmtk_add_nmethod_oop((void*) p); } public: void do_oop(oop* p) { do_oop_work(p, false); } From 351f8e2dd72f0fa2e15a83ef2e58605852c0f9e9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 11 Oct 2023 01:45:25 +0000 Subject: [PATCH 374/494] Optionally use discontig heap when compressed oops is disabled --- mmtk/Cargo.toml | 1 + mmtk/src/lib.rs | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 3116fdeb..4cac823e 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -85,3 +85,4 @@ log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] log_work_packets = ["mmtk/log_work_packets"] object_size_distribution = ["mmtk/object_size_distribution"] +discontig_space = [] diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index be47a084..3376bb9c 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -239,8 +239,11 @@ lazy_static! { }; pub static ref SINGLETON_UNCOMPRESSED: MMTK> = { assert!(!use_compressed_oops()); - let builder = BUILDER.lock().unwrap(); + let mut builder = BUILDER.lock().unwrap(); assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); + if cfg!(feature = "discontig_space") { + set_no_compressed_pointer_discontig_vm_layout(&mut builder); + } let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); unsafe { @@ -343,3 +346,47 @@ fn set_compressed_pointer_vm_layout(builder: &mut MMTKBuilder) { }; builder.set_vm_layout(constants); } + +fn set_no_compressed_pointer_discontig_vm_layout(builder: &mut MMTKBuilder) { + let max_heap_size = builder.options.gc_trigger.max_heap_size(); + assert!( + max_heap_size <= (32usize << LOG_BYTES_IN_GBYTE), + "Heap size is larger than 32 GB" + ); + let rounded_heap_size = (max_heap_size + (BYTES_IN_CHUNK - 1)) & !(BYTES_IN_CHUNK - 1); + let mut start: usize = 0x4000_0000; // block lowest 1G + let (end, mut small_chunk_space_size) = match rounded_heap_size { + // heap <= 2G; virtual = 3G; max-small-space=2G; min-small-space=1.5G + heap if heap <= 2 << 30 => { + let end = 4usize << 30; + let small_space = 2 << 30; + (end, small_space) + } + // heap <= 29G; virtual = 31G; max-small-space=29G; + heap if heap <= 29 << 30 => { + let end = 32usize << 30; + let small_space = usize::min(heap * 3 / 2, 29 << 30); + (end, small_space) + } + // heap > 29G; virtual = 32G - 1chunk; max-small-space=30G; start=0x200_0000_0000 + heap => { + // A workaround to avoid address conflict with the OpenJDK + // MetaSpace, which may start from 0x8_0000_0000 + start = 0x200_0000_0000; + let end = start + 0x8_0000_0000 - BYTES_IN_CHUNK; + let small_space = usize::min(heap * 3 / 2, 30 << 30); + (end, small_space) + } + }; + small_chunk_space_size = + (small_chunk_space_size + (BYTES_IN_CHUNK - 1)) & !(BYTES_IN_CHUNK - 1); + let constants = VMLayout { + log_address_space: 35, + heap_start: conversions::chunk_align_down(unsafe { Address::from_usize(start) }), + heap_end: conversions::chunk_align_up(unsafe { Address::from_usize(end) }), + log_space_extent: 31, + force_use_contiguous_spaces: false, + small_chunk_space_size: Some(small_chunk_space_size), + }; + builder.set_vm_layout(constants); +} From 366f12ab918689df7dd242e49d45d041048570b8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 11 Oct 2023 14:23:17 +0000 Subject: [PATCH 375/494] Update `MMTkCollectorPolicy::_heap_alignment` --- openjdk/mmtkCollectorPolicy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/mmtkCollectorPolicy.hpp b/openjdk/mmtkCollectorPolicy.hpp index 9dddbbe4..4fe4a32f 100644 --- a/openjdk/mmtkCollectorPolicy.hpp +++ b/openjdk/mmtkCollectorPolicy.hpp @@ -5,7 +5,7 @@ class MMTkCollectorPolicy : public CollectorPolicy { protected: virtual void initialize_alignments() { _space_alignment = 1 << 19; - _heap_alignment = _space_alignment; + _heap_alignment = 1 << 21; } public: MMTkCollectorPolicy() {} From d9e4a593a59950cb9178f6bcb5840a73bcae6e0b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 12 Oct 2023 15:32:45 +1100 Subject: [PATCH 376/494] Fix build error --- mmtk/Cargo.lock | 13 ++++++++++--- mmtk/Cargo.toml | 2 +- mmtk/src/reference_glue.rs | 4 ++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index f48ff4ac..6a2057eb 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -507,7 +507,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.19.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=50fbb1a9a0ec4cccdb2c8579d1be3ba833ebadf9#50fbb1a9a0ec4cccdb2c8579d1be3ba833ebadf9" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8f8cd43933618059d757d6cf6b870aa8f3484276#8f8cd43933618059d757d6cf6b870aa8f3484276" dependencies = [ "atomic", "atomic-traits", @@ -530,6 +530,7 @@ dependencies = [ "num-traits", "num_cpus", "pfm", + "portable-atomic", "regex", "spin", "strum", @@ -540,12 +541,12 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.19.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=50fbb1a9a0ec4cccdb2c8579d1be3ba833ebadf9#50fbb1a9a0ec4cccdb2c8579d1be3ba833ebadf9" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8f8cd43933618059d757d6cf6b870aa8f3484276#8f8cd43933618059d757d6cf6b870aa8f3484276" dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.29", ] [[package]] @@ -662,6 +663,12 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "portable-atomic" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" + [[package]] name = "proc-macro-error" version = "1.0.4" diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 4cac823e..f502c3cf 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -40,7 +40,7 @@ atomic = "0.5.1" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "50fbb1a9a0ec4cccdb2c8579d1be3ba833ebadf9" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8f8cd43933618059d757d6cf6b870aa8f3484276" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index ea952943..eba9b819 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -250,7 +250,7 @@ impl GCWork for ProcessDiscoveredList { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut trace = E::new(vec![], false, mmtk); + let mut trace = E::new(vec![], false, mmtk, WorkBucketStage::Unconstrained); trace.set_worker(worker); let retain = self.rt == ReferenceType::Soft && !mmtk.get_plan().is_emergency_collection(); let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { @@ -317,7 +317,7 @@ impl GCWork for ResurrectFinalizables { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut trace = E::new(vec![], false, mmtk); + let mut trace = E::new(vec![], false, mmtk, WorkBucketStage::Unconstrained); trace.set_worker(worker); let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { let reference = trace.trace_object(reference); From 5a15bacba5a3f2474e5da984605812b0e4150e74 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 12 Oct 2023 16:02:28 +1100 Subject: [PATCH 377/494] Update mmtk-core --- mmtk/Cargo.lock | 8 ++++---- mmtk/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 8c9b71fd..43cd5d06 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -516,8 +516,8 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" -version = "0.19.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8f8cd43933618059d757d6cf6b870aa8f3484276#8f8cd43933618059d757d6cf6b870aa8f3484276" +version = "0.20.0" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=edfca0ef54bf447fdbf9493928a196db5795924d#edfca0ef54bf447fdbf9493928a196db5795924d" dependencies = [ "atomic", "atomic-traits", @@ -550,8 +550,8 @@ dependencies = [ [[package]] name = "mmtk-macros" -version = "0.19.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8f8cd43933618059d757d6cf6b870aa8f3484276#8f8cd43933618059d757d6cf6b870aa8f3484276" +version = "0.20.0" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=edfca0ef54bf447fdbf9493928a196db5795924d#edfca0ef54bf447fdbf9493928a196db5795924d" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 528e3e86..6d19ea38 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8f8cd43933618059d757d6cf6b870aa8f3484276" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "edfca0ef54bf447fdbf9493928a196db5795924d" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From c5dfe062795a02eb033604444b963e92823c9624 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 16 Oct 2023 05:00:07 +0000 Subject: [PATCH 378/494] noop flags --- mmtk/Cargo.toml | 7 +++++++ mmtk/src/edges.rs | 10 ++++++++++ mmtk/src/lib.rs | 49 ++++++++++++++++++++++++++++++++++++----------- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 4cac823e..5438bac5 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -86,3 +86,10 @@ log_stages = ["mmtk/log_stages"] log_work_packets = ["mmtk/log_work_packets"] object_size_distribution = ["mmtk/object_size_distribution"] discontig_space = [] + +# Compressed pointer experiments +force_narrow_oop_mode = [] +narrow_oop_mode_32bit = ["force_narrow_oop_mode"] +narrow_oop_mode_shift = ["force_narrow_oop_mode"] +narrow_oop_mode_base = ["force_narrow_oop_mode"] +narrow_oop_mode_base_and_shift = ["force_narrow_oop_mode"] diff --git a/mmtk/src/edges.rs b/mmtk/src/edges.rs index 4b49d01a..7875ba72 100644 --- a/mmtk/src/edges.rs +++ b/mmtk/src/edges.rs @@ -34,13 +34,23 @@ pub fn use_compressed_oops() -> bool { } pub fn initialize_compressed_oops_base_and_shift() { + let heap_start = mmtk::memory_manager::starting_heap_address().as_usize(); let heap_end = mmtk::memory_manager::last_heap_address().as_usize(); + println!("heap_start: 0x{:x}", heap_start); + println!("heap_end: 0x{:x}", heap_end); if heap_end <= (4usize << 30) { BASE.store(Address::ZERO, Ordering::Relaxed); SHIFT.store(0, Ordering::Relaxed); } else if heap_end <= (32usize << 30) { BASE.store(Address::ZERO, Ordering::Relaxed); SHIFT.store(3, Ordering::Relaxed); + } else if cfg!(feature = "narrow_oop_mode_base") && (heap_end - heap_start) <= (4usize << 30) { + // set heap base as HEAP_START - 4096, to make sure null pointer value is not conflict with HEAP_START + BASE.store( + mmtk::memory_manager::starting_heap_address() - 4096, + Ordering::Relaxed, + ); + SHIFT.store(0, Ordering::Relaxed); } else { // set heap base as HEAP_START - 4096, to make sure null pointer value is not conflict with HEAP_START BASE.store( diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 3376bb9c..7ce36a85 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -311,27 +311,54 @@ fn set_compressed_pointer_vm_layout(builder: &mut MMTKBuilder) { ); let rounded_heap_size = (max_heap_size + (BYTES_IN_CHUNK - 1)) & !(BYTES_IN_CHUNK - 1); let mut start: usize = 0x4000_0000; // block lowest 1G - let (end, mut small_chunk_space_size) = match rounded_heap_size { - // heap <= 2G; virtual = 3G; max-small-space=2G; min-small-space=1.5G - heap if heap <= 2 << 30 => { + let (end, mut small_chunk_space_size) = if cfg!(feature = "force_narrow_oop_mode") { + assert!(rounded_heap_size <= (2 << 30)); + let heap = rounded_heap_size; + if cfg!(feature = "narrow_oop_mode_32bit") { let end = 4usize << 30; let small_space = 2 << 30; (end, small_space) - } - // heap <= 29G; virtual = 31G; max-small-space=29G; - heap if heap <= 29 << 30 => { + } else if cfg!(feature = "narrow_oop_mode_shift") { let end = 32usize << 30; let small_space = usize::min(heap * 3 / 2, 29 << 30); (end, small_space) - } - // heap > 29G; virtual = 32G - 1chunk; max-small-space=30G; start=0x200_0000_0000 - heap => { - // A workaround to avoid address conflict with the OpenJDK - // MetaSpace, which may start from 0x8_0000_0000 + } else if cfg!(feature = "narrow_oop_mode_base") { + // start = 0x200_0000_0000; + // let end = start + ((4usize << 30) - BYTES_IN_CHUNK); + // let small_space = 2 << 30; + // (end, small_space) + unreachable!() + } else if cfg!(feature = "narrow_oop_mode_base_and_shift") { start = 0x200_0000_0000; let end = start + 0x8_0000_0000 - BYTES_IN_CHUNK; let small_space = usize::min(heap * 3 / 2, 30 << 30); (end, small_space) + } else { + unreachable!() + } + } else { + match rounded_heap_size { + // heap <= 2G; virtual = 3G; max-small-space=2G; min-small-space=1.5G + heap if heap <= 2 << 30 => { + let end = 4usize << 30; + let small_space = 2 << 30; + (end, small_space) + } + // heap <= 29G; virtual = 31G; max-small-space=29G; + heap if heap <= 29 << 30 => { + let end = 32usize << 30; + let small_space = usize::min(heap * 3 / 2, 29 << 30); + (end, small_space) + } + // heap > 29G; virtual = 32G - 1chunk; max-small-space=30G; start=0x200_0000_0000 + heap => { + // A workaround to avoid address conflict with the OpenJDK + // MetaSpace, which may start from 0x8_0000_0000 + start = 0x200_0000_0000; + let end = start + 0x8_0000_0000 - BYTES_IN_CHUNK; + let small_space = usize::min(heap * 3 / 2, 30 << 30); + (end, small_space) + } } }; small_chunk_space_size = From 61f3be585492e3f8fb9cd9bd7396a3e7ae5e2faa Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 16 Oct 2023 05:06:04 +0000 Subject: [PATCH 379/494] minor --- mmtk/src/edges.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mmtk/src/edges.rs b/mmtk/src/edges.rs index 7875ba72..30d6c049 100644 --- a/mmtk/src/edges.rs +++ b/mmtk/src/edges.rs @@ -36,8 +36,10 @@ pub fn use_compressed_oops() -> bool { pub fn initialize_compressed_oops_base_and_shift() { let heap_start = mmtk::memory_manager::starting_heap_address().as_usize(); let heap_end = mmtk::memory_manager::last_heap_address().as_usize(); - println!("heap_start: 0x{:x}", heap_start); - println!("heap_end: 0x{:x}", heap_end); + if cfg!(feature = "force_narrow_oop_mode") { + println!("heap_start: 0x{:x}", heap_start); + println!("heap_end: 0x{:x}", heap_end); + } if heap_end <= (4usize << 30) { BASE.store(Address::ZERO, Ordering::Relaxed); SHIFT.store(0, Ordering::Relaxed); From 7181a1aff156fd8670e59ea09a9700351ac9d4e4 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 16 Oct 2023 13:44:46 +0000 Subject: [PATCH 380/494] minor --- mmtk/src/api.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index a2794a61..e03ed509 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -573,7 +573,11 @@ pub extern "C" fn mmtk_is_live(object: ObjectReference) -> usize { if object.is_null() { return 0; } - debug_assert!(object.to_raw_address().is_mapped()); + debug_assert!( + object.to_raw_address().is_mapped(), + "{:?} is not mapped", + object + ); object.is_live() as _ } From c88bc99ae25970311c658d7d1b20241057492d2e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 20 Oct 2023 00:56:08 +0000 Subject: [PATCH 381/494] Remove unnecessary bound check in allocation fast-path --- openjdk/mmtkBarrierSetAssembler_x86.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openjdk/mmtkBarrierSetAssembler_x86.cpp b/openjdk/mmtkBarrierSetAssembler_x86.cpp index 3f2ac1a6..9b3d7778 100644 --- a/openjdk/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/mmtkBarrierSetAssembler_x86.cpp @@ -94,8 +94,8 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa __ lea(end, Address(obj, var_size_in_bytes, Address::times_1)); } // slowpath if end < obj - __ cmpptr(end, obj); - __ jcc(Assembler::below, slow_case); + // __ cmpptr(end, obj); + // __ jcc(Assembler::below, slow_case); // slowpath if end > lab.limit __ cmpptr(end, limit); __ jcc(Assembler::above, slow_case); From 2ac045919031db1dbdebb0e46edf2d2a798ae3ff Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 21 Oct 2023 13:15:03 +0000 Subject: [PATCH 382/494] Remove rustflags `+sse4a` --- mmtk/.cargo/config.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mmtk/.cargo/config.toml b/mmtk/.cargo/config.toml index d038a028..9f12ddcd 100644 --- a/mmtk/.cargo/config.toml +++ b/mmtk/.cargo/config.toml @@ -1,2 +1,5 @@ [target.x86_64-unknown-linux-gnu] -rustflags = ["-C", "target-feature=+avx,+avx2,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+sse4a,+ssse3"] \ No newline at end of file +rustflags = [ + "-C", + "target-feature=+avx,+avx2,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3", +] From 2124a4792d676517ba0e14f1633a1ef91379970c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 31 Oct 2023 14:45:21 +1100 Subject: [PATCH 383/494] wip: barrier eager branch --- openjdk/barriers/mmtkFieldBarrier.cpp | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index ed02b587..d8752462 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -3,6 +3,7 @@ #include "runtime/interfaceSupport.inline.hpp" #define SOFT_REFERENCE_LOAD_BARRIER true +#define MMTK_BARRIER_EAGER_BRANCH true constexpr int kUnloggedValue = 1; @@ -22,8 +23,11 @@ void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, #if MMTK_ENABLE_BARRIER_FASTPATH intptr_t addr = ((intptr_t) (void*) slot); const volatile uint8_t * meta_addr = (const volatile uint8_t *) (side_metadata_base_address() + (addr >> (UseCompressedOops ? 5 : 6))); - intptr_t shift = (addr >> (UseCompressedOops ? 2 : 3)) & 0b111; uint8_t byte_val = *meta_addr; + #if MMTK_BARRIER_EAGER_BRANCH + if (byte_val == 0) return; + #endif + intptr_t shift = (addr >> (UseCompressedOops ? 2 : 3)) & 0b111; if (((byte_val >> shift) & 1) == kUnloggedValue) { ::mmtk_object_reference_write_slow((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) slot, (void*) target); } @@ -83,7 +87,11 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma __ lea(tmp3, dst); __ shrptr(tmp3, UseCompressedOops ? 5 : 6); __ movptr(tmp5, side_metadata_base_address()); - __ movb(tmp5, Address(tmp5, tmp3)); + __ movzbl(tmp5, Address(tmp5, tmp3)); + #if MMTK_BARRIER_EAGER_BRANCH + __ cmpl(tmp5, 0); + __ jcc(Assembler::equal, done); + #endif // tmp3 = (obj >> 3) & 7 __ lea(tmp3, dst); __ shrptr(tmp3, UseCompressedOops ? 2 : 3); @@ -257,6 +265,10 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op // uint8_t byte_val = *meta_addr; LIR_Opr byte_val = gen->new_register(T_INT); __ move(meta_addr, byte_val); + #if MMTK_BARRIER_EAGER_BRANCH + // __ cmp(lir_cond_equal, byte_val, LIR_OprFact::intConst(0)); + // __ branch(lir_cond_equal, T_BYTE, Ldone->label()); + #endif // intptr_t shift = (addr >> 3) & 0b111; LIR_Opr shift = gen->new_register(T_INT); __ move(addr, shift); @@ -291,18 +303,19 @@ static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* sl Node* addr = __ CastPX(__ ctrl(), slot); Node* meta_addr = __ AddP(no_base, __ ConP(side_metadata_base_address()), __ URShiftX(addr, __ ConI(UseCompressedOops ? 5 : 6))); Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); + #if MMTK_BARRIER_EAGER_BRANCH + __ if_then(byte, BoolTest::ne, zero, unlikely); + #endif Node* shift = __ URShiftX(addr, __ ConI(UseCompressedOops ? 2 : 3)); shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); - __ if_then(result, BoolTest::ne, zero, unlikely); { const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), "mmtk_barrier_call", src, slot, val); - // Looks like this is necessary - // See https://github.com/mmtk/openjdk/blob/c82e5c44adced4383162826c2c3933a83cfb139b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp#L288-L291 - Node* call = __ ctrl()->in(0); - call->add_req(slot); } __ end_if(); + #if MMTK_BARRIER_EAGER_BRANCH + __ end_if(); + #endif #else const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), "mmtk_barrier_call", src, slot, val); From a6e80ca37a3a081892e8401e6a691b8f4b9f6664 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 7 Nov 2023 14:13:29 +1100 Subject: [PATCH 384/494] wip --- openjdk/mmtkMutator.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp index c3070e03..b1fabd2e 100644 --- a/openjdk/mmtkMutator.hpp +++ b/openjdk/mmtkMutator.hpp @@ -57,6 +57,7 @@ struct ImmixAllocator { uint8_t line_opt_tag; uintptr_t line_opt; uintptr_t mutator_recycled_blocks; + uintptr_t local_blocks[4]; uintptr_t mutator_recycled_lines; uint8_t retry; }; From 0d43896f3513746550b5bdb63b332fb6292ef8d3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 8 Nov 2023 18:39:28 +1100 Subject: [PATCH 385/494] wip --- openjdk/mmtkMutator.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp index b1fabd2e..3bf0db55 100644 --- a/openjdk/mmtkMutator.hpp +++ b/openjdk/mmtkMutator.hpp @@ -56,6 +56,8 @@ struct ImmixAllocator { uint8_t _align[7]; uint8_t line_opt_tag; uintptr_t line_opt; + uint8_t block_opt_tag; + uintptr_t block_opt; uintptr_t mutator_recycled_blocks; uintptr_t local_blocks[4]; uintptr_t mutator_recycled_lines; From 07ea963241b798d6ebd9bd5d4a39efdd4cf9d870 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 9 Nov 2023 15:09:33 +1100 Subject: [PATCH 386/494] wip --- openjdk/mmtkMutator.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp index 3bf0db55..4f60b195 100644 --- a/openjdk/mmtkMutator.hpp +++ b/openjdk/mmtkMutator.hpp @@ -59,7 +59,7 @@ struct ImmixAllocator { uint8_t block_opt_tag; uintptr_t block_opt; uintptr_t mutator_recycled_blocks; - uintptr_t local_blocks[4]; + uintptr_t local_blocks[5]; uintptr_t mutator_recycled_lines; uint8_t retry; }; From 4bbae7ad910420ceecf76b822803b9bd459d0928 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 13 Nov 2023 16:53:31 +1100 Subject: [PATCH 387/494] wip --- openjdk/mmtkMutator.hpp | 2 +- openjdk/mmtkParallelCleaning.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp index 4f60b195..919bb67f 100644 --- a/openjdk/mmtkMutator.hpp +++ b/openjdk/mmtkMutator.hpp @@ -59,7 +59,7 @@ struct ImmixAllocator { uint8_t block_opt_tag; uintptr_t block_opt; uintptr_t mutator_recycled_blocks; - uintptr_t local_blocks[5]; + uintptr_t local_blocks[6]; uintptr_t mutator_recycled_lines; uint8_t retry; }; diff --git a/openjdk/mmtkParallelCleaning.cpp b/openjdk/mmtkParallelCleaning.cpp index eb9b4899..7e169a36 100644 --- a/openjdk/mmtkParallelCleaning.cpp +++ b/openjdk/mmtkParallelCleaning.cpp @@ -135,7 +135,7 @@ CodeCacheUnloadingTask::CodeCacheUnloadingTask(uint num_workers, BoolObjectClosu } CodeCacheUnloadingTask::~CodeCacheUnloadingTask() { - CodeCache::verify_clean_inline_caches(); + // CodeCache::verify_clean_inline_caches(); CodeCache::set_needs_cache_clean(false); guarantee(CodeCache::scavenge_root_nmethods() == NULL, "Must be"); From e66b8c20e3641633b5e0c6f7d0ab5bd817913fed Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 21 Nov 2023 16:18:30 +1100 Subject: [PATCH 388/494] Refactor weak-roots related code --- mmtk/Cargo.lock | 10 +++---- mmtk/Cargo.toml | 2 +- mmtk/src/edges.rs | 2 +- mmtk/src/gc_work.rs | 36 +++++++++-------------- openjdk/mmtkRootsClosure.hpp | 56 ------------------------------------ openjdk/mmtkUpcalls.cpp | 45 +++++++++++------------------ 6 files changed, 36 insertions(+), 115 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 9b59808b..f1e31c34 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,7 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=edfca0ef54bf447fdbf9493928a196db5795924d#edfca0ef54bf447fdbf9493928a196db5795924d" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=326a1b7213d11edee169b0e6df9be9cdc6401e7d#326a1b7213d11edee169b0e6df9be9cdc6401e7d" dependencies = [ "atomic", "atomic-traits", @@ -551,7 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=edfca0ef54bf447fdbf9493928a196db5795924d#edfca0ef54bf447fdbf9493928a196db5795924d" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=326a1b7213d11edee169b0e6df9be9cdc6401e7d#326a1b7213d11edee169b0e6df9be9cdc6401e7d" dependencies = [ "proc-macro-error", "proc-macro2", @@ -652,8 +652,7 @@ dependencies = [ [[package]] name = "pfm" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498f5ea94bbec98f1369c70a71178a1eb821370443ef393f1bbcd9950182758" +source = "git+https://github.com/wenyuzhao/pfm.git?branch=plm0#25e901836dbfa08a683e2b08a906a61ebf69f810" dependencies = [ "libc", "perf-event-open-sys", @@ -663,8 +662,7 @@ dependencies = [ [[package]] name = "pfm-sys" version = "0.0.16-libpfm4.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b520e7dec2e88bed569b487e68ba1ba4e729d5980b32b6c0243fbb1424f916cf" +source = "git+https://github.com/wenyuzhao/pfm.git?branch=plm0#25e901836dbfa08a683e2b08a906a61ebf69f810" dependencies = [ "bindgen", ] diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 1fb11246..53463992 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "edfca0ef54bf447fdbf9493928a196db5795924d" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "326a1b7213d11edee169b0e6df9be9cdc6401e7d" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/edges.rs b/mmtk/src/edges.rs index 372a925e..33f86cda 100644 --- a/mmtk/src/edges.rs +++ b/mmtk/src/edges.rs @@ -1,6 +1,6 @@ use std::{ ops::Range, - sync::atomic::{AtomicBool, AtomicUsize, Ordering, AtomicU32}, + sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}, }; use atomic::Atomic; diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index acdd2b69..667f9115 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -76,21 +76,12 @@ scan_roots_work!(ScanAOTLoaderRoots, scan_aot_loader_roots); scan_roots_work!(ScanSystemDictionaryRoots, scan_system_dictionary_roots); scan_roots_work!(ScanVMThreadRoots, scan_vm_thread_roots); -extern "C" fn report_edges_and_renew_buffer_cld< - E: Edge, - F: RootsWorkFactory, - const WEAK: bool, ->( +extern "C" fn report_edges_and_renew_buffer_cld>( ptr: *mut Address, length: usize, capacity: usize, factory_ptr: *mut libc::c_void, ) -> NewBuffer { - let root_kind = if WEAK { - RootKind::Weak - } else { - RootKind::Young - }; if !ptr.is_null() { let ptr = ptr as *mut E; let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; @@ -98,7 +89,7 @@ extern "C" fn report_edges_and_renew_buffer_cld< record_roots(buf.len()); } let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; - factory.create_process_edge_roots_work(buf, root_kind); + factory.create_process_edge_roots_work(buf, RootKind::Incomplete); } let (ptr, _, capacity) = { // TODO: Use Vec::into_raw_parts() when the method is available. @@ -110,11 +101,9 @@ extern "C" fn report_edges_and_renew_buffer_cld< NewBuffer { ptr, capacity } } -fn to_edges_closure_cld, const WEAK: bool>( - factory: &mut F, -) -> EdgesClosure { +fn to_edges_closure_cld>(factory: &mut F) -> EdgesClosure { EdgesClosure { - func: report_edges_and_renew_buffer_cld::, + func: report_edges_and_renew_buffer_cld::, data: factory as *mut F as *mut libc::c_void, } } @@ -144,10 +133,11 @@ impl> GCWork }; unsafe { ((*UPCALLS).scan_class_loader_data_graph_roots)( - to_edges_closure_cld::(&mut self.factory), - to_edges_closure_cld::(&mut self.factory), + to_edges_closure_cld::(&mut self.factory), + to_edges_closure_cld::(&mut self.factory), mmtk.get_plan() - .current_gc_should_scan_all_classloader_strong_roots(), + .current_gc_should_prepare_for_class_unloading() + || mmtk.get_plan().current_gc_should_perform_class_unloading(), ); } if cfg!(feature = "roots_breakdown") { @@ -187,7 +177,7 @@ impl> GCWork for slice in new_roots.chunks(mmtk::args::BUFFER_SIZE) { let slice = unsafe { std::mem::transmute::<&[Address], &[VM::VMEdge]>(slice) }; self.factory - .create_process_edge_roots_work(slice.to_vec(), RootKind::Weak); + .create_process_edge_roots_work(slice.to_vec(), RootKind::Incomplete); } new_roots.clear(); if cfg!(feature = "roots_breakdown") { @@ -223,7 +213,7 @@ impl> GCWork let mut edges = Vec::with_capacity(F::BUFFER_SIZE); let scan_all_roots = mmtk .get_plan() - .current_gc_should_scan_all_classloader_strong_roots() + .current_gc_should_prepare_for_class_unloading() || mmtk.get_plan().current_gc_should_perform_class_unloading(); let mut mature = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); let mut nursery_guard = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); @@ -240,7 +230,7 @@ impl> GCWork } self.factory.create_process_edge_roots_work( std::mem::take(&mut edges), - RootKind::Young, + RootKind::Incomplete, ); edges.reserve(F::BUFFER_SIZE); } @@ -257,7 +247,7 @@ impl> GCWork } self.factory.create_process_edge_roots_work( std::mem::take(&mut edges), - RootKind::Young, + RootKind::Incomplete, ); edges.reserve(F::BUFFER_SIZE); } @@ -269,7 +259,7 @@ impl> GCWork c += edges.len(); } self.factory - .create_process_edge_roots_work(edges, RootKind::Young); + .create_process_edge_roots_work(edges, RootKind::Incomplete); } if cfg!(feature = "roots_breakdown") { let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 45331294..4b26bebd 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -8,7 +8,6 @@ #include "utilities/globalDefinitions.hpp" #include "classfile/classLoaderData.inline.hpp" -template class MMTkRootsClosure : public OopClosure { EdgesClosure _edges_closure; void** _buffer; @@ -19,12 +18,6 @@ class MMTkRootsClosure : public OopClosure { void do_oop_work(T* p, bool narrow) { T heap_oop = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(heap_oop)) { - if (UNPINNED_ONLY) { - auto o = CompressedOops::decode(heap_oop); - if (3 == mmtk_get_rc((void*) o)) { - return; - } - } if (UseCompressedOops && !narrow) { guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); p = (T*) (uintptr_t(p) | (1ull << 63)); @@ -63,55 +56,6 @@ class MMTkRootsClosure : public OopClosure { virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } }; -template -class MMTkCollectRootObjects : public OopClosure { - EdgesClosure _edges_closure; - void** _buffer; - size_t _cap; - size_t _cursor; - - template - void do_oop_work(T* p, bool narrow) { - T heap_oop = RawAccess<>::oop_load(p); - if (!CompressedOops::is_null(heap_oop)) { - auto o = CompressedOops::decode(heap_oop); - if (UNPINNED_ONLY && 3 == mmtk_get_rc((void*) o)) { - return; - } - _buffer[_cursor++] = (void*) o; - if (_cursor >= _cap) { - flush(); - } - } - } - - void flush() { - if (_cursor > 0) { - NewBuffer buf = _edges_closure.invoke(_buffer, _cursor, _cap); - _buffer = buf.buf; - _cap = buf.cap; - _cursor = 0; - } - } - -public: - - MMTkCollectRootObjects(EdgesClosure edges_closure): _edges_closure(edges_closure), _cursor(0) { - NewBuffer buf = edges_closure.invoke(NULL, 0, 0); - _buffer = buf.buf; - _cap = buf.cap; - } - - ~MMTkCollectRootObjects() { - if (_cursor > 0) flush(); - if (_buffer != NULL) { - release_buffer(_buffer, _cursor, _cap); - } - } - - virtual void do_oop(oop* p) { do_oop_work(p, false); } - virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } -}; template class MMTkScanCLDClosure: public CLDClosure { diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index f3c6e249..a49dcd00 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -22,6 +22,7 @@ * */ +#include "mmtkBarrierSet.hpp" #include "precompiled.hpp" #include "classfile/stringTable.hpp" #include "code/nmethod.hpp" @@ -348,21 +349,21 @@ static void mmtk_get_mutators(MutatorClosure closure) { } static void mmtk_scan_roots_in_all_mutator_threads(EdgesClosure closure) { - MMTkRootsClosure<> cl(closure); + MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_roots_in_all_mutator_threads(cl); } static void mmtk_scan_roots_in_mutator_thread(EdgesClosure closure, void* tls) { ResourceMark rm; JavaThread* thread = (JavaThread*) tls; - MMTkRootsClosure<> cl(closure); + MMTkRootsClosure cl(closure); thread->oops_do(&cl, NULL); } static void mmtk_scan_multiple_thread_roots(EdgesClosure closure, void* ptr, size_t len) { ResourceMark rm; auto mutators = (JavaThread**) ptr; - MMTkRootsClosure<> cl(closure); + MMTkRootsClosure cl(closure); for (size_t i = 0; i < len; i++) mutators[i]->oops_do(&cl, NULL); } @@ -447,38 +448,26 @@ static void mmtk_schedule_finalizer() { MMTkHeap::heap()->schedule_finalizer(); } -static void mmtk_scan_universe_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_universe_roots(cl); } -static void mmtk_scan_jni_handle_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_jni_handle_roots(cl); } -static void mmtk_scan_object_synchronizer_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_object_synchronizer_roots(cl); } -static void mmtk_scan_management_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_management_roots(cl); } -static void mmtk_scan_jvmti_export_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_jvmti_export_roots(cl); } -static void mmtk_scan_aot_loader_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_aot_loader_roots(cl); } -static void mmtk_scan_system_dictionary_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_system_dictionary_roots(cl); } -static void mmtk_scan_code_cache_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } +static void mmtk_scan_universe_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_universe_roots(cl); } +static void mmtk_scan_jni_handle_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_jni_handle_roots(cl); } +static void mmtk_scan_object_synchronizer_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_object_synchronizer_roots(cl); } +static void mmtk_scan_management_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_management_roots(cl); } +static void mmtk_scan_jvmti_export_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_jvmti_export_roots(cl); } +static void mmtk_scan_aot_loader_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_aot_loader_roots(cl); } +static void mmtk_scan_system_dictionary_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_system_dictionary_roots(cl); } +static void mmtk_scan_code_cache_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } static void mmtk_scan_string_table_roots(EdgesClosure closure, bool rc_non_stuck_objs_only) { - if (rc_non_stuck_objs_only) { - MMTkRootsClosure cl(closure); - MMTkHeap::heap()->scan_string_table_roots(cl, &*par_state_string); - } else { - MMTkRootsClosure cl(closure); - MMTkHeap::heap()->scan_string_table_roots(cl, &*par_state_string); - } + UNREACHABLE(); } static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, EdgesClosure weak_closure, bool scan_all_strong_roots) { - MMTkRootsClosure cl(closure); - MMTkRootsClosure weak_cl(weak_closure); + MMTkRootsClosure cl(closure); + MMTkRootsClosure weak_cl(weak_closure); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl, weak_cl, scan_all_strong_roots); } static void mmtk_scan_weak_processor_roots(EdgesClosure closure, bool rc_non_stuck_objs_only) { - if (rc_non_stuck_objs_only) { - MMTkRootsClosure cl(closure); - MMTkHeap::heap()->scan_weak_processor_roots(cl); - } else { - MMTkRootsClosure cl(closure); - MMTkHeap::heap()->scan_weak_processor_roots(cl); - } + UNREACHABLE(); } -static void mmtk_scan_vm_thread_roots(EdgesClosure closure) { MMTkRootsClosure<> cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } +static void mmtk_scan_vm_thread_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } static size_t mmtk_number_of_mutators() { return Threads::number_of_threads(); From 6c52e71d73af14872cfa3945b13bff3b70620c04 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 Nov 2023 10:21:21 +1100 Subject: [PATCH 389/494] Fix JNICritical_lock safepoint check errors --- openjdk/mmtkHeap.cpp | 7 +++++++ openjdk/mmtkVMCompanionThread.cpp | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 2c8d14d0..ab73a198 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -305,9 +305,16 @@ bool MMTkHeap::card_mark_must_follow_store() const { //OK void MMTkHeap::collect(GCCause::Cause cause) {//later when gc is implemented in rust if (cause == GCCause::_gc_locker) { + #ifndef PRODUCT + auto safepoint_check_required = JNICritical_lock->_safepoint_check_required; + JNICritical_lock->_safepoint_check_required = Monitor::_safepoint_check_sometimes; + #endif MutexLockerEx locker(JNICritical_lock, Mutex::_no_safepoint_check_flag); // Notify the VMCompanionThread to trigger another VM_MMTkSTWOperation. JNICritical_lock->notify_all(); + #ifndef PRODUCT + JNICritical_lock->_safepoint_check_required = safepoint_check_required; + #endif } handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, cause != GCCause::_java_lang_system_gc); } diff --git a/openjdk/mmtkVMCompanionThread.cpp b/openjdk/mmtkVMCompanionThread.cpp index 8312f072..5704df0b 100644 --- a/openjdk/mmtkVMCompanionThread.cpp +++ b/openjdk/mmtkVMCompanionThread.cpp @@ -82,6 +82,9 @@ void MMTkVMCompanionThread::run() { #endif MutexLockerEx locker(JNICritical_lock, Mutex::_no_safepoint_check_flag); while (GCLocker::is_active_and_needs_gc()) { + #ifndef PRODUCT + JNICritical_lock->_safepoint_check_required = Monitor::_safepoint_check_sometimes; + #endif JNICritical_lock->wait(true); } #ifndef PRODUCT @@ -179,7 +182,7 @@ void MMTkVMCompanionThread::wait_for_reached(stw_state desired_state) { // This method will block until the GC requests start-the-world. void MMTkVMCompanionThread::reach_suspended_and_wait_for_resume() { assert(Thread::current()->is_VM_thread(), "reach_suspended_and_wait_for_resume can only be executed by the VM thread"); - + MutexLockerEx locker(_lock, Mutex::_no_safepoint_check_flag); // Tell the waiter thread that the world has stopped. From 986904832d37ab34866d1cd46485c7f22b5e27af Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 23 Nov 2023 15:00:41 +1100 Subject: [PATCH 390/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- openjdk/mmtkMutator.hpp | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index f1e31c34..e221e846 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,7 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=326a1b7213d11edee169b0e6df9be9cdc6401e7d#326a1b7213d11edee169b0e6df9be9cdc6401e7d" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=263b6e11b05fbd4c86dd42c1ad81f35e5b0a2753#263b6e11b05fbd4c86dd42c1ad81f35e5b0a2753" dependencies = [ "atomic", "atomic-traits", @@ -551,7 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=326a1b7213d11edee169b0e6df9be9cdc6401e7d#326a1b7213d11edee169b0e6df9be9cdc6401e7d" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=263b6e11b05fbd4c86dd42c1ad81f35e5b0a2753#263b6e11b05fbd4c86dd42c1ad81f35e5b0a2753" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 53463992..44a7128f 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "326a1b7213d11edee169b0e6df9be9cdc6401e7d" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "263b6e11b05fbd4c86dd42c1ad81f35e5b0a2753" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/openjdk/mmtkMutator.hpp b/openjdk/mmtkMutator.hpp index 919bb67f..2d21639f 100644 --- a/openjdk/mmtkMutator.hpp +++ b/openjdk/mmtkMutator.hpp @@ -58,6 +58,8 @@ struct ImmixAllocator { uintptr_t line_opt; uint8_t block_opt_tag; uintptr_t block_opt; + uint8_t large_block_opt_tag; + uintptr_t large_block_opt; uintptr_t mutator_recycled_blocks; uintptr_t local_blocks[6]; uintptr_t mutator_recycled_lines; From 8c87321c67572ef1db011c6ff0cdaf6ebb426997 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 5 Dec 2023 17:50:47 +1100 Subject: [PATCH 391/494] Pass `ConcGCThreads` to mmtk --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/api.rs | 7 +++++++ openjdk/mmtk.h | 1 + openjdk/mmtkHeap.cpp | 7 +++++++ openjdk/thirdPartyHeapArguments.cpp | 7 +++++++ 6 files changed, 25 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index e221e846..dfdfdfb1 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,7 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=263b6e11b05fbd4c86dd42c1ad81f35e5b0a2753#263b6e11b05fbd4c86dd42c1ad81f35e5b0a2753" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=af39191771244a3025fdbfe215af4ae8866370ec#af39191771244a3025fdbfe215af4ae8866370ec" dependencies = [ "atomic", "atomic-traits", @@ -551,7 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=263b6e11b05fbd4c86dd42c1ad81f35e5b0a2753#263b6e11b05fbd4c86dd42c1ad81f35e5b0a2753" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=af39191771244a3025fdbfe215af4ae8866370ec#af39191771244a3025fdbfe215af4ae8866370ec" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 44a7128f..ea8b3640 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "263b6e11b05fbd4c86dd42c1ad81f35e5b0a2753" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "af39191771244a3025fdbfe215af4ae8866370ec" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index e03ed509..be5a6f45 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -398,6 +398,13 @@ pub extern "C" fn mmtk_builder_set_threads(value: usize) { builder.options.threads.set(value); } +/// Pass hotspot `ConcGCThreads` flag to mmtk +#[no_mangle] +pub extern "C" fn mmtk_builder_set_conc_threads(value: usize) { + let mut builder = BUILDER.lock().unwrap(); + builder.options.conc_threads.set(value); +} + /// Pass hotspot `UseTransparentHugePages` flag to mmtk #[no_mangle] pub extern "C" fn mmtk_builder_set_transparent_hugepages(value: bool) { diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 997ca151..e4500bc2 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -280,6 +280,7 @@ inline uint8_t mmtk_get_rc(void* o) { extern void mmtk_builder_read_env_var_settings(); extern void mmtk_builder_set_threads(size_t value); +extern void mmtk_builder_set_conc_threads(size_t value); extern void mmtk_builder_set_transparent_hugepages(bool value); #ifdef __cplusplus diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index ab73a198..ae24e581 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -141,6 +141,9 @@ jint MMTkHeap::initialize() { Universe::set_narrow_oop_shift(mmtk_narrow_oop_shift()); if (mmtk_verbose() > 0) fprintf(stderr, "narrow_oop_mode: %s\n", Universe::narrow_oop_mode_to_string(Universe::narrow_oop_mode())); } + if (mmtk_verbose() > 0) { + fprintf(stderr, "ParallelGCThreads = %d; ConcGCThreads = %d\n", ParallelGCThreads, ConcGCThreads); + } initialize_reserved_region(_start, _end); @@ -182,6 +185,10 @@ void MMTkHeap::set_mmtk_options(bool set_defaults) { mmtk_builder_set_threads(ParallelGCThreads); } + if (FLAG_IS_DEFAULT(ConcGCThreads) == set_defaults) { + mmtk_builder_set_conc_threads(ConcGCThreads); + } + if (FLAG_IS_DEFAULT(UseTransparentHugePages) == set_defaults) { mmtk_builder_set_transparent_hugepages(UseTransparentHugePages); } diff --git a/openjdk/thirdPartyHeapArguments.cpp b/openjdk/thirdPartyHeapArguments.cpp index aba71fee..d8f2cf01 100644 --- a/openjdk/thirdPartyHeapArguments.cpp +++ b/openjdk/thirdPartyHeapArguments.cpp @@ -40,6 +40,11 @@ size_t ThirdPartyHeapArguments::conservative_max_heap_alignment() { return CollectorPolicy::compute_heap_alignment(); } +static uint scale_concurrent_worker_threads(uint num_gc_workers) { + auto v = (num_gc_workers + 2) / 4; + return v > 1 ? v : 1; +} + void ThirdPartyHeapArguments::initialize() { GCArguments::initialize(); assert(UseThirdPartyHeap , "Error, should UseThirdPartyHeap"); @@ -50,6 +55,8 @@ void ThirdPartyHeapArguments::initialize() { assert(!FLAG_IS_DEFAULT(ParallelGCThreads), "ParallelGCThreads should not be 0."); vm_exit_during_initialization("The flag -XX:+UseUseThirdPartyHeap can not be combined with -XX:ParallelGCThreads=0", NULL); } + uint marking_thread_num = scale_concurrent_worker_threads(ParallelGCThreads); + FLAG_SET_ERGO(uint, ConcGCThreads, marking_thread_num); // Note: If you add an option here that may be forwarded to an MMTk option, // make sure to add appropriate code to MMTkHeap::set_mmtk_options. } From b164b358ea760d36f27bda2dc4ab37386ef33652 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 5 Dec 2023 22:43:27 +1100 Subject: [PATCH 392/494] Add `same_stw_and_conc_threads` flag --- mmtk/Cargo.toml | 3 ++- mmtk/src/api.rs | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index ea8b3640..12431959 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -49,7 +49,7 @@ mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "af3919177124 built = { version = "0.5.1", features = ["git2"] } [features] -default = [] +default = ["same_stw_and_conc_threads"] release_debug_assertions = [] mmtk_extreme_assertions = ["mmtk/extreme_assertions"] nogc_lock_free = ["mmtk/nogc_lock_free"] @@ -59,6 +59,7 @@ nogc_no_zeroing = ["mmtk/nogc_no_zeroing"] vo_bit = ["mmtk/vo_bit"] no_fast_alloc = [] roots_breakdown = [] +same_stw_and_conc_threads = [] # This compile time constant places the mark bit in the header of the object instead of on the side. mark_bit_in_header = [] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index be5a6f45..af62cea1 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -402,7 +402,12 @@ pub extern "C" fn mmtk_builder_set_threads(value: usize) { #[no_mangle] pub extern "C" fn mmtk_builder_set_conc_threads(value: usize) { let mut builder = BUILDER.lock().unwrap(); - builder.options.conc_threads.set(value); + if cfg!(feature = "same_stw_and_conc_threads") { + let threads = *builder.options.threads; + builder.options.conc_threads.set(threads); + } else { + builder.options.conc_threads.set(value); + } } /// Pass hotspot `UseTransparentHugePages` flag to mmtk From 91538956fb283f61f2eb44b92054277955804aeb Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 5 Dec 2023 23:36:54 +1100 Subject: [PATCH 393/494] max priority for gc worker threads --- openjdk/mmtkCollectorThread.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/openjdk/mmtkCollectorThread.cpp b/openjdk/mmtkCollectorThread.cpp index 8333b2ff..ae3c3bae 100644 --- a/openjdk/mmtkCollectorThread.cpp +++ b/openjdk/mmtkCollectorThread.cpp @@ -33,5 +33,6 @@ MMTkCollectorThread::MMTkCollectorThread(void* context): NamedThread() { void MMTkCollectorThread::run() { this->initialize_named_thread(); + os::set_priority(this, NearMaxPriority); start_worker((void*) this, third_party_heap_collector); } From 9e67a7b3437da81191d638d27aed25cf2bfc2c75 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 12 Dec 2023 22:16:13 +1100 Subject: [PATCH 394/494] minor optimizations --- mmtk/src/object_model.rs | 12 ++---------- mmtk/src/object_scanning.rs | 14 +++++++------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 50753f63..faaa0284 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -48,11 +48,7 @@ impl ObjectModel> for VMObjectModel< copy: CopySemantics, copy_context: &mut GCWorkerCopyContext>, ) -> Option { - let bytes = if crate::use_compressed_oops() { - unsafe { Oop::from(from).size::() } - } else { - unsafe { Oop::from(from).size::() } - }; + let bytes = unsafe { Oop::from(from).size::() }; let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::(), 0, copy); if dst.is_zero() { return None; @@ -143,11 +139,7 @@ impl ObjectModel> for VMObjectModel< fn get_class_pointer(object: ObjectReference) -> Address { let oop: Oop = unsafe { std::mem::transmute(object) }; - if crate::use_compressed_oops() { - oop.klass_ptr::() - } else { - oop.klass_ptr::() - } + oop.klass_ptr::() } fn is_object_sane(object: ObjectReference) -> bool { diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 7ede0597..1fff2de4 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -282,6 +282,13 @@ fn oop_iterate( let instance_klass = unsafe { klass.cast::() }; instance_klass.oop_iterate::(oop, closure); } + KlassID::ObjArray => { + let array_klass = unsafe { klass.cast::() }; + array_klass.oop_iterate::(oop, closure); + } + KlassID::TypeArray => { + // Skip scanning primitive arrays as they contain no reference fields. + } KlassID::InstanceClassLoader => { let instance_klass = unsafe { klass.cast::() }; instance_klass.oop_iterate::(oop, closure); @@ -290,13 +297,6 @@ fn oop_iterate( let instance_klass = unsafe { klass.cast::() }; instance_klass.oop_iterate::(oop, closure); } - KlassID::ObjArray => { - let array_klass = unsafe { klass.cast::() }; - array_klass.oop_iterate::(oop, closure); - } - KlassID::TypeArray => { - // Skip scanning primitive arrays as they contain no reference fields. - } KlassID::InstanceRef => { let instance_klass = unsafe { klass.cast::() }; instance_klass.oop_iterate::(oop, closure); From 36cb147d39220847b28ba7da3ce5093bcdf347d0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 12 Dec 2023 22:32:10 +1100 Subject: [PATCH 395/494] Fix MMTK_CONC_THREADS --- mmtk/Cargo.toml | 2 +- openjdk/thirdPartyHeapArguments.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 12431959..fcad1ba8 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -49,7 +49,7 @@ mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "af3919177124 built = { version = "0.5.1", features = ["git2"] } [features] -default = ["same_stw_and_conc_threads"] +default = [] release_debug_assertions = [] mmtk_extreme_assertions = ["mmtk/extreme_assertions"] nogc_lock_free = ["mmtk/nogc_lock_free"] diff --git a/openjdk/thirdPartyHeapArguments.cpp b/openjdk/thirdPartyHeapArguments.cpp index d8f2cf01..3f843085 100644 --- a/openjdk/thirdPartyHeapArguments.cpp +++ b/openjdk/thirdPartyHeapArguments.cpp @@ -56,7 +56,7 @@ void ThirdPartyHeapArguments::initialize() { vm_exit_during_initialization("The flag -XX:+UseUseThirdPartyHeap can not be combined with -XX:ParallelGCThreads=0", NULL); } uint marking_thread_num = scale_concurrent_worker_threads(ParallelGCThreads); - FLAG_SET_ERGO(uint, ConcGCThreads, marking_thread_num); + FLAG_SET_DEFAULT(ConcGCThreads, marking_thread_num); // Note: If you add an option here that may be forwarded to an MMTk option, // make sure to add appropriate code to MMTkHeap::set_mmtk_options. } From c6fe72d46c17e4090910205a036fca9a8b4168f2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 12 Dec 2023 22:33:21 +1100 Subject: [PATCH 396/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index dfdfdfb1..94b2b62c 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,7 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=af39191771244a3025fdbfe215af4ae8866370ec#af39191771244a3025fdbfe215af4ae8866370ec" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=432c273e40d75e14a430d7fb2f0e1994a7383c25#432c273e40d75e14a430d7fb2f0e1994a7383c25" dependencies = [ "atomic", "atomic-traits", @@ -551,7 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=af39191771244a3025fdbfe215af4ae8866370ec#af39191771244a3025fdbfe215af4ae8866370ec" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=432c273e40d75e14a430d7fb2f0e1994a7383c25#432c273e40d75e14a430d7fb2f0e1994a7383c25" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index fcad1ba8..ec55e90c 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "af39191771244a3025fdbfe215af4ae8866370ec" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "432c273e40d75e14a430d7fb2f0e1994a7383c25" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 8c13fc93d467f9844c086048ab8f3707a0264d37 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 13 Dec 2023 00:15:42 +1100 Subject: [PATCH 397/494] minor --- mmtk/src/object_scanning.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 7ede0597..19f492d4 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -271,12 +271,12 @@ fn oop_iterate( oop.klass::() }; let klass_id = klass.id; - assert!( - klass_id as i32 >= 0 && (klass_id as i32) < 6, - "Invalid klass-id: {:x} for oop: {:x}", - klass_id as i32, - unsafe { mem::transmute::(oop) } - ); + // assert!( + // klass_id as i32 >= 0 && (klass_id as i32) < 6, + // "Invalid klass-id: {:x} for oop: {:x}", + // klass_id as i32, + // unsafe { mem::transmute::(oop) } + // ); match klass_id { KlassID::Instance => { let instance_klass = unsafe { klass.cast::() }; From b7c0f8ae65e50f4f8437e8fd94abb6ae0236a9ff Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 19 Dec 2023 15:05:54 +1100 Subject: [PATCH 398/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/abi.rs | 4 ++-- mmtk/src/object_scanning.rs | 31 +++++++++++++++++++++---------- mmtk/src/scanning.rs | 5 +++++ 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 94b2b62c..95103b6c 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,7 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=432c273e40d75e14a430d7fb2f0e1994a7383c25#432c273e40d75e14a430d7fb2f0e1994a7383c25" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=044d20404f46d2eadce0d4d8e51eb4352a7d3ec2#044d20404f46d2eadce0d4d8e51eb4352a7d3ec2" dependencies = [ "atomic", "atomic-traits", @@ -551,7 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=432c273e40d75e14a430d7fb2f0e1994a7383c25#432c273e40d75e14a430d7fb2f0e1994a7383c25" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=044d20404f46d2eadce0d4d8e51eb4352a7d3ec2#044d20404f46d2eadce0d4d8e51eb4352a7d3ec2" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index ec55e90c..2d09fb9b 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "432c273e40d75e14a430d7fb2f0e1994a7383c25" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "044d20404f46d2eadce0d4d8e51eb4352a7d3ec2" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index f46e9fed..45816c33 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -447,7 +447,7 @@ impl ArrayOopDesc { typesize_in_bytes / BYTES_IN_WORD } } - fn length(&self) -> i32 { + pub fn length(&self) -> i32 { unsafe { (Address::from_ref(self) + Self::length_offset::()).load::() } } fn base(&self, ty: BasicType) -> Address { @@ -528,7 +528,7 @@ impl ChunkedHandleList { if COMPRESSED { word = word | (1usize << 63); } - closure.visit_edge(E::from_address(Address::from_usize(word))) + closure.visit_edge(E::from_address(Address::from_usize(word)), true) } } } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 066877d3..a14b566b 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -6,6 +6,7 @@ use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::edge_shape::Edge; use mmtk::vm::EdgeVisitor; +use mmtk::vm::ObjectKind; use std::cell::UnsafeCell; use std::{mem, slice}; @@ -25,11 +26,10 @@ impl OopIterate for OopMapBlock { oop: Oop, closure: &mut impl EdgeVisitor>, ) { - let log_bytes_in_oop = if COMPRESSED { 2 } else { 3 }; let start = oop.get_field_address(self.offset); for i in 0..self.count as usize { - let edge = (start + (i << log_bytes_in_oop)).into(); - closure.visit_edge(edge); + let edge = (start + (i << E::::LOG_BYTES_IN_EDGE)).into(); + closure.visit_edge(edge, false); } } } @@ -80,13 +80,13 @@ impl OopIterate for InstanceMirrorKlass { let start: *const NarrowOop = start.to_ptr::(); let slice = unsafe { slice::from_raw_parts(start, len as _) }; for narrow_oop in slice { - closure.visit_edge(narrow_oop.slot().into()); + closure.visit_edge(narrow_oop.slot().into(), false); } } else { let start: *const Oop = start.to_ptr::(); let slice = unsafe { slice::from_raw_parts(start, len as _) }; for oop in slice { - closure.visit_edge(Address::from_ref(oop as &Oop).into()); + closure.visit_edge(Address::from_ref(oop as &Oop).into(), false); } } } @@ -123,11 +123,11 @@ impl OopIterate for ObjArrayKlass { let array = unsafe { oop.as_array_oop() }; if COMPRESSED { for narrow_oop in unsafe { array.data::(BasicType::T_OBJECT) } { - closure.visit_edge(narrow_oop.slot().into()); + closure.visit_edge(narrow_oop.slot().into(), false); } } else { for oop in unsafe { array.data::(BasicType::T_OBJECT) } { - closure.visit_edge(Address::from_ref(oop as &Oop).into()); + closure.visit_edge(Address::from_ref(oop as &Oop).into(), false); } } } @@ -205,9 +205,9 @@ impl InstanceRefKlass { closure: &mut impl EdgeVisitor>, ) { let referent_addr = Self::referent_address::(oop); - closure.visit_edge(referent_addr); + closure.visit_edge(referent_addr, false); let discovered_addr = Self::discovered_address::(oop); - closure.visit_edge(discovered_addr); + closure.visit_edge(discovered_addr, false); } fn discover_reference(oop: Oop, rt: ReferenceType) -> bool { // Do not discover new refs during reference processing. @@ -331,6 +331,17 @@ pub fn is_val_array(oop: Oop) -> bool { oop.klass::().id == KlassID::TypeArray } +pub fn get_obj_kind(oop: Oop) -> ObjectKind { + let cls_id = oop.klass::().id; + match cls_id { + KlassID::TypeArray => ObjectKind::ValArray, + KlassID::ObjArray => { + ObjectKind::ObjArray(unsafe { oop.as_array_oop().length::() as u32 }) + } + _ => ObjectKind::Scalar, + } +} + pub fn obj_array_data(oop: Oop) -> crate::OpenJDKEdgeRange { unsafe { let array = oop.as_array_oop(); @@ -350,7 +361,7 @@ pub unsafe extern "C" fn scan_object_fn< ) { let ptr: *mut u8 = CLOSURE.with(|x| *x.get()); let closure = &mut *(ptr as *mut V); - closure.visit_edge(edge.into()); + closure.visit_edge(edge.into(), false); } pub fn scan_object( diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 79d9bfb4..61391c5e 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -7,6 +7,7 @@ use mmtk::scheduler::RootKind; use mmtk::scheduler::WorkBucketStage; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; +use mmtk::vm::ObjectKind; use mmtk::vm::{EdgeVisitor, RootsWorkFactory, Scanning}; use mmtk::Mutator; use mmtk::MutatorContext; @@ -81,6 +82,10 @@ impl Scanning> for VMScanning { crate::object_scanning::is_val_array::(unsafe { std::mem::transmute(o) }) } + fn get_obj_kind(o: ObjectReference) -> ObjectKind { + crate::object_scanning::get_obj_kind::(unsafe { std::mem::transmute(o) }) + } + fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { // unimplemented!() // TODO From dbad26ed031f980faca5a279163b922012d0ef37 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 20 Dec 2023 10:43:37 +1100 Subject: [PATCH 399/494] LOG_CLS_UNLOAD --- openjdk/mmtk.h | 7 +++++++ openjdk/mmtkParallelCleaning.cpp | 6 ++++++ openjdk/mmtkUpcalls.cpp | 6 ++++++ 3 files changed, 19 insertions(+) diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index e4500bc2..fe53c996 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -5,6 +5,13 @@ #include #include +#define ENABLE_CLASS_UNLOADING_LOGS true +#if ENABLE_CLASS_UNLOADING_LOGS +#define LOG_CLS_UNLOAD(...) log_info(gc)(__VA_ARGS__) +#else +#define LOG_CLS_UNLOAD(...) +#endif // ENABLE_CLASS_UNLOADING_LOGS + #ifdef __cplusplus extern "C" { #endif diff --git a/openjdk/mmtkParallelCleaning.cpp b/openjdk/mmtkParallelCleaning.cpp index 7e169a36..d1a0b1df 100644 --- a/openjdk/mmtkParallelCleaning.cpp +++ b/openjdk/mmtkParallelCleaning.cpp @@ -334,6 +334,7 @@ ParallelCleaningTask::ParallelCleaningTask(BoolObjectClosure* is_alive, // The parallel work done by all worker threads. void ParallelCleaningTask::work(uint worker_id) { + LOG_CLS_UNLOAD("[complete_cleaning %d] _code_cache_task", worker_id); { // Do first pass of code cache cleaning. _code_cache_task.work_first_pass(worker_id); @@ -341,16 +342,19 @@ void ParallelCleaningTask::work(uint worker_id) { // Let the threads mark that the first pass is done. _code_cache_task.barrier_mark(worker_id); } + LOG_CLS_UNLOAD("[complete_cleaning %d] _string_symbol_task", worker_id); { // Clean the Strings and Symbols. _string_symbol_task.work(worker_id); } + LOG_CLS_UNLOAD("[complete_cleaning %d] _resolved_method_cleaning_task", worker_id); { // Clean unreferenced things in the ResolvedMethodTable _resolved_method_cleaning_task.work(); } + LOG_CLS_UNLOAD("[complete_cleaning %d] _code_cache_task2", worker_id); { // Wait for all workers to finish the first code cache cleaning pass. @@ -360,11 +364,13 @@ void ParallelCleaningTask::work(uint worker_id) { // the liveness information gathered during the first pass. _code_cache_task.work_second_pass(worker_id); } + LOG_CLS_UNLOAD("[complete_cleaning %d] _klass_cleaning_task", worker_id); { // Clean all klasses that were not unloaded. _klass_cleaning_task.work(); } + LOG_CLS_UNLOAD("[complete_cleaning %d] finish", worker_id); } } \ No newline at end of file diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index a49dcd00..e092088f 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -215,15 +215,21 @@ static void mmtk_update_weak_processor(bool lxr) { static void mmtk_unload_classes() { if (ClassUnloading) { + LOG_CLS_UNLOAD("[mmtk_unload_classes] start"); // Unload classes and purge SystemDictionary. + LOG_CLS_UNLOAD("[mmtk_unload_classes] SystemDictionary::do_unloading"); auto purged_classes = SystemDictionary::do_unloading(NULL, false /* Defer cleaning */); MMTkIsAliveClosure is_alive; MMTkForwardClosure forward; + LOG_CLS_UNLOAD("[mmtk_unload_classes] complete_cleaning"); MMTkHeap::heap()->complete_cleaning(&is_alive, &forward, purged_classes); + LOG_CLS_UNLOAD("[mmtk_unload_classes] ClassLoaderDataGraph::purge"); ClassLoaderDataGraph::purge(); + LOG_CLS_UNLOAD("[mmtk_unload_classes] compute_new_size"); // Resize and verify metaspace MetaspaceGC::compute_new_size(); MetaspaceUtils::verify_metrics(); + LOG_CLS_UNLOAD("[mmtk_unload_classes] end"); } } From f1014a69df322b3e12be8df0d37df8a86910300e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 20 Dec 2023 12:03:26 +1100 Subject: [PATCH 400/494] Skip klass cleaning task if no class was unloaded --- openjdk/mmtkParallelCleaning.cpp | 5 +++-- openjdk/mmtkParallelCleaning.hpp | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/openjdk/mmtkParallelCleaning.cpp b/openjdk/mmtkParallelCleaning.cpp index d1a0b1df..824b8de5 100644 --- a/openjdk/mmtkParallelCleaning.cpp +++ b/openjdk/mmtkParallelCleaning.cpp @@ -324,6 +324,7 @@ ParallelCleaningTask::ParallelCleaningTask(BoolObjectClosure* is_alive, uint num_workers, bool unloading_occurred) : AbstractGangTask("Parallel Cleaning"), + _unloading_occurred(unloading_occurred), _string_symbol_task(num_workers, is_alive, forward), _code_cache_task(num_workers, is_alive, unloading_occurred), _klass_cleaning_task(is_alive), @@ -364,9 +365,9 @@ void ParallelCleaningTask::work(uint worker_id) { // the liveness information gathered during the first pass. _code_cache_task.work_second_pass(worker_id); } - LOG_CLS_UNLOAD("[complete_cleaning %d] _klass_cleaning_task", worker_id); - { + if (_unloading_occurred) { + LOG_CLS_UNLOAD("[complete_cleaning %d] _klass_cleaning_task", worker_id); // Clean all klasses that were not unloaded. _klass_cleaning_task.work(); } diff --git a/openjdk/mmtkParallelCleaning.hpp b/openjdk/mmtkParallelCleaning.hpp index 4cdeb6d8..e3725250 100644 --- a/openjdk/mmtkParallelCleaning.hpp +++ b/openjdk/mmtkParallelCleaning.hpp @@ -149,6 +149,7 @@ class ResolvedMethodCleaningTask : public StackObj { // To minimize the remark pause times, the tasks below are done in parallel. class ParallelCleaningTask : public AbstractGangTask { private: + bool _unloading_occurred; StringSymbolTableUnlinkTask _string_symbol_task; CodeCacheUnloadingTask _code_cache_task; KlassCleaningTask _klass_cleaning_task; From e133065305c8b7ceaed872411919dd8ac3422d21 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 11 Jan 2024 10:39:34 +1100 Subject: [PATCH 401/494] force 12 or 6 gc worker threads --- mmtk/Cargo.toml | 2 ++ mmtk/src/api.rs | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 2d09fb9b..da7a7d6b 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -60,6 +60,8 @@ vo_bit = ["mmtk/vo_bit"] no_fast_alloc = [] roots_breakdown = [] same_stw_and_conc_threads = [] +lxr_6_conc_workers = [] +lxr_12_conc_workers = [] # This compile time constant places the mark bit in the header of the object instead of on the side. mark_bit_in_header = [] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index af62cea1..57036681 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -405,6 +405,10 @@ pub extern "C" fn mmtk_builder_set_conc_threads(value: usize) { if cfg!(feature = "same_stw_and_conc_threads") { let threads = *builder.options.threads; builder.options.conc_threads.set(threads); + } else if cfg!(feature = "lxr_6_conc_workers") { + builder.options.conc_threads.set(6); + } else if cfg!(feature = "lxr_12_conc_workers") { + builder.options.conc_threads.set(12); } else { builder.options.conc_threads.set(value); } From 73d7b96370cca10994bb202627676a7b5fe093b5 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 17 Jan 2024 05:01:56 +0000 Subject: [PATCH 402/494] Disable class unloading logging --- openjdk/mmtk.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index fe53c996..16c32505 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -5,7 +5,7 @@ #include #include -#define ENABLE_CLASS_UNLOADING_LOGS true +#define ENABLE_CLASS_UNLOADING_LOGS false #if ENABLE_CLASS_UNLOADING_LOGS #define LOG_CLS_UNLOAD(...) log_info(gc)(__VA_ARGS__) #else From c11e26bebe9fb905f3d59361084dd9bae20aa83a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 18 Jan 2024 03:54:20 +0000 Subject: [PATCH 403/494] WIP: Fix code cache unloading --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/gc_work.rs | 26 ++------------------------ openjdk/mmtkUpcalls.cpp | 6 +++++- 4 files changed, 10 insertions(+), 28 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 95103b6c..d008938c 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,7 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=044d20404f46d2eadce0d4d8e51eb4352a7d3ec2#044d20404f46d2eadce0d4d8e51eb4352a7d3ec2" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=919f427b26e5054654809f8f7a4815ea884d1900#919f427b26e5054654809f8f7a4815ea884d1900" dependencies = [ "atomic", "atomic-traits", @@ -551,7 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=044d20404f46d2eadce0d4d8e51eb4352a7d3ec2#044d20404f46d2eadce0d4d8e51eb4352a7d3ec2" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=919f427b26e5054654809f8f7a4815ea884d1900#919f427b26e5054654809f8f7a4815ea884d1900" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index da7a7d6b..f62666cc 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "044d20404f46d2eadce0d4d8e51eb4352a7d3ec2" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "919f427b26e5054654809f8f7a4815ea884d1900" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 667f9115..88100801 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -204,39 +204,17 @@ impl> ScanCodeCacheRoots { impl> GCWork for ScanCodeCacheRoots { - fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { let t = if cfg!(feature = "roots_breakdown") { Some(std::time::SystemTime::now()) } else { None }; let mut edges = Vec::with_capacity(F::BUFFER_SIZE); - let scan_all_roots = mmtk - .get_plan() - .current_gc_should_prepare_for_class_unloading() - || mmtk.get_plan().current_gc_should_perform_class_unloading(); let mut mature = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); let mut nursery_guard = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); let nursery = std::mem::take::>>(&mut nursery_guard); let mut c = 0; - if scan_all_roots { - // Collect all the mature cached roots - for roots in mature.values() { - for r in roots { - edges.push(VM::VMEdge::from_address(*r)); - if edges.len() >= F::BUFFER_SIZE { - if cfg!(feature = "roots_breakdown") { - c += edges.len(); - } - self.factory.create_process_edge_roots_work( - std::mem::take(&mut edges), - RootKind::Incomplete, - ); - edges.reserve(F::BUFFER_SIZE); - } - } - } - } // Young roots for (key, roots) in nursery { for r in &roots { @@ -247,7 +225,7 @@ impl> GCWork } self.factory.create_process_edge_roots_work( std::mem::take(&mut edges), - RootKind::Incomplete, + RootKind::YoungCodeCacheRoots, ); edges.reserve(F::BUFFER_SIZE); } diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index e092088f..69f9d5bb 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -221,6 +221,9 @@ static void mmtk_unload_classes() { auto purged_classes = SystemDictionary::do_unloading(NULL, false /* Defer cleaning */); MMTkIsAliveClosure is_alive; MMTkForwardClosure forward; + LOG_CLS_UNLOAD("[mmtk_unload_classes] forward code cache ptrs"); + CodeBlobToOopClosure cb_cl(&forward, true); + CodeCache::blobs_do(&cb_cl); LOG_CLS_UNLOAD("[mmtk_unload_classes] complete_cleaning"); MMTkHeap::heap()->complete_cleaning(&is_alive, &forward, purged_classes); LOG_CLS_UNLOAD("[mmtk_unload_classes] ClassLoaderDataGraph::purge"); @@ -363,7 +366,8 @@ static void mmtk_scan_roots_in_mutator_thread(EdgesClosure closure, void* tls) { ResourceMark rm; JavaThread* thread = (JavaThread*) tls; MMTkRootsClosure cl(closure); - thread->oops_do(&cl, NULL); + MarkingCodeBlobClosure cb_cl(&cl, !CodeBlobToOopClosure::FixRelocations); + thread->oops_do(&cl, &cb_cl); } static void mmtk_scan_multiple_thread_roots(EdgesClosure closure, void* ptr, size_t len) { From dd9886c1ddba4882068c6248b985fe468fa467c9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 18 Jan 2024 05:29:45 +0000 Subject: [PATCH 404/494] cleanup --- mmtk/src/api.rs | 2 -- mmtk/src/gc_work.rs | 4 +--- mmtk/src/lib.rs | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 57036681..64174b16 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -641,8 +641,6 @@ pub extern "C" fn mmtk_register_nmethod(nm: Address) { pub extern "C" fn mmtk_unregister_nmethod(nm: Address) { let mut roots = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); roots.remove(&nm); - let mut roots = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); - roots.remove(&nm); } #[no_mangle] diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 88100801..bed0ff75 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -211,12 +211,11 @@ impl> GCWork None }; let mut edges = Vec::with_capacity(F::BUFFER_SIZE); - let mut mature = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); let mut nursery_guard = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); let nursery = std::mem::take::>>(&mut nursery_guard); let mut c = 0; // Young roots - for (key, roots) in nursery { + for (_key, roots) in nursery { for r in &roots { edges.push(VM::VMEdge::from_address(*r)); if edges.len() >= F::BUFFER_SIZE { @@ -230,7 +229,6 @@ impl> GCWork edges.reserve(F::BUFFER_SIZE); } } - mature.insert(key, roots); } if !edges.is_empty() { if cfg!(feature = "roots_breakdown") { diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 5c395f6f..b04c2baa 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -278,7 +278,6 @@ pub static MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES: usize = lazy_static! { /// A global storage for all the cached CodeCache root pointers static ref NURSERY_CODE_CACHE_ROOTS: Mutex>> = Mutex::new(HashMap::new()); - static ref MATURE_CODE_CACHE_ROOTS: Mutex>> = Mutex::new(HashMap::new()); static ref NURSERY_WEAK_HANDLE_ROOTS: Mutex> = Mutex::new(Vec::new()); } From 80ac9deb5387f0bfebd73f847d8af4607c303587 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 19 Jan 2024 01:36:28 +0000 Subject: [PATCH 405/494] Fix class unloading --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/gc_work.rs | 56 +++++++++++++++++++++++++++++++++----------- openjdk/mmtkHeap.cpp | 6 ++--- 4 files changed, 48 insertions(+), 20 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index d008938c..d48a4910 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,7 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=919f427b26e5054654809f8f7a4815ea884d1900#919f427b26e5054654809f8f7a4815ea884d1900" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e5a40d44d1f71d43397a94c86cc1b6aa45d93993#e5a40d44d1f71d43397a94c86cc1b6aa45d93993" dependencies = [ "atomic", "atomic-traits", @@ -551,7 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=919f427b26e5054654809f8f7a4815ea884d1900#919f427b26e5054654809f8f7a4815ea884d1900" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e5a40d44d1f71d43397a94c86cc1b6aa45d93993#e5a40d44d1f71d43397a94c86cc1b6aa45d93993" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index f62666cc..d0df9806 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "919f427b26e5054654809f8f7a4815ea884d1900" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e5a40d44d1f71d43397a94c86cc1b6aa45d93993" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index bed0ff75..a09bed25 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -76,7 +76,12 @@ scan_roots_work!(ScanAOTLoaderRoots, scan_aot_loader_roots); scan_roots_work!(ScanSystemDictionaryRoots, scan_system_dictionary_roots); scan_roots_work!(ScanVMThreadRoots, scan_vm_thread_roots); -extern "C" fn report_edges_and_renew_buffer_cld>( +extern "C" fn report_edges_and_renew_buffer_cld< + E: Edge, + F: RootsWorkFactory, + const WEAK: bool, + const ALL_STRONG: bool, +>( ptr: *mut Address, length: usize, capacity: usize, @@ -89,7 +94,14 @@ extern "C" fn report_edges_and_renew_buffer_cld> record_roots(buf.len()); } let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; - factory.create_process_edge_roots_work(buf, RootKind::Incomplete); + let kind = if WEAK { + RootKind::YoungWeakCLDRoots + } else if ALL_STRONG { + RootKind::StrongCLDRoots + } else { + RootKind::YoungStrongCLDRoots + }; + factory.create_process_edge_roots_work(buf, kind); } let (ptr, _, capacity) = { // TODO: Use Vec::into_raw_parts() when the method is available. @@ -101,9 +113,16 @@ extern "C" fn report_edges_and_renew_buffer_cld> NewBuffer { ptr, capacity } } -fn to_edges_closure_cld>(factory: &mut F) -> EdgesClosure { +fn to_edges_closure_cld< + E: Edge, + F: RootsWorkFactory, + const WEAK: bool, + const ALL_STRONG: bool, +>( + factory: &mut F, +) -> EdgesClosure { EdgesClosure { - func: report_edges_and_renew_buffer_cld::, + func: report_edges_and_renew_buffer_cld::, data: factory as *mut F as *mut libc::c_void, } } @@ -131,14 +150,23 @@ impl> GCWork } else { None }; - unsafe { - ((*UPCALLS).scan_class_loader_data_graph_roots)( - to_edges_closure_cld::(&mut self.factory), - to_edges_closure_cld::(&mut self.factory), - mmtk.get_plan() - .current_gc_should_prepare_for_class_unloading() - || mmtk.get_plan().current_gc_should_perform_class_unloading(), - ); + let scan_all_strong_roots = mmtk.get_plan().current_gc_should_perform_class_unloading(); + if scan_all_strong_roots { + unsafe { + ((*UPCALLS).scan_class_loader_data_graph_roots)( + to_edges_closure_cld::(&mut self.factory), + to_edges_closure_cld::(&mut self.factory), + scan_all_strong_roots, + ); + } + } else { + unsafe { + ((*UPCALLS).scan_class_loader_data_graph_roots)( + to_edges_closure_cld::(&mut self.factory), + to_edges_closure_cld::(&mut self.factory), + scan_all_strong_roots, + ); + } } if cfg!(feature = "roots_breakdown") { let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; @@ -177,7 +205,7 @@ impl> GCWork for slice in new_roots.chunks(mmtk::args::BUFFER_SIZE) { let slice = unsafe { std::mem::transmute::<&[Address], &[VM::VMEdge]>(slice) }; self.factory - .create_process_edge_roots_work(slice.to_vec(), RootKind::Incomplete); + .create_process_edge_roots_work(slice.to_vec(), RootKind::YoungWeakHandleRoots); } new_roots.clear(); if cfg!(feature = "roots_breakdown") { @@ -235,7 +263,7 @@ impl> GCWork c += edges.len(); } self.factory - .create_process_edge_roots_work(edges, RootKind::Incomplete); + .create_process_edge_roots_work(edges, RootKind::YoungCodeCacheRoots); } if cfg!(feature = "roots_breakdown") { let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index ae24e581..93c5a5b0 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -493,9 +493,9 @@ void MMTkHeap::scan_string_table_roots(OopClosure& cl, OopStorage::ParState cld_cl(&cl); From e74ec86e4d9f9428b6df6c2f9030ba36e7680b40 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 19 Jan 2024 03:24:17 +0000 Subject: [PATCH 406/494] Reduce duplicated CLD scanning --- openjdk/mmtkHeap.cpp | 4 ++-- openjdk/mmtkRootsClosure.hpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 93c5a5b0..d47dfb36 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -498,8 +498,8 @@ void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& we ClassLoaderDataGraph::roots_cld_do(&cld_cl, &weak_cld_cl); } else if (scan_all_strong_roots) { // At the start of full heap trace, we want to scan all the strong CLD roots + all the modified CLDs. - MMTkScanCLDClosure cld_cl(&cl); - MMTkScanCLDClosure weak_cld_cl(&weak_cl); + MMTkScanCLDClosure cld_cl(&cl); + MMTkScanCLDClosure weak_cld_cl(&weak_cl); ClassLoaderDataGraph::roots_cld_do(&cld_cl, &weak_cld_cl); } else { // Normal RC pause: We simply apply increments to modified CLD roots. No decrements will be applied on these roots. diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp index 4b26bebd..d838f328 100644 --- a/openjdk/mmtkRootsClosure.hpp +++ b/openjdk/mmtkRootsClosure.hpp @@ -57,7 +57,7 @@ class MMTkRootsClosure : public OopClosure { }; -template +template class MMTkScanCLDClosure: public CLDClosure { private: OopClosure* _oop_closure; @@ -66,10 +66,10 @@ class MMTkScanCLDClosure: public CLDClosure { MMTkScanCLDClosure(OopClosure* c) : _oop_closure(c) { } void do_cld(ClassLoaderData* cld) { if (MODIFIED_ONLY) { - if (cld->has_modified_oops()) cld->oops_do(_oop_closure, false, /*clear_modified_oops*/true); + if (cld->has_modified_oops()) cld->oops_do(_oop_closure, CLAIM, /*clear_modified_oops*/true); } else { if (cld->has_modified_oops() || !WEAK) - cld->oops_do(_oop_closure, false, /*clear_modified_oops*/true); + cld->oops_do(_oop_closure, CLAIM, /*clear_modified_oops*/true); } } }; From c4065302cc8c0cbf317e7e65b54db2cd5ced848b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 19 Jan 2024 03:38:52 +0000 Subject: [PATCH 407/494] Fix code cache scanning performance --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/api.rs | 2 ++ mmtk/src/collection.rs | 1 + mmtk/src/gc_work.rs | 4 +++- mmtk/src/lib.rs | 17 +++++++++++++++++ openjdk/mmtkUpcalls.cpp | 6 +++--- 7 files changed, 29 insertions(+), 7 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index d48a4910..c6811777 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,7 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e5a40d44d1f71d43397a94c86cc1b6aa45d93993#e5a40d44d1f71d43397a94c86cc1b6aa45d93993" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=33c617e7d6cce202b48383418d5a1b6b4bbe585e#33c617e7d6cce202b48383418d5a1b6b4bbe585e" dependencies = [ "atomic", "atomic-traits", @@ -551,7 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e5a40d44d1f71d43397a94c86cc1b6aa45d93993#e5a40d44d1f71d43397a94c86cc1b6aa45d93993" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=33c617e7d6cce202b48383418d5a1b6b4bbe585e#33c617e7d6cce202b48383418d5a1b6b4bbe585e" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index d0df9806..c474ffb2 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e5a40d44d1f71d43397a94c86cc1b6aa45d93993" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "33c617e7d6cce202b48383418d5a1b6b4bbe585e" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 64174b16..57036681 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -641,6 +641,8 @@ pub extern "C" fn mmtk_register_nmethod(nm: Address) { pub extern "C" fn mmtk_unregister_nmethod(nm: Address) { let mut roots = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); roots.remove(&nm); + let mut roots = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); + roots.remove(&nm); } #[no_mangle] diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 3b163c4c..a4fda76e 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -109,6 +109,7 @@ impl Collection> for VMCollection { unsafe { if do_unloading { ((*UPCALLS).unload_classes)(); + crate::update_code_cache_roots::>(); } ((*UPCALLS).gc_epilogue)(); } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index a09bed25..76c9f004 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -239,11 +239,12 @@ impl> GCWork None }; let mut edges = Vec::with_capacity(F::BUFFER_SIZE); + let mut mature = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); let mut nursery_guard = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); let nursery = std::mem::take::>>(&mut nursery_guard); let mut c = 0; // Young roots - for (_key, roots) in nursery { + for (key, roots) in nursery { for r in &roots { edges.push(VM::VMEdge::from_address(*r)); if edges.len() >= F::BUFFER_SIZE { @@ -257,6 +258,7 @@ impl> GCWork edges.reserve(F::BUFFER_SIZE); } } + mature.insert(key, roots); } if !edges.is_empty() { if cfg!(feature = "roots_breakdown") { diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index b04c2baa..2694167d 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -278,9 +278,26 @@ pub static MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES: usize = lazy_static! { /// A global storage for all the cached CodeCache root pointers static ref NURSERY_CODE_CACHE_ROOTS: Mutex>> = Mutex::new(HashMap::new()); + static ref MATURE_CODE_CACHE_ROOTS: Mutex>> = Mutex::new(HashMap::new()); static ref NURSERY_WEAK_HANDLE_ROOTS: Mutex> = Mutex::new(Vec::new()); } +fn update_code_cache_roots() { + let roots = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); + for (_, roots) in &*roots { + for e in roots { + let e = VM::VMEdge::from_address(*e); + let o = e.load(); + if o.is_null() { + continue; + } + if let Some(f) = o.get_forwarded_object() { + e.store(f); + } + } + } +} + lazy_static! { static ref OBJ_COUNT: Mutex> = Mutex::new(HashMap::new()); } diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 69f9d5bb..c1963f37 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -221,9 +221,9 @@ static void mmtk_unload_classes() { auto purged_classes = SystemDictionary::do_unloading(NULL, false /* Defer cleaning */); MMTkIsAliveClosure is_alive; MMTkForwardClosure forward; - LOG_CLS_UNLOAD("[mmtk_unload_classes] forward code cache ptrs"); - CodeBlobToOopClosure cb_cl(&forward, true); - CodeCache::blobs_do(&cb_cl); + // LOG_CLS_UNLOAD("[mmtk_unload_classes] forward code cache ptrs"); + // CodeBlobToOopClosure cb_cl(&forward, true); + // CodeCache::blobs_do(&cb_cl); LOG_CLS_UNLOAD("[mmtk_unload_classes] complete_cleaning"); MMTkHeap::heap()->complete_cleaning(&is_alive, &forward, purged_classes); LOG_CLS_UNLOAD("[mmtk_unload_classes] ClassLoaderDataGraph::purge"); From a20da9c1e5970b72365e5df9554fba6dfe27f2e9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 20 Jan 2024 06:48:29 +0000 Subject: [PATCH 408/494] Minor --- mmtk/src/collection.rs | 2 +- openjdk/mmtkHeap.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index a4fda76e..c4708f72 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -108,8 +108,8 @@ impl Collection> for VMCollection { fn vm_release(do_unloading: bool) { unsafe { if do_unloading { - ((*UPCALLS).unload_classes)(); crate::update_code_cache_roots::>(); + ((*UPCALLS).unload_classes)(); } ((*UPCALLS).gc_epilogue)(); } diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index d47dfb36..9ceddf36 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -498,8 +498,8 @@ void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& we ClassLoaderDataGraph::roots_cld_do(&cld_cl, &weak_cld_cl); } else if (scan_all_strong_roots) { // At the start of full heap trace, we want to scan all the strong CLD roots + all the modified CLDs. - MMTkScanCLDClosure cld_cl(&cl); - MMTkScanCLDClosure weak_cld_cl(&weak_cl); + MMTkScanCLDClosure cld_cl(&cl); + MMTkScanCLDClosure weak_cld_cl(&weak_cl); ClassLoaderDataGraph::roots_cld_do(&cld_cl, &weak_cld_cl); } else { // Normal RC pause: We simply apply increments to modified CLD roots. No decrements will be applied on these roots. From 6cbb42be0dd883c5c941bb84b162232626e472f7 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 22 Jan 2024 03:41:35 +0000 Subject: [PATCH 409/494] Add more timers --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/collection.rs | 8 +++++++- openjdk/mmtk.h | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index c6811777..e9dc8c94 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,7 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=33c617e7d6cce202b48383418d5a1b6b4bbe585e#33c617e7d6cce202b48383418d5a1b6b4bbe585e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=225c7cc50a602f0401f503a54b46151f79fa0275#225c7cc50a602f0401f503a54b46151f79fa0275" dependencies = [ "atomic", "atomic-traits", @@ -551,7 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=33c617e7d6cce202b48383418d5a1b6b4bbe585e#33c617e7d6cce202b48383418d5a1b6b4bbe585e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=225c7cc50a602f0401f503a54b46151f79fa0275#225c7cc50a602f0401f503a54b46151f79fa0275" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index c474ffb2..94b91e25 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "33c617e7d6cce202b48383418d5a1b6b4bbe585e" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "225c7cc50a602f0401f503a54b46151f79fa0275" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index c4708f72..6459d52c 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -1,3 +1,4 @@ +use mmtk::gc_log; use mmtk::scheduler::{GCWorker, ProcessEdgesWork}; use mmtk::util::alloc::AllocationError; use mmtk::util::opaque_pointer::*; @@ -95,6 +96,10 @@ impl Collection> for VMCollection { } } + fn update_code_cache() { + crate::update_code_cache_roots::>() + } + fn clear_cld_claimed_marks() { unsafe { ((*UPCALLS).clear_claimed_marks)(); @@ -108,9 +113,10 @@ impl Collection> for VMCollection { fn vm_release(do_unloading: bool) { unsafe { if do_unloading { - crate::update_code_cache_roots::>(); + gc_log!(" - unload_classes"); ((*UPCALLS).unload_classes)(); } + gc_log!(" - gc_epilogue"); ((*UPCALLS).gc_epilogue)(); } } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 16c32505..fe53c996 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -5,7 +5,7 @@ #include #include -#define ENABLE_CLASS_UNLOADING_LOGS false +#define ENABLE_CLASS_UNLOADING_LOGS true #if ENABLE_CLASS_UNLOADING_LOGS #define LOG_CLS_UNLOAD(...) log_info(gc)(__VA_ARGS__) #else From fcec162b30daa467c133730c6476bd811a02c938 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 22 Jan 2024 17:18:17 +1100 Subject: [PATCH 410/494] revert --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/collection.rs | 6 ++---- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index e9dc8c94..6ca99235 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,7 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=225c7cc50a602f0401f503a54b46151f79fa0275#225c7cc50a602f0401f503a54b46151f79fa0275" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a71f52643d13c9e2dd3e13bdb74e64897b4e6fdd#a71f52643d13c9e2dd3e13bdb74e64897b4e6fdd" dependencies = [ "atomic", "atomic-traits", @@ -551,7 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=225c7cc50a602f0401f503a54b46151f79fa0275#225c7cc50a602f0401f503a54b46151f79fa0275" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a71f52643d13c9e2dd3e13bdb74e64897b4e6fdd#a71f52643d13c9e2dd3e13bdb74e64897b4e6fdd" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 94b91e25..84242d2c 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "225c7cc50a602f0401f503a54b46151f79fa0275" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "a71f52643d13c9e2dd3e13bdb74e64897b4e6fdd" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 6459d52c..82c33e9a 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -96,10 +96,6 @@ impl Collection> for VMCollection { } } - fn update_code_cache() { - crate::update_code_cache_roots::>() - } - fn clear_cld_claimed_marks() { unsafe { ((*UPCALLS).clear_claimed_marks)(); @@ -113,6 +109,8 @@ impl Collection> for VMCollection { fn vm_release(do_unloading: bool) { unsafe { if do_unloading { + gc_log!(" - update_code_cache_roots"); + crate::update_code_cache_roots::>(); gc_log!(" - unload_classes"); ((*UPCALLS).unload_classes)(); } From e0ad68425f2bc1669620bccac9fbf541d400115e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 23 Jan 2024 14:54:46 +1100 Subject: [PATCH 411/494] wip --- mmtk/src/gc_work.rs | 77 ++++++++++++++++++++++++++++++++ mmtk/src/scanning.rs | 6 +++ openjdk/mmtkParallelCleaning.cpp | 2 - openjdk/mmtkUpcalls.cpp | 3 +- 4 files changed, 85 insertions(+), 3 deletions(-) diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 76c9f004..95c0ee04 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -159,6 +159,12 @@ impl> GCWork scan_all_strong_roots, ); } + unsafe { + ((*UPCALLS).scan_string_table_roots)( + to_edges_closure_cld::(&mut self.factory), + false, + ); + } } else { unsafe { ((*UPCALLS).scan_class_loader_data_graph_roots)( @@ -273,3 +279,74 @@ impl> GCWork } } } + +extern "C" fn report_edges_and_renew_buffer_weak>( + ptr: *mut Address, + length: usize, + capacity: usize, + factory_ptr: *mut libc::c_void, +) -> NewBuffer { + if !ptr.is_null() { + let ptr = ptr as *mut E; + let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; + if cfg!(feature = "roots_breakdown") { + record_roots(buf.len()); + } + let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; + let kind = RootKind::Weak; + factory.create_process_edge_roots_work(buf, kind); + } + let (ptr, _, capacity) = { + // TODO: Use Vec::into_raw_parts() when the method is available. + use std::mem::ManuallyDrop; + let new_vec = Vec::with_capacity(F::BUFFER_SIZE); + let mut me = ManuallyDrop::new(new_vec); + (me.as_mut_ptr(), me.len(), me.capacity()) + }; + NewBuffer { ptr, capacity } +} + +fn to_edges_closure_weak>(factory: &mut F) -> EdgesClosure { + EdgesClosure { + func: report_edges_and_renew_buffer_weak::, + data: factory as *mut F as *mut libc::c_void, + } +} + +pub struct ScanWeakStringTableRoots> { + factory: F, + _p: PhantomData, +} + +impl> ScanWeakStringTableRoots { + pub fn new(factory: F) -> Self { + Self { + factory, + _p: PhantomData, + } + } +} + +impl> GCWork + for ScanWeakStringTableRoots +{ + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let t = if cfg!(feature = "roots_breakdown") { + Some(std::time::SystemTime::now()) + } else { + None + }; + let scan_all_strong_roots = mmtk.get_plan().current_gc_should_perform_class_unloading(); + assert!(scan_all_strong_roots); + unsafe { + ((*UPCALLS).scan_string_table_roots)( + to_edges_closure_weak::(&mut self.factory), + false, + ); + } + if cfg!(feature = "roots_breakdown") { + let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; + report_roots("WeakStringTableRoots", ms); + } + } +} diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 61391c5e..679cebc5 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -149,6 +149,12 @@ impl Scanning> for VMScanning { { w.push(Box::new(ScanNewWeakHandleRoots::new(factory.clone())) as _); } + if crate::singleton::() + .get_plan() + .current_gc_should_perform_class_unloading() + { + w.push(Box::new(ScanWeakStringTableRoots::new(factory.clone())) as _); + } memory_manager::add_work_packets( crate::singleton::(), WorkBucketStage::RCProcessIncs, diff --git a/openjdk/mmtkParallelCleaning.cpp b/openjdk/mmtkParallelCleaning.cpp index 824b8de5..23973e12 100644 --- a/openjdk/mmtkParallelCleaning.cpp +++ b/openjdk/mmtkParallelCleaning.cpp @@ -82,8 +82,6 @@ void StringSymbolTableUnlinkTask::work(uint worker_id) { int symbols_processed = 0; int symbols_removed = 0; // if (_process_strings) { - StringTable::possibly_parallel_oops_do(&_par_state_string_fwd, _forward); - barrier_wait(worker_id); StringTable::possibly_parallel_unlink(&_par_state_string, _is_alive, &strings_processed, &strings_removed); Atomic::add(strings_processed, &_strings_processed); Atomic::add(strings_removed, &_strings_removed); diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index c1963f37..199d981f 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -467,7 +467,8 @@ static void mmtk_scan_aot_loader_roots(EdgesClosure closure) { MMTkRootsClosure static void mmtk_scan_system_dictionary_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_system_dictionary_roots(cl); } static void mmtk_scan_code_cache_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } static void mmtk_scan_string_table_roots(EdgesClosure closure, bool rc_non_stuck_objs_only) { - UNREACHABLE(); + MMTkRootsClosure cl(closure); + MMTkHeap::heap()->scan_string_table_roots(cl, NULL); } static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, EdgesClosure weak_closure, bool scan_all_strong_roots) { MMTkRootsClosure cl(closure); From 98d0902297e5b9e51b39de7e41ee99c5e163cd97 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 23 Jan 2024 16:43:12 +1100 Subject: [PATCH 412/494] Fix WeakProcessor --- mmtk/src/gc_work.rs | 38 ++++++++++++++++++++++++++++++++++++++ mmtk/src/scanning.rs | 1 + openjdk/mmtkUpcalls.cpp | 8 ++++---- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 95c0ee04..7e5e45a8 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -350,3 +350,41 @@ impl> GCWork } } } + +pub struct ScanWeakProcessorRoots> { + factory: F, + _p: PhantomData, +} + +impl> ScanWeakProcessorRoots { + pub fn new(factory: F) -> Self { + Self { + factory, + _p: PhantomData, + } + } +} + +impl> GCWork + for ScanWeakProcessorRoots +{ + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let t = if cfg!(feature = "roots_breakdown") { + Some(std::time::SystemTime::now()) + } else { + None + }; + let scan_all_strong_roots = mmtk.get_plan().current_gc_should_perform_class_unloading(); + assert!(scan_all_strong_roots); + unsafe { + ((*UPCALLS).scan_weak_processor_roots)( + to_edges_closure_weak::(&mut self.factory), + false, + ); + } + if cfg!(feature = "roots_breakdown") { + let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; + report_roots("WeakProcessorRoots", ms); + } + } +} diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 679cebc5..65136b54 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -154,6 +154,7 @@ impl Scanning> for VMScanning { .current_gc_should_perform_class_unloading() { w.push(Box::new(ScanWeakStringTableRoots::new(factory.clone())) as _); + w.push(Box::new(ScanWeakProcessorRoots::new(factory.clone())) as _); } memory_manager::add_work_packets( crate::singleton::(), diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 199d981f..9d49924a 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -203,13 +203,12 @@ static void mmtk_clear_claimed_marks() { static void mmtk_update_weak_processor(bool lxr) { HandleMark hm; - MMTkForwardClosure forward; if (lxr) { MMTkLXRFastIsAliveClosure is_alive; - WeakProcessor::weak_oops_do(&is_alive, &forward); + WeakProcessor::weak_oops_do(&is_alive, &do_nothing_cl); } else { MMTkIsAliveClosure is_alive; - WeakProcessor::weak_oops_do(&is_alive, &forward); + WeakProcessor::weak_oops_do(&is_alive, &do_nothing_cl); } } @@ -476,7 +475,8 @@ static void mmtk_scan_class_loader_data_graph_roots(EdgesClosure closure, EdgesC MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl, weak_cl, scan_all_strong_roots); } static void mmtk_scan_weak_processor_roots(EdgesClosure closure, bool rc_non_stuck_objs_only) { - UNREACHABLE(); + MMTkRootsClosure cl(closure); + MMTkHeap::heap()->scan_weak_processor_roots(cl); } static void mmtk_scan_vm_thread_roots(EdgesClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } From 1d88404297c14c66eaf4df2b537ae1db7379f854 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 23 Jan 2024 16:50:19 +1100 Subject: [PATCH 413/494] Fix code cache roots --- mmtk/src/gc_work.rs | 59 +++++++++++++++++++++++++++++++++++++++++++- mmtk/src/lib.rs | 26 +++++++++---------- mmtk/src/scanning.rs | 1 + 3 files changed, 72 insertions(+), 14 deletions(-) diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 7e5e45a8..c8a65eea 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -275,7 +275,7 @@ impl> GCWork } if cfg!(feature = "roots_breakdown") { let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; - eprintln!(" - CodeCache roots count: {} ({:.3})", c, ms); + eprintln!(" - NewCodeCache roots count: {} ({:.3})", c, ms); } } } @@ -388,3 +388,60 @@ impl> GCWork } } } + +pub struct ScanWeakCodeCacheRoots> { + factory: F, + _p: PhantomData, +} + +impl> ScanWeakCodeCacheRoots { + pub fn new(factory: F) -> Self { + Self { + factory, + _p: PhantomData, + } + } +} + +impl> GCWork + for ScanWeakCodeCacheRoots +{ + fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { + let t = if cfg!(feature = "roots_breakdown") { + Some(std::time::SystemTime::now()) + } else { + None + }; + let scan_all_strong_roots = mmtk.get_plan().current_gc_should_perform_class_unloading(); + assert!(scan_all_strong_roots); + + let mut edges = Vec::with_capacity(F::BUFFER_SIZE); + let mature = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); + let mut c = 0; + // Young roots + for (_key, roots) in &*mature { + for r in roots { + edges.push(VM::VMEdge::from_address(*r)); + if edges.len() >= F::BUFFER_SIZE { + if cfg!(feature = "roots_breakdown") { + c += edges.len(); + } + self.factory + .create_process_edge_roots_work(std::mem::take(&mut edges), RootKind::Weak); + edges.reserve(F::BUFFER_SIZE); + } + } + } + if !edges.is_empty() { + if cfg!(feature = "roots_breakdown") { + c += edges.len(); + } + self.factory + .create_process_edge_roots_work(edges, RootKind::Weak); + } + if cfg!(feature = "roots_breakdown") { + let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; + eprintln!(" - WeakodeCacheRoots roots count: {} ({:.3})", c, ms); + } + } +} diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 2694167d..3be70f64 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -283,19 +283,19 @@ lazy_static! { } fn update_code_cache_roots() { - let roots = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); - for (_, roots) in &*roots { - for e in roots { - let e = VM::VMEdge::from_address(*e); - let o = e.load(); - if o.is_null() { - continue; - } - if let Some(f) = o.get_forwarded_object() { - e.store(f); - } - } - } + // let roots = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); + // for (_, roots) in &*roots { + // for e in roots { + // let e = VM::VMEdge::from_address(*e); + // let o = e.load(); + // if o.is_null() { + // continue; + // } + // if let Some(f) = o.get_forwarded_object() { + // e.store(f); + // } + // } + // } } lazy_static! { diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 65136b54..4c855dbb 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -155,6 +155,7 @@ impl Scanning> for VMScanning { { w.push(Box::new(ScanWeakStringTableRoots::new(factory.clone())) as _); w.push(Box::new(ScanWeakProcessorRoots::new(factory.clone())) as _); + w.push(Box::new(ScanWeakCodeCacheRoots::new(factory.clone())) as _); } memory_manager::add_work_packets( crate::singleton::(), From d942eab999663c1d5e2130a482a17b49f473491c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 24 Jan 2024 05:05:19 +0000 Subject: [PATCH 414/494] cleanup --- mmtk/src/collection.rs | 2 -- mmtk/src/gc_work.rs | 6 ------ mmtk/src/lib.rs | 16 ---------------- 3 files changed, 24 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 82c33e9a..b2629f2d 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -109,8 +109,6 @@ impl Collection> for VMCollection { fn vm_release(do_unloading: bool) { unsafe { if do_unloading { - gc_log!(" - update_code_cache_roots"); - crate::update_code_cache_roots::>(); gc_log!(" - unload_classes"); ((*UPCALLS).unload_classes)(); } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index c8a65eea..d32c1cf4 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -159,12 +159,6 @@ impl> GCWork scan_all_strong_roots, ); } - unsafe { - ((*UPCALLS).scan_string_table_roots)( - to_edges_closure_cld::(&mut self.factory), - false, - ); - } } else { unsafe { ((*UPCALLS).scan_class_loader_data_graph_roots)( diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 3be70f64..5c395f6f 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -282,22 +282,6 @@ lazy_static! { static ref NURSERY_WEAK_HANDLE_ROOTS: Mutex> = Mutex::new(Vec::new()); } -fn update_code_cache_roots() { - // let roots = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); - // for (_, roots) in &*roots { - // for e in roots { - // let e = VM::VMEdge::from_address(*e); - // let o = e.load(); - // if o.is_null() { - // continue; - // } - // if let Some(f) = o.get_forwarded_object() { - // e.store(f); - // } - // } - // } -} - lazy_static! { static ref OBJ_COUNT: Mutex> = Mutex::new(HashMap::new()); } From 27091d20417d24a1d96e11eb076f1189121664d8 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 24 Jan 2024 05:06:26 +0000 Subject: [PATCH 415/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 6ca99235..696b37bd 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,7 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a71f52643d13c9e2dd3e13bdb74e64897b4e6fdd#a71f52643d13c9e2dd3e13bdb74e64897b4e6fdd" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=de7c63292a5553fa3373da1eaa5810350a14c148#de7c63292a5553fa3373da1eaa5810350a14c148" dependencies = [ "atomic", "atomic-traits", @@ -551,7 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=a71f52643d13c9e2dd3e13bdb74e64897b4e6fdd#a71f52643d13c9e2dd3e13bdb74e64897b4e6fdd" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=de7c63292a5553fa3373da1eaa5810350a14c148#de7c63292a5553fa3373da1eaa5810350a14c148" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 84242d2c..64d17962 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "a71f52643d13c9e2dd3e13bdb74e64897b4e6fdd" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "de7c63292a5553fa3373da1eaa5810350a14c148" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 2cf7c70a637016d2354af86733896daf3970c0e5 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 30 Jan 2024 12:48:44 +1100 Subject: [PATCH 416/494] g3_compatibilities --- mmtk/Cargo.toml | 1 + mmtk/src/abi.rs | 2 ++ mmtk/src/build_info.rs | 8 ++++++++ 3 files changed, 11 insertions(+) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 64d17962..57172ece 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -62,6 +62,7 @@ roots_breakdown = [] same_stw_and_conc_threads = [] lxr_6_conc_workers = [] lxr_12_conc_workers = [] +g3_compatibilities = [] # This compile time constant places the mark bit in the header of the object instead of on the side. mark_bit_in_header = [] diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 45816c33..1e45fe99 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -346,6 +346,7 @@ impl fmt::Debug for OopDesc { /// 32-bit compressed klass pointers #[repr(transparent)] +#[allow(unused)] #[derive(Clone, Copy)] pub struct NarrowKlass(u32); @@ -353,6 +354,7 @@ pub type Oop = &'static OopDesc; /// 32-bit compressed reference pointers #[repr(transparent)] +#[allow(unused)] #[derive(Clone, Copy)] pub struct NarrowOop(u32); diff --git a/mmtk/src/build_info.rs b/mmtk/src/build_info.rs index 1dcc9ae1..0ddf4051 100644 --- a/mmtk/src/build_info.rs +++ b/mmtk/src/build_info.rs @@ -1,8 +1,16 @@ +#[cfg(not(feature = "g3_compatibilities"))] mod raw { // The include imports a full list of the constants in built.rs from https://docs.rs/built/latest/built/index.html include!(concat!(env!("OUT_DIR"), "/built.rs")); } +#[cfg(feature = "g3_compatibilities")] +mod raw { + pub const PKG_VERSION: &str = ""; + pub const GIT_COMMIT_HASH: Option<&str> = None; + pub const GIT_DIRTY: Option = None; +} + lazy_static! { // Owned string for the binding version, such as MMTk OpenJDK 0.14.0 (cfc755f-dirty) static ref BINDING_VERSION_STRING: String = match (raw::GIT_COMMIT_HASH, raw::GIT_DIRTY) { From 21a82fa4f75ef3478621c49b7a88fd70f9b5e86c Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 22 May 2024 10:42:33 +1000 Subject: [PATCH 417/494] bump mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 696b37bd..10eb7ec9 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,7 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=de7c63292a5553fa3373da1eaa5810350a14c148#de7c63292a5553fa3373da1eaa5810350a14c148" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eded7e1860d1fd80809280d7ddf1ec280b43caab#eded7e1860d1fd80809280d7ddf1ec280b43caab" dependencies = [ "atomic", "atomic-traits", @@ -551,7 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=de7c63292a5553fa3373da1eaa5810350a14c148#de7c63292a5553fa3373da1eaa5810350a14c148" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eded7e1860d1fd80809280d7ddf1ec280b43caab#eded7e1860d1fd80809280d7ddf1ec280b43caab" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 57172ece..27390bc3 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "de7c63292a5553fa3373da1eaa5810350a14c148" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "eded7e1860d1fd80809280d7ddf1ec280b43caab" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From b9df8b5d05ab9fe78eb93cd1dff8e939fa561fa6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 22 May 2024 10:49:57 +1000 Subject: [PATCH 418/494] Update Cargo.lock --- mmtk/Cargo.lock | 2 -- 1 file changed, 2 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 10eb7ec9..42ff6df5 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,7 +517,6 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eded7e1860d1fd80809280d7ddf1ec280b43caab#eded7e1860d1fd80809280d7ddf1ec280b43caab" dependencies = [ "atomic", "atomic-traits", @@ -551,7 +550,6 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eded7e1860d1fd80809280d7ddf1ec280b43caab#eded7e1860d1fd80809280d7ddf1ec280b43caab" dependencies = [ "proc-macro-error", "proc-macro2", From a40c38d135a1d1f23201619ae68ceb2b1812b36f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 24 May 2024 14:40:01 +1000 Subject: [PATCH 419/494] Fix WeakProcessor --- mmtk/src/gc_work.rs | 2 ++ mmtk/src/scanning.rs | 1 - openjdk/mmtk.h | 2 +- openjdk/mmtkUpcalls.cpp | 3 ++- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index d32c1cf4..93e3a689 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -345,12 +345,14 @@ impl> GCWork } } +#[allow(unused)] pub struct ScanWeakProcessorRoots> { factory: F, _p: PhantomData, } impl> ScanWeakProcessorRoots { + #[allow(unused)] pub fn new(factory: F) -> Self { Self { factory, diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 4c855dbb..b01f12df 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -154,7 +154,6 @@ impl Scanning> for VMScanning { .current_gc_should_perform_class_unloading() { w.push(Box::new(ScanWeakStringTableRoots::new(factory.clone())) as _); - w.push(Box::new(ScanWeakProcessorRoots::new(factory.clone())) as _); w.push(Box::new(ScanWeakCodeCacheRoots::new(factory.clone())) as _); } memory_manager::add_work_packets( diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index fe53c996..16c32505 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -5,7 +5,7 @@ #include #include -#define ENABLE_CLASS_UNLOADING_LOGS true +#define ENABLE_CLASS_UNLOADING_LOGS false #if ENABLE_CLASS_UNLOADING_LOGS #define LOG_CLS_UNLOAD(...) log_info(gc)(__VA_ARGS__) #else diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp index 9d49924a..32b717fd 100644 --- a/openjdk/mmtkUpcalls.cpp +++ b/openjdk/mmtkUpcalls.cpp @@ -208,7 +208,8 @@ static void mmtk_update_weak_processor(bool lxr) { WeakProcessor::weak_oops_do(&is_alive, &do_nothing_cl); } else { MMTkIsAliveClosure is_alive; - WeakProcessor::weak_oops_do(&is_alive, &do_nothing_cl); + MMTkForwardClosure forward; + WeakProcessor::weak_oops_do(&is_alive, &forward); } } From c0e5ee1918d03f113495f6bc3509d99f53c3e95f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 27 May 2024 14:05:44 +1000 Subject: [PATCH 420/494] Update barrier --- openjdk/barriers/mmtkFieldBarrier.cpp | 8 ++++---- openjdk/barriers/mmtkFieldBarrier.hpp | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index d8752462..9abb32f5 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -127,10 +127,10 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma void MMTkFieldBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { if (type == T_OBJECT || type == T_ARRAY) { - // Label slow, done; + Label done; // // Bailout if count is zero - // __ cmpptr(count, 0); - // __ jcc(Assembler::equal, done); + __ cmpptr(count, 0); + __ jcc(Assembler::equal, done); // // Fast path if count is one // __ cmpptr(count, 1); // __ jcc(Assembler::notEqual, slow); @@ -147,13 +147,13 @@ void MMTkFieldBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Deco // assert(count == rdx, "expected"); // __ call_VM_leaf(CAST_FROM_FN_PTR(address, MMTkFieldBarrierSetRuntime::record_array_copy_slow), src, dst, count); // __ popa(); - // __ bind(done); __ pusha(); __ movptr(c_rarg0, src); __ movptr(c_rarg1, dst); __ movptr(c_rarg2, count); __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_pre_call), 3); __ popa(); + __ bind(done); } } diff --git a/openjdk/barriers/mmtkFieldBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp index c8888a7d..c092a8d6 100644 --- a/openjdk/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -21,11 +21,12 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { // Interfaces called by `MMTkBarrierSet::AccessBarrier` virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const override { + if (count == 0) return; ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) dst, count); } virtual void load_reference(DecoratorSet decorators, oop value) const override; virtual void clone_pre(DecoratorSet decorators, oop value) const override { - ::mmtk_object_reference_clone_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); + // ::mmtk_object_reference_clone_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); }; virtual void object_probable_write(oop new_obj) const override; }; From 46f70a5da616b4980ef5f7bb85ede21fe316dc60 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 27 May 2024 15:02:20 +1000 Subject: [PATCH 421/494] barrier experiments --- mmtk/Cargo.toml | 6 +++++ mmtk/src/lib.rs | 12 ++++++++++ openjdk/barriers/mmtkFieldBarrier.cpp | 32 +++++++++++++-------------- openjdk/barriers/mmtkFieldBarrier.hpp | 1 + openjdk/mmtk.h | 5 +++++ openjdk/mmtkBarrierSet.cpp | 3 +++ 6 files changed, 42 insertions(+), 17 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 27390bc3..14353e78 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -98,3 +98,9 @@ narrow_oop_mode_32bit = ["force_narrow_oop_mode"] narrow_oop_mode_shift = ["force_narrow_oop_mode"] narrow_oop_mode_base = ["force_narrow_oop_mode"] narrow_oop_mode_base_and_shift = ["force_narrow_oop_mode"] + +field_barrier_no_eager_branch = [] +field_barrier_no_arraycopy = [] +field_barrier_no_arraycopy_slow = [] +field_barrier_no_c2_slow_call = [] +field_barrier_no_c2_rust_call = [] diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 5c395f6f..7f5f1c5a 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -186,6 +186,18 @@ pub static DISABLE_ALLOCATION_FAST_PATH: i32 = pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = std::mem::size_of::>>(); +#[no_mangle] +pub static FIELD_BARRIER_NO_EAGER_BRANCH: u8 = cfg!(feature = "field_barrier_no_eager_branch") as _; +#[no_mangle] +pub static FIELD_BARRIER_NO_ARRAYCOPY: u8 = cfg!(feature = "field_barrier_no_arraycopy") as _; +#[no_mangle] +pub static FIELD_BARRIER_NO_ARRAYCOPY_SLOW: u8 = + cfg!(feature = "field_barrier_no_arraycopy_slow") as _; +#[no_mangle] +pub static FIELD_BARRIER_NO_C2_SLOW_CALL: u8 = cfg!(feature = "field_barrier_no_c2_slow_call") as _; +#[no_mangle] +pub static FIELD_BARRIER_NO_C2_RUST_CALL: u8 = cfg!(feature = "field_barrier_no_c2_rust_call") as _; + #[no_mangle] pub static mut CONCURRENT_MARKING_ACTIVE: u8 = 0; diff --git a/openjdk/barriers/mmtkFieldBarrier.cpp b/openjdk/barriers/mmtkFieldBarrier.cpp index 9abb32f5..f3c9231f 100644 --- a/openjdk/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/barriers/mmtkFieldBarrier.cpp @@ -3,7 +3,6 @@ #include "runtime/interfaceSupport.inline.hpp" #define SOFT_REFERENCE_LOAD_BARRIER true -#define MMTK_BARRIER_EAGER_BRANCH true constexpr int kUnloggedValue = 1; @@ -24,9 +23,8 @@ void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, intptr_t addr = ((intptr_t) (void*) slot); const volatile uint8_t * meta_addr = (const volatile uint8_t *) (side_metadata_base_address() + (addr >> (UseCompressedOops ? 5 : 6))); uint8_t byte_val = *meta_addr; - #if MMTK_BARRIER_EAGER_BRANCH - if (byte_val == 0) return; - #endif + if (!FIELD_BARRIER_NO_EAGER_BRANCH) + if (byte_val == 0) return; intptr_t shift = (addr >> (UseCompressedOops ? 2 : 3)) & 0b111; if (((byte_val >> shift) & 1) == kUnloggedValue) { ::mmtk_object_reference_write_slow((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) slot, (void*) target); @@ -88,10 +86,10 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma __ shrptr(tmp3, UseCompressedOops ? 5 : 6); __ movptr(tmp5, side_metadata_base_address()); __ movzbl(tmp5, Address(tmp5, tmp3)); - #if MMTK_BARRIER_EAGER_BRANCH - __ cmpl(tmp5, 0); - __ jcc(Assembler::equal, done); - #endif + if (!FIELD_BARRIER_NO_EAGER_BRANCH) { + __ cmpl(tmp5, 0); + __ jcc(Assembler::equal, done); + } // tmp3 = (obj >> 3) & 7 __ lea(tmp3, dst); __ shrptr(tmp3, UseCompressedOops ? 2 : 3); @@ -126,6 +124,7 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma } void MMTkFieldBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { + if (FIELD_BARRIER_NO_ARRAYCOPY) return; if (type == T_OBJECT || type == T_ARRAY) { Label done; // // Bailout if count is zero @@ -265,10 +264,10 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op // uint8_t byte_val = *meta_addr; LIR_Opr byte_val = gen->new_register(T_INT); __ move(meta_addr, byte_val); - #if MMTK_BARRIER_EAGER_BRANCH + // #if MMTK_BARRIER_EAGER_BRANCH // __ cmp(lir_cond_equal, byte_val, LIR_OprFact::intConst(0)); // __ branch(lir_cond_equal, T_BYTE, Ldone->label()); - #endif + // #endif // intptr_t shift = (addr >> 3) & 0b111; LIR_Opr shift = gen->new_register(T_INT); __ move(addr, shift); @@ -303,19 +302,18 @@ static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* sl Node* addr = __ CastPX(__ ctrl(), slot); Node* meta_addr = __ AddP(no_base, __ ConP(side_metadata_base_address()), __ URShiftX(addr, __ ConI(UseCompressedOops ? 5 : 6))); Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); - #if MMTK_BARRIER_EAGER_BRANCH - __ if_then(byte, BoolTest::ne, zero, unlikely); - #endif + if (!FIELD_BARRIER_NO_EAGER_BRANCH) + __ if_then(byte, BoolTest::ne, zero, unlikely); Node* shift = __ URShiftX(addr, __ ConI(UseCompressedOops ? 2 : 3)); shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); __ if_then(result, BoolTest::ne, zero, unlikely); { const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), "mmtk_barrier_call", src, slot, val); + if (!FIELD_BARRIER_NO_C2_SLOW_CALL) + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), "mmtk_barrier_call", src, slot, val); } __ end_if(); - #if MMTK_BARRIER_EAGER_BRANCH - __ end_if(); - #endif + if (!FIELD_BARRIER_NO_EAGER_BRANCH) + __ end_if(); #else const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), "mmtk_barrier_call", src, slot, val); diff --git a/openjdk/barriers/mmtkFieldBarrier.hpp b/openjdk/barriers/mmtkFieldBarrier.hpp index c092a8d6..63102a5c 100644 --- a/openjdk/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/barriers/mmtkFieldBarrier.hpp @@ -21,6 +21,7 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { // Interfaces called by `MMTkBarrierSet::AccessBarrier` virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const override { + if (FIELD_BARRIER_NO_ARRAYCOPY) return; if (count == 0) return; ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) dst, count); } diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 16c32505..79a139ff 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -34,6 +34,11 @@ extern const uintptr_t IMMIX_ALLOCATOR_SIZE; extern uint8_t CONCURRENT_MARKING_ACTIVE; extern uint8_t RC_ENABLED; extern uint8_t REQUIRES_WEAK_HANDLE_BARRIER; +extern uint8_t FIELD_BARRIER_NO_EAGER_BRANCH; +extern uint8_t FIELD_BARRIER_NO_ARRAYCOPY; +extern uint8_t FIELD_BARRIER_NO_ARRAYCOPY_SLOW; +extern uint8_t FIELD_BARRIER_NO_C2_SLOW_CALL; +extern uint8_t FIELD_BARRIER_NO_C2_RUST_CALL; inline bool disable_fast_alloc() { return DISABLE_ALLOCATION_FAST_PATH != 0; diff --git a/openjdk/mmtkBarrierSet.cpp b/openjdk/mmtkBarrierSet.cpp index 6abd031c..fd41ccf2 100644 --- a/openjdk/mmtkBarrierSet.cpp +++ b/openjdk/mmtkBarrierSet.cpp @@ -148,14 +148,17 @@ void MMTkBarrierSetRuntime::object_reference_write_post_call(void* src, void* sl } void MMTkBarrierSetRuntime::object_reference_write_slow_call(void* src, void* slot, void* target) { + if (FIELD_BARRIER_NO_C2_RUST_CALL) return; ::mmtk_object_reference_write_slow((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, target); } void MMTkBarrierSetRuntime::object_reference_array_copy_pre_call(void* src, void* dst, size_t count) { + if (FIELD_BARRIER_NO_ARRAYCOPY_SLOW) return; ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); } void MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, void* dst, size_t count) { + if (FIELD_BARRIER_NO_ARRAYCOPY_SLOW) return; ::mmtk_array_copy_post((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); } From 00ee1edb55d40c3fba867b261a5c16946a4d5ecd Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 21 Jul 2024 22:04:04 +1000 Subject: [PATCH 422/494] initial prefetch support --- mmtk/src/edges.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mmtk/src/edges.rs b/mmtk/src/edges.rs index 33f86cda..4361fbba 100644 --- a/mmtk/src/edges.rs +++ b/mmtk/src/edges.rs @@ -385,4 +385,9 @@ impl MemorySlice for OpenJDKEdgeRange { Range::
::copy(&src.clone().into(), &tgt.clone().into()) } } + + fn get(&self, index: usize) -> Self::Edge { + let addr = self.range.start.addr + (index << OpenJDKEdge::::LOG_BYTES_IN_EDGE); + addr.into() + } } From 53107d5cfd7886fe9de0e5ffd97f611a818011bb Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 1 Aug 2024 13:45:51 +1000 Subject: [PATCH 423/494] Update hash --- mmtk/Cargo.lock | 2 ++ mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 42ff6df5..bc7a0ddd 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,6 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b41321c36dc659a29a3add93c4ad3e19e3f7e409#b41321c36dc659a29a3add93c4ad3e19e3f7e409" dependencies = [ "atomic", "atomic-traits", @@ -550,6 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b41321c36dc659a29a3add93c4ad3e19e3f7e409#b41321c36dc659a29a3add93c4ad3e19e3f7e409" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 14353e78..482020f3 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -41,7 +41,7 @@ memoffset = "0.9.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "eded7e1860d1fd80809280d7ddf1ec280b43caab" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "b41321c36dc659a29a3add93c4ad3e19e3f7e409" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From aad4d0e41878432036f618830f33a59a268d49b3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 1 Aug 2024 13:46:30 +1000 Subject: [PATCH 424/494] Update hash --- mmtk/Cargo.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 42ff6df5..10eb7ec9 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -517,6 +517,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.20.0" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eded7e1860d1fd80809280d7ddf1ec280b43caab#eded7e1860d1fd80809280d7ddf1ec280b43caab" dependencies = [ "atomic", "atomic-traits", @@ -550,6 +551,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.20.0" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=eded7e1860d1fd80809280d7ddf1ec280b43caab#eded7e1860d1fd80809280d7ddf1ec280b43caab" dependencies = [ "proc-macro-error", "proc-macro2", From 45299476d6b020b99bec5636cb4f7bfff299459e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 1 Aug 2024 13:49:38 +1000 Subject: [PATCH 425/494] Update hash --- mmtk/Cargo.lock | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 9b59808b..693f6a30 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -52,17 +52,17 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bindgen" -version = "0.68.1" +version = "0.69.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" dependencies = [ "bitflags 2.4.0", "cexpr", "clang-sys", + "itertools", "lazy_static", "lazycell", "log", - "peeking_take_while", "prettyplease", "proc-macro2", "quote", @@ -617,12 +617,6 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "percent-encoding" version = "2.3.0" @@ -651,20 +645,21 @@ dependencies = [ [[package]] name = "pfm" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498f5ea94bbec98f1369c70a71178a1eb821370443ef393f1bbcd9950182758" +checksum = "6a7cff01e1f62b10f9b3428330a600b79d380b3b893fca8d3f397fff629b183d" dependencies = [ "libc", + "log", "perf-event-open-sys", "pfm-sys", ] [[package]] name = "pfm-sys" -version = "0.0.16-libpfm4.13.0" +version = "0.0.18+libpfm4.13.0-42d6d0c3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b520e7dec2e88bed569b487e68ba1ba4e729d5980b32b6c0243fbb1424f916cf" +checksum = "8ecea6f8eee06cb96295a16d0cb86ba6165c2e9c0e77331b444cccf9e00c6590" dependencies = [ "bindgen", ] From 6f9d50ab47fcc789f2445b934f901425da5a96fb Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 1 Aug 2024 13:50:27 +1000 Subject: [PATCH 426/494] Update hash --- mmtk/Cargo.lock | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 5a0968dd..bc7a0ddd 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -52,17 +52,17 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bindgen" -version = "0.69.4" +version = "0.68.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078" dependencies = [ "bitflags 2.4.0", "cexpr", "clang-sys", - "itertools", "lazy_static", "lazycell", "log", + "peeking_take_while", "prettyplease", "proc-macro2", "quote", @@ -617,6 +617,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.3.0" @@ -649,7 +655,6 @@ version = "0.1.0" source = "git+https://github.com/wenyuzhao/pfm.git?branch=plm0#25e901836dbfa08a683e2b08a906a61ebf69f810" dependencies = [ "libc", - "log", "perf-event-open-sys", "pfm-sys", ] From 5f79679d2fb4573bdc8cdd4b9172f7dbc2d49bdb Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 6 Aug 2024 23:32:24 +1000 Subject: [PATCH 427/494] Clean up --- mmtk/src/abi.rs | 14 ++++++++++---- mmtk/src/lib.rs | 4 ++++ openjdk/mmtk.h | 2 ++ openjdk/mmtkHeap.cpp | 1 + 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index a42453a4..0a9bccb9 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -190,24 +190,29 @@ impl InstanceKlass { const HEADER_SIZE: usize = mem::size_of::() / BYTES_IN_WORD; const VTABLE_START_OFFSET: usize = Self::HEADER_SIZE * BYTES_IN_WORD; const MISC_IS_ANONYMOUS: u16 = 1 << 5; + fn start_of_vtable(&self) -> *const usize { (Address::from_ref(self) + Self::VTABLE_START_OFFSET).to_ptr() } + fn start_of_itable(&self) -> *const usize { unsafe { self.start_of_vtable().add(self.klass.vtable_len as _) } } + fn nonstatic_oop_map_count(&self) -> usize { let oop_map_block_size = mem::size_of::(); let oop_map_block_size_up = mmtk::util::conversions::raw_align_up(oop_map_block_size, BYTES_IN_WORD); self.nonstatic_oop_map_size as usize / (oop_map_block_size_up >> LOG_BYTES_IN_WORD) } + pub fn nonstatic_oop_maps(&self) -> &'static [OopMapBlock] { let start_of_itable = self.start_of_itable(); let start = unsafe { start_of_itable.add(self.itable_len as _) as *const OopMapBlock }; let count = self.nonstatic_oop_map_count(); unsafe { slice::from_raw_parts(start, count) } } + pub fn is_anonymous(&self) -> bool { self.misc_flags & Self::MISC_IS_ANONYMOUS != 0 } @@ -382,6 +387,7 @@ impl From for &OopDesc { } } +/// Convert Oop to ObjectReference impl Into for &OopDesc { fn into(self) -> ObjectReference { unsafe { mem::transmute::<&OopDesc, _>(self) } @@ -467,10 +473,10 @@ impl ArrayOopDesc { let base_offset_in_bytes = Self::header_size::(ty) * BYTES_IN_WORD; Address::from_ref(self) + base_offset_in_bytes } - // This provides an easy way to access the array data in Rust. However, the array data - // is Java types, so we have to map Java types to Rust types. The caller needs to guarantee: - // 1. matches the actual Java type - // 2. matches the argument, BasicType `ty` + /// This provides an easy way to access the array data in Rust. However, the array data + /// is Java types, so we have to map Java types to Rust types. The caller needs to guarantee: + /// 1. `` matches the actual Java type + /// 2. `` matches the argument, BasicType `ty` pub unsafe fn data(&self, ty: BasicType) -> &[T] { slice::from_raw_parts( self.base::(ty).to_ptr(), diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 2da13245..e913d95a 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -154,6 +154,10 @@ lazy_static! { pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); +#[no_mangle] +pub static GLOBAL_SIDE_METADATA_BASE_ADDRESS: uintptr_t = + mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_BASE_ADDRESS.as_usize(); + #[no_mangle] pub static GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS: uintptr_t = mmtk::util::metadata::side_metadata::GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS.as_usize(); diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h index 5ef85a6f..06eb6b07 100644 --- a/openjdk/mmtk.h +++ b/openjdk/mmtk.h @@ -25,6 +25,7 @@ typedef enum { MmapOutOfMemory, } MMTkAllocationError; +extern const uintptr_t GLOBAL_SIDE_METADATA_BASE_ADDRESS; extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS_COMPRESSED; extern const uintptr_t RC_TABLE_BASE_ADDRESS; @@ -130,6 +131,7 @@ extern char* mmtk_active_barrier(); extern void initialize_collection(void *tls); extern void gc_init(size_t heap_size); extern bool will_never_move(void* object); +extern bool process(char* name, char* value); extern bool process_bulk(char* options); extern void scan_region(); extern void handle_user_collection_request(void *tls, bool force); diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index fbdc4421..2b594fb1 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -65,6 +65,7 @@ object iterator??!! //mmtkGCTaskManager* MMTkHeap::_mmtk_gc_task_manager = NULL; + MMTkHeap* MMTkHeap::_heap = NULL; MMTkHeap::MMTkHeap(MMTkCollectorPolicy* policy) : From 689439a15af70d5d615aa02128dae02eea374b97 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 8 Aug 2024 14:29:49 +1000 Subject: [PATCH 428/494] Fix safepoint trigger --- openjdk/mmtkVMCompanionThread.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/openjdk/mmtkVMCompanionThread.cpp b/openjdk/mmtkVMCompanionThread.cpp index f5def4ec..7fb5241d 100644 --- a/openjdk/mmtkVMCompanionThread.cpp +++ b/openjdk/mmtkVMCompanionThread.cpp @@ -100,16 +100,6 @@ void MMTkVMCompanionThread::run() { // calls request(_threads_resumed). VMThread::execute(&op); } - - // Tell the waiter thread that the world has resumed. - log_trace(gc)("MMTkVMCompanionThread: Notifying threads resumption..."); - if (!_wait_for_gc_locker) { - MutexLockerEx locker(_lock, Mutex::_no_safepoint_check_flag); - assert(_desired_state == _threads_resumed, "start-the-world should be requested."); - assert(_reached_state == _threads_suspended, "Threads should still be suspended at this moment."); - _reached_state = _threads_resumed; - _lock->notify_all(); - } } } From 9b39764a9ae02f31f0b11570f4d00ffef3f433be Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 8 Aug 2024 14:30:11 +1000 Subject: [PATCH 429/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 1ef887b5..65a05219 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -528,7 +528,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.26.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ada2732e3e7ba519f93a7346ea09143f75d4030f#ada2732e3e7ba519f93a7346ea09143f75d4030f" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b383f7c46bce5e0dea78b9a11333faf40e24622e#b383f7c46bce5e0dea78b9a11333faf40e24622e" dependencies = [ "atomic", "atomic-traits", @@ -563,7 +563,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.26.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=ada2732e3e7ba519f93a7346ea09143f75d4030f#ada2732e3e7ba519f93a7346ea09143f75d4030f" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b383f7c46bce5e0dea78b9a11333faf40e24622e#b383f7c46bce5e0dea78b9a11333faf40e24622e" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 4982eb07..a9734c4e 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -43,7 +43,7 @@ cfg-if = "1.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "ada2732e3e7ba519f93a7346ea09143f75d4030f" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "b383f7c46bce5e0dea78b9a11333faf40e24622e" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From d348aec8e3df663f4db0b0331baecf6d66c22101 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 24 Sep 2024 12:29:01 +1000 Subject: [PATCH 430/494] Minor refactorings --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/reference_glue.rs | 2 ++ mmtk/src/scanning.rs | 3 +-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 65a05219..41dd9ae8 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -528,7 +528,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.26.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b383f7c46bce5e0dea78b9a11333faf40e24622e#b383f7c46bce5e0dea78b9a11333faf40e24622e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=10da2ea18e596fa22e5d0fe3e62c1aad201c50f7#10da2ea18e596fa22e5d0fe3e62c1aad201c50f7" dependencies = [ "atomic", "atomic-traits", @@ -563,7 +563,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.26.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=b383f7c46bce5e0dea78b9a11333faf40e24622e#b383f7c46bce5e0dea78b9a11333faf40e24622e" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=10da2ea18e596fa22e5d0fe3e62c1aad201c50f7#10da2ea18e596fa22e5d0fe3e62c1aad201c50f7" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index a9734c4e..d9129a43 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -43,7 +43,7 @@ cfg-if = "1.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "b383f7c46bce5e0dea78b9a11333faf40e24622e" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "10da2ea18e596fa22e5d0fe3e62c1aad201c50f7" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index a8627908..014bfff3 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -320,6 +320,7 @@ impl GCWork unsafe { *slot = ObjectReference::NULL }; } } + trace.process_slots(); trace.flush(); } } @@ -360,6 +361,7 @@ impl GCWork let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; set_next_reference::(tail, Some(old_head)); } + trace.process_slots(); trace.flush(); } } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index ea4c7b80..7ad91edf 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -4,7 +4,6 @@ use crate::{NewBuffer, OpenJDKSlot, UPCALLS}; use crate::{OpenJDK, SlotsClosure}; use mmtk::memory_manager; use mmtk::scheduler::RootKind; -use mmtk::scheduler::WorkBucketStage; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::ObjectKind; @@ -158,7 +157,7 @@ impl Scanning> for VMScanning { } memory_manager::add_work_packets( crate::singleton::(), - WorkBucketStage::RCProcessIncs, + factory.roots_stage(), w, ); } From a0c83638f8da90ed51791a789d6253e016866b9d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 24 Sep 2024 13:16:36 +1000 Subject: [PATCH 431/494] Fix rust 1.80 warnings --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 5 ++++- mmtk/rust-toolchain | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 41dd9ae8..d3476883 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -528,7 +528,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.26.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=10da2ea18e596fa22e5d0fe3e62c1aad201c50f7#10da2ea18e596fa22e5d0fe3e62c1aad201c50f7" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=138c204e98ba896e519d6a0c3ebddde9132732bc#138c204e98ba896e519d6a0c3ebddde9132732bc" dependencies = [ "atomic", "atomic-traits", @@ -563,7 +563,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.26.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=10da2ea18e596fa22e5d0fe3e62c1aad201c50f7#10da2ea18e596fa22e5d0fe3e62c1aad201c50f7" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=138c204e98ba896e519d6a0c3ebddde9132732bc#138c204e98ba896e519d6a0c3ebddde9132732bc" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index d9129a43..64556401 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -43,7 +43,7 @@ cfg-if = "1.0" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "10da2ea18e596fa22e5d0fe3e62c1aad201c50f7" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "138c204e98ba896e519d6a0c3ebddde9132732bc" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } @@ -85,6 +85,9 @@ gencopy = [] marksweep = [] pageprotect = [] immix = [] +genimmix = [] +stickyimmix = [] +markcompact = [] sanity = ["mmtk/sanity"] barrier_measurement = ["mmtk/barrier_measurement"] diff --git a/mmtk/rust-toolchain b/mmtk/rust-toolchain index 79e15fd4..dbd41264 100644 --- a/mmtk/rust-toolchain +++ b/mmtk/rust-toolchain @@ -1 +1 @@ -1.77.0 +1.81.0 From 09f87beba0231e0e3cde4fdb254c275f5b7e704e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 10 Jan 2025 00:42:33 +0000 Subject: [PATCH 432/494] Minor --- openjdk/mmtkHeap.cpp | 2 +- openjdk/mmtkMutator.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openjdk/mmtkHeap.cpp b/openjdk/mmtkHeap.cpp index 2b594fb1..be34da60 100644 --- a/openjdk/mmtkHeap.cpp +++ b/openjdk/mmtkHeap.cpp @@ -566,7 +566,7 @@ static inline HeapWord* alloc_fast(size_t bytes, Allocator allocator) { allocator = AllocatorLos; } else if (allocator == AllocatorDefault) { AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; - if (selector.tag == TAG_IMMIX) { + if (selector.tag == TAG_IMMIX && !disable_fast_alloc()) { auto& allocator = Thread::current()->third_party_heap_mutator.allocators.immix[selector.index]; auto cursor = uintptr_t(allocator.cursor); auto limit = uintptr_t(allocator.limit); diff --git a/openjdk/mmtkMutator.cpp b/openjdk/mmtkMutator.cpp index 06a118b8..4dde9391 100644 --- a/openjdk/mmtkMutator.cpp +++ b/openjdk/mmtkMutator.cpp @@ -33,7 +33,7 @@ HeapWord* MMTkMutatorContext::alloc(size_t bytes, Allocator allocator) { allocator = AllocatorLos; } else { AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; - if (selector.tag == TAG_IMMIX) { + if (selector.tag == TAG_IMMIX && !disable_fast_alloc()) { auto& allocator = allocators.immix[selector.index]; auto cursor = uintptr_t(allocator.cursor); auto limit = uintptr_t(allocator.limit); From 01177c106b705631df4730d982154b0130fb76f0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 22 Apr 2025 11:44:46 +1000 Subject: [PATCH 433/494] fmt and fix warnings --- mmtk/src/lib.rs | 3 +-- mmtk/src/object_scanning.rs | 1 - mmtk/src/scanning.rs | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 413d6d0f..f76d7933 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -2,9 +2,8 @@ extern crate lazy_static; extern crate atomic; extern crate once_cell; -extern crate spin; -#[macro_use] extern crate probe; +extern crate spin; use std::collections::HashMap; use std::ptr::null_mut; diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 3f656091..31d5542a 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -1,7 +1,6 @@ use super::abi::*; use super::UPCALLS; use crate::reference_glue::DISCOVERED_LISTS; -use crate::OpenJDK; use crate::OpenJDKSlot; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 7907b799..a3bea96d 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -13,6 +13,7 @@ use mmtk::MutatorContext; pub struct VMScanning {} +#[allow(unused)] pub(crate) const WORK_PACKET_CAPACITY: usize = mmtk::scheduler::EDGES_WORK_BUFFER_SIZE; extern "C" fn report_slots_and_renew_buffer>( From 5065111cab83ac28f417c325c4e8a5bd681f907a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 22 Apr 2025 11:55:39 +1000 Subject: [PATCH 434/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 4290e179..3e774a8c 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -555,7 +555,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.30.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=fe3430b1aa0b76b4998acf382c22f617d1e567a5#fe3430b1aa0b76b4998acf382c22f617d1e567a5" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=432ce914cf78ab933ebd77ae53492019eba663ad#432ce914cf78ab933ebd77ae53492019eba663ad" dependencies = [ "atomic", "atomic-traits", @@ -593,7 +593,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.30.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=fe3430b1aa0b76b4998acf382c22f617d1e567a5#fe3430b1aa0b76b4998acf382c22f617d1e567a5" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=432ce914cf78ab933ebd77ae53492019eba663ad#432ce914cf78ab933ebd77ae53492019eba663ad" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 08fca340..b2294c79 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -44,7 +44,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "fe3430b1aa0b76b4998acf382c22f617d1e567a5" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "432ce914cf78ab933ebd77ae53492019eba663ad" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 46546ed41d28dd101eb57dca5422b73148f6ee06 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Mon, 28 Apr 2025 00:13:02 +0000 Subject: [PATCH 435/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 3e774a8c..5c6f2a4f 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -555,7 +555,7 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mmtk" version = "0.30.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=432ce914cf78ab933ebd77ae53492019eba663ad#432ce914cf78ab933ebd77ae53492019eba663ad" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=97443c056874fb3c9aa447e42d264c28b30af84d#97443c056874fb3c9aa447e42d264c28b30af84d" dependencies = [ "atomic", "atomic-traits", @@ -593,7 +593,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.30.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=432ce914cf78ab933ebd77ae53492019eba663ad#432ce914cf78ab933ebd77ae53492019eba663ad" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=97443c056874fb3c9aa447e42d264c28b30af84d#97443c056874fb3c9aa447e42d264c28b30af84d" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index b2294c79..45065d1b 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -44,7 +44,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "432ce914cf78ab933ebd77ae53492019eba663ad" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "97443c056874fb3c9aa447e42d264c28b30af84d" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 3e27334f2cc589c41cf1b99835d96665cf02fdbf Mon Sep 17 00:00:00 2001 From: Hayley Patton <43771738+no-defun-allowed@users.noreply.github.com> Date: Tue, 2 Dec 2025 13:01:07 +1100 Subject: [PATCH 436/494] Demote re-registered nmethods (#336) If an nmethod is in the mature roots and the nmethod is re-registered due to code patching, the nmethod will appear in both the nursery roots and the mature roots for one collection, causing some roots to be scanned twice. This PR removes an nmethod from the mature roots, when the nmethod is re-registered, so that the nmethod will only be present in the nursery. --- mmtk/src/api.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 5520e88b..12df8df6 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -512,6 +512,13 @@ pub extern "C" fn mmtk_register_nmethod(nm: Address) { NMETHOD_SLOTS.with_borrow_mut(|slots| { if !slots.is_empty() { let mut roots = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); + let mut mature_roots = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); + // The nmethod might already be in the mature roots, if we + // are re-registering the nmethod due to code patching. + // If the nmethod is already in the mature roots, we demote + // the nmethod to the nursery roots, as the nmethod might + // now refer to a young object. + mature_roots.remove(&nm); roots.insert(nm, std::mem::take(slots)); } }); From 007d6e054b1609919dfb75922056659a80f5d808 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 9 Jan 2026 17:05:57 +1300 Subject: [PATCH 437/494] Move to Rust 1.92 (#338) Co-authored-by: mmtkgc-bot --- .github/workflows/stylecheck.yml | 2 +- .github/workflows/test-pr.yml | 2 +- mmtk/Cargo.lock | 7 +++---- mmtk/Cargo.toml | 4 ++-- mmtk/rust-toolchain | 2 +- mmtk/src/slots.rs | 8 ++++++-- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/stylecheck.yml b/.github/workflows/stylecheck.yml index 92ad4063..f8b6645d 100644 --- a/.github/workflows/stylecheck.yml +++ b/.github/workflows/stylecheck.yml @@ -3,7 +3,7 @@ name: Style check on: pull_request: branches: - - master + - jdk-11 concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml index c56080bc..80e4542d 100644 --- a/.github/workflows/test-pr.yml +++ b/.github/workflows/test-pr.yml @@ -3,7 +3,7 @@ name: DaCapo Tests on: pull_request: branches: - - master + - jdk-11 concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 943bf1c0..06681c8b 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "aho-corasick" @@ -491,7 +491,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.31.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=e25ad8bb969f5d650de3f1946582075923bafcc9#e25ad8bb969f5d650de3f1946582075923bafcc9" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=e30d0a0866fe66d352eb09eb7f9e9466dc649172#e30d0a0866fe66d352eb09eb7f9e9466dc649172" dependencies = [ "atomic", "atomic-traits", @@ -505,7 +505,6 @@ dependencies = [ "downcast-rs", "enum-map", "env_logger", - "idna_adapter", "is-terminal", "itertools", "lazy_static", @@ -530,7 +529,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.31.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=e25ad8bb969f5d650de3f1946582075923bafcc9#e25ad8bb969f5d650de3f1946582075923bafcc9" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=e30d0a0866fe66d352eb09eb7f9e9466dc649172#e30d0a0866fe66d352eb09eb7f9e9466dc649172" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index a5e16855..00e76f8d 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -2,7 +2,7 @@ name = "mmtk_openjdk" version = "0.31.0" authors = [" <>"] -rust-version = "1.74.1" +rust-version = "1.84.0" build = "build.rs" edition = "2021" @@ -36,7 +36,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "e25ad8bb969f5d650de3f1946582075923bafcc9" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "e30d0a0866fe66d352eb09eb7f9e9466dc649172" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } diff --git a/mmtk/rust-toolchain b/mmtk/rust-toolchain index 6b4de0a4..7f229af9 100644 --- a/mmtk/rust-toolchain +++ b/mmtk/rust-toolchain @@ -1 +1 @@ -1.83.0 +1.92.0 diff --git a/mmtk/src/slots.rs b/mmtk/src/slots.rs index e299247e..02896836 100644 --- a/mmtk/src/slots.rs +++ b/mmtk/src/slots.rs @@ -94,7 +94,9 @@ impl OpenJDKSlot { } fn x86_read_unaligned(&self) -> T { - debug_assert!(cfg!(any(target_arch = "x86", target_arch = "x86_64"))); + const { + assert!(cfg!(any(target_arch = "x86", target_arch = "x86_64"))); + } // Workaround: On x86 (including x86_64), machine instructions may contain pointers as // immediates, and they may be unaligned. It is an undefined behavior in Rust to // dereference unaligned pointers. We have to explicitly use unaligned memory access @@ -113,7 +115,9 @@ impl OpenJDKSlot { } fn x86_write_unaligned(&self, v: T) { - debug_assert!(cfg!(any(target_arch = "x86", target_arch = "x86_64"))); + const { + assert!(cfg!(any(target_arch = "x86", target_arch = "x86_64"))); + } unsafe { let slot = if UNTAG { self.untagged_address() From 7b3017263dff6e7edbedb875a81ad3b22228a7b9 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 6 Feb 2026 14:12:34 +1300 Subject: [PATCH 438/494] Bump version to v0.32.0 (#341) Co-authored-by: mmtkgc-bot --- CHANGELOG.md | 25 +++++ mmtk/Cargo.lock | 246 +++++++++++++++++++++++------------------------- mmtk/Cargo.toml | 4 +- 3 files changed, 147 insertions(+), 128 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6367cc2..f34df5da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,28 @@ +0.32.0 (2025-02-04) +=== + +## What's Changed +* Update to MMTk core PR #1308 by @qinsoon in https://github.com/mmtk/mmtk-openjdk/pull/308 +* Fix reference processing crash by @wks in https://github.com/mmtk/mmtk-openjdk/pull/309 +* Add Compressor tests by @k-sareen in https://github.com/mmtk/mmtk-openjdk/pull/310 +* Enable reference processing by default in CI by @k-sareen in https://github.com/mmtk/mmtk-openjdk/pull/312 +* Acquire and release Heap_lock on VM_MMTkSTWOperation by @qinsoon in https://github.com/mmtk/mmtk-openjdk/pull/316 +* Fix bugs related to `Reference.discovered` by @wks in https://github.com/mmtk/mmtk-openjdk/pull/323 +* Control barrier fast paths via env var by @wks in https://github.com/mmtk/mmtk-openjdk/pull/326 +* Warn instead of panic when `object_iterate` is called. by @wks in https://github.com/mmtk/mmtk-openjdk/pull/325 +* Do not apply write barriers when enqueuing refs by @wks in https://github.com/mmtk/mmtk-openjdk/pull/328 +* Enable compressed pointers for Compressor by @k-sareen in https://github.com/mmtk/mmtk-openjdk/pull/327 +* Concurrent Immix by @tianleq in https://github.com/mmtk/mmtk-openjdk/pull/311 +* Reduce the test heap size for concurrent immix by @qinsoon in https://github.com/mmtk/mmtk-openjdk/pull/330 +* Refactor barrier implementations by @wks in https://github.com/mmtk/mmtk-openjdk/pull/332 +* Demote re-registered nmethods by @no-defun-allowed in https://github.com/mmtk/mmtk-openjdk/pull/336 +* Move to Rust 1.92 by @qinsoon in https://github.com/mmtk/mmtk-openjdk/pull/338 + +## New Contributors +* @no-defun-allowed made their first contribution in https://github.com/mmtk/mmtk-openjdk/pull/336 + +**Full Changelog**: https://github.com/mmtk/mmtk-openjdk/compare/v0.31.0...v0.32.0 + 0.31.0 (2025-04-17) === diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 06681c8b..99a18e97 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -107,6 +107,15 @@ dependencies = [ "git2", ] +[[package]] +name = "built" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" +dependencies = [ + "git2", +] + [[package]] name = "bytemuck" version = "1.23.1" @@ -151,12 +160,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - [[package]] name = "crossbeam" version = "0.8.4" @@ -490,15 +493,14 @@ dependencies = [ [[package]] name = "mmtk" -version = "0.31.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=e30d0a0866fe66d352eb09eb7f9e9466dc649172#e30d0a0866fe66d352eb09eb7f9e9466dc649172" +version = "0.32.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=105661429473dc141f940e981c04dc86e995daaa#105661429473dc141f940e981c04dc86e995daaa" dependencies = [ "atomic", "atomic-traits", "atomic_refcell", - "built", + "built 0.8.0", "bytemuck", - "bytemuck_derive", "cfg-if", "crossbeam", "delegate", @@ -516,7 +518,6 @@ dependencies = [ "num_cpus", "portable-atomic", "probe", - "rayon-core", "regex", "rustversion", "spin", @@ -528,8 +529,8 @@ dependencies = [ [[package]] name = "mmtk-macros" -version = "0.31.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=e30d0a0866fe66d352eb09eb7f9e9466dc649172#e30d0a0866fe66d352eb09eb7f9e9466dc649172" +version = "0.32.0" +source = "git+https://github.com/mmtk/mmtk-core.git?rev=105661429473dc141f940e981c04dc86e995daaa#105661429473dc141f940e981c04dc86e995daaa" dependencies = [ "proc-macro-error", "proc-macro2", @@ -539,10 +540,10 @@ dependencies = [ [[package]] name = "mmtk_openjdk" -version = "0.31.0" +version = "0.32.0" dependencies = [ "atomic", - "built", + "built 0.7.7", "cfg-if", "lazy_static", "libc", @@ -581,6 +582,25 @@ dependencies = [ "libc", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -674,26 +694,6 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "regex" version = "1.12.2" @@ -794,9 +794,9 @@ checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "spin" -version = "0.9.8" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" dependencies = [ "lock_api", ] @@ -848,15 +848,15 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.33.1" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +checksum = "252800745060e7b9ffb7b2badbd8b31cfa4aa2e61af879d0a3bf2a317c20217d" dependencies = [ - "core-foundation-sys", "libc", "memchr", "ntapi", - "rayon", + "objc2-core-foundation", + "objc2-io-kit", "windows", ] @@ -971,31 +971,55 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.57.0" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" dependencies = [ "windows-core", - "windows-targets 0.52.6", ] [[package]] name = "windows-core" -version = "0.57.0" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement", "windows-interface", + "windows-link 0.1.3", "windows-result", - "windows-targets 0.52.6", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core", + "windows-link 0.1.3", + "windows-threading", ] [[package]] name = "windows-implement" -version = "0.57.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", @@ -1004,62 +1028,71 @@ dependencies = [ [[package]] name = "windows-interface" -version = "0.57.0" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", "syn 2.0.109", ] +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core", + "windows-link 0.1.3", +] + [[package]] name = "windows-result" -version = "0.1.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ - "windows-targets 0.52.6", + "windows-link 0.1.3", ] [[package]] -name = "windows-sys" -version = "0.60.2" +name = "windows-strings" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ - "windows-targets 0.53.5", + "windows-link 0.1.3", ] [[package]] name = "windows-sys" -version = "0.61.2" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-link", + "windows-targets", ] [[package]] -name = "windows-targets" -version = "0.52.6" +name = "windows-sys" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", + "windows-link 0.2.1", ] [[package]] @@ -1068,22 +1101,25 @@ version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows-link 0.2.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" +name = "windows-threading" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] [[package]] name = "windows_aarch64_gnullvm" @@ -1091,84 +1127,42 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - [[package]] name = "windows_aarch64_msvc" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - [[package]] name = "windows_i686_gnu" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - [[package]] name = "windows_i686_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - [[package]] name = "windows_i686_msvc" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - [[package]] name = "windows_x86_64_gnu" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - [[package]] name = "windows_x86_64_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "windows_x86_64_msvc" version = "0.53.1" diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 00e76f8d..4873e737 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mmtk_openjdk" -version = "0.31.0" +version = "0.32.0" authors = [" <>"] rust-version = "1.84.0" build = "build.rs" @@ -36,7 +36,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "e30d0a0866fe66d352eb09eb7f9e9466dc649172" } +mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "105661429473dc141f940e981c04dc86e995daaa" } # Uncomment the following to build locally # mmtk = { path = "../repos/mmtk-core" } From 87c25ebab09bb93fe90786f9088413fb975bf880 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 3 Mar 2026 19:05:51 +1100 Subject: [PATCH 439/494] LXR without class unloading and weak processor working --- mmtk/src/abi.rs | 22 +++---- mmtk/src/gc_work.rs | 4 +- mmtk/src/scanning.rs | 23 ++++--- openjdk/CompileThirdPartyHeap.gmk | 10 ++- openjdk/share/barriers/mmtkFieldBarrier.cpp | 72 +++++++++++++-------- 5 files changed, 78 insertions(+), 53 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 69abcbe1..df78b103 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -89,6 +89,7 @@ pub struct Klass { pub access_flags: i32, // AccessFlags #[cfg(feature = "jfr")] pub trace_id: u64, // JFR_ONLY(traceid _trace_id;) + pub _shared_class_path_index: u16, pub shared_class_flags: u16, pub archived_mirror_index: i32, pub padding: i32, @@ -543,8 +544,6 @@ pub fn validate_memory_layouts() { #[repr(C)] #[derive(Debug)] struct Chunk { - #[cfg(debug_assertions)] - vptr: OpaquePointer, data: [*mut OopDesc; 32], size: AtomicU32, next: *mut Chunk, @@ -595,17 +594,14 @@ impl ChunkedHandleList { #[repr(C)] #[derive(Debug)] pub struct ClassLoaderData { - #[cfg(debug_assertions)] - vptr: OpaquePointer, - holder: OpaquePointer, // WeakHandle - class_loader: OpaquePointer, // OopHandle - metaspace: OpaquePointer, // ClassLoaderMetaspace*volatile - metaspace_lock: OpaquePointer, // Mutex* - unloading: bool, // bool - is_anonymous: bool, // bool - modified_oops: bool, // bool - accumulated_modified_oops: bool, // bool - keep_alive: i16, + holder: OpaquePointer, // WeakHandle + class_loader: OpaquePointer, // OopHandle + metaspace: OpaquePointer, // ClassLoaderMetaspace*volatile + metaspace_lock: OpaquePointer, // Mutex* + unloading: u8, // bool + is_anonymous: u8, // bool + modified_oops: u8, // bool + keep_alive: i32, claimed: AtomicU32, handles: ChunkedHandleList, } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 2bb5e3cb..7abcf325 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -399,8 +399,8 @@ impl> GCWork } else { None }; - let scan_all_strong_roots = mmtk.get_plan().current_gc_should_perform_class_unloading(); - assert!(scan_all_strong_roots); + // let scan_all_strong_roots = mmtk.get_plan().current_gc_should_perform_class_unloading(); + // assert!(scan_all_strong_roots); unsafe { ((*UPCALLS).scan_weak_processor_roots)(to_slots_closure_weak::( &mut self.factory, diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index dd15be93..d3ef28f1 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -134,17 +134,20 @@ impl Scanning> for VMScanning { factory: impl RootsWorkFactory>, ) { let mut w: Vec>>> = vec![ - Box::new(ScanCodeCacheRoots::new(factory.clone())) as _, - Box::new(ScanClassLoaderDataGraphRoots::new(factory.clone())) as _, - Box::new(ScanOopStorageSetRoots::new(factory.clone())) as _, - Box::new(ScanVMThreadRoots::new(factory.clone())) as _, + Box::new(ScanCodeCacheRoots::new(factory.clone())), + Box::new(ScanClassLoaderDataGraphRoots::new(factory.clone())), + Box::new(ScanOopStorageSetRoots::new(factory.clone())), + Box::new(ScanVMThreadRoots::new(factory.clone())), ]; - if crate::singleton::() - .get_plan() - .requires_weak_root_scanning() - { - w.push(Box::new(ScanNewWeakHandleRoots::new(factory.clone())) as _); - } + // if crate::singleton::() + // .get_plan() + // .requires_weak_root_scanning() + // { + // w.push(Box::new(ScanNewWeakHandleRoots::new(factory.clone())) as _); + // } + w.push(Box::new( + ScanWeakProcessorRoots::, _>::new(factory.clone()), + )); // if crate::singleton::() // .get_plan() // .current_gc_should_perform_class_unloading() diff --git a/openjdk/CompileThirdPartyHeap.gmk b/openjdk/CompileThirdPartyHeap.gmk index 35ff2dc6..0c331b97 100644 --- a/openjdk/CompileThirdPartyHeap.gmk +++ b/openjdk/CompileThirdPartyHeap.gmk @@ -15,8 +15,16 @@ OPENJDK_LOCAL_VERSION=`git rev-parse HEAD` # https://doc.rust-lang.org/rustc/profile-guided-optimization.html#a-complete-cargo-workflow HOST_TRIPLE=`rustc -vV | grep host | cut -d' ' -f2` +ifdef GC_FEATURES + GC_FEATURES:=--features $(strip $(GC_FEATURES)) +endif + ifdef MMTK_PLAN - GC_FEATURES=--features $(MMTK_PLAN) + ifndef GC_FEATURES + GC_FEATURES=--features $(MMTK_PLAN) + else + GC_FEATURES:=$(strip $(GC_FEATURES))",$(MMTK_PLAN)" + endif endif ifeq ($(MMTK_VO_BIT), 1) diff --git a/openjdk/share/barriers/mmtkFieldBarrier.cpp b/openjdk/share/barriers/mmtkFieldBarrier.cpp index 4f40581b..6a681436 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.cpp @@ -54,17 +54,21 @@ void MMTkFieldBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet de #if SOFT_REFERENCE_LOAD_BARRIER if (on_oop && on_reference) { Label done; + + assert_different_registers(dst, tmp1); + // No slow-call if SATB is not active - Register tmp = rscratch1; - Register tmp2 = rscratch2; - __ movptr(tmp, intptr_t(&CONCURRENT_MARKING_ACTIVE)); - __ xorq(tmp2, tmp2); - __ movb(tmp2, Address(tmp, 0)); - __ cmpptr(tmp2, 1); - __ jcc(Assembler::notEqual, done); - // No slow-call if dst is NULL - __ cmpptr(dst, 0); - __ jcc(Assembler::equal, done); + // intptr_t tmp1_q = CONCURRENT_MARKING_ACTIVE; + __ movptr(tmp1, intptr_t(&CONCURRENT_MARKING_ACTIVE)); + // Load with zero extension to 32 bits. + // uint32_t tmp1_l = (uint32_t)(*(unt8_t*)tmp1_q); + __ movzbl(tmp1, Address(tmp1, 0)); + // if (tmp1_l == 0) goto done; + __ testl(tmp1, tmp1); + __ jcc(Assembler::zero, done); + // if (dst == 0) goto done; + __ testptr(dst, dst); + __ jcc(Assembler::zero, done); // Do slow-call __ pusha(); __ mov(c_rarg0, dst); @@ -401,6 +405,12 @@ static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool em } static void reference_load_barrier_for_unknown_load(GraphKit* kit, Node* base_oop, Node* offset, Node* slot, Node* val, bool need_mem_bar) { + // Note: This function is copied from G1BarrierSetC2::insert_pre_barrier, + // and ShenandoahBarrierSetC2::insert_pre_barrier is probably copied from G1 as well. + // It basically implements BarrierSetC1::generate_referent_check in C2 IR. + // TODO: If another barrier needs weak reference load barrier, + // consider hoisting this function to a superclass. + // We could be accessing the referent field of a reference object. If so, when G1 // is enabled, we need to log the value in the referent field in an SATB buffer. // This routine performs some compile time filters and generates suitable @@ -412,26 +422,26 @@ static void reference_load_barrier_for_unknown_load(GraphKit* kit, Node* base_oo // If offset is a constant, is it java_lang_ref_Reference::_reference_offset? const TypeX* otype = offset->find_intptr_t_type(); - if (otype != NULL && otype->is_con() && - otype->get_con() != java_lang_ref_Reference::referent_offset) { + if (otype != nullptr && otype->is_con() && + otype->get_con() != java_lang_ref_Reference::referent_offset()) { // Constant offset but not the reference_offset so just return return; } // We only need to generate the runtime guards for instances. const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr(); - if (btype != NULL) { + if (btype != nullptr) { if (btype->isa_aryptr()) { // Array type so nothing to do return; } const TypeInstPtr* itype = btype->isa_instptr(); - if (itype != NULL) { + if (itype != nullptr) { // Can the klass of base_oop be statically determined to be // _not_ a sub-class of Reference and _not_ Object? - ciKlass* klass = itype->klass(); - if ( klass->is_loaded() && + ciKlass* klass = itype->instance_klass(); + if (klass->is_loaded() && !klass->is_subtype_of(kit->env()->Reference_klass()) && !kit->env()->Object_klass()->is_subtype_of(klass)) { return; @@ -444,7 +454,7 @@ static void reference_load_barrier_for_unknown_load(GraphKit* kit, Node* base_oo IdealKit ideal(kit); - Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset); + Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset()); __ if_then(offset, BoolTest::eq, referent_off, unlikely); { // Update graphKit memory and control from IdealKit. @@ -454,7 +464,7 @@ static void reference_load_barrier_for_unknown_load(GraphKit* kit, Node* base_oo // Update IdealKit memory and control from graphKit. __ sync_kit(kit); Node* one = __ ConI(1); - // is_instof == 0 if base_oop == NULL + // is_instof == 0 if base_oop == nullptr __ if_then(is_instof, BoolTest::eq, one, unlikely); { // Update graphKit from IdeakKit. kit->sync_kit(ideal); @@ -475,24 +485,23 @@ static void reference_load_barrier_for_unknown_load(GraphKit* kit, Node* base_oo } Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { - DecoratorSet decorators = access.decorators(); - C2ParseAccess& parse_access = static_cast(access); - GraphKit* kit = parse_access.kit(); - Node* adr = access.addr().node(); Node* obj = access.base(); bool mismatched = (decorators & C2_MISMATCHED) != 0; bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0; bool in_heap = (decorators & IN_HEAP) != 0; + bool in_native = (decorators & IN_NATIVE) != 0; bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; + bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; bool is_unordered = (decorators & MO_UNORDERED) != 0; - bool need_cpu_mem_bar = !is_unordered || mismatched || !in_heap; + bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0; + bool is_mixed = !in_heap && !in_native; + bool need_cpu_mem_bar = !is_unordered || mismatched || is_mixed; Node* top = Compile::current()->top(); - Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) :top; - Node* load = BarrierSetC2::load_at_resolved(access, val_type); + Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top; // If we are reading the value of the referent field of a Reference // object (either by using Unsafe directly or through reflection) @@ -500,12 +509,21 @@ Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_ // SATB log buffer using the pre-barrier mechanism. // Also we need to add memory barrier to prevent commoning reads // from this field across safepoint since GC can change its value. - bool need_read_barrier = in_heap && (on_weak || (unknown && offset != top && obj != top)); + bool need_read_barrier = (((on_weak || on_phantom) && !no_keepalive) || + (in_heap && unknown && offset != top && obj != top)); if (!access.is_oop() || !need_read_barrier) { - return load; + return BarrierSetC2::load_at_resolved(access, val_type); } + // The other access "opt_access" is only used in arraycopy barriers. + // OpenJDK doesn't have weak arrays, so it must be "parse_access". + assert(access.is_parse_access(), "entry not supported at optimization time"); + + C2ParseAccess& parse_access = static_cast(access); + GraphKit* kit = parse_access.kit(); + Node* load = BarrierSetC2::load_at_resolved(access, val_type); + #if SOFT_REFERENCE_LOAD_BARRIER if (on_weak) { reference_load_barrier(kit, adr, load, true); From 44e8b97a8957a955d9d23d224b5121a900303131 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 4 Mar 2026 11:49:43 +1100 Subject: [PATCH 440/494] Optimizations: Reduce code cache roots & improve parallelism --- mmtk/src/api.rs | 12 ++++++------ mmtk/src/gc_work.rs | 15 +++++++++------ mmtk/src/lib.rs | 2 +- mmtk/src/scanning.rs | 8 ++++---- openjdk/share/mmtk.h | 2 +- openjdk/share/mmtkHeap.cpp | 12 ++++++++++-- openjdk/share/mmtkHeap.hpp | 18 ++++++++++++++++++ openjdk/share/mmtkUpcalls.cpp | 35 ++++++++++++----------------------- 8 files changed, 61 insertions(+), 43 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 7b9ffde9..e6a728bc 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -624,10 +624,10 @@ pub extern "C" fn mmtk_verbose() -> usize { #[no_mangle] pub unsafe extern "C" fn mmtk_register_new_weak_handle(oop: *const Oop) { - let addr = if crate::use_compressed_oops() { - Address::from_usize(oop as usize | (1 << 63)) - } else { - Address::from_ptr(oop) - }; - crate::NURSERY_WEAK_HANDLE_ROOTS.lock().unwrap().push(addr); + // let addr = if crate::use_compressed_oops() { + // Address::from_usize(oop as usize | (1 << 63)) + // } else { + // Address::from_ptr(oop) + // }; + // crate::NURSERY_WEAK_HANDLE_ROOTS.lock().unwrap().push(addr); } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 7abcf325..bbb41287 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -6,6 +6,7 @@ use crate::OpenJDKSlot; use crate::Slot; use crate::SlotsClosure; use crate::UPCALLS; +use mmtk::plan::immix::Pause; use mmtk::scheduler::*; use mmtk::util::Address; use mmtk::vm::RootsWorkFactory; @@ -241,6 +242,10 @@ impl>> .get_plan() .downcast_ref::>>() .is_some(); + let is_rc_pause = mmtk + .get_plan() + .downcast_ref::>>() + .is_some_and(|lxr| lxr.current_pause() == Some(Pause::RefCount)); let class_unloading_enabled = unsafe { crate::CLASS_UNLOADING_ENABLED } == 1; let mut slots = Vec::with_capacity(scanning::WORK_PACKET_CAPACITY); @@ -271,7 +276,7 @@ impl>> let mut mature = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); // Only scan mature roots in full-heap collections. - if !is_current_gc_nursery && !(is_lxr && class_unloading_enabled) { + if !is_current_gc_nursery && !(is_lxr && (class_unloading_enabled || is_rc_pause)) { for (key, roots) in mature.iter() { mature_slots += roots.len(); add_roots(roots); @@ -308,7 +313,7 @@ impl>> .create_process_roots_work(slots, RootKind::Strong); } - if moves_object { + if moves_object && !nmethods_to_fix.is_empty() { // Note: If the current GC doesn't move objects at all, we don't need to fix relocation. // FIXME: Even during copying GC, some GC algorithms (such as Immix) don't move every // single object. We only need to call `fix_oop_relocations` on nmethods that actually @@ -487,10 +492,8 @@ impl GCWork> for FixRelocations { _mmtk: &'static MMTK>, ) { let num_nmethods = self.nmethods.len(); - for nmethod in self.nmethods.iter().copied() { - unsafe { - ((*UPCALLS).fix_oop_relocations)(nmethod.to_mut_ptr()); - } + unsafe { + ((*UPCALLS).fix_oop_relocations)(self.nmethods.as_mut_ptr() as *mut _, num_nmethods); } // probe!(mmtk_openjdk, fix_relocations, num_nmethods); } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 27ee56c0..4d40f767 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -134,7 +134,7 @@ pub struct OpenJDK_Upcalls { pub swap_reference_pending_list: extern "C" fn(objects: ObjectReference) -> ObjectReference, pub java_lang_class_klass_offset_in_bytes: extern "C" fn() -> usize, pub java_lang_classloader_loader_data_offset: extern "C" fn() -> usize, - pub fix_oop_relocations: extern "C" fn(nmethod: *mut libc::c_void), + pub fix_oop_relocations: extern "C" fn(nmethods: *mut libc::c_void, len: usize), // pub nmethod_fix_relocation: extern "C" fn(Address), pub clear_claimed_marks: extern "C" fn(), pub unload_classes: extern "C" fn(), diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index d3ef28f1..6d1f98db 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -136,18 +136,18 @@ impl Scanning> for VMScanning { let mut w: Vec>>> = vec![ Box::new(ScanCodeCacheRoots::new(factory.clone())), Box::new(ScanClassLoaderDataGraphRoots::new(factory.clone())), - Box::new(ScanOopStorageSetRoots::new(factory.clone())), Box::new(ScanVMThreadRoots::new(factory.clone())), ]; + for _ in 0..*crate::singleton::().get_options().threads { + w.push(Box::new(ScanOopStorageSetRoots::new(factory.clone()))); + w.push(Box::new(ScanWeakProcessorRoots::new(factory.clone()))); + } // if crate::singleton::() // .get_plan() // .requires_weak_root_scanning() // { // w.push(Box::new(ScanNewWeakHandleRoots::new(factory.clone())) as _); // } - w.push(Box::new( - ScanWeakProcessorRoots::, _>::new(factory.clone()), - )); // if crate::singleton::() // .get_plan() // .current_gc_should_perform_class_unloading() diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index fedcd498..b6446318 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -219,7 +219,7 @@ typedef struct { void* (*swap_reference_pending_list)(void* objects); size_t (*java_lang_class_klass_offset_in_bytes)(); size_t (*java_lang_classloader_loader_data_offset)(); - void (*fix_oop_relocations)(void* nmethod); + void (*fix_oop_relocations)(void* nmethods, size_t len); void (*clear_claimed_marks)(); void (*unload_classes)(); void (*gc_epilogue)(); diff --git a/openjdk/share/mmtkHeap.cpp b/openjdk/share/mmtkHeap.cpp index 2244ac3b..ecdfe8a4 100644 --- a/openjdk/share/mmtkHeap.cpp +++ b/openjdk/share/mmtkHeap.cpp @@ -31,6 +31,7 @@ #include "gc/shared/gcLocker.inline.hpp" #include "gc/shared/gcWhen.hpp" #include "gc/shared/oopStorageSet.inline.hpp" +#include "gc/shared/oopStorageSetParState.inline.hpp" #include "gc/shared/scavengableNMethods.hpp" #include "gc/shared/strongRootsScope.hpp" #include "gc/shared/weakProcessor.hpp" @@ -65,6 +66,9 @@ using namespace JavaClassFile; +MaybeUninit> oop_storage_set_strong_par_state; +MaybeUninit> oop_storage_set_weak_par_state; + /* needed support from rust heap capacity @@ -493,11 +497,15 @@ void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& we } } void MMTkHeap::scan_oop_storage_set_roots(OopClosure& cl) { - OopStorageSet::strong_oops_do(&cl); + for (auto id : EnumRange()) { + oop_storage_set_strong_par_state->par_state(id)->oops_do(&cl); + } } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { ResourceMark rm; - WeakProcessor::oops_do(&cl); + for (auto id : EnumRange()) { + oop_storage_set_weak_par_state->par_state(id)->weak_oops_do(&cl); + } } void MMTkHeap::scan_vm_thread_roots(OopClosure& cl) { ResourceMark rm; diff --git a/openjdk/share/mmtkHeap.hpp b/openjdk/share/mmtkHeap.hpp index 47546c6f..abe02046 100644 --- a/openjdk/share/mmtkHeap.hpp +++ b/openjdk/share/mmtkHeap.hpp @@ -31,6 +31,7 @@ #include "gc/shared/gcWhen.hpp" #include "gc/shared/oopStorage.hpp" #include "gc/shared/oopStorageParState.hpp" +#include "gc/shared/oopStorageSetParState.hpp" #include "gc/shared/space.hpp" #include "gc/shared/strongRootsScope.hpp" #include "gc/shared/workerThread.hpp" @@ -44,6 +45,23 @@ #define WORKER_STACK_SIZE (64 * 1024 * 1024) +template +struct MaybeUninit { + MaybeUninit() {} + T* operator->() { + return (T*) &_data; + } + T& operator*() { + return *((T*) &_data); + } + template + void init(Args... args) { + new (&_data) T(args...); + } +private: + char _data[sizeof(T)]; +}; + class GCMemoryManager; class MemoryPool; //class mmtkGCTaskManager; diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index 1b9af14f..7c87120d 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -159,24 +159,8 @@ class MMTkLXRFastUpdateClosure : public OopClosure { } }; -template -struct MaybeUninit { - MaybeUninit() {} - T* operator->() { - return (T*) &_data; - } - T& operator*() { - return *((T*) &_data); - } - template - void init(Args... args) { - new (&_data) T(args...); - } -private: - char _data[sizeof(T)]; -}; - -static MaybeUninit> par_state_string; +extern MaybeUninit> oop_storage_set_strong_par_state; +extern MaybeUninit> oop_storage_set_weak_par_state; static void mmtk_stop_all_mutators(void *tls, MutatorClosure closure, bool current_gc_should_unload_classes) { log_debug(gc)("Requesting the VM to suspend all mutators..."); @@ -184,7 +168,8 @@ static void mmtk_stop_all_mutators(void *tls, MutatorClosure closure, bool curre log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); MMTkHeap::heap()->set_is_gc_active(true); - // par_state_string.init(StringTable::weak_storage()); + oop_storage_set_strong_par_state.init(); + oop_storage_set_weak_par_state.init(); mmtk_report_gc_start(); if (ClassUnloading && current_gc_should_unload_classes) { @@ -212,7 +197,8 @@ static void mmtk_update_weak_processor(bool lxr) { // HandleMark hm(THREAD); if (lxr) { MMTkLXRFastIsAliveClosure is_alive; - WeakProcessor::weak_oops_do(&is_alive, &do_nothing_cl); + MMTkForwardClosure forward; + WeakProcessor::weak_oops_do(&is_alive, &forward); } else { MMTkIsAliveClosure is_alive; MMTkForwardClosure forward; @@ -247,6 +233,7 @@ static void mmtk_unload_classes() { // MMTkHeap::heap()->bulk_unregister_nmethods(); ctx.free_code_blobs(); ClassLoaderDataGraph::purge(true /* at_safepoint */); + DEBUG_ONLY(MetaspaceUtils::verify();) LOG_CLS_UNLOAD("[mmtk_unload_classes] compute_new_size"); // Resize and verify metaspace MetaspaceGC::compute_new_size(); @@ -576,9 +563,11 @@ static size_t mmtk_java_lang_classloader_loader_data_offset() { return v; } -void mmtk_fix_oop_relocations(void *nmptr) { - nmethod* nm = (nmethod*)nmptr; - nm->fix_oop_relocations(); +void mmtk_fix_oop_relocations(void* nmethods, size_t len) { + for (size_t i = 0; i < len; i++) { + nmethod* nm = (nmethod*) ((nmethod**) nmethods)[i]; + nm->fix_oop_relocations(); + } } OpenJDK_Upcalls mmtk_upcalls = { From 1b8e9a85c8149387120c104a0a2f44be009a9444 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 10 Mar 2026 16:18:20 +1100 Subject: [PATCH 441/494] update_reserved_argument_area_size --- openjdk/share/barriers/mmtkFieldBarrier.hpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openjdk/share/barriers/mmtkFieldBarrier.hpp b/openjdk/share/barriers/mmtkFieldBarrier.hpp index 53327534..63435b50 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.hpp @@ -34,7 +34,9 @@ struct MMTkC1FieldBarrierStub: CodeStub { LIR_PatchCode patch_code; // Enable code patching? LIR_Opr scratch = NULL; // Scratch register for the resolved field - MMTkC1FieldBarrierStub(LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info = NULL, LIR_PatchCode patch_code = lir_patch_none): src(src), slot(slot), new_val(new_val), info(info), patch_code(patch_code) {} + MMTkC1FieldBarrierStub(LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info = NULL, LIR_PatchCode patch_code = lir_patch_none): src(src), slot(slot), new_val(new_val), info(info), patch_code(patch_code) { + FrameMap* f = Compilation::current()->frame_map(); + } virtual void emit_code(LIR_Assembler* ce) override; From 6d99555fbf06f0c0c36b6136a6c140758a43422e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 10 Mar 2026 20:34:50 +1100 Subject: [PATCH 442/494] Fix update_reserved_argument_area_size --- openjdk/share/barriers/mmtkFieldBarrier.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/openjdk/share/barriers/mmtkFieldBarrier.hpp b/openjdk/share/barriers/mmtkFieldBarrier.hpp index 63435b50..4ffb0b79 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.hpp @@ -36,6 +36,7 @@ struct MMTkC1FieldBarrierStub: CodeStub { MMTkC1FieldBarrierStub(LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info = NULL, LIR_PatchCode patch_code = lir_patch_none): src(src), slot(slot), new_val(new_val), info(info), patch_code(patch_code) { FrameMap* f = Compilation::current()->frame_map(); + f->update_reserved_argument_area_size(3 * BytesPerWord); } virtual void emit_code(LIR_Assembler* ce) override; From 16bd5f9299e98b42309d8e96d8df273099c58518 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 11 Mar 2026 18:32:27 +1100 Subject: [PATCH 443/494] Fix array_copy_requires_gc_barriers --- openjdk/share/barriers/mmtkFieldBarrier.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openjdk/share/barriers/mmtkFieldBarrier.hpp b/openjdk/share/barriers/mmtkFieldBarrier.hpp index 4ffb0b79..c9170efd 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.hpp @@ -70,8 +70,8 @@ class MMTkFieldBarrierSetC2: public MMTkBarrierSetC2 { protected: virtual void object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const override; public: - virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, bool is_clone_instance, ArrayCopyPhase phase) const { - return false; + virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, bool is_clone_instance, ArrayCopyPhase phase) const override { + return is_reference_type(type) && !tightly_coupled_alloc && !is_clone; } virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const override; }; From 31a6fb8fc331bf1d436260599ec95f8165b3eb3e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 20 Mar 2026 00:11:24 +0000 Subject: [PATCH 444/494] optimize --- mmtk/src/api.rs | 15 +++++++++++++++ openjdk/share/barriers/mmtkFieldBarrier.cpp | 20 +++++++++++++------- openjdk/share/mmtk.h | 1 + openjdk/share/mmtkBarrierSet.cpp | 5 ++++- openjdk/share/mmtkBarrierSet.hpp | 3 +++ 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index e6a728bc..2b618e96 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -510,6 +510,21 @@ pub extern "C" fn mmtk_array_copy_pre( }) } +#[no_mangle] +pub extern "C" fn mmtk_array_copy_pre2( + src: Address, + dst: Address, + count: usize, + mutator: *mut libc::c_void, +) { + let bytes = count << log_bytes_in_slot(); + with_mutator!(|mutator| { + mutator + .barrier() + .memory_region_copy_pre((src..src + bytes).into(), (dst..dst + bytes).into()); + }) +} + /// Array-copy post-barrier #[no_mangle] pub extern "C" fn mmtk_array_copy_post( diff --git a/openjdk/share/barriers/mmtkFieldBarrier.cpp b/openjdk/share/barriers/mmtkFieldBarrier.cpp index 6a681436..1a70b72b 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.cpp @@ -136,17 +136,23 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma void MMTkFieldBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { if (FIELD_BARRIER_NO_ARRAYCOPY) return; + bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; + if (dest_uninitialized) return; if (type == T_OBJECT || type == T_ARRAY) { Label done; - // // Bailout if count is zero + Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); + // Bailout if count is zero __ cmpptr(count, 0); __ jcc(Assembler::equal, done); - __ pusha(); - __ movptr(c_rarg0, src); - __ movptr(c_rarg1, dst); - __ movptr(c_rarg2, count); - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_pre_call), 3); - __ popa(); + __ push_call_clobbered_registers(false /* save_fpu */); + assert_different_registers(c_rarg0, dst, count); + assert_different_registers(c_rarg1, count); + if (c_rarg0 != src) __ movptr(c_rarg0, src); + if (c_rarg1 != dst) __ movptr(c_rarg1, dst); + if (c_rarg2 != count) __ movptr(c_rarg2, count); + __ lea(c_rarg3, mutator); + __ call_VM_leaf_base(FN_ADDR(mmtk_array_copy_pre2), 4); + __ pop_call_clobbered_registers(false /* save_fpu */); __ bind(done); } } diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index b6446318..e509ad75 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -89,6 +89,7 @@ extern void mmtk_object_reference_write_slow(MMTk_Mutator mutator, void* src, vo /// Full array-copy pre-barrier extern void mmtk_array_copy_pre(MMTk_Mutator mutator, void* src, void* dst, size_t count); +extern void mmtk_array_copy_pre2(void* src, void* dst, size_t count, void* mutator); /// Full array-copy post-barrier extern void mmtk_array_copy_post(MMTk_Mutator mutator, void* src, void* dst, size_t count); diff --git a/openjdk/share/mmtkBarrierSet.cpp b/openjdk/share/mmtkBarrierSet.cpp index 1203817f..f67d88a1 100644 --- a/openjdk/share/mmtkBarrierSet.cpp +++ b/openjdk/share/mmtkBarrierSet.cpp @@ -164,10 +164,13 @@ void MMTkBarrierSetRuntime::object_reference_write_slow_call(void* src, void* sl } void MMTkBarrierSetRuntime::object_reference_array_copy_pre_call(void* src, void* dst, size_t count) { - if (FIELD_BARRIER_NO_ARRAYCOPY_SLOW) return; ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); } +void MMTkBarrierSetRuntime::object_reference_array_copy_pre_call2(void* src, void* dst, size_t count, void* mutator) { + ::mmtk_array_copy_pre((MMTk_Mutator) mutator, src, dst, count); +} + void MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, void* dst, size_t count) { if (FIELD_BARRIER_NO_ARRAYCOPY_SLOW) return; ::mmtk_array_copy_post((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); diff --git a/openjdk/share/mmtkBarrierSet.hpp b/openjdk/share/mmtkBarrierSet.hpp index c5dc6373..b2f08df7 100644 --- a/openjdk/share/mmtkBarrierSet.hpp +++ b/openjdk/share/mmtkBarrierSet.hpp @@ -79,6 +79,7 @@ class MMTkBarrierSetRuntime: public CHeapObj { static void object_reference_write_slow_call(void* src, void* slot, void* target); /// Generic arraycopy post-barrier. Called by fast-paths. static void object_reference_array_copy_pre_call(void* src, void* dst, size_t count); + static void object_reference_array_copy_pre_call2(void* src, void* dst, size_t count, void* mutator); /// Generic arraycopy pre-barrier. Called by fast-paths. static void object_reference_array_copy_post_call(void* src, void* dst, size_t count); static void object_reference_clone_pre_call(void* obj); @@ -88,6 +89,8 @@ class MMTkBarrierSetRuntime: public CHeapObj { || call == CAST_FROM_FN_PTR(address, object_reference_write_post_call) || call == CAST_FROM_FN_PTR(address, object_reference_write_slow_call) || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call) + || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call2) + || call == CAST_FROM_FN_PTR(address, mmtk_array_copy_pre2) || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_post_call) || call == CAST_FROM_FN_PTR(address, load_reference_call) || call == CAST_FROM_FN_PTR(address, object_reference_clone_pre_call); From 376df85617097209579541486e77eed3a4613af1 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 20 Mar 2026 00:23:54 +0000 Subject: [PATCH 445/494] More optimizations --- openjdk/share/barriers/mmtkFieldBarrier.cpp | 8 +++++--- openjdk/share/mmtkBarrierSet.hpp | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/openjdk/share/barriers/mmtkFieldBarrier.cpp b/openjdk/share/barriers/mmtkFieldBarrier.cpp index 1a70b72b..55eee6cd 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.cpp @@ -351,8 +351,10 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* slot, Node* val) { + Node* no_base = __ top(); + Node* tls = __ thread(); + Node* mutator = __ AddP(no_base, tls, __ ConX(in_bytes(JavaThread::third_party_heap_mutator_offset()))); if (mmtk_enable_barrier_fastpath) { - Node* no_base = __ top(); float unlikely = PROB_UNLIKELY(0.999); Node* zero = __ ConI(0); @@ -365,9 +367,9 @@ static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* sl shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); __ if_then(result, BoolTest::ne, zero, unlikely); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + const TypeFunc* tf = __ func_type(TypeRawPtr::NOTNULL, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); if (!FIELD_BARRIER_NO_C2_SLOW_CALL) - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), "mmtk_barrier_call", src, slot, val); + Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_object_reference_write_slow), "mmtk_barrier_call", mutator, src, slot, val); } __ end_if(); if (!FIELD_BARRIER_NO_EAGER_BRANCH) __ end_if(); diff --git a/openjdk/share/mmtkBarrierSet.hpp b/openjdk/share/mmtkBarrierSet.hpp index b2f08df7..8b434f86 100644 --- a/openjdk/share/mmtkBarrierSet.hpp +++ b/openjdk/share/mmtkBarrierSet.hpp @@ -91,6 +91,7 @@ class MMTkBarrierSetRuntime: public CHeapObj { || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call) || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call2) || call == CAST_FROM_FN_PTR(address, mmtk_array_copy_pre2) + || call == CAST_FROM_FN_PTR(address, mmtk_object_reference_write_slow) || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_post_call) || call == CAST_FROM_FN_PTR(address, load_reference_call) || call == CAST_FROM_FN_PTR(address, object_reference_clone_pre_call); From 53fb390f4926596b27abda81f829fe99125ec027 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 20 Mar 2026 02:31:59 +0000 Subject: [PATCH 446/494] cleanup --- mmtk/src/api.rs | 15 --------------- openjdk/share/barriers/mmtkFieldBarrier.cpp | 2 +- openjdk/share/barriers/mmtkFieldBarrier.hpp | 2 +- openjdk/share/barriers/mmtkSATBBarrier.hpp | 2 +- openjdk/share/mmtk.h | 3 +-- openjdk/share/mmtkBarrierSet.cpp | 7 ++----- openjdk/share/mmtkBarrierSet.hpp | 4 +--- 7 files changed, 7 insertions(+), 28 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 2b618e96..7f34625d 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -497,21 +497,6 @@ fn log_bytes_in_slot() -> usize { /// Array-copy pre-barrier #[no_mangle] pub extern "C" fn mmtk_array_copy_pre( - mutator: *mut libc::c_void, - src: Address, - dst: Address, - count: usize, -) { - let bytes = count << log_bytes_in_slot(); - with_mutator!(|mutator| { - mutator - .barrier() - .memory_region_copy_pre((src..src + bytes).into(), (dst..dst + bytes).into()); - }) -} - -#[no_mangle] -pub extern "C" fn mmtk_array_copy_pre2( src: Address, dst: Address, count: usize, diff --git a/openjdk/share/barriers/mmtkFieldBarrier.cpp b/openjdk/share/barriers/mmtkFieldBarrier.cpp index 55eee6cd..959ba2af 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.cpp @@ -151,7 +151,7 @@ void MMTkFieldBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Deco if (c_rarg1 != dst) __ movptr(c_rarg1, dst); if (c_rarg2 != count) __ movptr(c_rarg2, count); __ lea(c_rarg3, mutator); - __ call_VM_leaf_base(FN_ADDR(mmtk_array_copy_pre2), 4); + __ call_VM_leaf_base(FN_ADDR(mmtk_array_copy_pre), 4); __ pop_call_clobbered_registers(false /* save_fpu */); __ bind(done); } diff --git a/openjdk/share/barriers/mmtkFieldBarrier.hpp b/openjdk/share/barriers/mmtkFieldBarrier.hpp index c9170efd..731a09f3 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.hpp @@ -22,7 +22,7 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const override { if (FIELD_BARRIER_NO_ARRAYCOPY || count == 0) return; - ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) dst, count); + ::mmtk_array_copy_pre((void*) src, (void*) dst, count, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } virtual void object_probable_write(oop new_obj) const override; virtual void load_reference(DecoratorSet decorators, oop value) const override; diff --git a/openjdk/share/barriers/mmtkSATBBarrier.hpp b/openjdk/share/barriers/mmtkSATBBarrier.hpp index 7869caff..98caad4e 100644 --- a/openjdk/share/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/share/barriers/mmtkSATBBarrier.hpp @@ -30,7 +30,7 @@ class MMTkSATBBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const override { if (count == 0) return; - ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) dst, count); + ::mmtk_array_copy_pre((void*) src, (void*) dst, count, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } virtual void object_probable_write(oop new_obj) const override; virtual void load_reference(DecoratorSet decorators, oop value) const override; diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index e509ad75..ec1cf499 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -88,8 +88,7 @@ extern void mmtk_object_reference_write_post(MMTk_Mutator mutator, void* src, vo extern void mmtk_object_reference_write_slow(MMTk_Mutator mutator, void* src, void* slot, void* target); /// Full array-copy pre-barrier -extern void mmtk_array_copy_pre(MMTk_Mutator mutator, void* src, void* dst, size_t count); -extern void mmtk_array_copy_pre2(void* src, void* dst, size_t count, void* mutator); +extern void mmtk_array_copy_pre(void* src, void* dst, size_t count, MMTk_Mutator mutator); /// Full array-copy post-barrier extern void mmtk_array_copy_post(MMTk_Mutator mutator, void* src, void* dst, size_t count); diff --git a/openjdk/share/mmtkBarrierSet.cpp b/openjdk/share/mmtkBarrierSet.cpp index f67d88a1..ae7c617e 100644 --- a/openjdk/share/mmtkBarrierSet.cpp +++ b/openjdk/share/mmtkBarrierSet.cpp @@ -164,11 +164,8 @@ void MMTkBarrierSetRuntime::object_reference_write_slow_call(void* src, void* sl } void MMTkBarrierSetRuntime::object_reference_array_copy_pre_call(void* src, void* dst, size_t count) { - ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); -} - -void MMTkBarrierSetRuntime::object_reference_array_copy_pre_call2(void* src, void* dst, size_t count, void* mutator) { - ::mmtk_array_copy_pre((MMTk_Mutator) mutator, src, dst, count); + if (FIELD_BARRIER_NO_ARRAYCOPY_SLOW) return; + ::mmtk_array_copy_pre(src, dst, count, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } void MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, void* dst, size_t count) { diff --git a/openjdk/share/mmtkBarrierSet.hpp b/openjdk/share/mmtkBarrierSet.hpp index 8b434f86..4e25158a 100644 --- a/openjdk/share/mmtkBarrierSet.hpp +++ b/openjdk/share/mmtkBarrierSet.hpp @@ -79,7 +79,6 @@ class MMTkBarrierSetRuntime: public CHeapObj { static void object_reference_write_slow_call(void* src, void* slot, void* target); /// Generic arraycopy post-barrier. Called by fast-paths. static void object_reference_array_copy_pre_call(void* src, void* dst, size_t count); - static void object_reference_array_copy_pre_call2(void* src, void* dst, size_t count, void* mutator); /// Generic arraycopy pre-barrier. Called by fast-paths. static void object_reference_array_copy_post_call(void* src, void* dst, size_t count); static void object_reference_clone_pre_call(void* obj); @@ -89,8 +88,7 @@ class MMTkBarrierSetRuntime: public CHeapObj { || call == CAST_FROM_FN_PTR(address, object_reference_write_post_call) || call == CAST_FROM_FN_PTR(address, object_reference_write_slow_call) || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call) - || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call2) - || call == CAST_FROM_FN_PTR(address, mmtk_array_copy_pre2) + || call == CAST_FROM_FN_PTR(address, mmtk_array_copy_pre) || call == CAST_FROM_FN_PTR(address, mmtk_object_reference_write_slow) || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_post_call) || call == CAST_FROM_FN_PTR(address, load_reference_call) From f676c1916a3c084b8a7a985c8b265f3535f5c4ae Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 20 Mar 2026 02:46:12 +0000 Subject: [PATCH 447/494] More barrier optimizations --- mmtk/src/api.rs | 4 +-- openjdk/share/barriers/mmtkFieldBarrier.cpp | 30 ++++++++++++--------- openjdk/share/barriers/mmtkSATBBarrier.cpp | 2 +- openjdk/share/mmtk.h | 4 +-- openjdk/share/mmtkBarrierSet.cpp | 4 +-- openjdk/share/mmtkBarrierSet.hpp | 1 + 6 files changed, 26 insertions(+), 19 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 7f34625d..3242396a 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -429,7 +429,7 @@ pub extern "C" fn executable() -> bool { } #[no_mangle] -pub extern "C" fn mmtk_load_reference(mutator: *mut libc::c_void, o: ObjectReference) { +pub extern "C" fn mmtk_load_reference(o: ObjectReference, mutator: *mut libc::c_void) { with_mutator!(|mutator| mutator.barrier().load_weak_reference(o)) } @@ -474,10 +474,10 @@ pub extern "C" fn mmtk_object_reference_write_post( /// Barrier slow-path call #[no_mangle] pub extern "C" fn mmtk_object_reference_write_slow( - mutator: *mut libc::c_void, src: NullableObjectReference, slot: Address, target: NullableObjectReference, + mutator: *mut libc::c_void, ) { with_mutator!(|mutator| { mutator diff --git a/openjdk/share/barriers/mmtkFieldBarrier.cpp b/openjdk/share/barriers/mmtkFieldBarrier.cpp index 959ba2af..6695f667 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.cpp @@ -17,7 +17,7 @@ static inline intptr_t side_metadata_base_address() { void MMTkFieldBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const { #if SOFT_REFERENCE_LOAD_BARRIER if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL && mmtk_get_rc((void*) value) != 0) - ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); + ::mmtk_load_reference((void*) value, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); #endif }; @@ -36,7 +36,7 @@ void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, if (byte_val == 0) return; intptr_t shift = (addr >> (UseCompressedOops ? 2 : 3)) & 0b111; if (((byte_val >> shift) & 1) == kUnloggedValue) { - ::mmtk_object_reference_write_slow((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) slot, (void*) target); + ::mmtk_object_reference_write_slow((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } } else { ::mmtk_object_reference_write_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) slot, (void*) target); @@ -70,10 +70,12 @@ void MMTkFieldBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet de __ testptr(dst, dst); __ jcc(Assembler::zero, done); // Do slow-call - __ pusha(); + __ push_call_clobbered_registers(false /* save_fpu */); __ mov(c_rarg0, dst); - __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); - __ popa(); + Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); + __ lea(c_rarg1, mutator); + __ MacroAssembler::call_VM_leaf_base(FN_ADDR(mmtk_load_reference), 2); + __ pop_call_clobbered_registers(false /* save_fpu */); __ bind(done); } #endif @@ -110,15 +112,17 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma __ jcc(Assembler::notEqual, done); // TODO: Spill fewer registers - __ pusha(); + __ push_call_clobbered_registers(false /* save_fpu */); __ movptr(c_rarg0, dst.base()); __ lea(c_rarg1, dst); if (val == noreg) __ movptr(c_rarg2, NULL_WORD); else __ movptr(c_rarg2, val); - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); - __ popa(); + Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); + __ lea(c_rarg3, mutator); + __ call_VM_leaf_base(FN_ADDR(mmtk_object_reference_write_slow), 4); + __ pop_call_clobbered_registers(false /* save_fpu */); __ bind(done); } else { @@ -367,9 +371,9 @@ static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* sl shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); __ if_then(result, BoolTest::ne, zero, unlikely); { - const TypeFunc* tf = __ func_type(TypeRawPtr::NOTNULL, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeRawPtr::NOTNULL); if (!FIELD_BARRIER_NO_C2_SLOW_CALL) - Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_object_reference_write_slow), "mmtk_barrier_call", mutator, src, slot, val); + Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_object_reference_write_slow), "mmtk_barrier_call", src, slot, val, mutator); } __ end_if(); if (!FIELD_BARRIER_NO_EAGER_BRANCH) __ end_if(); @@ -396,6 +400,8 @@ void MMTkFieldBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool emit_barrier) { MMTkIdealKit ideal(kit, true); Node* no_base = __ top(); + Node* tls = __ thread(); + Node* mutator = __ AddP(no_base, tls, __ ConX(in_bytes(JavaThread::third_party_heap_mutator_offset()))); float unlikely = PROB_UNLIKELY(0.999); Node* zero = __ ConI(0); Node* cm_flag = __ load(__ ctrl(), __ ConP(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); @@ -403,8 +409,8 @@ static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool em __ if_then(cm_flag, BoolTest::ne, zero, unlikely); { // No slow-call if dst is NULL __ if_then(val, BoolTest::ne, kit->null()); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", val); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeRawPtr::NOTNULL); + Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_load_reference), "mmtk_barrier_call", val, mutator); } __ end_if(); } __ end_if(); kit->sync_kit(ideal); diff --git a/openjdk/share/barriers/mmtkSATBBarrier.cpp b/openjdk/share/barriers/mmtkSATBBarrier.cpp index 7f08df63..da139e2b 100644 --- a/openjdk/share/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/share/barriers/mmtkSATBBarrier.cpp @@ -7,7 +7,7 @@ void MMTkSATBBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const { if (mmtk_enable_reference_load_barrier) { if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) - ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); + ::mmtk_load_reference((void*) value, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } }; diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index ec1cf499..a992c716 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -76,7 +76,7 @@ extern void post_alloc(MMTk_Mutator mutator, void* refer, size_t bytes, int allocator); /// java.lang.Reference load barrier -extern void mmtk_load_reference(MMTk_Mutator mutator, void* obj); +extern void mmtk_load_reference(void* obj, MMTk_Mutator mutator); /// Full pre-barrier extern void mmtk_object_reference_write_pre(MMTk_Mutator mutator, void* src, void* slot, void* target); @@ -85,7 +85,7 @@ extern void mmtk_object_reference_write_pre(MMTk_Mutator mutator, void* src, voi extern void mmtk_object_reference_write_post(MMTk_Mutator mutator, void* src, void* slot, void* target); /// Generic slow-path -extern void mmtk_object_reference_write_slow(MMTk_Mutator mutator, void* src, void* slot, void* target); +extern void mmtk_object_reference_write_slow(void* src, void* slot, void* target, MMTk_Mutator mutator); /// Full array-copy pre-barrier extern void mmtk_array_copy_pre(void* src, void* dst, size_t count, MMTk_Mutator mutator); diff --git a/openjdk/share/mmtkBarrierSet.cpp b/openjdk/share/mmtkBarrierSet.cpp index ae7c617e..e3ba0c90 100644 --- a/openjdk/share/mmtkBarrierSet.cpp +++ b/openjdk/share/mmtkBarrierSet.cpp @@ -160,7 +160,7 @@ void MMTkBarrierSetRuntime::object_reference_write_post_call(void* src, void* sl void MMTkBarrierSetRuntime::object_reference_write_slow_call(void* src, void* slot, void* target) { if (FIELD_BARRIER_NO_C2_RUST_CALL) return; - ::mmtk_object_reference_write_slow((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, target); + ::mmtk_object_reference_write_slow(src, slot, target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } void MMTkBarrierSetRuntime::object_reference_array_copy_pre_call(void* src, void* dst, size_t count) { @@ -174,7 +174,7 @@ void MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, voi } void MMTkBarrierSetRuntime::load_reference_call(void* ref) { - ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); + ::mmtk_load_reference(ref, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } void MMTkBarrierSetRuntime::object_reference_clone_pre_call(void* ref) { diff --git a/openjdk/share/mmtkBarrierSet.hpp b/openjdk/share/mmtkBarrierSet.hpp index 4e25158a..37b71311 100644 --- a/openjdk/share/mmtkBarrierSet.hpp +++ b/openjdk/share/mmtkBarrierSet.hpp @@ -90,6 +90,7 @@ class MMTkBarrierSetRuntime: public CHeapObj { || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call) || call == CAST_FROM_FN_PTR(address, mmtk_array_copy_pre) || call == CAST_FROM_FN_PTR(address, mmtk_object_reference_write_slow) + || call == CAST_FROM_FN_PTR(address, mmtk_load_reference) || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_post_call) || call == CAST_FROM_FN_PTR(address, load_reference_call) || call == CAST_FROM_FN_PTR(address, object_reference_clone_pre_call); From 3e25f1ee492a05b39af7360df05ac959b65064d3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 22 Mar 2026 12:57:53 +0000 Subject: [PATCH 448/494] Fix h2o --- mmtk/src/reference_glue.rs | 212 +++++++++++++++++++++++++++++++--- openjdk/share/mmtkUpcalls.cpp | 7 -- 2 files changed, 198 insertions(+), 21 deletions(-) diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 879a81a9..a2871565 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -12,7 +12,7 @@ use mmtk::util::opaque_pointer::VMWorkerThread; use mmtk::util::ObjectReference; use mmtk::vm::slot::Slot; use mmtk::vm::ReferenceGlue; -use mmtk::MMTK; +use mmtk::{gc_log, MMTK}; fn set_referent(reff: ObjectReference, referent: Option) { let oop = Oop::from(reff); @@ -180,23 +180,53 @@ impl DiscoveredLists { clear: bool, ) { let mut packets = vec![]; - for i in 0..lists.len() { - let head = unsafe { *lists[i].head.get() }; - if clear { - unsafe { *lists[i].head.get() = ObjectReference::NULL }; + let mut packets_step2 = vec![]; + if rt == ReferenceType::Weak || rt == ReferenceType::Soft { + for i in 0..lists.len() { + let head = unsafe { *lists[i].head.get() }; + if clear { + unsafe { *lists[i].head.get() = ObjectReference::NULL }; + } + if head.is_none() { + continue; + } + let w = ProcessDiscoveredListForSoftAndWeakRefs::<_, COMPRESSED> { + head: head.unwrap(), + rt, + _p: PhantomData::, + }; + packets.push(Box::new(w) as Box>); + let w2 = ProcessDeadRefs::<_, COMPRESSED> { + list_index: i, + head: head.unwrap(), + rt, + _p: PhantomData::, + }; + packets_step2.push(Box::new(w2) as Box>); } - if head.is_none() { - continue; + } else { + for i in 0..lists.len() { + let head = unsafe { *lists[i].head.get() }; + if clear { + unsafe { *lists[i].head.get() = ObjectReference::NULL }; + } + if head.is_none() { + continue; + } + let w = ProcessDiscoveredList::<_, COMPRESSED> { + list_index: i, + head: head.unwrap(), + rt, + _p: PhantomData::, + }; + packets.push(Box::new(w) as Box>); } - let w = ProcessDiscoveredList::<_, COMPRESSED> { - list_index: i, - head: head.unwrap(), - rt, - _p: PhantomData::, - }; - packets.push(Box::new(w) as Box>); } worker.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets); + if rt == ReferenceType::Weak || rt == ReferenceType::Soft { + worker.scheduler().work_buckets[WorkBucketStage::PhantomRefClosure] + .bulk_add(packets_step2); + } } pub fn reconsider_soft_refs(&self, _worker: &mut GCWorker) {} @@ -321,6 +351,160 @@ impl GCWork } } +pub struct ProcessDiscoveredListForSoftAndWeakRefs { + head: ObjectReference, + rt: ReferenceType, + _p: PhantomData, +} + +impl GCWork + for ProcessDiscoveredListForSoftAndWeakRefs +{ + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { + let mut trace = E::new(vec![], false, mmtk, WorkBucketStage::Unconstrained); + trace.set_worker(worker); + let retain = self.rt == ReferenceType::Soft && !mmtk.is_emergency_collection(); + + let mut reference_opt = Some(self.head); + + while let Some(reference) = reference_opt { + let next_ref = get_next_reference::(reference); + let next_ref = next_ref + .map(|x| x.get_forwarded_object()) + .flatten() + .or_else(|| next_ref); + if let Some(o) = next_ref { + debug_assert!(o.get_forwarded_object().is_none()); + } + // Reaches the end of the list? + let end_of_list = next_ref == Some(reference) || next_ref.is_none(); + reference_opt = if end_of_list { None } else { next_ref }; + + let reference = trace.trace_object(reference); + let referent = get_referent::(reference); + if referent.is_none() { + continue; + } + let referent = referent.unwrap(); + gc_log!( + "REF Process: {:?} retain={:?} referent.is_reachable={:?} referent={:?}", + reference.range::(), + retain, + referent.is_reachable(), + referent.range::() + ); + if referent.is_reachable() || retain { + // unreachable!(); + // Keep this referent + let forwarded = trace.trace_object(referent); + debug_assert!(forwarded.get_forwarded_object().is_none()); + debug_assert!(reference.get_forwarded_object().is_none()); + set_referent::(reference, Some(forwarded)); + // Remove from the discovered list + gc_log!("REF Remove2: {:?}->{:?}", referent, forwarded); + } else { + gc_log!( + "REF Enqueue: {:?}->{:?} is_reachable={:?} setnull={:?} ", + reference, + referent, + referent.is_reachable(), + self.rt != ReferenceType::Final, + // should_scan + ); + } + // reference_opt = next_ref; + } + trace.process_slots(); + trace.flush(); + } +} + +pub struct ProcessDeadRefs { + list_index: usize, + head: ObjectReference, + rt: ReferenceType, + _p: PhantomData, +} + +impl GCWork for ProcessDeadRefs { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { + let mut trace = E::new(vec![], false, mmtk, WorkBucketStage::Unconstrained); + trace.set_worker(worker); + let retain = self.rt == ReferenceType::Soft && !mmtk.is_emergency_collection(); + let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { + debug_assert!( + get_next_reference::(reference).is_none(), + "next must be null. ref={:?} {:?} bin={}", + reference, + self.rt, + self.list_index + ); + let referent = get_referent::(reference); + if referent.is_none() { + // Remove from the discovered list + gc_log!("REF2 Remove1: {:?}", referent); + return DiscoveredListIterationResult::Remove; + } + let referent = referent.unwrap(); + gc_log!( + "REF2 Process: {:?} retain={:?} referent.is_reachable={:?} referent={:?}", + reference.range::(), + retain, + referent.is_reachable(), + referent.range::() + ); + if referent.is_reachable() || retain { + // Keep this referent + let forwarded = referent.get_forwarded_object().unwrap_or(referent); + debug_assert!(forwarded.get_forwarded_object().is_none()); + debug_assert!(reference.get_forwarded_object().is_none()); + if forwarded != referent { + set_referent::(reference, Some(forwarded)); + } + // Remove from the discovered list + gc_log!("REF2 Remove2: {:?}->{:?}", referent, forwarded); + return DiscoveredListIterationResult::Remove; + } else { + gc_log!( + "REF2 Enqueue: {:?}->{:?} is_reachable={:?} setnull={:?} ", + reference, + referent, + referent.is_reachable(), + self.rt != ReferenceType::Final, + // should_scan + ); + if self.rt != ReferenceType::Final { + // Clear this referent + set_referent::(reference, ObjectReference::NULL); + // set_referent(reference, ObjectReference::NULL); + } + // Keep the reference + return DiscoveredListIterationResult::Enqueue(reference); + } + }); + // Flush the list to the Universe::pending_list + if let Some((head, tail)) = new_list { + // debug_assert!(!head.is_null() && !tail.is_null()); + // debug_assert_eq!(ObjectReference::NULL, get_next_reference(tail)); + if self.rt == ReferenceType::Final { + let slot = DISCOVERED_LISTS.r#final[self.list_index].head.get(); + unsafe { *slot = Some(head) }; + } else { + let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; + set_next_reference::(tail, Some(old_head)); + } + } else { + if self.rt == ReferenceType::Final { + let slot = DISCOVERED_LISTS.r#final[self.list_index].head.get(); + unsafe { *slot = ObjectReference::NULL }; + } + } + // assert!(trace.is_empty()); + trace.process_slots(); + trace.flush(); + } +} + pub struct ResurrectFinalizables { list_index: usize, head: ObjectReference, diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index 7c87120d..00ed62b5 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -271,13 +271,6 @@ static void mmtk_resume_mutators(void *tls) { } log_debug(gc)("Notifying mutators blocking on Heap_lock for reference pending list..."); - // Note: That's the ReferenceHandler thread. - { - MutexLocker x(Heap_lock, Mutex::_no_safepoint_check_flag); - if (Universe::has_reference_pending_list()) { - Heap_lock->notify_all(); - } - } } static const int GC_THREAD_KIND_WORKER = 1; From 6fee9fccf99e6692a0c0987f2750f819399f23c2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 22 Mar 2026 21:17:36 +0000 Subject: [PATCH 449/494] Debug --- openjdk/share/mmtkUpcalls.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index 00ed62b5..7c87120d 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -271,6 +271,13 @@ static void mmtk_resume_mutators(void *tls) { } log_debug(gc)("Notifying mutators blocking on Heap_lock for reference pending list..."); + // Note: That's the ReferenceHandler thread. + { + MutexLocker x(Heap_lock, Mutex::_no_safepoint_check_flag); + if (Universe::has_reference_pending_list()) { + Heap_lock->notify_all(); + } + } } static const int GC_THREAD_KIND_WORKER = 1; From c6732635c39b77b2d9b091908c81f83d59b7ea78 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 25 Mar 2026 03:25:13 +0000 Subject: [PATCH 450/494] cleanup --- mmtk/src/reference_glue.rs | 39 +------------------------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index a2871565..6d4cd191 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -12,7 +12,7 @@ use mmtk::util::opaque_pointer::VMWorkerThread; use mmtk::util::ObjectReference; use mmtk::vm::slot::Slot; use mmtk::vm::ReferenceGlue; -use mmtk::{gc_log, MMTK}; +use mmtk::MMTK; fn set_referent(reff: ObjectReference, referent: Option) { let oop = Oop::from(reff); @@ -386,33 +386,13 @@ impl GCWork continue; } let referent = referent.unwrap(); - gc_log!( - "REF Process: {:?} retain={:?} referent.is_reachable={:?} referent={:?}", - reference.range::(), - retain, - referent.is_reachable(), - referent.range::() - ); if referent.is_reachable() || retain { - // unreachable!(); // Keep this referent let forwarded = trace.trace_object(referent); debug_assert!(forwarded.get_forwarded_object().is_none()); debug_assert!(reference.get_forwarded_object().is_none()); set_referent::(reference, Some(forwarded)); - // Remove from the discovered list - gc_log!("REF Remove2: {:?}->{:?}", referent, forwarded); - } else { - gc_log!( - "REF Enqueue: {:?}->{:?} is_reachable={:?} setnull={:?} ", - reference, - referent, - referent.is_reachable(), - self.rt != ReferenceType::Final, - // should_scan - ); } - // reference_opt = next_ref; } trace.process_slots(); trace.flush(); @@ -442,17 +422,9 @@ impl GCWork for ProcessDeadR let referent = get_referent::(reference); if referent.is_none() { // Remove from the discovered list - gc_log!("REF2 Remove1: {:?}", referent); return DiscoveredListIterationResult::Remove; } let referent = referent.unwrap(); - gc_log!( - "REF2 Process: {:?} retain={:?} referent.is_reachable={:?} referent={:?}", - reference.range::(), - retain, - referent.is_reachable(), - referent.range::() - ); if referent.is_reachable() || retain { // Keep this referent let forwarded = referent.get_forwarded_object().unwrap_or(referent); @@ -462,17 +434,8 @@ impl GCWork for ProcessDeadR set_referent::(reference, Some(forwarded)); } // Remove from the discovered list - gc_log!("REF2 Remove2: {:?}->{:?}", referent, forwarded); return DiscoveredListIterationResult::Remove; } else { - gc_log!( - "REF2 Enqueue: {:?}->{:?} is_reachable={:?} setnull={:?} ", - reference, - referent, - referent.is_reachable(), - self.rt != ReferenceType::Final, - // should_scan - ); if self.rt != ReferenceType::Final { // Clear this referent set_referent::(reference, ObjectReference::NULL); From 4cf4303034ec3c1816de0c0cc037d61752f9db10 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 29 Mar 2026 22:15:31 +1100 Subject: [PATCH 451/494] optimize --- mmtk/src/reference_glue.rs | 14 ++++++++--- openjdk/share/mmtkUpcalls.cpp | 45 +++++++++++++++++++++-------------- 2 files changed, 38 insertions(+), 21 deletions(-) diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 6d4cd191..1e14c0e8 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -295,7 +295,10 @@ impl GCWork fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { let mut trace = E::new(vec![], false, mmtk, WorkBucketStage::Unconstrained); trace.set_worker(worker); - let retain = self.rt == ReferenceType::Soft && !mmtk.is_emergency_collection(); + let retain = self.rt == ReferenceType::Soft + && !mmtk.is_emergency_collection() + && !mmtk.is_user_triggered_collection(); + let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { debug_assert!( get_next_reference::(reference).is_none(), @@ -363,7 +366,9 @@ impl GCWork fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { let mut trace = E::new(vec![], false, mmtk, WorkBucketStage::Unconstrained); trace.set_worker(worker); - let retain = self.rt == ReferenceType::Soft && !mmtk.is_emergency_collection(); + let retain = self.rt == ReferenceType::Soft + && !mmtk.is_emergency_collection() + && !mmtk.is_user_triggered_collection(); let mut reference_opt = Some(self.head); @@ -410,7 +415,10 @@ impl GCWork for ProcessDeadR fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { let mut trace = E::new(vec![], false, mmtk, WorkBucketStage::Unconstrained); trace.set_worker(worker); - let retain = self.rt == ReferenceType::Soft && !mmtk.is_emergency_collection(); + let retain = self.rt == ReferenceType::Soft + && !mmtk.is_emergency_collection() + && !mmtk.is_user_triggered_collection(); + let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { debug_assert!( get_next_reference::(reference).is_none(), diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index 7c87120d..22a7fb42 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -115,7 +115,7 @@ class MMTkLXRFastIsAliveClosure : public BoolObjectClosure { return MMTkForwardClosure::is_forwarded(MMTkForwardClosure::read_forwarding_word(o)); } - inline virtual bool do_object_b(oop o) { + inline virtual bool do_object_b(oop o) override { const uintptr_t v = uintptr_t((void*) o); // if (v >= 0x220000000000ULL || v < 0x20000000000ULL) return false; return o != NULL && (rc_live(o) || is_forwarded(o)); @@ -123,15 +123,11 @@ class MMTkLXRFastIsAliveClosure : public BoolObjectClosure { }; class MMTkLXRFastUpdateClosure : public OopClosure { - uintptr_t heap_start = HEAP_START; - uintptr_t heap_end = HEAP_END; - public: - inline virtual void do_oop(oop* slot) { + inline virtual void do_oop(oop* slot) override { const auto o = *slot; const uintptr_t v = uintptr_t((void*) o); - if (v >= heap_end || v < heap_start) { - *slot = NULL; + if (o == NULL) { return; } const auto status = MMTkForwardClosure::read_forwarding_word(o); @@ -141,15 +137,11 @@ class MMTkLXRFastUpdateClosure : public OopClosure { *slot = NULL; } } - inline virtual void do_oop(narrowOop* slot) { + inline virtual void do_oop(narrowOop* slot) override { narrowOop heap_oop = RawAccess<>::oop_load(slot); if (CompressedOops::is_null(heap_oop)) return; oop o = CompressedOops::decode_not_null(heap_oop); const uintptr_t v = uintptr_t((void*) o); - if (v >= heap_end || v < heap_start) { - RawAccess<>::oop_store(slot, CompressedOops::encode(oop(NULL))); - return; - } const auto status = MMTkForwardClosure::read_forwarding_word(o); if (MMTkForwardClosure::is_forwarded(status)) { RawAccess<>::oop_store(slot, CompressedOops::encode_not_null(MMTkForwardClosure::extract_forwarding_pointer(status))); @@ -159,6 +151,25 @@ class MMTkLXRFastUpdateClosure : public OopClosure { } }; +class MMTkUpdateClosure : public OopClosure { + public: + inline virtual void do_oop(oop* slot) override { + const auto o = *slot; + const uintptr_t v = uintptr_t((void*) o); + if (o == NULL) { + return; + } + if (mmtk_is_live((void*) o) == 0) { + *slot = NULL; + } else if (MMTkForwardClosure::is_forwarded(MMTkForwardClosure::read_forwarding_word(o))) { + *slot = MMTkForwardClosure::extract_forwarding_pointer(MMTkForwardClosure::read_forwarding_word(o)); + } + } + inline virtual void do_oop(narrowOop* slot) override { + guarantee(false, "unreachable"); + } +}; + extern MaybeUninit> oop_storage_set_strong_par_state; extern MaybeUninit> oop_storage_set_weak_par_state; @@ -196,13 +207,11 @@ static void mmtk_clear_claimed_marks() { static void mmtk_update_weak_processor(bool lxr) { // HandleMark hm(THREAD); if (lxr) { - MMTkLXRFastIsAliveClosure is_alive; - MMTkForwardClosure forward; - WeakProcessor::weak_oops_do(&is_alive, &forward); + MMTkLXRFastUpdateClosure cl; + WeakProcessor::oops_do(&cl); } else { - MMTkIsAliveClosure is_alive; - MMTkForwardClosure forward; - WeakProcessor::weak_oops_do(&is_alive, &forward); + MMTkUpdateClosure cl; + WeakProcessor::oops_do(&cl); } } From 88f5b8075f708f6c5c6e82829261005fcdfdcad3 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 31 Mar 2026 15:18:01 +1100 Subject: [PATCH 452/494] class unloading working --- mmtk/src/collection.rs | 1 + mmtk/src/gc_work.rs | 100 +++++++++++----------------------- mmtk/src/lib.rs | 3 +- openjdk/share/mmtk.h | 3 +- openjdk/share/mmtkHeap.cpp | 3 +- openjdk/share/mmtkUpcalls.cpp | 32 ++++++++--- 6 files changed, 63 insertions(+), 79 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 9346b979..df82631a 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -115,6 +115,7 @@ impl Collection> for VMCollection { fn vm_release(do_unloading: bool) { unsafe { + ((*UPCALLS).oops_do_marking_epilogue)(); if do_unloading { gc_log!(" - unload_classes"); ((*UPCALLS).unload_classes)(); diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index bbb41287..47c003e5 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -238,14 +238,11 @@ impl>> .get_plan() .generational() .is_some_and(|gen| gen.is_current_gc_nursery()); - let is_lxr = mmtk - .get_plan() - .downcast_ref::>>() - .is_some(); - let is_rc_pause = mmtk + let lxr = mmtk .get_plan() - .downcast_ref::>>() - .is_some_and(|lxr| lxr.current_pause() == Some(Pause::RefCount)); + .downcast_ref::>>(); + let is_lxr = lxr.is_some(); + let is_rc_pause = lxr.is_some_and(|lxr| lxr.current_pause() == Some(Pause::RefCount)); let class_unloading_enabled = unsafe { crate::CLASS_UNLOADING_ENABLED } == 1; let mut slots = Vec::with_capacity(scanning::WORK_PACKET_CAPACITY); @@ -286,6 +283,17 @@ impl>> } } + let pause = lxr.map(|x| x.current_pause()).unwrap_or(None); + + if is_lxr + && class_unloading_enabled + && (pause == Some(Pause::FinalMark) || pause == Some(Pause::Full)) + { + for key in mature.keys() { + nmethods_to_fix.push(*key); + } + } + { let mut nursery = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); for (key, roots) in nursery.drain() { @@ -339,7 +347,7 @@ impl>> // For scavenging GCs, the mmtk-openjdk binding reports the *slots* of nmethods as // roots. They will be traced at unspecified times during the Closure stage. // SoftRefClosure is the first safe place to call fix_oop_relocations. - WorkBucketStage::Release + WorkBucketStage::FixRelocations }; worker.scheduler().work_buckets[stage].bulk_add(packets); } @@ -379,7 +387,6 @@ fn to_slots_closure_weak>(factory: &mut F) -> Sl } } -#[allow(unused)] pub struct ScanWeakProcessorRoots> { factory: F, _p: PhantomData, @@ -418,63 +425,6 @@ impl> GCWork } } -pub struct ScanWeakCodeCacheRoots> { - factory: F, - _p: PhantomData, -} - -impl> ScanWeakCodeCacheRoots { - pub fn new(factory: F) -> Self { - Self { - factory, - _p: PhantomData, - } - } -} - -impl> GCWork - for ScanWeakCodeCacheRoots -{ - fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { - let t = if cfg!(feature = "roots_breakdown") { - Some(std::time::SystemTime::now()) - } else { - None - }; - let scan_all_strong_roots = mmtk.get_plan().current_gc_should_perform_class_unloading(); - assert!(scan_all_strong_roots); - - let mut slots = Vec::with_capacity(F::BUFFER_SIZE); - let mature = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); - let mut c = 0; - // Young roots - for (_key, roots) in &*mature { - for r in roots { - slots.push(VM::VMSlot::from_address(*r)); - if slots.len() >= F::BUFFER_SIZE { - if cfg!(feature = "roots_breakdown") { - c += slots.len(); - } - self.factory - .create_process_roots_work(std::mem::take(&mut slots), RootKind::Weak); - slots.reserve(F::BUFFER_SIZE); - } - } - } - if !slots.is_empty() { - if cfg!(feature = "roots_breakdown") { - c += slots.len(); - } - self.factory - .create_process_roots_work(slots, RootKind::Weak); - } - if cfg!(feature = "roots_breakdown") { - let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; - eprintln!(" - WeakodeCacheRoots roots count: {} ({:.3})", c, ms); - } - } -} - struct FixRelocations { nmethods: Vec
, } @@ -489,11 +439,25 @@ impl GCWork> for FixRelocations { fn do_work( &mut self, _worker: &mut GCWorker>, - _mmtk: &'static MMTK>, + mmtk: &'static MMTK>, ) { + let is_lxr = mmtk + .get_plan() + .downcast_ref::>>() + .is_some(); + let is_rc_pause = mmtk + .get_plan() + .downcast_ref::>>() + .is_some_and(|lxr| lxr.current_pause() == Some(Pause::RefCount)); + let num_nmethods = self.nmethods.len(); + let lxr_rc_pause = is_lxr && is_rc_pause; unsafe { - ((*UPCALLS).fix_oop_relocations)(self.nmethods.as_mut_ptr() as *mut _, num_nmethods); + ((*UPCALLS).fix_oop_relocations)( + lxr_rc_pause, + self.nmethods.as_mut_ptr() as *mut _, + num_nmethods, + ); } // probe!(mmtk_openjdk, fix_relocations, num_nmethods); } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 4d40f767..0f64007b 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -134,11 +134,12 @@ pub struct OpenJDK_Upcalls { pub swap_reference_pending_list: extern "C" fn(objects: ObjectReference) -> ObjectReference, pub java_lang_class_klass_offset_in_bytes: extern "C" fn() -> usize, pub java_lang_classloader_loader_data_offset: extern "C" fn() -> usize, - pub fix_oop_relocations: extern "C" fn(nmethods: *mut libc::c_void, len: usize), + pub fix_oop_relocations: extern "C" fn(lxr: bool, nmethods: *mut libc::c_void, len: usize), // pub nmethod_fix_relocation: extern "C" fn(Address), pub clear_claimed_marks: extern "C" fn(), pub unload_classes: extern "C" fn(), pub gc_epilogue: extern "C" fn(), + pub oops_do_marking_epilogue: extern "C" fn(), } lazy_static! { diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index a992c716..53ad54f6 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -219,10 +219,11 @@ typedef struct { void* (*swap_reference_pending_list)(void* objects); size_t (*java_lang_class_klass_offset_in_bytes)(); size_t (*java_lang_classloader_loader_data_offset)(); - void (*fix_oop_relocations)(void* nmethods, size_t len); + void (*fix_oop_relocations)(bool lxr, void* nmethods, size_t len); void (*clear_claimed_marks)(); void (*unload_classes)(); void (*gc_epilogue)(); + void (*oops_do_marking_epilogue)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); diff --git a/openjdk/share/mmtkHeap.cpp b/openjdk/share/mmtkHeap.cpp index ecdfe8a4..1107daea 100644 --- a/openjdk/share/mmtkHeap.cpp +++ b/openjdk/share/mmtkHeap.cpp @@ -509,7 +509,8 @@ void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { } void MMTkHeap::scan_vm_thread_roots(OopClosure& cl) { ResourceMark rm; - VMThread::vm_thread()->oops_do(&cl, NULL); + MarkingCodeBlobClosure cb_cl(&cl, false, true); + VMThread::vm_thread()->oops_do(&cl, &cb_cl); } void MMTkHeap::scan_roots_in_all_mutator_threads(OopClosure& cl) { diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index 22a7fb42..4c4359fc 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -215,9 +215,12 @@ static void mmtk_update_weak_processor(bool lxr) { } } +static void mmtk_oops_do_marking_epilogue() { + nmethod::oops_do_marking_epilogue(); +} + static void mmtk_unload_classes() { if (ClassUnloading) { - LOG_CLS_UNLOAD("[mmtk_unload_classes] start"); // Unload classes and purge SystemDictionary. // LOG_CLS_UNLOAD("[mmtk_unload_classes] SystemDictionary::do_unloading"); // auto purged_classes = SystemDictionary::do_unloading(NULL); @@ -252,7 +255,6 @@ static void mmtk_unload_classes() { } static void mmtk_gc_epilogue() { - nmethod::oops_do_marking_epilogue(); Universe::heap()->update_capacity_and_used_at_gc(); // CodeCache::gc_epilogue(); // JvmtiExport::gc_epilogue(); @@ -260,6 +262,7 @@ static void mmtk_gc_epilogue() { #if COMPILER2_OR_JVMCI DerivedPointerTable::update_pointers(); #endif + CodeCache::arm_all_nmethods(); } static void mmtk_resume_mutators(void *tls) { @@ -389,15 +392,16 @@ static void mmtk_scan_roots_in_mutator_thread(SlotsClosure closure, void* tls) { JavaThread* thread = (JavaThread*) tls; MMTkRootsClosure cl(closure); MarkingCodeBlobClosure cb_cl(&cl, false, true); - thread->oops_do(&cl, NULL); + thread->oops_do(&cl, &cb_cl); } static void mmtk_scan_multiple_thread_roots(SlotsClosure closure, void* ptr, size_t len) { ResourceMark rm; auto mutators = (JavaThread**) ptr; MMTkRootsClosure cl(closure); + MarkingCodeBlobClosure cb_cl(&cl, false, true); for (size_t i = 0; i < len; i++) - mutators[i]->oops_do(&cl, NULL); + mutators[i]->oops_do(&cl, &cb_cl); } static void mmtk_scan_object(void* trace, void* object, void* tls, bool follow_clds, bool claim_clds) { @@ -572,10 +576,21 @@ static size_t mmtk_java_lang_classloader_loader_data_offset() { return v; } -void mmtk_fix_oop_relocations(void* nmethods, size_t len) { - for (size_t i = 0; i < len; i++) { - nmethod* nm = (nmethod*) ((nmethod**) nmethods)[i]; - nm->fix_oop_relocations(); +void mmtk_fix_oop_relocations(bool lxr, void* nmethods, size_t len) { + if (lxr) { + MMTkLXRFastUpdateClosure cl; + for (size_t i = 0; i < len; i++) { + nmethod* nm = (nmethod*) ((nmethod**) nmethods)[i]; + nm->oops_do(&cl); + nm->fix_oop_relocations(); + } + } else { + MMTkUpdateClosure cl; + for (size_t i = 0; i < len; i++) { + nmethod* nm = (nmethod*) ((nmethod**) nmethods)[i]; + nm->oops_do(&cl); + nm->fix_oop_relocations(); + } } } @@ -619,4 +634,5 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_clear_claimed_marks, mmtk_unload_classes, mmtk_gc_epilogue, + mmtk_oops_do_marking_epilogue, }; From 3d90d3a1cb228b66a528acde77d87177a315389e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 31 Mar 2026 22:24:41 +1100 Subject: [PATCH 453/494] Enhance fix_oop_relocations to support forward-only mode during class unloading --- mmtk/src/gc_work.rs | 18 ++++++++++++++--- mmtk/src/lib.rs | 2 +- openjdk/share/mmtk.h | 2 +- openjdk/share/mmtkUpcalls.cpp | 37 +++++++++++++++++++++++++++++++++-- 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 47c003e5..f3d24c09 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -244,6 +244,7 @@ impl>> let is_lxr = lxr.is_some(); let is_rc_pause = lxr.is_some_and(|lxr| lxr.current_pause() == Some(Pause::RefCount)); let class_unloading_enabled = unsafe { crate::CLASS_UNLOADING_ENABLED } == 1; + let pause = lxr.map(|x| x.current_pause()).unwrap_or(None); let mut slots = Vec::with_capacity(scanning::WORK_PACKET_CAPACITY); @@ -283,8 +284,6 @@ impl>> } } - let pause = lxr.map(|x| x.current_pause()).unwrap_or(None); - if is_lxr && class_unloading_enabled && (pause == Some(Pause::FinalMark) || pause == Some(Pause::Full)) @@ -321,6 +320,14 @@ impl>> .create_process_roots_work(slots, RootKind::Strong); } + // When class unloading is pending, use forward-only mode: forward alive oops + // but don't null dead oops. This allows has_dead_oop() during class unloading + // to detect nmethods that should be unloaded. + let forward_only = (is_lxr + && class_unloading_enabled + && (pause == Some(Pause::FinalMark) || pause == Some(Pause::Full))) + || !is_current_gc_nursery; + if moves_object && !nmethods_to_fix.is_empty() { // Note: If the current GC doesn't move objects at all, we don't need to fix relocation. // FIXME: Even during copying GC, some GC algorithms (such as Immix) don't move every @@ -331,7 +338,10 @@ impl>> .chunks(FixRelocations::NMETHODS_PER_PACKET) .map(|chunk| { let nmethods = chunk.to_vec(); - Box::new(FixRelocations { nmethods }) as _ + Box::new(FixRelocations { + nmethods, + forward_only, + }) as _ }) .collect(); @@ -427,6 +437,7 @@ impl> GCWork struct FixRelocations { nmethods: Vec
, + forward_only: bool, } impl FixRelocations { @@ -455,6 +466,7 @@ impl GCWork> for FixRelocations { unsafe { ((*UPCALLS).fix_oop_relocations)( lxr_rc_pause, + self.forward_only, self.nmethods.as_mut_ptr() as *mut _, num_nmethods, ); diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 0f64007b..54c28e72 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -134,7 +134,7 @@ pub struct OpenJDK_Upcalls { pub swap_reference_pending_list: extern "C" fn(objects: ObjectReference) -> ObjectReference, pub java_lang_class_klass_offset_in_bytes: extern "C" fn() -> usize, pub java_lang_classloader_loader_data_offset: extern "C" fn() -> usize, - pub fix_oop_relocations: extern "C" fn(lxr: bool, nmethods: *mut libc::c_void, len: usize), + pub fix_oop_relocations: extern "C" fn(lxr: bool, forward_only: bool, nmethods: *mut libc::c_void, len: usize), // pub nmethod_fix_relocation: extern "C" fn(Address), pub clear_claimed_marks: extern "C" fn(), pub unload_classes: extern "C" fn(), diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index 53ad54f6..479a63dc 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -219,7 +219,7 @@ typedef struct { void* (*swap_reference_pending_list)(void* objects); size_t (*java_lang_class_klass_offset_in_bytes)(); size_t (*java_lang_classloader_loader_data_offset)(); - void (*fix_oop_relocations)(bool lxr, void* nmethods, size_t len); + void (*fix_oop_relocations)(bool lxr, bool forward_only, void* nmethods, size_t len); void (*clear_claimed_marks)(); void (*unload_classes)(); void (*gc_epilogue)(); diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index 4c4359fc..f03dc4c8 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -151,6 +151,30 @@ class MMTkLXRFastUpdateClosure : public OopClosure { } }; +// Forward-only closure: forwards alive oops but does NOT null dead oops. +// Used during fix_oop_relocations when class unloading is pending, so that +// has_dead_oop() can still detect nmethods that should be unloaded. +class MMTkForwardOnlyClosure : public OopClosure { + public: + inline virtual void do_oop(oop* slot) override { + const auto o = *slot; + if (o == NULL) return; + const auto status = MMTkForwardClosure::read_forwarding_word(o); + if (MMTkForwardClosure::is_forwarded(status)) { + *slot = MMTkForwardClosure::extract_forwarding_pointer(status); + } + } + inline virtual void do_oop(narrowOop* slot) override { + narrowOop heap_oop = RawAccess<>::oop_load(slot); + if (CompressedOops::is_null(heap_oop)) return; + oop o = CompressedOops::decode_not_null(heap_oop); + const auto status = MMTkForwardClosure::read_forwarding_word(o); + if (MMTkForwardClosure::is_forwarded(status)) { + RawAccess<>::oop_store(slot, CompressedOops::encode_not_null(MMTkForwardClosure::extract_forwarding_pointer(status))); + } + } +}; + class MMTkUpdateClosure : public OopClosure { public: inline virtual void do_oop(oop* slot) override { @@ -576,8 +600,17 @@ static size_t mmtk_java_lang_classloader_loader_data_offset() { return v; } -void mmtk_fix_oop_relocations(bool lxr, void* nmethods, size_t len) { - if (lxr) { +void mmtk_fix_oop_relocations(bool lxr, bool forward_only, void* nmethods, size_t len) { + if (forward_only) { + // Forward-only: forward alive oops but don't null dead oops. + // Used when class unloading is pending so has_dead_oop() can detect dead nmethods. + MMTkForwardOnlyClosure cl; + for (size_t i = 0; i < len; i++) { + nmethod* nm = (nmethod*) ((nmethod**) nmethods)[i]; + nm->oops_do(&cl); + nm->fix_oop_relocations(); + } + } else if (lxr) { MMTkLXRFastUpdateClosure cl; for (size_t i = 0; i < len; i++) { nmethod* nm = (nmethod*) ((nmethod**) nmethods)[i]; From 2b09d2da0dc7cd59152da95fcec42ac53235ac46 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 31 Mar 2026 22:29:33 +1100 Subject: [PATCH 454/494] Minor --- mmtk/src/gc_work.rs | 15 +++++++++------ openjdk/share/mmtkUpcalls.cpp | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index f3d24c09..5e45c3ff 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -456,16 +456,19 @@ impl GCWork> for FixRelocations { .get_plan() .downcast_ref::>>() .is_some(); - let is_rc_pause = mmtk - .get_plan() - .downcast_ref::>>() - .is_some_and(|lxr| lxr.current_pause() == Some(Pause::RefCount)); + let fast_mode = is_lxr + && mmtk + .get_plan() + .downcast_ref::>>() + .is_some_and(|lxr| { + lxr.current_pause() == Some(Pause::RefCount) + || lxr.current_pause() == Some(Pause::InitialMark) + }); let num_nmethods = self.nmethods.len(); - let lxr_rc_pause = is_lxr && is_rc_pause; unsafe { ((*UPCALLS).fix_oop_relocations)( - lxr_rc_pause, + fast_mode, self.forward_only, self.nmethods.as_mut_ptr() as *mut _, num_nmethods, diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index f03dc4c8..1412a4db 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -286,7 +286,7 @@ static void mmtk_gc_epilogue() { #if COMPILER2_OR_JVMCI DerivedPointerTable::update_pointers(); #endif - CodeCache::arm_all_nmethods(); + // CodeCache::arm_all_nmethods(); } static void mmtk_resume_mutators(void *tls) { From 1aca87847c22a7b03a026b5bc82ee8f142ade976 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 31 Mar 2026 23:39:57 +1100 Subject: [PATCH 455/494] optimize --- mmtk/src/gc_work.rs | 75 +---- mmtk/src/lib.rs | 2 +- openjdk/share/mmtk.h | 2 +- openjdk/share/mmtkHeap.cpp | 5 +- openjdk/share/mmtkHeap.hpp | 2 +- openjdk/share/mmtkParallelCleaning.cpp | 395 ++++++------------------- openjdk/share/mmtkParallelCleaning.hpp | 94 +----- openjdk/share/mmtkUpcalls.cpp | 47 +-- 8 files changed, 131 insertions(+), 491 deletions(-) diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 5e45c3ff..4e7991fb 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -172,46 +172,6 @@ impl> GCWork } } -pub struct ScanNewWeakHandleRoots> { - factory: F, - _p: PhantomData, -} - -impl> ScanNewWeakHandleRoots { - pub fn new(factory: F) -> Self { - Self { - factory, - _p: PhantomData, - } - } -} - -impl> GCWork - for ScanNewWeakHandleRoots -{ - fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { - let t = if cfg!(feature = "roots_breakdown") { - Some(std::time::SystemTime::now()) - } else { - None - }; - let mut new_roots = crate::NURSERY_WEAK_HANDLE_ROOTS.lock().unwrap(); - if cfg!(feature = "roots_breakdown") { - record_roots(new_roots.len()); - } - for slice in new_roots.chunks(F::BUFFER_SIZE) { - let slice = unsafe { std::mem::transmute::<&[Address], &[VM::VMSlot]>(slice) }; - self.factory - .create_process_roots_work(slice.to_vec(), RootKind::YoungWeakHandleRoots); - } - new_roots.clear(); - if cfg!(feature = "roots_breakdown") { - let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; - report_roots("NewWeakHandleRoots", ms); - } - } -} - #[allow(unused)] pub struct ScanCodeCacheRoots>> { @@ -269,6 +229,8 @@ impl>> // That includes all nmethods with moved children. // In nursery GCs, that means nmethods added since the previous GC. let mut nmethods_to_fix = Vec::new(); + let no_fix_relocation = class_unloading_enabled + && (pause == Some(Pause::FinalMark) || pause == Some(Pause::Full)); { let mut mature = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); @@ -284,16 +246,14 @@ impl>> } } - if is_lxr - && class_unloading_enabled - && (pause == Some(Pause::FinalMark) || pause == Some(Pause::Full)) - { - for key in mature.keys() { - nmethods_to_fix.push(*key); + if no_fix_relocation { + let mut nursery = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); + for (key, roots) in nursery.drain() { + nursery_slots += roots.len(); + add_roots(&roots); + mature.insert(key, roots); } - } - - { + } else { let mut nursery = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); for (key, roots) in nursery.drain() { nursery_slots += roots.len(); @@ -320,14 +280,6 @@ impl>> .create_process_roots_work(slots, RootKind::Strong); } - // When class unloading is pending, use forward-only mode: forward alive oops - // but don't null dead oops. This allows has_dead_oop() during class unloading - // to detect nmethods that should be unloaded. - let forward_only = (is_lxr - && class_unloading_enabled - && (pause == Some(Pause::FinalMark) || pause == Some(Pause::Full))) - || !is_current_gc_nursery; - if moves_object && !nmethods_to_fix.is_empty() { // Note: If the current GC doesn't move objects at all, we don't need to fix relocation. // FIXME: Even during copying GC, some GC algorithms (such as Immix) don't move every @@ -338,10 +290,7 @@ impl>> .chunks(FixRelocations::NMETHODS_PER_PACKET) .map(|chunk| { let nmethods = chunk.to_vec(); - Box::new(FixRelocations { - nmethods, - forward_only, - }) as _ + Box::new(FixRelocations { nmethods }) as _ }) .collect(); @@ -357,7 +306,7 @@ impl>> // For scavenging GCs, the mmtk-openjdk binding reports the *slots* of nmethods as // roots. They will be traced at unspecified times during the Closure stage. // SoftRefClosure is the first safe place to call fix_oop_relocations. - WorkBucketStage::FixRelocations + WorkBucketStage::Release }; worker.scheduler().work_buckets[stage].bulk_add(packets); } @@ -437,7 +386,6 @@ impl> GCWork struct FixRelocations { nmethods: Vec
, - forward_only: bool, } impl FixRelocations { @@ -469,7 +417,6 @@ impl GCWork> for FixRelocations { unsafe { ((*UPCALLS).fix_oop_relocations)( fast_mode, - self.forward_only, self.nmethods.as_mut_ptr() as *mut _, num_nmethods, ); diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 54c28e72..0f64007b 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -134,7 +134,7 @@ pub struct OpenJDK_Upcalls { pub swap_reference_pending_list: extern "C" fn(objects: ObjectReference) -> ObjectReference, pub java_lang_class_klass_offset_in_bytes: extern "C" fn() -> usize, pub java_lang_classloader_loader_data_offset: extern "C" fn() -> usize, - pub fix_oop_relocations: extern "C" fn(lxr: bool, forward_only: bool, nmethods: *mut libc::c_void, len: usize), + pub fix_oop_relocations: extern "C" fn(lxr: bool, nmethods: *mut libc::c_void, len: usize), // pub nmethod_fix_relocation: extern "C" fn(Address), pub clear_claimed_marks: extern "C" fn(), pub unload_classes: extern "C" fn(), diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index 479a63dc..53ad54f6 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -219,7 +219,7 @@ typedef struct { void* (*swap_reference_pending_list)(void* objects); size_t (*java_lang_class_klass_offset_in_bytes)(); size_t (*java_lang_classloader_loader_data_offset)(); - void (*fix_oop_relocations)(bool lxr, bool forward_only, void* nmethods, size_t len); + void (*fix_oop_relocations)(bool lxr, void* nmethods, size_t len); void (*clear_claimed_marks)(); void (*unload_classes)(); void (*gc_epilogue)(); diff --git a/openjdk/share/mmtkHeap.cpp b/openjdk/share/mmtkHeap.cpp index 1107daea..35032e37 100644 --- a/openjdk/share/mmtkHeap.cpp +++ b/openjdk/share/mmtkHeap.cpp @@ -580,11 +580,10 @@ HeapWord* MMTkHeap::mem_allocate_nonmove(size_t size, bool* gc_overhead_limit_wa return alloc_fast(size << LogHeapWordSize, AllocatorLos); } -void MMTkHeap::complete_cleaning(BoolObjectClosure* is_alive, OopClosure* forward, bool purged_classes) { +void MMTkHeap::complete_cleaning(bool purged_classes) { ResourceMark rm; - // HandleMark hm(THREAD); uint num_workers = _workers->active_workers(); - mmtk::ParallelCleaningTask unlink_task(is_alive, forward, num_workers, purged_classes); + mmtk::ParallelCleaningTask unlink_task(num_workers, purged_classes); _workers->run_task(&unlink_task); } diff --git a/openjdk/share/mmtkHeap.hpp b/openjdk/share/mmtkHeap.hpp index abe02046..7a9b72b2 100644 --- a/openjdk/share/mmtkHeap.hpp +++ b/openjdk/share/mmtkHeap.hpp @@ -218,7 +218,7 @@ class MMTkHeap : public CollectedHeap { void scan_weak_processor_roots(OopClosure& cl); void scan_vm_thread_roots(OopClosure& cl); - void complete_cleaning(BoolObjectClosure* is_alive, OopClosure* forward, bool purged_classes); + void complete_cleaning(bool purged_classes); }; diff --git a/openjdk/share/mmtkParallelCleaning.cpp b/openjdk/share/mmtkParallelCleaning.cpp index 3474476e..73fef671 100644 --- a/openjdk/share/mmtkParallelCleaning.cpp +++ b/openjdk/share/mmtkParallelCleaning.cpp @@ -34,334 +34,129 @@ #include "gc/shared/gcTraceTime.inline.hpp" #include "mmtkParallelCleaning.hpp" #include "mmtk.h" - #include "runtime/atomic.hpp" -#if INCLUDE_JVMCI -#include "jvmci/jvmci.hpp" -#endif +#include "oops/access.inline.hpp" +#include "oops/compressedOops.inline.hpp" +#include "oops/oop.inline.hpp" +#include "code/nmethod.hpp" using namespace JavaClassFile; namespace mmtk { +MMTkCodeCacheUnloadingTask::MMTkCodeCacheUnloadingTask(uint num_workers, bool unloading_occurred) : + _unloading_occurred(unloading_occurred), + _num_workers(num_workers), + _first_nmethod(nullptr), + _claimed_nmethod(nullptr) { + // Get first alive nmethod + CompiledMethodIterator iter(CompiledMethodIterator::all_blobs); + if(iter.next()) { + _first_nmethod = iter.method(); + } + _claimed_nmethod = _first_nmethod; +} -// StringSymbolTableUnlinkTask::StringSymbolTableUnlinkTask(uint num_workers, BoolObjectClosure* is_alive, OopClosure* forward) : -// AbstractGangTask("String/Symbol Unlinking"), -// _is_alive(is_alive), -// _forward(forward), -// _par_state_string_fwd(StringTable::weak_storage()), -// _par_state_string(StringTable::weak_storage()), -// _strings_processed(0), _strings_removed(0), -// _symbols_processed(0), _symbols_removed(0), -// _num_workers(num_workers), _num_entered_barrier(0) { - -// _initial_string_table_size = (int) StringTable::the_table()->table_size(); -// _initial_symbol_table_size = SymbolTable::the_table()->table_size(); -// // if (process_symbols) { -// SymbolTable::clear_parallel_claimed_index(); -// // } -// // if (process_strings) { -// StringTable::reset_dead_counter(); -// // } -// } - -// StringSymbolTableUnlinkTask::~StringSymbolTableUnlinkTask() { -// guarantee(!_process_symbols || SymbolTable::parallel_claimed_index() >= _initial_symbol_table_size, -// "claim value %d after unlink less than initial symbol table size %d", -// SymbolTable::parallel_claimed_index(), _initial_symbol_table_size); - -// log_info(gc, stringtable)( -// "Cleaned string and symbol table, " -// "strings: " SIZE_FORMAT " processed, " SIZE_FORMAT " removed, " -// "symbols: " SIZE_FORMAT " processed, " SIZE_FORMAT " removed", -// strings_processed(), strings_removed(), -// symbols_processed(), symbols_removed()); - -// // if (_process_strings) { -// StringTable::finish_dead_counter(); -// // } -// } - -// void StringSymbolTableUnlinkTask::work(uint worker_id) { -// int strings_processed = 0; -// int strings_removed = 0; -// int symbols_processed = 0; -// int symbols_removed = 0; -// // if (_process_strings) { -// StringTable::possibly_parallel_unlink(&_par_state_string, _is_alive, &strings_processed, &strings_removed); -// Atomic::add(strings_processed, &_strings_processed); -// Atomic::add(strings_removed, &_strings_removed); -// // } -// // if (_process_symbols) { -// SymbolTable::possibly_parallel_unlink(&symbols_processed, &symbols_removed); -// Atomic::add(symbols_processed, &_symbols_processed); -// Atomic::add(symbols_removed, &_symbols_removed); -// // } -// } - -// size_t StringSymbolTableUnlinkTask::strings_processed() const { return (size_t)_strings_processed; } -// size_t StringSymbolTableUnlinkTask::strings_removed() const { return (size_t)_strings_removed; } - -// size_t StringSymbolTableUnlinkTask::symbols_processed() const { return (size_t)_symbols_processed; } -// size_t StringSymbolTableUnlinkTask::symbols_removed() const { return (size_t)_symbols_removed; } - -// void StringSymbolTableUnlinkTask::barrier_wait(uint worker_id) { -// MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag); -// _num_entered_barrier++; -// if (_num_entered_barrier >= _num_workers) { -// ml.notify_all(); -// } -// while (_num_entered_barrier < _num_workers) { -// ml.wait(Mutex::_no_safepoint_check_flag, 0, false); -// } -// } - - -// Monitor* StringSymbolTableUnlinkTask::_lock = new Monitor(Mutex::leaf, "String Table Unload lock", false, Monitor::_safepoint_check_never); -// Monitor* CodeCacheUnloadingTask::_lock = new Monitor(Mutex::leaf, "Code Cache Unload lock", false, Monitor::_safepoint_check_never); - -// CodeCacheUnloadingTask::CodeCacheUnloadingTask(uint num_workers, BoolObjectClosure* is_alive, bool unloading_occurred) : -// _is_alive(is_alive), -// _unloading_occurred(unloading_occurred), -// _num_workers(num_workers), -// _first_nmethod(NULL), -// _claimed_nmethod(NULL), -// _postponed_list(NULL), -// _num_entered_barrier(0) -// { -// CompiledMethod::increase_unloading_clock(); -// // Get first alive nmethod -// CompiledMethodIterator iter = CompiledMethodIterator(); -// if(iter.next_alive()) { -// _first_nmethod = iter.method(); -// } -// _claimed_nmethod = (volatile CompiledMethod*)_first_nmethod; -// } - -// CodeCacheUnloadingTask::~CodeCacheUnloadingTask() { -// // CodeCache::verify_clean_inline_caches(); - -// CodeCache::set_needs_cache_clean(false); -// guarantee(CodeCache::scavenge_root_nmethods() == NULL, "Must be"); - -// CodeCache::verify_icholder_relocations(); -// } - -// void CodeCacheUnloadingTask::add_to_postponed_list(CompiledMethod* nm) { -// CompiledMethod* old; -// do { -// old = (CompiledMethod*)_postponed_list; -// nm->set_unloading_next(old); -// } while ((CompiledMethod*)Atomic::cmpxchg(nm, &_postponed_list, old) != old); -// } - -// void CodeCacheUnloadingTask::clean_nmethod(CompiledMethod* nm) { -// bool postponed = nm->do_unloading_parallel(_is_alive, _unloading_occurred); - -// if (postponed) { -// // This nmethod referred to an nmethod that has not been cleaned/unloaded yet. -// add_to_postponed_list(nm); -// } - -// // Mark that this thread has been cleaned/unloaded. -// // After this call, it will be safe to ask if this nmethod was unloaded or not. -// nm->set_unloading_clock(CompiledMethod::global_unloading_clock()); -// } - -// void CodeCacheUnloadingTask::clean_nmethod_postponed(CompiledMethod* nm) { -// nm->do_unloading_parallel_postponed(); -// } - -// void CodeCacheUnloadingTask::claim_nmethods(CompiledMethod** claimed_nmethods, int *num_claimed_nmethods) { -// CompiledMethod* first; -// CompiledMethodIterator last; - -// do { -// *num_claimed_nmethods = 0; - -// first = (CompiledMethod*)_claimed_nmethod; -// last = CompiledMethodIterator(first); - -// if (first != NULL) { - -// for (int i = 0; i < MaxClaimNmethods; i++) { -// if (!last.next_alive()) { -// break; -// } -// claimed_nmethods[i] = last.method(); -// (*num_claimed_nmethods)++; -// } -// } - -// } while ((CompiledMethod*)Atomic::cmpxchg(last.method(), &_claimed_nmethod, first) != first); -// } - -// CompiledMethod* CodeCacheUnloadingTask::claim_postponed_nmethod() { -// CompiledMethod* claim; -// CompiledMethod* next; - -// do { -// claim = (CompiledMethod*)_postponed_list; -// if (claim == NULL) { -// return NULL; -// } - -// next = claim->unloading_next(); - -// } while ((CompiledMethod*)Atomic::cmpxchg(next, &_postponed_list, claim) != claim); - -// return claim; -// } - -// // Mark that we're done with the first pass of nmethod cleaning. -// void CodeCacheUnloadingTask::barrier_mark(uint worker_id) { -// MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag); -// _num_entered_barrier++; -// if (_num_entered_barrier == _num_workers) { -// ml.notify_all(); -// } -// } - -// // See if we have to wait for the other workers to -// // finish their first-pass nmethod cleaning work. -// void CodeCacheUnloadingTask::barrier_wait(uint worker_id) { -// if (_num_entered_barrier < _num_workers) { -// MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag); -// while (_num_entered_barrier < _num_workers) { -// ml.wait(Mutex::_no_safepoint_check_flag, 0, false); -// } -// } -// } - -// // Cleaning and unloading of nmethods. Some work has to be postponed -// // to the second pass, when we know which nmethods survive. -// void CodeCacheUnloadingTask::work_first_pass(uint worker_id) { -// // The first nmethods is claimed by the first worker. -// if (worker_id == 0 && _first_nmethod != NULL) { -// clean_nmethod(_first_nmethod); -// _first_nmethod = NULL; -// } - -// int num_claimed_nmethods; -// CompiledMethod* claimed_nmethods[MaxClaimNmethods]; - -// while (true) { -// claim_nmethods(claimed_nmethods, &num_claimed_nmethods); - -// if (num_claimed_nmethods == 0) { -// break; -// } - -// for (int i = 0; i < num_claimed_nmethods; i++) { -// clean_nmethod(claimed_nmethods[i]); -// } -// } -// } - -// void CodeCacheUnloadingTask::work_second_pass(uint worker_id) { -// CompiledMethod* nm; -// // Take care of postponed nmethods. -// while ((nm = claim_postponed_nmethod()) != NULL) { -// clean_nmethod_postponed(nm); -// } -// } - -// KlassCleaningTask::KlassCleaningTask(BoolObjectClosure* is_alive) : -// _is_alive(is_alive), -// _clean_klass_tree_claimed(0), -// _klass_iterator() { -// } - -// bool KlassCleaningTask::claim_clean_klass_tree_task() { -// if (_clean_klass_tree_claimed) { -// return false; -// } - -// return Atomic::cmpxchg(1, &_clean_klass_tree_claimed, 0) == 0; -// } - -// InstanceKlass* KlassCleaningTask::claim_next_klass() { -// Klass* klass; -// do { -// klass =_klass_iterator.next_klass(); -// } while (klass != NULL && !klass->is_instance_klass()); - -// // this can be null so don't call InstanceKlass::cast -// return static_cast(klass); -// } +MMTkCodeCacheUnloadingTask::~MMTkCodeCacheUnloadingTask() { + CodeCache::verify_clean_inline_caches(); + CodeCache::verify_icholder_relocations(); +} -// void KlassCleaningTask::clean_klass(InstanceKlass* ik) { -// ik->clean_weak_instanceklass_links(); -// } +void MMTkCodeCacheUnloadingTask::claim_nmethods(CompiledMethod** claimed_nmethods, int *num_claimed_nmethods) { + CompiledMethod* first; + CompiledMethodIterator last(CompiledMethodIterator::all_blobs); -// void KlassCleaningTask::work() { -// ResourceMark rm; + do { + *num_claimed_nmethods = 0; -// // One worker will clean the subklass/sibling klass tree. -// if (claim_clean_klass_tree_task()) { -// Klass::clean_subklass_tree(); -// } + first = _claimed_nmethod; + last = CompiledMethodIterator(CompiledMethodIterator::all_blobs, first); -// // All workers will help cleaning the classes, -// InstanceKlass* klass; -// while ((klass = claim_next_klass()) != NULL) { -// clean_klass(klass); -// } -// } + if (first != nullptr) { -// bool ResolvedMethodCleaningTask::claim_resolved_method_task() { -// if (_resolved_method_task_claimed) { -// return false; -// } -// return Atomic::cmpxchg(1, &_resolved_method_task_claimed, 0) == 0; -// } + for (int i = 0; i < MaxClaimNmethods; i++) { + if (!last.next()) { + break; + } + claimed_nmethods[i] = last.method(); + (*num_claimed_nmethods)++; + } + } -// // These aren't big, one thread can do it all. -// void ResolvedMethodCleaningTask::work() { -// if (claim_resolved_method_task()) { -// ResolvedMethodTable::unlink(); -// } -// } -#if INCLUDE_JVMCI -JVMCICleaningTask::JVMCICleaningTask() : - _cleaning_claimed(0) { + } while (Atomic::cmpxchg(&_claimed_nmethod, first, last.method()) != first); } -bool JVMCICleaningTask::claim_cleaning_task() { - if (_cleaning_claimed) { - return false; +class MMTkForwardClosure : public OopClosure { + public: + inline static size_t read_forwarding_word(oop o) { + return *((size_t*) (void*) o); + } + inline static oop extract_forwarding_pointer(size_t status) { + return (oop) (void*) (status << 8 >> 8); + } + inline static bool is_forwarded(size_t status) { + return (status >> 56) != 0; + } + inline virtual void do_oop(oop* slot) override { + const auto o = *slot; + if (o == NULL) return; + const auto status = read_forwarding_word(o); + if (is_forwarded(status)) { + *slot = extract_forwarding_pointer(status); + } + } + inline virtual void do_oop(narrowOop* slot) override { + narrowOop heap_oop = RawAccess<>::oop_load(slot); + if (CompressedOops::is_null(heap_oop)) return; + oop o = CompressedOops::decode_not_null(heap_oop); + const auto status = read_forwarding_word(o); + if (is_forwarded(status)) { + RawAccess<>::oop_store(slot, CompressedOops::encode_not_null(extract_forwarding_pointer(status))); + } + } +}; + +void MMTkCodeCacheUnloadingTask::work(uint worker_id) { + MMTkForwardClosure cl; + // The first nmethods is claimed by the first worker. + if (worker_id == 0 && _first_nmethod != nullptr) { + _first_nmethod->do_unloading(_unloading_occurred); + _first_nmethod = nullptr; } - return Atomic::cmpxchg(&_cleaning_claimed, 0, 1) == 0; -} - -void JVMCICleaningTask::work(bool unloading_occurred) { - // One worker will clean JVMCI metadata handles. - if (unloading_occurred && EnableJVMCI && claim_cleaning_task()) { - JVMCI::do_unloading(unloading_occurred); + int num_claimed_nmethods; + CompiledMethod* claimed_nmethods[MaxClaimNmethods]; + + while (true) { + claim_nmethods(claimed_nmethods, &num_claimed_nmethods); + + if (num_claimed_nmethods == 0) { + break; + } + + for (int i = 0; i < num_claimed_nmethods; i++) { + if (claimed_nmethods[i]->is_nmethod()) { + auto nm = (nmethod*) claimed_nmethods[i]; + nm->oops_do(&cl); + nm->fix_oop_relocations(); + } + } + for (int i = 0; i < num_claimed_nmethods; i++) { + claimed_nmethods[i]->do_unloading(_unloading_occurred); + } } } -#endif -ParallelCleaningTask::ParallelCleaningTask(BoolObjectClosure* is_alive, - OopClosure* forward, - uint num_workers, - bool unloading_occurred) : + +ParallelCleaningTask::ParallelCleaningTask(uint num_workers, bool unloading_occurred) : WorkerTask("Parallel Cleaning"), _unloading_occurred(unloading_occurred), _code_cache_task(num_workers, unloading_occurred), - JVMCI_ONLY(_jvmci_cleaning_task() COMMA) _klass_cleaning_task() { } // The parallel work done by all worker threads. void ParallelCleaningTask::work(uint worker_id) { - // Clean JVMCI metadata handles. - // Execute this task first because it is serial task. - JVMCI_ONLY(_jvmci_cleaning_task.work(_unloading_occurred);) - - // Do first pass of code cache cleaning. _code_cache_task.work(worker_id); - // Clean all klasses that were not unloaded. // The weak metadata in klass doesn't need to be // processed if there was no unloading. diff --git a/openjdk/share/mmtkParallelCleaning.hpp b/openjdk/share/mmtkParallelCleaning.hpp index f6530edf..3c5dde88 100644 --- a/openjdk/share/mmtkParallelCleaning.hpp +++ b/openjdk/share/mmtkParallelCleaning.hpp @@ -31,100 +31,38 @@ namespace mmtk { -// class CodeCacheUnloadingTask { -// private: -// static Monitor* _lock; +class MMTkCodeCacheUnloadingTask { + const bool _unloading_occurred; + const uint _num_workers; -// BoolObjectClosure* const _is_alive; -// const bool _unloading_occurred; -// const uint _num_workers; + // Variables used to claim nmethods. + CompiledMethod* _first_nmethod; + CompiledMethod* volatile _claimed_nmethod; -// // Variables used to claim nmethods. -// CompiledMethod* _first_nmethod; -// volatile CompiledMethod* _claimed_nmethod; - -// // The list of nmethods that need to be processed by the second pass. -// volatile CompiledMethod* _postponed_list; -// volatile uint _num_entered_barrier; - -// public: -// CodeCacheUnloadingTask(uint num_workers, BoolObjectClosure* is_alive, bool unloading_occurred); -// ~CodeCacheUnloadingTask(); - -// private: -// void add_to_postponed_list(CompiledMethod* nm); - -// void clean_nmethod(CompiledMethod* nm); - -// void clean_nmethod_postponed(CompiledMethod* nm); - -// static const int MaxClaimNmethods = 16; - -// void claim_nmethods(CompiledMethod** claimed_nmethods, int *num_claimed_nmethods); - -// CompiledMethod* claim_postponed_nmethod(); - -// public: -// // Mark that we're done with the first pass of nmethod cleaning. -// void barrier_mark(uint worker_id); - -// // See if we have to wait for the other workers to -// // finish their first-pass nmethod cleaning work. -// void barrier_wait(uint worker_id); - -// // Cleaning and unloading of nmethods. Some work has to be postponed -// // to the second pass, when we know which nmethods survive. -// void work_first_pass(uint worker_id); - -// void work_second_pass(uint worker_id); -// }; - -// class KlassCleaningTask : public StackObj { -// BoolObjectClosure* _is_alive; -// volatile int _clean_klass_tree_claimed; -// ClassLoaderDataGraphKlassIteratorAtomic _klass_iterator; - -// public: -// KlassCleaningTask(BoolObjectClosure* is_alive); - -// private: -// bool claim_clean_klass_tree_task(); - -// InstanceKlass* claim_next_klass(); - -// public: +public: + MMTkCodeCacheUnloadingTask(uint num_workers, bool unloading_occurred); -// void clean_klass(InstanceKlass* ik); + ~MMTkCodeCacheUnloadingTask(); -// void work(); -// }; -#if INCLUDE_JVMCI -class JVMCICleaningTask : public StackObj { - volatile int _cleaning_claimed; +private: + static const int MaxClaimNmethods = 16; + void claim_nmethods(CompiledMethod** claimed_nmethods, int *num_claimed_nmethods); public: - JVMCICleaningTask(); - // Clean JVMCI metadata handles. - void work(bool unloading_occurred); - -private: - bool claim_cleaning_task(); + // Cleaning and unloading of nmethods. + void work(uint worker_id); }; -#endif // To minimize the remark pause times, the tasks below are done in parallel. class ParallelCleaningTask : public WorkerTask { private: bool _unloading_occurred; - CodeCacheUnloadingTask _code_cache_task; -#if INCLUDE_JVMCI - JVMCICleaningTask _jvmci_cleaning_task; -#endif + MMTkCodeCacheUnloadingTask _code_cache_task; KlassCleaningTask _klass_cleaning_task; public: // The constructor is run in the VMThread. - ParallelCleaningTask(BoolObjectClosure* is_alive, OopClosure* forward, uint num_workers, bool unloading_occurred); + ParallelCleaningTask(uint num_workers, bool unloading_occurred); // The parallel work done by all worker threads. void work(uint worker_id); diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index 1412a4db..bdd0af17 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -151,30 +151,6 @@ class MMTkLXRFastUpdateClosure : public OopClosure { } }; -// Forward-only closure: forwards alive oops but does NOT null dead oops. -// Used during fix_oop_relocations when class unloading is pending, so that -// has_dead_oop() can still detect nmethods that should be unloaded. -class MMTkForwardOnlyClosure : public OopClosure { - public: - inline virtual void do_oop(oop* slot) override { - const auto o = *slot; - if (o == NULL) return; - const auto status = MMTkForwardClosure::read_forwarding_word(o); - if (MMTkForwardClosure::is_forwarded(status)) { - *slot = MMTkForwardClosure::extract_forwarding_pointer(status); - } - } - inline virtual void do_oop(narrowOop* slot) override { - narrowOop heap_oop = RawAccess<>::oop_load(slot); - if (CompressedOops::is_null(heap_oop)) return; - oop o = CompressedOops::decode_not_null(heap_oop); - const auto status = MMTkForwardClosure::read_forwarding_word(o); - if (MMTkForwardClosure::is_forwarded(status)) { - RawAccess<>::oop_store(slot, CompressedOops::encode_not_null(MMTkForwardClosure::extract_forwarding_pointer(status))); - } - } -}; - class MMTkUpdateClosure : public OopClosure { public: inline virtual void do_oop(oop* slot) override { @@ -252,17 +228,11 @@ static void mmtk_unload_classes() { true /* unregister_nmethods_during_purge */, false /* lock_codeblob_free_separately */); MMTkIsAliveClosure is_alive; - MMTkForwardClosure forward; { CodeCache::UnlinkingScope scope(&is_alive); bool unloading_occurred = SystemDictionary::do_unloading(NULL); - MMTkHeap::heap()->complete_cleaning(&is_alive, &forward, unloading_occurred); + MMTkHeap::heap()->complete_cleaning(unloading_occurred); } - // LOG_CLS_UNLOAD("[mmtk_unload_classes] forward code cache ptrs"); - // CodeBlobToOopClosure cb_cl(&forward, true); - // CodeCache::blobs_do(&cb_cl); - // LOG_CLS_UNLOAD("[mmtk_unload_classes] complete_cleaning"); - // MMTkHeap::heap()->complete_cleaning(&is_alive, &forward, purged_classes); LOG_CLS_UNLOAD("[mmtk_unload_classes] ClassLoaderDataGraph::purge"); ctx.purge_nmethods(); @@ -286,7 +256,7 @@ static void mmtk_gc_epilogue() { #if COMPILER2_OR_JVMCI DerivedPointerTable::update_pointers(); #endif - // CodeCache::arm_all_nmethods(); + CodeCache::arm_all_nmethods(); } static void mmtk_resume_mutators(void *tls) { @@ -600,17 +570,8 @@ static size_t mmtk_java_lang_classloader_loader_data_offset() { return v; } -void mmtk_fix_oop_relocations(bool lxr, bool forward_only, void* nmethods, size_t len) { - if (forward_only) { - // Forward-only: forward alive oops but don't null dead oops. - // Used when class unloading is pending so has_dead_oop() can detect dead nmethods. - MMTkForwardOnlyClosure cl; - for (size_t i = 0; i < len; i++) { - nmethod* nm = (nmethod*) ((nmethod**) nmethods)[i]; - nm->oops_do(&cl); - nm->fix_oop_relocations(); - } - } else if (lxr) { +void mmtk_fix_oop_relocations(bool lxr, void* nmethods, size_t len) { + if (lxr) { MMTkLXRFastUpdateClosure cl; for (size_t i = 0; i < len; i++) { nmethod* nm = (nmethod*) ((nmethod**) nmethods)[i]; From 522bfa92ea43991e5cdc3192408477c2bc00fc27 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 1 Apr 2026 14:22:27 +1100 Subject: [PATCH 456/494] cleanup --- mmtk/src/collection.rs | 3 --- mmtk/src/lib.rs | 5 ----- 2 files changed, 8 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index df82631a..fbe97c1a 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -31,9 +31,6 @@ impl Collection> for VMCollection { } fn resume_mutators(tls: VMWorkerThread) { - if cfg!(feature = "object_size_distribution") { - crate::dump_and_reset_obj_dist(); - } DISCOVERED_LISTS.enable_discover(); if *crate::singleton::().get_options().plan == mmtk::util::options::PlanSelector::ConcurrentImmix diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 0f64007b..6d2cf1dd 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -314,11 +314,6 @@ fn record_alloc(size: usize) { .or_insert((1, size)); } -extern "C" fn dump_and_reset_obj_dist() { - assert!(cfg!(feature = "object_size_distribution")); - mmtk::dump_and_reset_obj_dist("Dynamic", &mut OBJ_COUNT.lock().unwrap()); -} - fn set_compressed_pointer_vm_layout(builder: &mut MMTKBuilder) { let max_heap_size = builder.options.gc_trigger.max_heap_size(); assert!( From eaff406c77ef16d133eebd4e3d554971c4e419d9 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 1 Apr 2026 14:44:11 +1100 Subject: [PATCH 457/494] fix --- mmtk/src/reference_glue.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 1e14c0e8..3749db45 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -17,7 +17,6 @@ use mmtk::MMTK; fn set_referent(reff: ObjectReference, referent: Option) { let oop = Oop::from(reff); let slot = InstanceRefKlass::referent_address::(oop); - mmtk::plan::lxr::record_slot_for_validation(slot, referent); slot.store(referent) } @@ -42,7 +41,6 @@ fn set_next_reference( next: Option, ) { let slot = get_next_reference_slot::(object); - mmtk::plan::lxr::record_slot_for_validation(slot, next); slot.store(next) } From 05a1d5526c43a9507bc4942ba6d5b5b1218a653d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 1 Apr 2026 15:02:27 +1100 Subject: [PATCH 458/494] cleanup --- mmtk/Cargo.toml | 1 - mmtk/src/api.rs | 3 --- mmtk/src/lib.rs | 15 +-------------- 3 files changed, 1 insertion(+), 18 deletions(-) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 08ca0a89..926ea398 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -100,7 +100,6 @@ instrumentation = ["mmtk/instrumentation", "no_fast_alloc"] log_gc = ["mmtk/log_gc"] log_stages = ["mmtk/log_stages"] log_work_packets = ["mmtk/log_work_packets"] -object_size_distribution = ["mmtk/object_size_distribution"] discontig_space = [] # Compressed pointer experiments diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 3242396a..cecbc36f 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -189,9 +189,6 @@ pub extern "C" fn alloc( offset: usize, allocator: AllocationSemantics, ) -> Address { - if cfg!(feature = "object_size_distribution") { - crate::record_alloc(size); - } with_mutator!(|mutator| memory_manager::alloc(mutator, size, align, offset, allocator)) } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 6d2cf1dd..bb58ca5e 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -180,8 +180,7 @@ pub static FREE_LIST_ALLOCATOR_SIZE: uintptr_t = std::mem::size_of::>>(); #[no_mangle] -pub static DISABLE_ALLOCATION_FAST_PATH: i32 = - (cfg!(feature = "no_fast_alloc") || cfg!(feature = "object_size_distribution")) as _; +pub static DISABLE_ALLOCATION_FAST_PATH: i32 = cfg!(feature = "no_fast_alloc") as _; #[no_mangle] pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = @@ -302,18 +301,6 @@ lazy_static! { static ref OBJ_COUNT: Mutex> = Mutex::new(HashMap::new()); } -fn record_alloc(size: usize) { - assert!(cfg!(feature = "object_size_distribution")); - let mut counts = OBJ_COUNT.lock().unwrap(); - counts - .entry(size.next_power_of_two()) - .and_modify(|x| { - x.0 += 1; - x.1 += size; - }) - .or_insert((1, size)); -} - fn set_compressed_pointer_vm_layout(builder: &mut MMTKBuilder) { let max_heap_size = builder.options.gc_trigger.max_heap_size(); assert!( From 6bd3a646dc9aef289f4400337585424ef8fb20c1 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 1 Apr 2026 15:34:39 +1100 Subject: [PATCH 459/494] Fix --- mmtk/src/api.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index cecbc36f..48e012c5 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -553,7 +553,7 @@ pub extern "C" fn mmtk_is_live(object: NullableObjectReference) -> usize { "{:?} is not mapped", object ); - object.is_live2() as _ + object.is_live() as _ } /// If the object is non-null and forwarded, return the forwarded pointer. Otherwise, return the original pointer. @@ -565,7 +565,7 @@ pub extern "C" fn mmtk_get_forwarded_ref( let Some(o) = o else { return ObjectReference::NULL.into(); }; - match o.get_forwarded_object2() { + match o.get_forwarded_object() { Some(o) => Some(o).into(), None => object, } From c7018d077e67f5a53c8df103c4915697f869ff6a Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 1 Apr 2026 16:56:56 +1100 Subject: [PATCH 460/494] Fix --- mmtk/src/lib.rs | 108 +++--------------------------------------------- 1 file changed, 6 insertions(+), 102 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index bb58ca5e..08c7b4f8 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -254,11 +254,8 @@ lazy_static! { }; pub static ref SINGLETON_UNCOMPRESSED: MMTK> = { assert!(!use_compressed_oops()); - let mut builder = BUILDER.lock().unwrap(); + let builder = BUILDER.lock().unwrap(); assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); - if cfg!(feature = "discontig_space") { - set_no_compressed_pointer_discontig_vm_layout(&mut builder); - } let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); unsafe { @@ -307,111 +304,18 @@ fn set_compressed_pointer_vm_layout(builder: &mut MMTKBuilder) { max_heap_size <= (32usize << LOG_BYTES_IN_GBYTE), "Heap size is larger than 32 GB" ); - let rounded_heap_size = (max_heap_size + (BYTES_IN_CHUNK - 1)) & !(BYTES_IN_CHUNK - 1); - let mut start: usize = 0x4000_0000; // block lowest 1G - let (end, mut small_chunk_space_size) = if cfg!(feature = "force_narrow_oop_mode") { - assert!(rounded_heap_size <= (2 << 30)); - let heap = rounded_heap_size; - if cfg!(feature = "narrow_oop_mode_32bit") { - let end = 4usize << 30; - let small_space = 2 << 30; - (end, small_space) - } else if cfg!(feature = "narrow_oop_mode_shift") { - let end = 32usize << 30; - let small_space = usize::min(heap * 3 / 2, 29 << 30); - (end, small_space) - } else if cfg!(feature = "narrow_oop_mode_base") { - // start = 0x200_0000_0000; - // let end = start + ((4usize << 30) - BYTES_IN_CHUNK); - // let small_space = 2 << 30; - // (end, small_space) - unreachable!() - } else if cfg!(feature = "narrow_oop_mode_base_and_shift") { - start = 0x200_0000_0000; - let end = start + 0x8_0000_0000 - BYTES_IN_CHUNK; - let small_space = usize::min(heap * 3 / 2, 30 << 30); - (end, small_space) - } else { - unreachable!() - } - } else { - match rounded_heap_size { - // heap <= 2G; virtual = 3G; max-small-space=2G; min-small-space=1.5G - heap if heap <= 2 << 30 => { - let end = 4usize << 30; - let small_space = 2 << 30; - (end, small_space) - } - // heap <= 29G; virtual = 31G; max-small-space=29G; - heap if heap <= 29 << 30 => { - let end = 32usize << 30; - let small_space = usize::min(heap * 3 / 2, 29 << 30); - (end, small_space) - } - // heap > 29G; virtual = 32G - 1chunk; max-small-space=30G; start=0x200_0000_0000 - heap => { - // A workaround to avoid address conflict with the OpenJDK - // MetaSpace, which may start from 0x8_0000_0000 - start = 0x200_0000_0000; - let end = start + 0x8_0000_0000 - BYTES_IN_CHUNK; - let small_space = usize::min(heap * 3 / 2, 30 << 30); - (end, small_space) - } - } - }; - small_chunk_space_size = - (small_chunk_space_size + (BYTES_IN_CHUNK - 1)) & !(BYTES_IN_CHUNK - 1); - let constants = VMLayout { - log_address_space: 35, - heap_start: conversions::chunk_align_down(unsafe { Address::from_usize(start) }), - heap_end: conversions::chunk_align_up(unsafe { Address::from_usize(end) }), - log_space_extent: 31, - force_use_contiguous_spaces: false, - small_chunk_space_size: Some(small_chunk_space_size), - }; - builder.set_vm_layout(constants); -} - -fn set_no_compressed_pointer_discontig_vm_layout(builder: &mut MMTKBuilder) { - let max_heap_size = builder.options.gc_trigger.max_heap_size(); - assert!( - max_heap_size <= (32usize << LOG_BYTES_IN_GBYTE), - "Heap size is larger than 32 GB" - ); - let rounded_heap_size = (max_heap_size + (BYTES_IN_CHUNK - 1)) & !(BYTES_IN_CHUNK - 1); - let mut start: usize = 0x4000_0000; // block lowest 1G - let (end, mut small_chunk_space_size) = match rounded_heap_size { - // heap <= 2G; virtual = 3G; max-small-space=2G; min-small-space=1.5G - heap if heap <= 2 << 30 => { - let end = 4usize << 30; - let small_space = 2 << 30; - (end, small_space) - } - // heap <= 29G; virtual = 31G; max-small-space=29G; - heap if heap <= 29 << 30 => { - let end = 32usize << 30; - let small_space = usize::min(heap * 3 / 2, 29 << 30); - (end, small_space) - } - // heap > 29G; virtual = 32G - 1chunk; max-small-space=30G; start=0x200_0000_0000 - heap => { - // A workaround to avoid address conflict with the OpenJDK - // MetaSpace, which may start from 0x8_0000_0000 - start = 0x200_0000_0000; - let end = start + 0x8_0000_0000 - BYTES_IN_CHUNK; - let small_space = usize::min(heap * 3 / 2, 30 << 30); - (end, small_space) - } + let start = 0x4000_0000; + let end = match start + max_heap_size { + end if end <= (4usize << 30) => 4usize << 30, + end if end <= (32usize << 30) => 32usize << 30, + _ => 0x4000_0000 + (32usize << 30), }; - small_chunk_space_size = - (small_chunk_space_size + (BYTES_IN_CHUNK - 1)) & !(BYTES_IN_CHUNK - 1); let constants = VMLayout { log_address_space: 35, heap_start: conversions::chunk_align_down(unsafe { Address::from_usize(start) }), heap_end: conversions::chunk_align_up(unsafe { Address::from_usize(end) }), log_space_extent: 31, force_use_contiguous_spaces: false, - small_chunk_space_size: Some(small_chunk_space_size), }; builder.set_vm_layout(constants); } From f3d407bd1224f71c640d7e13a159aee06ee8eba6 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 1 Apr 2026 21:35:43 +1100 Subject: [PATCH 461/494] Fix assertions --- mmtk/src/collection.rs | 1 - openjdk/share/mmtkParallelCleaning.cpp | 1 - openjdk/share/mmtkUpcalls.cpp | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index fbe97c1a..85cf03fa 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -112,7 +112,6 @@ impl Collection> for VMCollection { fn vm_release(do_unloading: bool) { unsafe { - ((*UPCALLS).oops_do_marking_epilogue)(); if do_unloading { gc_log!(" - unload_classes"); ((*UPCALLS).unload_classes)(); diff --git a/openjdk/share/mmtkParallelCleaning.cpp b/openjdk/share/mmtkParallelCleaning.cpp index 73fef671..024a9ff0 100644 --- a/openjdk/share/mmtkParallelCleaning.cpp +++ b/openjdk/share/mmtkParallelCleaning.cpp @@ -57,7 +57,6 @@ MMTkCodeCacheUnloadingTask::MMTkCodeCacheUnloadingTask(uint num_workers, bool un } MMTkCodeCacheUnloadingTask::~MMTkCodeCacheUnloadingTask() { - CodeCache::verify_clean_inline_caches(); CodeCache::verify_icholder_relocations(); } diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index bdd0af17..bfaffc74 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -260,6 +260,7 @@ static void mmtk_gc_epilogue() { } static void mmtk_resume_mutators(void *tls) { + nmethod::oops_do_marking_epilogue(); // Note: we don't have to hold gc_lock to increment the counter. // The increment has to be done before mutators can be resumed (from `block_for_gc` or yieldpoints). // Otherwise, mutators might see an outdated start-the-world count. From 8e2fac3ffeff31c8342e19cbfad0a12b741ab883 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 3 Apr 2026 19:08:43 +1100 Subject: [PATCH 462/494] Revert block allocator changes --- openjdk/share/mmtkMutator.hpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/openjdk/share/mmtkMutator.hpp b/openjdk/share/mmtkMutator.hpp index 602a0f98..b0f092d8 100644 --- a/openjdk/share/mmtkMutator.hpp +++ b/openjdk/share/mmtkMutator.hpp @@ -56,12 +56,7 @@ struct ImmixAllocator { uint8_t _align[7]; uint8_t line_opt_tag; uintptr_t line_opt; - uint8_t block_opt_tag; - uintptr_t block_opt; - uint8_t large_block_opt_tag; - uintptr_t large_block_opt; uintptr_t mutator_recycled_blocks; - uintptr_t local_blocks[6]; uintptr_t mutator_recycled_lines; uint8_t retry; }; From cf6fbf22577d8b461031dfa58449ff0bdeb862ae Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Fri, 3 Apr 2026 23:50:22 +1100 Subject: [PATCH 463/494] cleanup --- mmtk/src/api.rs | 58 +++++-------------- mmtk/src/reference_glue.rs | 12 +++- mmtk/src/slots.rs | 23 +++++--- .../cpu/x86/mmtkBarrierSetAssembler_x86.cpp | 2 +- openjdk/share/mmtk.h | 6 -- openjdk/share/mmtkBarrierSet.cpp | 6 +- openjdk/share/mmtkBarrierSet.hpp | 9 +-- openjdk/share/mmtkHeap.cpp | 13 +---- 8 files changed, 48 insertions(+), 81 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 48e012c5..4fe7a73a 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -1,4 +1,3 @@ -use crate::abi::Oop; use crate::slots::OpenJDKSlot; use crate::OpenJDK; use crate::OpenJDK_Upcalls; @@ -257,12 +256,6 @@ pub extern "C" fn total_bytes() -> usize { with_singleton!(|singleton| memory_manager::total_bytes(singleton)) } -#[no_mangle] -#[cfg(feature = "sanity")] -pub extern "C" fn scan_region() { - with_singleton!(|singleton| memory_manager::scan_region(singleton)) -} - #[no_mangle] pub extern "C" fn handle_user_collection_request(tls: VMMutatorThread, force: bool) { with_singleton!(|singleton| { @@ -430,26 +423,18 @@ pub extern "C" fn mmtk_load_reference(o: ObjectReference, mutator: *mut libc::c_ with_mutator!(|mutator| mutator.barrier().load_weak_reference(o)) } -#[no_mangle] -pub extern "C" fn mmtk_object_reference_clone_pre( - mutator: *mut libc::c_void, - obj: ObjectReference, -) { - with_mutator!(|mutator| mutator.barrier().object_reference_clone_pre(obj)) -} - /// Full pre barrier #[no_mangle] pub extern "C" fn mmtk_object_reference_write_pre( mutator: *mut libc::c_void, - src: NullableObjectReference, + src: ObjectReference, slot: Address, target: NullableObjectReference, ) { with_mutator!(|mutator| { mutator .barrier() - .object_reference_write_pre(src.into(), slot.into(), target.into()); + .object_reference_write_pre(src, slot.into(), target.into()); }) } @@ -457,21 +442,21 @@ pub extern "C" fn mmtk_object_reference_write_pre( #[no_mangle] pub extern "C" fn mmtk_object_reference_write_post( mutator: *mut libc::c_void, - src: NullableObjectReference, + src: ObjectReference, slot: Address, target: NullableObjectReference, ) { with_mutator!(|mutator| { mutator .barrier() - .object_reference_write_post(src.into(), slot.into(), target.into()); + .object_reference_write_post(src, slot.into(), target.into()); }) } /// Barrier slow-path call #[no_mangle] pub extern "C" fn mmtk_object_reference_write_slow( - src: NullableObjectReference, + src: ObjectReference, slot: Address, target: NullableObjectReference, mutator: *mut libc::c_void, @@ -479,7 +464,7 @@ pub extern "C" fn mmtk_object_reference_write_slow( with_mutator!(|mutator| { mutator .barrier() - .object_reference_write_slow(src.into(), slot.into(), target.into()); + .object_reference_write_slow(src, slot.into(), target.into()); }) } @@ -531,8 +516,8 @@ pub extern "C" fn mmtk_object_probable_write(mutator: *mut libc::c_void, obj: Ob // finalization #[no_mangle] -pub extern "C" fn add_finalizer(_object: ObjectReference) { - // unreachable!() +pub extern "C" fn add_finalizer(object: ObjectReference) { + with_singleton!(|singleton| memory_manager::add_finalizer(singleton, object)); } #[no_mangle] @@ -608,23 +593,12 @@ pub extern "C" fn mmtk_register_nmethod(nm: Address) { /// Unregister an nmethod. #[no_mangle] pub extern "C" fn mmtk_unregister_nmethod(nm: Address) { - let mut roots = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); - roots.remove(&nm); - let mut roots = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); - roots.remove(&nm); -} - -#[no_mangle] -pub extern "C" fn mmtk_verbose() -> usize { - with_singleton!(|singleton| *singleton.options.verbose) -} - -#[no_mangle] -pub unsafe extern "C" fn mmtk_register_new_weak_handle(oop: *const Oop) { - // let addr = if crate::use_compressed_oops() { - // Address::from_usize(oop as usize | (1 << 63)) - // } else { - // Address::from_ptr(oop) - // }; - // crate::NURSERY_WEAK_HANDLE_ROOTS.lock().unwrap().push(addr); + { + let mut roots = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); + roots.remove(&nm); + } + { + let mut roots = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); + roots.remove(&nm); + } } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index 3749db45..d44848de 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -17,7 +17,11 @@ use mmtk::MMTK; fn set_referent(reff: ObjectReference, referent: Option) { let oop = Oop::from(reff); let slot = InstanceRefKlass::referent_address::(oop); - slot.store(referent) + if let Some(o) = referent { + slot.store(o); + } else { + slot.store_null(); + } } fn get_referent(object: ObjectReference) -> Option { @@ -41,7 +45,11 @@ fn set_next_reference( next: Option, ) { let slot = get_next_reference_slot::(object); - slot.store(next) + if let Some(o) = next { + slot.store(o); + } else { + slot.store_null(); + } } pub struct VMReferenceGlue {} diff --git a/mmtk/src/slots.rs b/mmtk/src/slots.rs index 5f435f4e..37e0f2db 100644 --- a/mmtk/src/slots.rs +++ b/mmtk/src/slots.rs @@ -142,10 +142,7 @@ impl OpenJDKSlot { } /// encode an object pointer to its u32 compressed form - fn compress(o: Option) -> u32 { - let Some(o) = o else { - return 0; - }; + fn compress(o: ObjectReference) -> u32 { ((o.to_raw_address() - BASE.load(Ordering::Relaxed)) >> SHIFT.load(Ordering::Relaxed)) as u32 } @@ -204,16 +201,16 @@ impl Slot for OpenJDKSlot { } } - fn store(&self, object: Option) { + fn store(&self, object: ObjectReference) { if cfg!(any(target_arch = "x86", target_arch = "x86_64")) { if COMPRESSED { if self.is_compressed() { self.x86_write_unaligned::(Self::compress(object)) } else { - self.x86_write_unaligned::, true>(object) + self.x86_write_unaligned::(object) } } else { - self.x86_write_unaligned::, false>(object) + self.x86_write_unaligned::(object) } } else { debug_assert!(!COMPRESSED); @@ -242,8 +239,16 @@ impl Slot for OpenJDKSlot { ) -> Result, Option> { if COMPRESSED { if self.is_compressed() { - let old_value = Self::compress(old_object); - let new_value = Self::compress(new_object); + let old_value = if let Some(o) = old_object { + Self::compress(o) + } else { + 0 + }; + let new_value = if let Some(o) = new_object { + Self::compress(o) + } else { + 0 + }; let slot = self.untagged_address(); unsafe { match slot.compare_exchange::(old_value, new_value, success, failure) diff --git a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp index b9573b47..70332064 100644 --- a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp @@ -165,7 +165,7 @@ void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sa if (do_code_patch) { // We don't know the field offset when a code patching is required. // As a temporary fix, we apply field barrier to all fields in this object. - __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_clone_pre_call), c_rarg0); + __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_probable_write_pre_call), c_rarg0); } else { __ call_VM_leaf_base(entry_point, 3); } diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index 53ad54f6..f7a1294d 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -93,8 +93,6 @@ extern void mmtk_array_copy_pre(void* src, void* dst, size_t count, MMTk_Mutator /// Full array-copy post-barrier extern void mmtk_array_copy_post(MMTk_Mutator mutator, void* src, void* dst, size_t count); -extern void mmtk_object_reference_clone_pre(MMTk_Mutator mutator, void* obj); - /// C2 slowpath allocation barrier extern void mmtk_object_probable_write(MMTk_Mutator mutator, void* obj); @@ -147,10 +145,6 @@ extern size_t mmtk_add_nmethod_oop(void* object); extern size_t mmtk_register_nmethod(void* nm); extern size_t mmtk_unregister_nmethod(void* nm); -extern size_t mmtk_register_new_weak_handle(void* entry); - -extern size_t mmtk_verbose(); - /** * VM Accounting */ diff --git a/openjdk/share/mmtkBarrierSet.cpp b/openjdk/share/mmtkBarrierSet.cpp index e3ba0c90..7254d33c 100644 --- a/openjdk/share/mmtkBarrierSet.cpp +++ b/openjdk/share/mmtkBarrierSet.cpp @@ -177,6 +177,6 @@ void MMTkBarrierSetRuntime::load_reference_call(void* ref) { ::mmtk_load_reference(ref, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } -void MMTkBarrierSetRuntime::object_reference_clone_pre_call(void* ref) { - ::mmtk_object_reference_clone_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); -} +void MMTkBarrierSetRuntime::object_probable_write_pre_call(void* ref) { + ::mmtk_object_probable_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); +} \ No newline at end of file diff --git a/openjdk/share/mmtkBarrierSet.hpp b/openjdk/share/mmtkBarrierSet.hpp index 37b71311..90e3a05c 100644 --- a/openjdk/share/mmtkBarrierSet.hpp +++ b/openjdk/share/mmtkBarrierSet.hpp @@ -81,7 +81,7 @@ class MMTkBarrierSetRuntime: public CHeapObj { static void object_reference_array_copy_pre_call(void* src, void* dst, size_t count); /// Generic arraycopy pre-barrier. Called by fast-paths. static void object_reference_array_copy_post_call(void* src, void* dst, size_t count); - static void object_reference_clone_pre_call(void* obj); + static void object_probable_write_pre_call(void* obj); /// Check if the address is a slow-path function. virtual bool is_slow_path_call(address call) const { return call == CAST_FROM_FN_PTR(address, object_reference_write_pre_call) @@ -91,9 +91,9 @@ class MMTkBarrierSetRuntime: public CHeapObj { || call == CAST_FROM_FN_PTR(address, mmtk_array_copy_pre) || call == CAST_FROM_FN_PTR(address, mmtk_object_reference_write_slow) || call == CAST_FROM_FN_PTR(address, mmtk_load_reference) + || call == CAST_FROM_FN_PTR(address, object_probable_write_pre_call) || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_post_call) - || call == CAST_FROM_FN_PTR(address, load_reference_call) - || call == CAST_FROM_FN_PTR(address, object_reference_clone_pre_call); + || call == CAST_FROM_FN_PTR(address, load_reference_call); } /// Full pre-barrier @@ -106,8 +106,6 @@ class MMTkBarrierSetRuntime: public CHeapObj { virtual void object_reference_array_copy_post(oop* src, oop* dst, size_t count) const {}; /// java.lang.Reference load barrier virtual void load_reference(DecoratorSet decorators, oop value) const {}; - /// Object clone pre-barrier - virtual void clone_pre(DecoratorSet decorators, oop value) const {}; /// Called at the end of every C2 slowpath allocation. /// Deoptimization can happen after C2 slowpath allocation, and the newly allocated object can be promoted. /// So this callback is requierd for any generational collectors. @@ -278,7 +276,6 @@ class MMTkBarrierSet : public BarrierSet { static void clone_in_heap(oop src, oop dst, size_t size) { // TODO: We don't need clone barriers at the moment. - runtime()->clone_pre(decorators, dst); Raw::clone_in_heap(src, dst, size); } }; diff --git a/openjdk/share/mmtkHeap.cpp b/openjdk/share/mmtkHeap.cpp index 35032e37..2f469b93 100644 --- a/openjdk/share/mmtkHeap.cpp +++ b/openjdk/share/mmtkHeap.cpp @@ -206,13 +206,6 @@ jint MMTkHeap::initialize() { fprintf(stderr, "Failed to create thread"); guarantee(false, "panic"); } - if (mmtk_verbose() > 0) { - if (ClassUnloading) { - fprintf(stderr, "ClassUnloading Enabled\n"); - } else { - fprintf(stderr, "ClassUnloading Disabled\n"); - } - } unsigned int ncpus = (unsigned int) os::initial_active_processor_count(); _workers = new WorkerThreads("GC Thread", ncpus); @@ -587,11 +580,7 @@ void MMTkHeap::complete_cleaning(bool purged_classes) { _workers->run_task(&unlink_task); } -void MMTkHeap::register_new_weak_handle(oop* handle) { - if (REQUIRES_WEAK_HANDLE_BARRIER) { - mmtk_register_new_weak_handle((void*) handle); - } -} +void MMTkHeap::register_new_weak_handle(oop* handle) {} bool MMTkHeap::requires_barriers(stackChunkOop obj) const { ShouldNotReachHere(); From ae7b8442aa2577211e852ba63d1e80ebd5092ddb Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 4 Apr 2026 15:35:27 +1100 Subject: [PATCH 464/494] Revert class unloading support --- mmtk/src/abi.rs | 100 +-------------- mmtk/src/build_info.rs | 8 -- mmtk/src/collection.rs | 22 +--- mmtk/src/gc_work.rs | 170 ++----------------------- mmtk/src/lib.rs | 43 +------ mmtk/src/object_scanning.rs | 98 ++------------ mmtk/src/scanning.rs | 20 +-- mmtk/src/slots.rs | 12 -- openjdk/share/mmtk.h | 10 +- openjdk/share/mmtkHeap.cpp | 28 +--- openjdk/share/mmtkHeap.hpp | 4 +- openjdk/share/mmtkParallelCleaning.cpp | 167 ------------------------ openjdk/share/mmtkParallelCleaning.hpp | 73 ----------- openjdk/share/mmtkUpcalls.cpp | 64 +--------- 14 files changed, 47 insertions(+), 772 deletions(-) delete mode 100644 openjdk/share/mmtkParallelCleaning.cpp delete mode 100644 openjdk/share/mmtkParallelCleaning.hpp diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index df78b103..daa2146c 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -1,17 +1,13 @@ use super::UPCALLS; use crate::OpenJDKSlot; -use crate::Slot; use atomic::Atomic; use atomic::Ordering; use mmtk::util::constants::*; use mmtk::util::conversions; use mmtk::util::ObjectReference; use mmtk::util::{Address, OpaquePointer}; -use mmtk::vm::SlotVisitor; use std::ffi::CStr; use std::fmt; -use std::sync::atomic::AtomicPtr; -use std::sync::atomic::AtomicU32; use std::sync::atomic::AtomicUsize; use std::{mem, slice}; @@ -67,7 +63,7 @@ pub enum BasicType { #[repr(C)] pub struct Klass { vptr: OpaquePointer, - #[cfg(all(debug_assertions, not(feature = "release_debug_assertions")))] + #[cfg(debug_assertions)] valid: i32, pub layout_helper: i32, pub kind: KlassKind, @@ -82,7 +78,7 @@ pub struct Klass { pub subklass: &'static Klass, pub next_sibling: &'static Klass, pub next_link: &'static Klass, - pub class_loader_data: &'static ClassLoaderData, + pub class_loader_data: OpaquePointer, // ClassLoaderData* pub bitmap: usize, pub hash_slot: u8, pub vtable_len: i32, @@ -164,7 +160,7 @@ pub struct InstanceKlass { // #if INCLUDE_JVMTI pub jvmti_cached_class_field_map: OpaquePointer, // JvmtiCachedClassFieldMap* // #endif - #[cfg(all(debug_assertions, not(feature = "release_debug_assertions")))] + #[cfg(debug_assertions)] verify_count: i32, #[cfg(debug_assertions)] _shared_class_load_count: i32, @@ -192,7 +188,6 @@ pub enum ReferenceType { impl InstanceKlass { const HEADER_SIZE: usize = mem::size_of::() / BYTES_IN_WORD; const VTABLE_START_OFFSET: usize = Self::HEADER_SIZE * BYTES_IN_WORD; - const MISC_IS_ANONYMOUS: u16 = 1 << 5; fn start_of_vtable(&self) -> *const usize { (Address::from_ref(self) + Self::VTABLE_START_OFFSET).to_ptr() @@ -215,10 +210,6 @@ impl InstanceKlass { let count = self.nonstatic_oop_map_count(); unsafe { slice::from_raw_parts(start, count) } } - - pub fn is_anonymous(&self) -> bool { - self.misc_flags & Self::MISC_IS_ANONYMOUS != 0 - } } #[repr(C)] @@ -373,7 +364,6 @@ impl fmt::Debug for OopDesc { /// 32-bit compressed klass pointers #[repr(transparent)] -#[allow(unused)] #[derive(Clone, Copy)] pub struct NarrowKlass(u32); @@ -381,7 +371,6 @@ pub type Oop = &'static OopDesc; /// 32-bit compressed reference pointers #[repr(transparent)] -#[allow(unused)] #[derive(Clone, Copy)] pub struct NarrowOop(u32); @@ -540,86 +529,3 @@ pub fn validate_memory_layouts() { } assert_eq!(vm_checksum, binding_checksum); } - -#[repr(C)] -#[derive(Debug)] -struct Chunk { - data: [*mut OopDesc; 32], - size: AtomicU32, - next: *mut Chunk, -} - -#[repr(C)] -#[derive(Debug)] -struct ChunkedHandleList { - head: AtomicPtr, -} - -impl ChunkedHandleList { - unsafe fn oops_do_chunk, const COMPRESSED: bool>( - &self, - chunk: &'static Chunk, - size: u32, - closure: &mut V, - ) { - for i in 0..size { - if !chunk.data[i as usize].is_null() { - let mut word = - Address::from_ref::<*mut OopDesc>(&chunk.data[i as usize]).as_usize(); - if COMPRESSED { - word = word | (1usize << 63); - } - closure.visit_slot(S::from_address(Address::from_usize(word)), true) - } - } - } - - fn oops_do, const COMPRESSED: bool>(&self, closure: &mut V) { - let head = self.head.load(Ordering::Acquire); - if !head.is_null() { - let head = unsafe { &*head }; - let size = head.size.load(Ordering::Acquire); - unsafe { self.oops_do_chunk::<_, _, COMPRESSED>(head, size, closure) }; - let mut c = head.next; - while !c.is_null() { - let chunk = unsafe { &*c }; - let size = chunk.size.load(Ordering::Relaxed); - unsafe { self.oops_do_chunk::<_, _, COMPRESSED>(chunk, size, closure) }; - c = chunk.next; - } - } - } -} - -#[repr(C)] -#[derive(Debug)] -pub struct ClassLoaderData { - holder: OpaquePointer, // WeakHandle - class_loader: OpaquePointer, // OopHandle - metaspace: OpaquePointer, // ClassLoaderMetaspace*volatile - metaspace_lock: OpaquePointer, // Mutex* - unloading: u8, // bool - is_anonymous: u8, // bool - modified_oops: u8, // bool - keep_alive: i32, - claimed: AtomicU32, - handles: ChunkedHandleList, -} - -impl ClassLoaderData { - fn claim(&self) -> bool { - if self.claimed.load(Ordering::Relaxed) == 1 { - return false; - } - self.claimed - .compare_exchange(0, 1, Ordering::SeqCst, Ordering::SeqCst) - .is_ok() - } - - pub fn oops_do, const COMPRESSED: bool>(&self, closure: &mut V) { - if closure.should_claim_clds() && !self.claim() { - return; - } - self.handles.oops_do::<_, _, COMPRESSED>(closure); - } -} diff --git a/mmtk/src/build_info.rs b/mmtk/src/build_info.rs index 0ddf4051..1dcc9ae1 100644 --- a/mmtk/src/build_info.rs +++ b/mmtk/src/build_info.rs @@ -1,16 +1,8 @@ -#[cfg(not(feature = "g3_compatibilities"))] mod raw { // The include imports a full list of the constants in built.rs from https://docs.rs/built/latest/built/index.html include!(concat!(env!("OUT_DIR"), "/built.rs")); } -#[cfg(feature = "g3_compatibilities")] -mod raw { - pub const PKG_VERSION: &str = ""; - pub const GIT_COMMIT_HASH: Option<&str> = None; - pub const GIT_DIRTY: Option = None; -} - lazy_static! { // Owned string for the binding version, such as MMTk OpenJDK 0.14.0 (cfc755f-dirty) static ref BINDING_VERSION_STRING: String = match (raw::GIT_COMMIT_HASH, raw::GIT_DIRTY) { diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 85cf03fa..047289fa 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -1,4 +1,3 @@ -use mmtk::gc_log; use mmtk::scheduler::{GCWorker, ProcessEdgesWork}; use mmtk::util::alloc::AllocationError; use mmtk::util::opaque_pointer::*; @@ -14,18 +13,14 @@ pub struct VMCollection {} const GC_THREAD_KIND_WORKER: libc::c_int = 1; impl Collection> for VMCollection { - fn stop_all_mutators( - tls: VMWorkerThread, - mut mutator_visitor: F, - current_gc_should_unload_classes: bool, - ) where + fn stop_all_mutators(tls: VMWorkerThread, mut mutator_visitor: F) + where F: FnMut(&'static mut Mutator>), { unsafe { ((*UPCALLS).stop_all_mutators)( tls, MutatorClosure::from_rust_closure::<_, COMPRESSED>(&mut mutator_visitor), - current_gc_should_unload_classes, ); } } @@ -100,23 +95,12 @@ impl Collection> for VMCollection { } } - fn clear_cld_claimed_marks() { - unsafe { - ((*UPCALLS).clear_claimed_marks)(); - } - } - fn set_concurrent_marking_state(active: bool) { unsafe { crate::CONCURRENT_MARKING_ACTIVE = if active { 1 } else { 0 } } } - fn vm_release(do_unloading: bool) { + fn vm_release() { unsafe { - if do_unloading { - gc_log!(" - unload_classes"); - ((*UPCALLS).unload_classes)(); - } - gc_log!(" - gc_epilogue"); ((*UPCALLS).gc_epilogue)(); } } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 4e7991fb..df3c8c7a 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -13,26 +13,6 @@ use mmtk::vm::RootsWorkFactory; use mmtk::vm::*; use mmtk::MMTK; use std::marker::PhantomData; -use std::sync::atomic::{AtomicUsize, Ordering}; - -thread_local! { - pub static COUNT: AtomicUsize = AtomicUsize::new(0); -} - -pub fn record_roots(len: usize) { - super::gc_work::COUNT.with(|x| { - let c = x.load(Ordering::Relaxed); - x.store(c + len, Ordering::Relaxed); - }); -} - -fn report_roots(name: &str, ms: f32) { - super::gc_work::COUNT.with(|x| { - let c = x.load(Ordering::Relaxed); - eprintln!(" - {} roots count: {} ({:.3}ms)", name, c, ms); - x.store(0, Ordering::Relaxed); - }); -} macro_rules! scan_roots_work { ($struct_name: ident, $func_name: ident) => { @@ -52,126 +32,21 @@ macro_rules! scan_roots_work { impl> GCWork for $struct_name { fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { - let t = if cfg!(feature = "roots_breakdown") { - Some(std::time::SystemTime::now()) - } else { - None - }; unsafe { ((*UPCALLS).$func_name)(to_slots_closure(&mut self.factory)); } - if cfg!(feature = "roots_breakdown") { - let name = stringify!($struct_name); - let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; - report_roots(&name[4..name.len() - 5], ms); - } } } }; } +scan_roots_work!( + ScanClassLoaderDataGraphRoots, + scan_class_loader_data_graph_roots +); scan_roots_work!(ScanOopStorageSetRoots, scan_oop_storage_set_roots); scan_roots_work!(ScanVMThreadRoots, scan_vm_thread_roots); -extern "C" fn report_slots_and_renew_buffer_cld< - S: Slot, - F: RootsWorkFactory, - const WEAK: bool, - const ALL_STRONG: bool, ->( - ptr: *mut Address, - length: usize, - capacity: usize, - factory_ptr: *mut libc::c_void, -) -> NewBuffer { - if !ptr.is_null() { - let ptr = ptr as *mut S; - let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; - if cfg!(feature = "roots_breakdown") { - record_roots(buf.len()); - } - let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; - let kind = if WEAK { - RootKind::YoungWeakCLDRoots - } else if ALL_STRONG { - RootKind::StrongCLDRoots - } else { - RootKind::YoungStrongCLDRoots - }; - factory.create_process_roots_work(buf, kind); - } - let (ptr, _, capacity) = { - // TODO: Use Vec::into_raw_parts() when the method is available. - use std::mem::ManuallyDrop; - let new_vec = Vec::with_capacity(F::BUFFER_SIZE); - let mut me = ManuallyDrop::new(new_vec); - (me.as_mut_ptr(), me.len(), me.capacity()) - }; - NewBuffer { ptr, capacity } -} - -fn to_slots_closure_cld< - S: Slot, - F: RootsWorkFactory, - const WEAK: bool, - const ALL_STRONG: bool, ->( - factory: &mut F, -) -> SlotsClosure { - SlotsClosure { - func: report_slots_and_renew_buffer_cld::, - data: factory as *mut F as *mut libc::c_void, - } -} - -pub struct ScanClassLoaderDataGraphRoots> { - factory: F, - _p: PhantomData, -} - -impl> ScanClassLoaderDataGraphRoots { - pub fn new(factory: F) -> Self { - Self { - factory, - _p: PhantomData, - } - } -} - -impl> GCWork - for ScanClassLoaderDataGraphRoots -{ - fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { - let t = if cfg!(feature = "roots_breakdown") { - Some(std::time::SystemTime::now()) - } else { - None - }; - let scan_all_strong_roots = mmtk.get_plan().current_gc_should_perform_class_unloading(); - if scan_all_strong_roots { - unsafe { - ((*UPCALLS).scan_class_loader_data_graph_roots)( - to_slots_closure_cld::(&mut self.factory), - to_slots_closure_cld::(&mut self.factory), - scan_all_strong_roots, - ); - } - } else { - unsafe { - ((*UPCALLS).scan_class_loader_data_graph_roots)( - to_slots_closure_cld::(&mut self.factory), - to_slots_closure_cld::(&mut self.factory), - scan_all_strong_roots, - ); - } - } - if cfg!(feature = "roots_breakdown") { - let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; - report_roots("ClassLoaderDataGraph", ms); - } - } -} - #[allow(unused)] pub struct ScanCodeCacheRoots>> { @@ -203,8 +78,6 @@ impl>> .downcast_ref::>>(); let is_lxr = lxr.is_some(); let is_rc_pause = lxr.is_some_and(|lxr| lxr.current_pause() == Some(Pause::RefCount)); - let class_unloading_enabled = unsafe { crate::CLASS_UNLOADING_ENABLED } == 1; - let pause = lxr.map(|x| x.current_pause()).unwrap_or(None); let mut slots = Vec::with_capacity(scanning::WORK_PACKET_CAPACITY); @@ -229,14 +102,12 @@ impl>> // That includes all nmethods with moved children. // In nursery GCs, that means nmethods added since the previous GC. let mut nmethods_to_fix = Vec::new(); - let no_fix_relocation = class_unloading_enabled - && (pause == Some(Pause::FinalMark) || pause == Some(Pause::Full)); { let mut mature = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); // Only scan mature roots in full-heap collections. - if !is_current_gc_nursery && !(is_lxr && (class_unloading_enabled || is_rc_pause)) { + if !is_current_gc_nursery && !(is_lxr && is_rc_pause) { for (key, roots) in mature.iter() { mature_slots += roots.len(); add_roots(roots); @@ -246,14 +117,7 @@ impl>> } } - if no_fix_relocation { - let mut nursery = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); - for (key, roots) in nursery.drain() { - nursery_slots += roots.len(); - add_roots(&roots); - mature.insert(key, roots); - } - } else { + { let mut nursery = crate::NURSERY_CODE_CACHE_ROOTS.lock().unwrap(); for (key, roots) in nursery.drain() { nursery_slots += roots.len(); @@ -279,6 +143,10 @@ impl>> self.factory .create_process_roots_work(slots, RootKind::Strong); } + // Use the following code to scan CodeCache directly, instead of scanning the "remembered set". + // unsafe { + // ((*UPCALLS).scan_code_cache_roots)(to_slots_closure(&mut self.factory)); + // } if moves_object && !nmethods_to_fix.is_empty() { // Note: If the current GC doesn't move objects at all, we don't need to fix relocation. @@ -322,9 +190,6 @@ extern "C" fn report_slots_and_renew_buffer_weak if !ptr.is_null() { let ptr = ptr as *mut S; let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; - if cfg!(feature = "roots_breakdown") { - record_roots(buf.len()); - } let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; let kind = RootKind::Weak; factory.create_process_roots_work(buf, kind); @@ -364,23 +229,12 @@ impl> ScanWeakProcessorRoots { impl> GCWork for ScanWeakProcessorRoots { - fn do_work(&mut self, _worker: &mut GCWorker, mmtk: &'static MMTK) { - let t = if cfg!(feature = "roots_breakdown") { - Some(std::time::SystemTime::now()) - } else { - None - }; - // let scan_all_strong_roots = mmtk.get_plan().current_gc_should_perform_class_unloading(); - // assert!(scan_all_strong_roots); + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { unsafe { ((*UPCALLS).scan_weak_processor_roots)(to_slots_closure_weak::( &mut self.factory, )); } - if cfg!(feature = "roots_breakdown") { - let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; - report_roots("WeakProcessorRoots", ms); - } } } @@ -421,6 +275,6 @@ impl GCWork> for FixRelocations { num_nmethods, ); } - // probe!(mmtk_openjdk, fix_relocations, num_nmethods); + probe!(mmtk_openjdk, fix_relocations, num_nmethods); } } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 08c7b4f8..5a876895 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -14,7 +14,7 @@ use libc::{c_char, c_void, uintptr_t}; use mmtk::plan::lxr::LXR; use mmtk::util::alloc::AllocationError; use mmtk::util::constants::LOG_BYTES_IN_GBYTE; -use mmtk::util::heap::vm_layout::{VMLayout, BYTES_IN_CHUNK}; +use mmtk::util::heap::vm_layout::VMLayout; use mmtk::util::{conversions, opaque_pointer::*}; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::slot::Slot; @@ -86,23 +86,13 @@ pub struct SlotsClosure { #[repr(C)] pub struct OpenJDK_Upcalls { - pub stop_all_mutators: extern "C" fn( - tls: VMWorkerThread, - closure: MutatorClosure, - current_gc_should_unload_classes: bool, - ), + pub stop_all_mutators: extern "C" fn(tls: VMWorkerThread, closure: MutatorClosure), pub resume_mutators: extern "C" fn(tls: VMWorkerThread), pub spawn_gc_thread: extern "C" fn(tls: VMThread, kind: libc::c_int, ctx: *mut libc::c_void), pub block_for_gc: extern "C" fn(), pub out_of_memory: extern "C" fn(tls: VMThread, err_kind: AllocationError), pub get_mutators: extern "C" fn(closure: MutatorClosure), - pub scan_object: extern "C" fn( - trace: *mut c_void, - object: ObjectReference, - tls: OpaquePointer, - follow_clds: bool, - claim_clds: bool, - ), + pub scan_object: extern "C" fn(trace: *mut c_void, object: ObjectReference, tls: OpaquePointer), pub dump_object: extern "C" fn(object: ObjectReference), pub get_object_size: extern "C" fn(object: ObjectReference) -> usize, pub get_mmtk_mutator: extern "C" fn(tls: VMMutatorThread) -> *mut libc::c_void, @@ -120,11 +110,9 @@ pub struct OpenJDK_Upcalls { pub scan_multiple_thread_roots: extern "C" fn(closure: SlotsClosure, ptr: OpaquePointer, len: usize), pub scan_code_cache_roots: extern "C" fn(closure: SlotsClosure), - pub scan_class_loader_data_graph_roots: - extern "C" fn(closure: SlotsClosure, weak_closure: SlotsClosure, scan_weak: bool), + pub scan_class_loader_data_graph_roots: extern "C" fn(closure: SlotsClosure), pub scan_oop_storage_set_roots: extern "C" fn(closure: SlotsClosure), pub scan_weak_processor_roots: extern "C" fn(closure: SlotsClosure), - // pub scan_string_table_roots: extern "C" fn(closure: SlotsClosure, rc_non_stuck_objs_only: bool), pub scan_vm_thread_roots: extern "C" fn(closure: SlotsClosure), pub number_of_mutators: extern "C" fn() -> usize, pub schedule_finalizer: extern "C" fn(), @@ -132,21 +120,9 @@ pub struct OpenJDK_Upcalls { pub update_weak_processor: extern "C" fn(lxr: bool), pub enqueue_references: extern "C" fn(objects: *const ObjectReference, len: usize), pub swap_reference_pending_list: extern "C" fn(objects: ObjectReference) -> ObjectReference, - pub java_lang_class_klass_offset_in_bytes: extern "C" fn() -> usize, - pub java_lang_classloader_loader_data_offset: extern "C" fn() -> usize, pub fix_oop_relocations: extern "C" fn(lxr: bool, nmethods: *mut libc::c_void, len: usize), - // pub nmethod_fix_relocation: extern "C" fn(Address), pub clear_claimed_marks: extern "C" fn(), - pub unload_classes: extern "C" fn(), pub gc_epilogue: extern "C" fn(), - pub oops_do_marking_epilogue: extern "C" fn(), -} - -lazy_static! { - pub static ref JAVA_LANG_CLASS_KLASS_OFFSET_IN_BYTES: usize = - unsafe { ((*UPCALLS).java_lang_class_klass_offset_in_bytes)() }; - pub static ref JAVA_LANG_CLASSLOADER_LOADER_DATA_OFFSET: usize = - unsafe { ((*UPCALLS).java_lang_classloader_loader_data_offset)() }; } pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); @@ -207,9 +183,6 @@ pub static mut RC_ENABLED: u8 = 0; #[no_mangle] pub static mut REQUIRES_WEAK_HANDLE_BARRIER: u8 = 0; -#[no_mangle] -pub static mut CLASS_UNLOADING_ENABLED: u8 = 0; - #[derive(Default)] pub struct OpenJDK; @@ -288,14 +261,10 @@ pub static MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES: usize = mmtk::util::alloc::MarkCompactAllocator::>::HEADER_RESERVED_IN_BYTES; lazy_static! { - /// A global storage for all the cached CodeCache root pointers + /// A global storage for all the cached CodeCache roots added since the last GC. static ref NURSERY_CODE_CACHE_ROOTS: Mutex>> = Mutex::new(HashMap::new()); + /// A global storage for all the cached CodeCache roots added before the last GC. static ref MATURE_CODE_CACHE_ROOTS: Mutex>> = Mutex::new(HashMap::new()); - static ref NURSERY_WEAK_HANDLE_ROOTS: Mutex> = Mutex::new(Vec::new()); -} - -lazy_static! { - static ref OBJ_COUNT: Mutex> = Mutex::new(HashMap::new()); } fn set_compressed_pointer_vm_layout(builder: &mut MMTKBuilder) { diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index d96f53fb..6d4864da 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -29,7 +29,7 @@ impl OopIterate for OopMapBlock { let start = oop.get_field_address(self.offset); for i in 0..self.count as usize { let slot = (start + (i << S::::LOG_BYTES_IN_SLOT)).into(); - closure.visit_slot(slot, false); + closure.visit_slot(slot); } } } @@ -40,9 +40,6 @@ impl OopIterate for InstanceKlass { oop: Oop, closure: &mut impl SlotVisitor>, ) { - if closure.should_follow_clds() { - do_klass::<_, _, COMPRESSED>(oop.klass::(), closure); - } let oop_maps = self.nonstatic_oop_maps(); for map in oop_maps { map.oop_iterate::(oop, closure) @@ -57,21 +54,6 @@ impl OopIterate for InstanceMirrorKlass { closure: &mut impl SlotVisitor>, ) { self.instance_klass.oop_iterate::(oop, closure); - if closure.should_follow_clds() { - let klass = unsafe { - (oop.start() + *crate::JAVA_LANG_CLASS_KLASS_OFFSET_IN_BYTES).load::<*mut Klass>() - }; - if !klass.is_null() { - let klass = unsafe { &mut *klass }; - if klass.is_instance_klass() - && (unsafe { klass.cast::().is_anonymous() }) - { - do_cld::<_, _, COMPRESSED>(&klass.class_loader_data, closure) - } else { - do_klass::<_, _, COMPRESSED>(klass, closure) - } - } - } // static fields let start = Self::start_of_static_fields(oop); @@ -80,13 +62,13 @@ impl OopIterate for InstanceMirrorKlass { let start: *const NarrowOop = start.to_ptr::(); let slice = unsafe { slice::from_raw_parts(start, len as _) }; for narrow_oop in slice { - closure.visit_slot(narrow_oop.slot().into(), false); + closure.visit_slot(narrow_oop.slot().into()); } } else { let start: *const Oop = start.to_ptr::(); let slice = unsafe { slice::from_raw_parts(start, len as _) }; for oop in slice { - closure.visit_slot(Address::from_ref(oop as &Oop).into(), false); + closure.visit_slot(Address::from_ref(oop as &Oop).into()); } } } @@ -99,15 +81,6 @@ impl OopIterate for InstanceClassLoaderKlass { closure: &mut impl SlotVisitor>, ) { self.instance_klass.oop_iterate::(oop, closure); - if closure.should_follow_clds() { - let cld = unsafe { - (oop.start() + *crate::JAVA_LANG_CLASSLOADER_LOADER_DATA_OFFSET) - .load::<*mut ClassLoaderData>() - }; - if !cld.is_null() { - do_cld::<_, _, COMPRESSED>(unsafe { &*cld }, closure); - } - } } } @@ -117,17 +90,14 @@ impl OopIterate for ObjArrayKlass { oop: Oop, closure: &mut impl SlotVisitor>, ) { - if closure.should_follow_clds() { - do_klass::<_, _, COMPRESSED>(oop.klass::(), closure); - } let array = unsafe { oop.as_array_oop() }; if COMPRESSED { for narrow_oop in unsafe { array.data::(BasicType::T_OBJECT) } { - closure.visit_slot(narrow_oop.slot().into(), false); + closure.visit_slot(narrow_oop.slot().into()); } } else { for oop in unsafe { array.data::(BasicType::T_OBJECT) } { - closure.visit_slot(Address::from_ref(oop as &Oop).into(), false); + closure.visit_slot(Address::from_ref(oop as &Oop).into()); } } } @@ -202,9 +172,9 @@ impl InstanceRefKlass { closure: &mut impl SlotVisitor>, ) { let referent_addr = Self::referent_address::(oop); - closure.visit_slot(referent_addr, false); + closure.visit_slot(referent_addr); let discovered_addr = Self::discovered_address::(oop); - closure.visit_slot(discovered_addr, false); + closure.visit_slot(discovered_addr); } fn discover_reference(oop: Oop, rt: ReferenceType) -> bool { // Do not discover new refs during reference processing. @@ -253,22 +223,12 @@ fn oop_iterate_slow>>( ), mem::transmute::<&OopDesc, ObjectReference>(oop), tls, - closure.should_follow_clds(), - closure.should_claim_clds(), ); } } -fn oop_iterate( - oop: Oop, - closure: &mut impl SlotVisitor>, - klass: Option
, -) { - let klass = if let Some(klass) = klass { - unsafe { &*(klass.as_usize() as *const Klass) } - } else { - oop.klass::() - }; +fn oop_iterate(oop: Oop, closure: &mut impl SlotVisitor>) { + let klass = oop.klass::(); let klass_id = klass.kind; assert!( klass_id as i32 >= 0 && (klass_id as i32) < KlassKind::Unknown as i32, @@ -309,23 +269,6 @@ fn oop_iterate( } } -fn do_cld, const COMPRESSED: bool>( - cld: &ClassLoaderData, - closure: &mut V, -) { - if !closure.should_follow_clds() { - return; - } - cld.oops_do::<_, _, COMPRESSED>(closure) -} - -fn do_klass, const COMPRESSED: bool>(klass: &Klass, closure: &mut V) { - if !closure.should_follow_clds() { - return; - } - do_cld::<_, _, COMPRESSED>(klass.class_loader_data, closure) -} - pub fn is_obj_array(oop: Oop) -> bool { oop.klass::().kind == KlassKind::ObjArray } @@ -364,7 +307,7 @@ pub unsafe extern "C" fn scan_object_fn< ) { let ptr: *mut u8 = CLOSURE.with(|x| *x.get()); let closure = &mut *(ptr as *mut V); - closure.visit_slot(slot.into(), false); + closure.visit_slot(slot.into()); } pub fn scan_object( @@ -373,25 +316,6 @@ pub fn scan_object( _tls: VMWorkerThread, ) { unsafe { - oop_iterate::( - mem::transmute::(object), - closure, - None, - ) - } -} - -pub fn scan_object_with_klass( - object: ObjectReference, - closure: &mut impl SlotVisitor>, - _tls: VMWorkerThread, - klass: Address, -) { - unsafe { - oop_iterate::( - mem::transmute::(object), - closure, - Some(klass), - ) + oop_iterate::(mem::transmute::(object), closure) } } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 6d1f98db..466922a1 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -13,7 +13,6 @@ use mmtk::MutatorContext; pub struct VMScanning {} -#[allow(unused)] pub(crate) const WORK_PACKET_CAPACITY: usize = mmtk::scheduler::EDGES_WORK_BUFFER_SIZE; extern "C" fn report_slots_and_renew_buffer>( @@ -26,16 +25,13 @@ extern "C" fn report_slots_and_renew_buffer>( // Note: Currently OpenJDKSlot has the same layout as Address. If the layout changes, we // should fix the Rust-to-C interface. let buf = unsafe { Vec::::from_raw_parts(ptr as _, length, capacity) }; - if cfg!(feature = "roots_breakdown") { - super::gc_work::record_roots(buf.len()); - } let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; factory.create_process_roots_work(buf, RootKind::Strong); } let (ptr, _, capacity) = { // TODO: Use Vec::into_raw_parts() when the method is available. use std::mem::ManuallyDrop; - let new_vec = Vec::with_capacity(F::BUFFER_SIZE); + let new_vec = Vec::with_capacity(WORK_PACKET_CAPACITY); let mut me = ManuallyDrop::new(new_vec); (me.as_mut_ptr(), me.len(), me.capacity()) }; @@ -58,20 +54,6 @@ impl Scanning> for VMScanning { crate::object_scanning::scan_object::(object, slot_visitor, tls); } - fn scan_object_with_klass( - tls: VMWorkerThread, - object: ObjectReference, - slot_visitor: &mut impl SlotVisitor>, - klass: Address, - ) { - crate::object_scanning::scan_object_with_klass::( - object, - slot_visitor, - tls, - klass, - ); - } - fn obj_array_data(o: ObjectReference) -> crate::OpenJDKSlotRange { crate::object_scanning::obj_array_data::(unsafe { std::mem::transmute(o) }) } diff --git a/mmtk/src/slots.rs b/mmtk/src/slots.rs index 37e0f2db..bd19d9c2 100644 --- a/mmtk/src/slots.rs +++ b/mmtk/src/slots.rs @@ -37,25 +37,13 @@ pub fn use_compressed_oops() -> bool { /// Set compressed pointer base and shift based on heap range pub fn initialize_compressed_oops_base_and_shift() { - let heap_start = mmtk::memory_manager::starting_heap_address().as_usize(); let heap_end = mmtk::memory_manager::last_heap_address().as_usize(); - if cfg!(feature = "force_narrow_oop_mode") { - println!("heap_start: 0x{:x}", heap_start); - println!("heap_end: 0x{:x}", heap_end); - } if heap_end <= (4usize << 30) { BASE.store(Address::ZERO, Ordering::Relaxed); SHIFT.store(0, Ordering::Relaxed); } else if heap_end <= (32usize << 30) { BASE.store(Address::ZERO, Ordering::Relaxed); SHIFT.store(3, Ordering::Relaxed); - } else if cfg!(feature = "narrow_oop_mode_base") && (heap_end - heap_start) <= (4usize << 30) { - // set heap base as HEAP_START - 4096, to make sure null pointer value is not conflict with HEAP_START - BASE.store( - mmtk::memory_manager::starting_heap_address() - 4096, - Ordering::Relaxed, - ); - SHIFT.store(0, Ordering::Relaxed); } else { // set heap base as HEAP_START - 4096, to make sure null pointer value does not conflict with HEAP_START BASE.store( diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index f7a1294d..55ddf9c0 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -178,13 +178,13 @@ struct SlotsClosure { * OpenJDK-specific */ typedef struct { - void (*stop_all_mutators) (void *tls, MutatorClosure closure, bool current_gc_should_unload_classes); + void (*stop_all_mutators) (void *tls, MutatorClosure closure); void (*resume_mutators) (void *tls); void (*spawn_gc_thread) (void *tls, int kind, void *ctx); void (*block_for_gc) (); void (*out_of_memory) (void *tls, MMTkAllocationError err_kind); void (*get_mutators) (MutatorClosure closure); - void (*scan_object) (void* trace, void* object, void* tls, bool follow_clds, bool claim_clds); + void (*scan_object) (void* trace, void* object, void* tls); void (*dump_object) (void* object); size_t (*get_object_size) (void* object); void* (*get_mmtk_mutator) (void* tls); @@ -201,7 +201,7 @@ typedef struct { void (*scan_roots_in_mutator_thread)(SlotsClosure closure, void* tls); void (*scan_multiple_thread_roots)(SlotsClosure closure, void* ptr, size_t len); void (*scan_code_cache_roots) (SlotsClosure closure); - void (*scan_class_loader_data_graph_roots) (SlotsClosure closure, SlotsClosure weak_closure, bool scan_weak); + void (*scan_class_loader_data_graph_roots) (SlotsClosure closure); void (*scan_oop_storage_set_roots) (SlotsClosure closure); void (*scan_weak_processor_roots) (SlotsClosure closure); void (*scan_vm_thread_roots) (SlotsClosure closure); @@ -211,13 +211,9 @@ typedef struct { void (*mmtk_update_weak_processor)(bool lxr); void (*enqueue_references)(void** objects, size_t len); void* (*swap_reference_pending_list)(void* objects); - size_t (*java_lang_class_klass_offset_in_bytes)(); - size_t (*java_lang_classloader_loader_data_offset)(); void (*fix_oop_relocations)(bool lxr, void* nmethods, size_t len); void (*clear_claimed_marks)(); - void (*unload_classes)(); void (*gc_epilogue)(); - void (*oops_do_marking_epilogue)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); diff --git a/openjdk/share/mmtkHeap.cpp b/openjdk/share/mmtkHeap.cpp index 2f469b93..55db417a 100644 --- a/openjdk/share/mmtkHeap.cpp +++ b/openjdk/share/mmtkHeap.cpp @@ -60,7 +60,6 @@ #include "services/memoryManager.hpp" #include "services/memTracker.hpp" #include "utilities/vmError.hpp" -#include "mmtkParallelCleaning.hpp" #include "mmtkRootsClosure.hpp" #include "gc/shared/workerPolicy.hpp" @@ -149,7 +148,6 @@ jint MMTkHeap::initialize() { const size_t max_heap_size = MaxHeapSize; if (UseCompressedOops) mmtk_enable_compressed_oops(); - CLASS_UNLOADING_ENABLED = ClassUnloading ? 1 : 0; // Note that MMTk options may be set from several different sources, with increasing priorities: // 1. Default values defined in mmtk::util::options::Options @@ -472,22 +470,9 @@ void MMTkHeap::scan_code_cache_roots(OopClosure& cl) { CodeCache::blobs_do(&cb_cl); } -void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& weak_cl, bool scan_all_strong_roots) { - if (!ClassUnloading) { - CLDToOopClosure cld_cl(&cl, false); - CLDToOopClosure weak_cld_cl(&weak_cl, false); - ClassLoaderDataGraph::roots_cld_do(&cld_cl, &weak_cld_cl); - } else if (scan_all_strong_roots) { - // At the start of full heap trace, we want to scan all the strong CLD roots + all the modified CLDs. - MMTkScanCLDClosure cld_cl(&cl); - MMTkScanCLDClosure weak_cld_cl(&weak_cl); - ClassLoaderDataGraph::roots_cld_do(&cld_cl, &weak_cld_cl); - } else { - // Normal RC pause: We simply apply increments to modified CLD roots. No decrements will be applied on these roots. - MMTkScanCLDClosure cld_cl(&cl); - MMTkScanCLDClosure weak_cld_cl(&weak_cl); - ClassLoaderDataGraph::roots_cld_do(&cld_cl, &weak_cld_cl); - } +void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl) { + CLDToOopClosure cld_cl(&cl, false); + ClassLoaderDataGraph::cld_do(&cld_cl); } void MMTkHeap::scan_oop_storage_set_roots(OopClosure& cl) { for (auto id : EnumRange()) { @@ -573,13 +558,6 @@ HeapWord* MMTkHeap::mem_allocate_nonmove(size_t size, bool* gc_overhead_limit_wa return alloc_fast(size << LogHeapWordSize, AllocatorLos); } -void MMTkHeap::complete_cleaning(bool purged_classes) { - ResourceMark rm; - uint num_workers = _workers->active_workers(); - mmtk::ParallelCleaningTask unlink_task(num_workers, purged_classes); - _workers->run_task(&unlink_task); -} - void MMTkHeap::register_new_weak_handle(oop* handle) {} bool MMTkHeap::requires_barriers(stackChunkOop obj) const { diff --git a/openjdk/share/mmtkHeap.hpp b/openjdk/share/mmtkHeap.hpp index 7a9b72b2..6f245515 100644 --- a/openjdk/share/mmtkHeap.hpp +++ b/openjdk/share/mmtkHeap.hpp @@ -213,12 +213,10 @@ class MMTkHeap : public CollectedHeap { void scan_roots_in_all_mutator_threads(OopClosure& cl); void scan_code_cache_roots(OopClosure& cl); - void scan_class_loader_data_graph_roots(OopClosure& cl, OopClosure& weak_cl, bool scan_all_strong_roots); + void scan_class_loader_data_graph_roots(OopClosure& cl); void scan_oop_storage_set_roots(OopClosure& cl); void scan_weak_processor_roots(OopClosure& cl); void scan_vm_thread_roots(OopClosure& cl); - - void complete_cleaning(bool purged_classes); }; diff --git a/openjdk/share/mmtkParallelCleaning.cpp b/openjdk/share/mmtkParallelCleaning.cpp deleted file mode 100644 index 024a9ff0..00000000 --- a/openjdk/share/mmtkParallelCleaning.cpp +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "classfile/symbolTable.hpp" -#include "classfile/stringTable.hpp" -#include "code/codeCache.hpp" -#include "memory/resourceArea.hpp" -#include "prims/resolvedMethodTable.hpp" -#include "logging/log.hpp" -#include "gc/shared/gcCause.hpp" -#include "gc/shared/gcTraceTime.hpp" -#include "gc/shared/gcTraceTime.inline.hpp" -#include "mmtkParallelCleaning.hpp" -#include "mmtk.h" -#include "runtime/atomic.hpp" -#include "oops/access.inline.hpp" -#include "oops/compressedOops.inline.hpp" -#include "oops/oop.inline.hpp" -#include "code/nmethod.hpp" - -using namespace JavaClassFile; - -namespace mmtk { -MMTkCodeCacheUnloadingTask::MMTkCodeCacheUnloadingTask(uint num_workers, bool unloading_occurred) : - _unloading_occurred(unloading_occurred), - _num_workers(num_workers), - _first_nmethod(nullptr), - _claimed_nmethod(nullptr) { - // Get first alive nmethod - CompiledMethodIterator iter(CompiledMethodIterator::all_blobs); - if(iter.next()) { - _first_nmethod = iter.method(); - } - _claimed_nmethod = _first_nmethod; -} - -MMTkCodeCacheUnloadingTask::~MMTkCodeCacheUnloadingTask() { - CodeCache::verify_icholder_relocations(); -} - -void MMTkCodeCacheUnloadingTask::claim_nmethods(CompiledMethod** claimed_nmethods, int *num_claimed_nmethods) { - CompiledMethod* first; - CompiledMethodIterator last(CompiledMethodIterator::all_blobs); - - do { - *num_claimed_nmethods = 0; - - first = _claimed_nmethod; - last = CompiledMethodIterator(CompiledMethodIterator::all_blobs, first); - - if (first != nullptr) { - - for (int i = 0; i < MaxClaimNmethods; i++) { - if (!last.next()) { - break; - } - claimed_nmethods[i] = last.method(); - (*num_claimed_nmethods)++; - } - } - - } while (Atomic::cmpxchg(&_claimed_nmethod, first, last.method()) != first); -} - -class MMTkForwardClosure : public OopClosure { - public: - inline static size_t read_forwarding_word(oop o) { - return *((size_t*) (void*) o); - } - inline static oop extract_forwarding_pointer(size_t status) { - return (oop) (void*) (status << 8 >> 8); - } - inline static bool is_forwarded(size_t status) { - return (status >> 56) != 0; - } - inline virtual void do_oop(oop* slot) override { - const auto o = *slot; - if (o == NULL) return; - const auto status = read_forwarding_word(o); - if (is_forwarded(status)) { - *slot = extract_forwarding_pointer(status); - } - } - inline virtual void do_oop(narrowOop* slot) override { - narrowOop heap_oop = RawAccess<>::oop_load(slot); - if (CompressedOops::is_null(heap_oop)) return; - oop o = CompressedOops::decode_not_null(heap_oop); - const auto status = read_forwarding_word(o); - if (is_forwarded(status)) { - RawAccess<>::oop_store(slot, CompressedOops::encode_not_null(extract_forwarding_pointer(status))); - } - } -}; - -void MMTkCodeCacheUnloadingTask::work(uint worker_id) { - MMTkForwardClosure cl; - // The first nmethods is claimed by the first worker. - if (worker_id == 0 && _first_nmethod != nullptr) { - _first_nmethod->do_unloading(_unloading_occurred); - _first_nmethod = nullptr; - } - - int num_claimed_nmethods; - CompiledMethod* claimed_nmethods[MaxClaimNmethods]; - - while (true) { - claim_nmethods(claimed_nmethods, &num_claimed_nmethods); - - if (num_claimed_nmethods == 0) { - break; - } - - for (int i = 0; i < num_claimed_nmethods; i++) { - if (claimed_nmethods[i]->is_nmethod()) { - auto nm = (nmethod*) claimed_nmethods[i]; - nm->oops_do(&cl); - nm->fix_oop_relocations(); - } - } - for (int i = 0; i < num_claimed_nmethods; i++) { - claimed_nmethods[i]->do_unloading(_unloading_occurred); - } - } -} - - -ParallelCleaningTask::ParallelCleaningTask(uint num_workers, bool unloading_occurred) : - WorkerTask("Parallel Cleaning"), - _unloading_occurred(unloading_occurred), - _code_cache_task(num_workers, unloading_occurred), - _klass_cleaning_task() { -} - -// The parallel work done by all worker threads. -void ParallelCleaningTask::work(uint worker_id) { - _code_cache_task.work(worker_id); - // Clean all klasses that were not unloaded. - // The weak metadata in klass doesn't need to be - // processed if there was no unloading. - if (_unloading_occurred) { - _klass_cleaning_task.work(); - } -} - -} \ No newline at end of file diff --git a/openjdk/share/mmtkParallelCleaning.hpp b/openjdk/share/mmtkParallelCleaning.hpp deleted file mode 100644 index 3c5dde88..00000000 --- a/openjdk/share/mmtkParallelCleaning.hpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef MMTK_OPENJDK_MMTK_PARALLELCLEANING_HPP -#define MMTK_OPENJDK_MMTK_PARALLELCLEANING_HPP - -#include "gc/shared/oopStorageParState.hpp" -#include "gc/shared/workerThread.hpp" -#include "gc/shared/parallelCleaning.hpp" - -namespace mmtk { - -class MMTkCodeCacheUnloadingTask { - const bool _unloading_occurred; - const uint _num_workers; - - // Variables used to claim nmethods. - CompiledMethod* _first_nmethod; - CompiledMethod* volatile _claimed_nmethod; - -public: - MMTkCodeCacheUnloadingTask(uint num_workers, bool unloading_occurred); - - ~MMTkCodeCacheUnloadingTask(); - -private: - static const int MaxClaimNmethods = 16; - void claim_nmethods(CompiledMethod** claimed_nmethods, int *num_claimed_nmethods); - -public: - // Cleaning and unloading of nmethods. - void work(uint worker_id); -}; - -// To minimize the remark pause times, the tasks below are done in parallel. -class ParallelCleaningTask : public WorkerTask { -private: - bool _unloading_occurred; - MMTkCodeCacheUnloadingTask _code_cache_task; - KlassCleaningTask _klass_cleaning_task; - -public: - // The constructor is run in the VMThread. - ParallelCleaningTask(uint num_workers, bool unloading_occurred); - - // The parallel work done by all worker threads. - void work(uint worker_id); -}; - -} - -#endif // MMTK_OPENJDK_MMTK_PARALLELCLEANING_HPP diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index bfaffc74..29409f65 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -173,7 +173,7 @@ class MMTkUpdateClosure : public OopClosure { extern MaybeUninit> oop_storage_set_strong_par_state; extern MaybeUninit> oop_storage_set_weak_par_state; -static void mmtk_stop_all_mutators(void *tls, MutatorClosure closure, bool current_gc_should_unload_classes) { +static void mmtk_stop_all_mutators(void *tls, MutatorClosure closure) { log_debug(gc)("Requesting the VM to suspend all mutators..."); MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_suspended, true); log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); @@ -183,9 +183,6 @@ static void mmtk_stop_all_mutators(void *tls, MutatorClosure closure, bool curre oop_storage_set_weak_par_state.init(); mmtk_report_gc_start(); - if (ClassUnloading && current_gc_should_unload_classes) { - ClassLoaderDataGraph::clear_claimed_marks(); - } // CodeCache::gc_prologue(); #if COMPILER2_OR_JVMCI DerivedPointerTable::clear(); @@ -215,39 +212,6 @@ static void mmtk_update_weak_processor(bool lxr) { } } -static void mmtk_oops_do_marking_epilogue() { - nmethod::oops_do_marking_epilogue(); -} - -static void mmtk_unload_classes() { - if (ClassUnloading) { - // Unload classes and purge SystemDictionary. - // LOG_CLS_UNLOAD("[mmtk_unload_classes] SystemDictionary::do_unloading"); - // auto purged_classes = SystemDictionary::do_unloading(NULL); - ClassUnloadingContext ctx(MMTkHeap::heap()->workers()->active_workers(), - true /* unregister_nmethods_during_purge */, - false /* lock_codeblob_free_separately */); - MMTkIsAliveClosure is_alive; - { - CodeCache::UnlinkingScope scope(&is_alive); - bool unloading_occurred = SystemDictionary::do_unloading(NULL); - MMTkHeap::heap()->complete_cleaning(unloading_occurred); - } - LOG_CLS_UNLOAD("[mmtk_unload_classes] ClassLoaderDataGraph::purge"); - - ctx.purge_nmethods(); - // MMTkHeap::heap()->bulk_unregister_nmethods(); - ctx.free_code_blobs(); - ClassLoaderDataGraph::purge(true /* at_safepoint */); - DEBUG_ONLY(MetaspaceUtils::verify();) - LOG_CLS_UNLOAD("[mmtk_unload_classes] compute_new_size"); - // Resize and verify metaspace - MetaspaceGC::compute_new_size(); - // MetaspaceUtils::verify_metrics(); - LOG_CLS_UNLOAD("[mmtk_unload_classes] end"); - } -} - static void mmtk_gc_epilogue() { Universe::heap()->update_capacity_and_used_at_gc(); // CodeCache::gc_epilogue(); @@ -399,8 +363,8 @@ static void mmtk_scan_multiple_thread_roots(SlotsClosure closure, void* ptr, siz mutators[i]->oops_do(&cl, &cb_cl); } -static void mmtk_scan_object(void* trace, void* object, void* tls, bool follow_clds, bool claim_clds) { - MMTkScanObjectClosure cl(trace, follow_clds, claim_clds); +static void mmtk_scan_object(void* trace, void* object, void* tls) { + MMTkScanObjectClosure cl(trace, false, false); ((oop) object)->oop_iterate(&cl); } @@ -490,11 +454,7 @@ static void mmtk_schedule_finalizer() { } static void mmtk_scan_code_cache_roots(SlotsClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } -static void mmtk_scan_class_loader_data_graph_roots(SlotsClosure closure, SlotsClosure weak_closure, bool scan_all_strong_roots) { - MMTkRootsClosure cl(closure); - MMTkRootsClosure weak_cl(weak_closure); - MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl, weak_cl, scan_all_strong_roots); -} +static void mmtk_scan_class_loader_data_graph_roots(SlotsClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl); } static void mmtk_scan_oop_storage_set_roots(SlotsClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_oop_storage_set_roots(cl); } static void mmtk_scan_weak_processor_roots(SlotsClosure closure) { MMTkRootsClosure cl(closure); @@ -559,18 +519,6 @@ static void* mmtk_swap_reference_pending_list(void* object) { return Universe::swap_reference_pending_list((oop) object); } -static size_t mmtk_java_lang_class_klass_offset_in_bytes() { - auto v = java_lang_Class::klass_offset(); - guarantee(v != 0 && v != -1, "checking"); - return v; -} - -static size_t mmtk_java_lang_classloader_loader_data_offset() { - auto v = java_lang_ClassLoader::loader_data_offset(); - guarantee(v != 0 && v != -1, "checking"); - return v; -} - void mmtk_fix_oop_relocations(bool lxr, void* nmethods, size_t len) { if (lxr) { MMTkLXRFastUpdateClosure cl; @@ -623,11 +571,7 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_update_weak_processor, mmtk_enqueue_references, mmtk_swap_reference_pending_list, - mmtk_java_lang_class_klass_offset_in_bytes, - mmtk_java_lang_classloader_loader_data_offset, mmtk_fix_oop_relocations, mmtk_clear_claimed_marks, - mmtk_unload_classes, mmtk_gc_epilogue, - mmtk_oops_do_marking_epilogue, }; From 85819a31127e80c9abddf62818896bfdf276f890 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 7 Apr 2026 11:57:52 +1000 Subject: [PATCH 465/494] Revert weak refs --- mmtk/src/abi.rs | 6 +- mmtk/src/collection.rs | 25 +- mmtk/src/lib.rs | 2 - mmtk/src/object_scanning.rs | 87 +--- mmtk/src/reference_glue.rs | 586 +------------------------- openjdk/share/mmtk.h | 2 - openjdk/share/mmtkCollectorThread.cpp | 2 - openjdk/share/mmtkHeap.cpp | 38 +- openjdk/share/mmtkHeap.hpp | 8 +- openjdk/share/mmtkMutator.cpp | 16 - openjdk/share/mmtkRootsClosure.hpp | 49 +-- openjdk/share/mmtkUpcalls.cpp | 11 +- 12 files changed, 64 insertions(+), 768 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index daa2146c..5fd56ea6 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -388,9 +388,9 @@ impl From for &OopDesc { } /// Convert Oop to ObjectReference -impl Into for &OopDesc { - fn into(self) -> ObjectReference { - unsafe { mem::transmute::<&OopDesc, _>(self) } +impl From<&OopDesc> for ObjectReference { + fn from(o: &OopDesc) -> Self { + unsafe { mem::transmute(o) } } } diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 047289fa..dbf155e6 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -1,10 +1,8 @@ -use mmtk::scheduler::{GCWorker, ProcessEdgesWork}; use mmtk::util::alloc::AllocationError; use mmtk::util::opaque_pointer::*; use mmtk::vm::{Collection, GCThreadContext}; use mmtk::Mutator; -use crate::reference_glue::DISCOVERED_LISTS; use crate::{singleton, UPCALLS}; use crate::{MutatorClosure, OpenJDK}; @@ -26,7 +24,6 @@ impl Collection> for VMCollection { } fn resume_mutators(tls: VMWorkerThread) { - DISCOVERED_LISTS.enable_discover(); if *crate::singleton::().get_options().plan == mmtk::util::options::PlanSelector::ConcurrentImmix { @@ -68,25 +65,9 @@ impl Collection> for VMCollection { } fn schedule_finalization(_tls: VMWorkerThread) { - unreachable!() - } - - fn process_weak_refs>>( - worker: &mut GCWorker>, - ) { - DISCOVERED_LISTS.process_soft_weak_final_refs::(worker) - } - - fn process_final_refs>>( - worker: &mut GCWorker>, - ) { - DISCOVERED_LISTS.resurrect_final_refs::(worker) - } - - fn process_phantom_refs>>( - worker: &mut GCWorker>, - ) { - DISCOVERED_LISTS.process_phantom_refs::(worker) + unsafe { + ((*UPCALLS).schedule_finalizer)(); + } } fn update_weak_processor(lxr: bool) { diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 5a876895..3c3f9d13 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -119,9 +119,7 @@ pub struct OpenJDK_Upcalls { pub prepare_for_roots_re_scanning: extern "C" fn(), pub update_weak_processor: extern "C" fn(lxr: bool), pub enqueue_references: extern "C" fn(objects: *const ObjectReference, len: usize), - pub swap_reference_pending_list: extern "C" fn(objects: ObjectReference) -> ObjectReference, pub fix_oop_relocations: extern "C" fn(lxr: bool, nmethods: *mut libc::c_void, len: usize), - pub clear_claimed_marks: extern "C" fn(), pub gc_epilogue: extern "C" fn(), } diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index 6d4864da..c9126f94 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -1,10 +1,9 @@ +use crate::OpenJDKSlot; + use super::abi::*; use super::UPCALLS; -use crate::reference_glue::DISCOVERED_LISTS; -use crate::OpenJDKSlot; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; -use mmtk::vm::slot::Slot; use mmtk::vm::ObjectKind; use mmtk::vm::SlotVisitor; use std::cell::UnsafeCell; @@ -26,9 +25,10 @@ impl OopIterate for OopMapBlock { oop: Oop, closure: &mut impl SlotVisitor>, ) { + let log_bytes_in_oop = if COMPRESSED { 2 } else { 3 }; let start = oop.get_field_address(self.offset); for i in 0..self.count as usize { - let slot = (start + (i << S::::LOG_BYTES_IN_SLOT)).into(); + let slot = (start + (i << log_bytes_in_oop)).into(); closure.visit_slot(slot); } } @@ -121,23 +121,29 @@ impl OopIterate for InstanceRefKlass { closure: &mut impl SlotVisitor>, ) { use crate::abi::*; + use crate::api::{add_phantom_candidate, add_soft_candidate, add_weak_candidate}; self.instance_klass.oop_iterate::(oop, closure); - let disable_discovery = !closure.should_discover_references(); + // Unlike OpenJDK's built-in collectors, we do not use the `discovered` field for + // recording discovered references (we use `add_{soft,weak,phantom}_candidate`). + // The `discovered` field links all `Reference` instances that are either in the + // global "reference pending list" or given to the `ReferenceHandler` thread. + // We treat it as a strong field. + let discovered_addr: OpenJDKSlot = Self::discovered_address::(oop); + closure.visit_slot(discovered_addr); - if Self::should_discover_refs::( - self.instance_klass.reference_type, - disable_discovery, - ) { + if Self::should_scan_weak_refs::() { + let reference = ObjectReference::from(oop); match self.instance_klass.reference_type { ReferenceType::None => { panic!("oop_iterate on InstanceRefKlass with reference_type as None") } - rt => { - if !Self::discover_reference::(oop, rt) { - Self::process_ref_as_strong(oop, closure) - } - } + ReferenceType::Weak => add_weak_candidate(reference), + ReferenceType::Soft => add_soft_candidate(reference), + ReferenceType::Phantom => add_phantom_candidate(reference), + // Process final reference normally. + // We will handle final reference later + ReferenceType::Final => Self::process_ref_as_strong(oop, closure), } } else { Self::process_ref_as_strong(oop, closure); @@ -146,26 +152,10 @@ impl OopIterate for InstanceRefKlass { } impl InstanceRefKlass { - fn should_discover_refs( - rt: ReferenceType, - disable_discovery: bool, - ) -> bool { - if disable_discovery { - return false; - } - if *crate::singleton::().get_options().no_finalizer - && rt == ReferenceType::Final - { - return false; - } - if *crate::singleton::() + fn should_scan_weak_refs() -> bool { + !*crate::singleton::() .get_options() .no_reference_types - && rt != ReferenceType::Final - { - return false; - } - true } fn process_ref_as_strong( oop: Oop, @@ -173,39 +163,6 @@ impl InstanceRefKlass { ) { let referent_addr = Self::referent_address::(oop); closure.visit_slot(referent_addr); - let discovered_addr = Self::discovered_address::(oop); - closure.visit_slot(discovered_addr); - } - fn discover_reference(oop: Oop, rt: ReferenceType) -> bool { - // Do not discover new refs during reference processing. - if !DISCOVERED_LISTS.allow_discover() { - return false; - } - // Do not discover if the referent is live. - let addr = InstanceRefKlass::referent_address::(oop); - let Some(referent) = addr.load() else { - return false; - }; - // Skip live or null referents - if referent.is_reachable() { - return false; - } - // Skip young referents - let reference: ObjectReference = oop.into(); - if !crate::singleton::() - .get_plan() - .should_process_reference(reference, referent) - { - return false; - } - if rt == ReferenceType::Final && DISCOVERED_LISTS.is_discovered::(reference) { - return false; - } - // Add to reference list - DISCOVERED_LISTS - .get(rt) - .add::(reference, referent); - true } } diff --git a/mmtk/src/reference_glue.rs b/mmtk/src/reference_glue.rs index d44848de..179335de 100644 --- a/mmtk/src/reference_glue.rs +++ b/mmtk/src/reference_glue.rs @@ -1,589 +1,31 @@ -use std::cell::UnsafeCell; -use std::marker::PhantomData; -use std::sync::atomic::AtomicBool; -use std::sync::Mutex; - -use crate::abi::{InstanceRefKlass, Oop, ReferenceType}; -use crate::slots::OpenJDKSlot; +use crate::abi::{InstanceRefKlass, Oop}; use crate::OpenJDK; -use atomic::Ordering; -use mmtk::scheduler::{GCWork, GCWorker, ProcessEdgesWork, WorkBucketStage}; +use crate::UPCALLS; use mmtk::util::opaque_pointer::VMWorkerThread; use mmtk::util::ObjectReference; use mmtk::vm::slot::Slot; use mmtk::vm::ReferenceGlue; -use mmtk::MMTK; - -fn set_referent(reff: ObjectReference, referent: Option) { - let oop = Oop::from(reff); - let slot = InstanceRefKlass::referent_address::(oop); - if let Some(o) = referent { - slot.store(o); - } else { - slot.store_null(); - } -} - -fn get_referent(object: ObjectReference) -> Option { - let oop = Oop::from(object); - InstanceRefKlass::referent_address::(oop).load() -} - -fn get_next_reference_slot( - object: ObjectReference, -) -> OpenJDKSlot { - let oop = Oop::from(object); - InstanceRefKlass::discovered_address::(oop) -} - -fn get_next_reference(object: ObjectReference) -> Option { - get_next_reference_slot::(object).load() -} - -fn set_next_reference( - object: ObjectReference, - next: Option, -) { - let slot = get_next_reference_slot::(object); - if let Some(o) = next { - slot.store(o); - } else { - slot.store_null(); - } -} pub struct VMReferenceGlue {} impl ReferenceGlue> for VMReferenceGlue { type FinalizableType = ObjectReference; - fn set_referent(_reff: ObjectReference, _referent: ObjectReference) { - unreachable!(); - } - fn get_referent(_object: ObjectReference) -> Option { - unreachable!(); - } - fn enqueue_references(_references: &[ObjectReference], _tls: VMWorkerThread) {} - fn clear_referent(_new_reference: ObjectReference) {} -} - -pub struct DiscoveredList { - head: UnsafeCell>, - _rt: ReferenceType, -} - -impl DiscoveredList { - fn new(rt: ReferenceType) -> Self { - Self { - head: UnsafeCell::new(ObjectReference::NULL), - _rt: rt, - } - } - - pub fn add( - &self, - reference: ObjectReference, - referent: ObjectReference, - ) { - // Keep reference and referent alive during SATB - crate::singleton::() - .get_plan() - .discover_reference(reference, referent); - // Add to the corresponding list - // Note that the list is a singly-linked list, and the tail object should point to itself. - let oop = Oop::from(reference); - let head = unsafe { *self.head.get() }; - let addr = InstanceRefKlass::discovered_address::(oop); - let next_discovered = if head.is_none() { - reference - } else { - head.unwrap() - }; - // debug_assert!(!next_discovered.is_null()); - if addr - .compare_exchange( - ObjectReference::NULL, - Some(next_discovered), - Ordering::SeqCst, - Ordering::SeqCst, - ) - .is_ok() - { - unsafe { - *self.head.get() = Some(reference); - } - } - debug_assert!(!get_next_reference::(reference).is_none()); - } -} - -pub struct DiscoveredLists { - pub soft: Vec, - pub weak: Vec, - pub r#final: Vec, - pub phantom: Vec, - allow_discover: AtomicBool, -} - -unsafe impl Sync for DiscoveredLists {} - -impl DiscoveredLists { - pub fn new() -> Self { - macro_rules! with_singleton { - (|$x: ident| $($expr:tt)*) => { - if crate::use_compressed_oops() { - let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_COMPRESSED; - $($expr)* - } else { - let $x: &'static mmtk::MMTK> = &*crate::SINGLETON_UNCOMPRESSED; - $($expr)* - } - }; - } - let workers = with_singleton!(|singleton| singleton.scheduler.num_workers()); - let build_lists = |rt| (0..workers).map(|_| DiscoveredList::new(rt)).collect(); - Self { - soft: build_lists(ReferenceType::Soft), - weak: build_lists(ReferenceType::Weak), - r#final: build_lists(ReferenceType::Final), - phantom: build_lists(ReferenceType::Phantom), - allow_discover: AtomicBool::new(true), - } - } - - pub fn enable_discover(&self) { - self.allow_discover.store(true, Ordering::SeqCst) - } - - pub fn allow_discover(&self) -> bool { - self.allow_discover.load(Ordering::SeqCst) - } - - pub fn is_discovered(&self, reference: ObjectReference) -> bool { - !get_next_reference::(reference).is_none() - } - - pub fn get_by_rt_and_index(&self, rt: ReferenceType, index: usize) -> &DiscoveredList { - match rt { - ReferenceType::Soft => &self.soft[index], - ReferenceType::Weak => &self.weak[index], - ReferenceType::Phantom => &self.phantom[index], - ReferenceType::Final => &self.r#final[index], - _ => unimplemented!(), - } - } - - pub fn get(&self, rt: ReferenceType) -> &DiscoveredList { - let id = mmtk::scheduler::worker::current_worker_ordinal().unwrap(); - self.get_by_rt_and_index(rt, id) + fn set_referent(reff: ObjectReference, referent: ObjectReference) { + let oop = Oop::from(reff); + InstanceRefKlass::referent_address::(oop).store(referent); } - - fn process_lists( - &self, - worker: &mut GCWorker, - rt: ReferenceType, - lists: &[DiscoveredList], - clear: bool, - ) { - let mut packets = vec![]; - let mut packets_step2 = vec![]; - if rt == ReferenceType::Weak || rt == ReferenceType::Soft { - for i in 0..lists.len() { - let head = unsafe { *lists[i].head.get() }; - if clear { - unsafe { *lists[i].head.get() = ObjectReference::NULL }; - } - if head.is_none() { - continue; - } - let w = ProcessDiscoveredListForSoftAndWeakRefs::<_, COMPRESSED> { - head: head.unwrap(), - rt, - _p: PhantomData::, - }; - packets.push(Box::new(w) as Box>); - let w2 = ProcessDeadRefs::<_, COMPRESSED> { - list_index: i, - head: head.unwrap(), - rt, - _p: PhantomData::, - }; - packets_step2.push(Box::new(w2) as Box>); - } - } else { - for i in 0..lists.len() { - let head = unsafe { *lists[i].head.get() }; - if clear { - unsafe { *lists[i].head.get() = ObjectReference::NULL }; - } - if head.is_none() { - continue; - } - let w = ProcessDiscoveredList::<_, COMPRESSED> { - list_index: i, - head: head.unwrap(), - rt, - _p: PhantomData::, - }; - packets.push(Box::new(w) as Box>); - } - } - worker.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets); - if rt == ReferenceType::Weak || rt == ReferenceType::Soft { - worker.scheduler().work_buckets[WorkBucketStage::PhantomRefClosure] - .bulk_add(packets_step2); - } + fn get_referent(object: ObjectReference) -> Option { + let oop = Oop::from(object); + InstanceRefKlass::referent_address::(oop).load() } - - pub fn reconsider_soft_refs(&self, _worker: &mut GCWorker) {} - - pub fn process_soft_weak_final_refs( - &self, - worker: &mut GCWorker, - ) { - self.allow_discover.store(false, Ordering::SeqCst); - if !*worker.mmtk.get_options().no_reference_types { - self.process_lists::(worker, ReferenceType::Soft, &self.soft, true); - self.process_lists::(worker, ReferenceType::Weak, &self.weak, true); - } - if !*worker.mmtk.get_options().no_finalizer { - self.process_lists::(worker, ReferenceType::Final, &self.r#final, false); - } - } - - pub fn resurrect_final_refs( - &self, - worker: &mut GCWorker, - ) { - assert!(!*worker.mmtk.get_options().no_finalizer); - let lists = &self.r#final; - let mut packets = vec![]; - for i in 0..lists.len() { - let head = unsafe { *lists[i].head.get() }; - unsafe { *lists[i].head.get() = ObjectReference::NULL }; - if head.is_none() { - continue; - } - let w = ResurrectFinalizables::<_, COMPRESSED> { - list_index: i, - head: head.unwrap(), - _p: PhantomData::, - }; - packets.push(Box::new(w) as Box>); - } - worker.scheduler().work_buckets[WorkBucketStage::Unconstrained].bulk_add(packets); - } - - pub fn process_phantom_refs( - &self, - worker: &mut GCWorker, - ) { - assert!(!*worker.mmtk.get_options().no_reference_types); - self.process_lists::(worker, ReferenceType::Phantom, &self.phantom, true); - } -} - -lazy_static! { - pub static ref LOCK: Mutex<()> = Mutex::new(()); - pub static ref DISCOVERED_LISTS: DiscoveredLists = DiscoveredLists::new(); -} - -pub struct ProcessDiscoveredList { - list_index: usize, - head: ObjectReference, - rt: ReferenceType, - _p: PhantomData, -} - -impl GCWork - for ProcessDiscoveredList -{ - fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut trace = E::new(vec![], false, mmtk, WorkBucketStage::Unconstrained); - trace.set_worker(worker); - let retain = self.rt == ReferenceType::Soft - && !mmtk.is_emergency_collection() - && !mmtk.is_user_triggered_collection(); - - let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { - debug_assert!( - get_next_reference::(reference).is_none(), - "next must be null. ref={:?} {:?} bin={}", - reference, - self.rt, - self.list_index - ); - let reference = trace.trace_object(reference); - let referent = get_referent::(reference); - if referent.is_none() { - // Remove from the discovered list - return DiscoveredListIterationResult::Remove; - } - let referent = referent.unwrap(); - if referent.is_reachable() || retain { - // Keep this referent - let forwarded = trace.trace_object(referent); - debug_assert!(forwarded.get_forwarded_object().is_none()); - debug_assert!(reference.get_forwarded_object().is_none()); - set_referent::(reference, Some(forwarded)); - // Remove from the discovered list - return DiscoveredListIterationResult::Remove; - } else { - if self.rt != ReferenceType::Final { - // Clear this referent - set_referent::(reference, ObjectReference::NULL); - // set_referent(reference, ObjectReference::NULL); - } - // Keep the reference - return DiscoveredListIterationResult::Enqueue(reference); - } - }); - // Flush the list to the Universe::pending_list - if let Some((head, tail)) = new_list { - // debug_assert!(!head.is_null() && !tail.is_null()); - // debug_assert_eq!(ObjectReference::NULL, get_next_reference(tail)); - if self.rt == ReferenceType::Final { - let slot = DISCOVERED_LISTS.r#final[self.list_index].head.get(); - unsafe { *slot = Some(head) }; - } else { - let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; - set_next_reference::(tail, Some(old_head)); - } - } else { - if self.rt == ReferenceType::Final { - let slot = DISCOVERED_LISTS.r#final[self.list_index].head.get(); - unsafe { *slot = ObjectReference::NULL }; - } - } - trace.process_slots(); - trace.flush(); - } -} - -pub struct ProcessDiscoveredListForSoftAndWeakRefs { - head: ObjectReference, - rt: ReferenceType, - _p: PhantomData, -} - -impl GCWork - for ProcessDiscoveredListForSoftAndWeakRefs -{ - fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut trace = E::new(vec![], false, mmtk, WorkBucketStage::Unconstrained); - trace.set_worker(worker); - let retain = self.rt == ReferenceType::Soft - && !mmtk.is_emergency_collection() - && !mmtk.is_user_triggered_collection(); - - let mut reference_opt = Some(self.head); - - while let Some(reference) = reference_opt { - let next_ref = get_next_reference::(reference); - let next_ref = next_ref - .map(|x| x.get_forwarded_object()) - .flatten() - .or_else(|| next_ref); - if let Some(o) = next_ref { - debug_assert!(o.get_forwarded_object().is_none()); - } - // Reaches the end of the list? - let end_of_list = next_ref == Some(reference) || next_ref.is_none(); - reference_opt = if end_of_list { None } else { next_ref }; - - let reference = trace.trace_object(reference); - let referent = get_referent::(reference); - if referent.is_none() { - continue; - } - let referent = referent.unwrap(); - if referent.is_reachable() || retain { - // Keep this referent - let forwarded = trace.trace_object(referent); - debug_assert!(forwarded.get_forwarded_object().is_none()); - debug_assert!(reference.get_forwarded_object().is_none()); - set_referent::(reference, Some(forwarded)); - } - } - trace.process_slots(); - trace.flush(); - } -} - -pub struct ProcessDeadRefs { - list_index: usize, - head: ObjectReference, - rt: ReferenceType, - _p: PhantomData, -} - -impl GCWork for ProcessDeadRefs { - fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut trace = E::new(vec![], false, mmtk, WorkBucketStage::Unconstrained); - trace.set_worker(worker); - let retain = self.rt == ReferenceType::Soft - && !mmtk.is_emergency_collection() - && !mmtk.is_user_triggered_collection(); - - let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { - debug_assert!( - get_next_reference::(reference).is_none(), - "next must be null. ref={:?} {:?} bin={}", - reference, - self.rt, - self.list_index - ); - let referent = get_referent::(reference); - if referent.is_none() { - // Remove from the discovered list - return DiscoveredListIterationResult::Remove; - } - let referent = referent.unwrap(); - if referent.is_reachable() || retain { - // Keep this referent - let forwarded = referent.get_forwarded_object().unwrap_or(referent); - debug_assert!(forwarded.get_forwarded_object().is_none()); - debug_assert!(reference.get_forwarded_object().is_none()); - if forwarded != referent { - set_referent::(reference, Some(forwarded)); - } - // Remove from the discovered list - return DiscoveredListIterationResult::Remove; - } else { - if self.rt != ReferenceType::Final { - // Clear this referent - set_referent::(reference, ObjectReference::NULL); - // set_referent(reference, ObjectReference::NULL); - } - // Keep the reference - return DiscoveredListIterationResult::Enqueue(reference); - } - }); - // Flush the list to the Universe::pending_list - if let Some((head, tail)) = new_list { - // debug_assert!(!head.is_null() && !tail.is_null()); - // debug_assert_eq!(ObjectReference::NULL, get_next_reference(tail)); - if self.rt == ReferenceType::Final { - let slot = DISCOVERED_LISTS.r#final[self.list_index].head.get(); - unsafe { *slot = Some(head) }; - } else { - let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; - set_next_reference::(tail, Some(old_head)); - } - } else { - if self.rt == ReferenceType::Final { - let slot = DISCOVERED_LISTS.r#final[self.list_index].head.get(); - unsafe { *slot = ObjectReference::NULL }; - } - } - // assert!(trace.is_empty()); - trace.process_slots(); - trace.flush(); - } -} - -pub struct ResurrectFinalizables { - list_index: usize, - head: ObjectReference, - _p: PhantomData, -} - -impl GCWork - for ResurrectFinalizables -{ - fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut trace = E::new(vec![], false, mmtk, WorkBucketStage::Unconstrained); - trace.set_worker(worker); - let new_list = iterate_list::<_, COMPRESSED>(self.head, |reference| { - let reference = trace.trace_object(reference); - let referent = get_referent::(reference); - let forwarded = match referent { - Some(o) => Some(trace.trace_object(o)), - None => None, - }; - set_referent::(reference, forwarded); - if let Some(forwarded) = forwarded { - debug_assert!(forwarded.get_forwarded_object().is_none()); - } - return DiscoveredListIterationResult::Enqueue(reference); - }); - - if let Some((head, tail)) = new_list { - // debug_assert!(!head.is_null() && !tail.is_null()); - // debug_assert_eq!(ObjectReference::NULL, get_next_reference(tail)); - let slot = DISCOVERED_LISTS.r#final[self.list_index].head.get(); - unsafe { *slot = ObjectReference::NULL }; - let old_head = unsafe { ((*crate::UPCALLS).swap_reference_pending_list)(head) }; - set_next_reference::(tail, Some(old_head)); - } - trace.process_slots(); - trace.flush(); - } -} - -enum DiscoveredListIterationResult { - Remove, - Enqueue(ObjectReference), -} - -fn iterate_list< - F: FnMut(ObjectReference) -> DiscoveredListIterationResult, - const COMPRESSED: bool, ->( - head: ObjectReference, - mut visitor: F, -) -> Option<(ObjectReference, ObjectReference)> { - let mut new_head: Option = None; - let mut new_tail: Option = None; - let mut reference = head; - loop { - // debug_assert!(reference.is_live()); - // Update reference forwarding pointer - if let Some(forwarded) = reference.get_forwarded_object() { - reference = forwarded; - } - debug_assert!(reference.get_forwarded_object().is_none()); - debug_assert!(reference.is_reachable()); - // Update next_ref forwarding pointer - let next_ref = get_next_reference::(reference); - let next_ref = next_ref - .map(|x| x.get_forwarded_object()) - .flatten() - .or_else(|| next_ref); - if let Some(o) = next_ref { - debug_assert!(o.get_forwarded_object().is_none()); - } - // Reaches the end of the list? - let end_of_list = next_ref == Some(reference) || next_ref.is_none(); - // Remove `reference` from current list - set_next_reference::(reference, ObjectReference::NULL); - if let Some(forwarded_ref) = reference.get_forwarded_object() { - set_next_reference::(forwarded_ref, ObjectReference::NULL); - } - // Process reference - let result = visitor(reference); - match result { - DiscoveredListIterationResult::Remove => {} - DiscoveredListIterationResult::Enqueue(reference) => { - // Add to new list - if let Some(new_head) = new_head { - set_next_reference::(reference, Some(new_head)); - } else { - new_tail = Some(reference); - } - new_head = Some(reference); - } - } - // Reached the end of the list? - if end_of_list { - break; + fn enqueue_references(references: &[ObjectReference], _tls: VMWorkerThread) { + unsafe { + ((*UPCALLS).enqueue_references)(references.as_ptr(), references.len()); } - // Move to next - reference = next_ref.unwrap(); } - if new_head.is_none() { - None - } else { - Some((new_head.unwrap(), new_tail.unwrap())) + fn clear_referent(new_reference: ObjectReference) { + let oop = Oop::from(new_reference); + InstanceRefKlass::referent_address::(oop).store_null(); } } diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index 55ddf9c0..0a491bca 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -210,9 +210,7 @@ typedef struct { void (*prepare_for_roots_re_scanning)(); void (*mmtk_update_weak_processor)(bool lxr); void (*enqueue_references)(void** objects, size_t len); - void* (*swap_reference_pending_list)(void* objects); void (*fix_oop_relocations)(bool lxr, void* nmethods, size_t len); - void (*clear_claimed_marks)(); void (*gc_epilogue)(); } OpenJDK_Upcalls; diff --git a/openjdk/share/mmtkCollectorThread.cpp b/openjdk/share/mmtkCollectorThread.cpp index 103c4ef0..7d22d35c 100644 --- a/openjdk/share/mmtkCollectorThread.cpp +++ b/openjdk/share/mmtkCollectorThread.cpp @@ -32,7 +32,5 @@ MMTkCollectorThread::MMTkCollectorThread(void* context): NamedThread() { } void MMTkCollectorThread::run() { - // this->initialize_named_thread(); - os::set_priority(this, NearMaxPriority); start_worker((void*) this, third_party_heap_collector); } diff --git a/openjdk/share/mmtkHeap.cpp b/openjdk/share/mmtkHeap.cpp index 55db417a..e4f184b1 100644 --- a/openjdk/share/mmtkHeap.cpp +++ b/openjdk/share/mmtkHeap.cpp @@ -458,6 +458,9 @@ void MMTkHeap::register_nmethod(nmethod* nm) { mmtk_register_nmethod((void*) nm); } +void MMTkHeap::verify_nmethod(nmethod* nm) { +} + void MMTkHeap::unregister_nmethod(nmethod* nm) { mmtk_unregister_nmethod((void*) nm); } @@ -469,7 +472,6 @@ void MMTkHeap::scan_code_cache_roots(OopClosure& cl) { MarkingCodeBlobClosure cb_cl(&cl, false, true); CodeCache::blobs_do(&cb_cl); } - void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl) { CLDToOopClosure cld_cl(&cl, false); ClassLoaderDataGraph::cld_do(&cld_cl); @@ -521,41 +523,13 @@ void MMTkHeap::scan_roots(OopClosure& cl) { WeakProcessor::oops_do(&cl); } -static inline HeapWord* alloc_fast(size_t bytes, Allocator allocator) { - // All allocations with size larger than max non-los bytes will get to this slowpath here. - // We will use LOS for those. - assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); - if (bytes >= MMTkMutatorContext::max_non_los_default_alloc_bytes) { - allocator = AllocatorLos; - } else if (allocator == AllocatorDefault) { - AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; - if (selector.tag == TAG_IMMIX && !disable_fast_alloc()) { - auto& allocator = Thread::current()->third_party_heap_mutator.allocators.immix[selector.index]; - auto cursor = uintptr_t(allocator.cursor); - auto limit = uintptr_t(allocator.limit); - if (cursor + bytes <= limit) { - allocator.cursor = (void*) (cursor + bytes); - return (HeapWord*) cursor; - } - } - } - - // FIXME: Proper use of slow-path api - HeapWord* o = (HeapWord*) ::alloc((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, bytes, HeapWordSize, 0, allocator); - // Post allocation hooks. Note that we can get a nullptr from mmtk core in the case of OOM. - // Hence, only call post allocation hooks if we have a proper object. - if (o != nullptr && allocator != AllocatorDefault) { - ::post_alloc((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, o, bytes, allocator); - } - return o; -} - HeapWord* MMTkHeap::mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) { - return alloc_fast(size << LogHeapWordSize, AllocatorDefault); + HeapWord* obj = Thread::current()->third_party_heap_mutator.alloc(size << LogHeapWordSize); + return obj; } HeapWord* MMTkHeap::mem_allocate_nonmove(size_t size, bool* gc_overhead_limit_was_exceeded) { - return alloc_fast(size << LogHeapWordSize, AllocatorLos); + return Thread::current()->third_party_heap_mutator.alloc(size << LogHeapWordSize, AllocatorLos); } void MMTkHeap::register_new_weak_handle(oop* handle) {} diff --git a/openjdk/share/mmtkHeap.hpp b/openjdk/share/mmtkHeap.hpp index 6f245515..7c7d5ecc 100644 --- a/openjdk/share/mmtkHeap.hpp +++ b/openjdk/share/mmtkHeap.hpp @@ -159,8 +159,6 @@ class MMTkHeap : public CollectedHeap { virtual void collect_as_vm_thread(GCCause::Cause cause) override; - // Return the CollectorPolicy for the heap - virtual SoftRefPolicy* soft_ref_policy() override; virtual GrowableArray memory_managers() override; @@ -200,7 +198,11 @@ class MMTkHeap : public CollectedHeap { virtual void register_nmethod(nmethod* nm) override; virtual void unregister_nmethod(nmethod* nm) override; - virtual void verify_nmethod(nmethod* nm) override {} + + virtual void verify_nmethod(nmethod* nm) override; + + // An object is scavengable if its location may move during a scavenge. + // (A scavenge is a GC which is not a full GC.) inline bool is_scavengable(oop obj) { return true; } // Heap verification diff --git a/openjdk/share/mmtkMutator.cpp b/openjdk/share/mmtkMutator.cpp index a9f273a0..1d51ad4a 100644 --- a/openjdk/share/mmtkMutator.cpp +++ b/openjdk/share/mmtkMutator.cpp @@ -31,17 +31,6 @@ HeapWord* MMTkMutatorContext::alloc(size_t bytes, Allocator allocator) { assert(MMTkMutatorContext::max_non_los_default_alloc_bytes != 0, "max_non_los_default_alloc_bytes hasn't been initialized"); if (bytes >= MMTkMutatorContext::max_non_los_default_alloc_bytes) { allocator = AllocatorLos; - } else { - AllocatorSelector selector = MMTkHeap::heap()->default_allocator_selector; - if (selector.tag == TAG_IMMIX && !disable_fast_alloc()) { - auto& allocator = allocators.immix[selector.index]; - auto cursor = uintptr_t(allocator.cursor); - auto limit = uintptr_t(allocator.limit); - if (cursor + bytes <= limit) { - allocator.cursor = (void*) (cursor + bytes); - return (HeapWord*) cursor; - } - } } // FIXME: Proper use of slow-path api @@ -60,9 +49,4 @@ void MMTkMutatorContext::flush() { void MMTkMutatorContext::destroy() { ::destroy_mutator((MMTk_Mutator) this); - // if (original_rust_mutator_pointer != NULL) { - // *original_rust_mutator_pointer = *this; - // release_mutator(original_rust_mutator_pointer); - // original_rust_mutator_pointer = NULL; - // } } diff --git a/openjdk/share/mmtkRootsClosure.hpp b/openjdk/share/mmtkRootsClosure.hpp index a0bbfc46..f46ffb94 100644 --- a/openjdk/share/mmtkRootsClosure.hpp +++ b/openjdk/share/mmtkRootsClosure.hpp @@ -56,28 +56,9 @@ class MMTkRootsClosure : public OopClosure { virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } }; - -template -class MMTkScanCLDClosure: public CLDClosure { - private: - OopClosure* _oop_closure; - protected: - public: - MMTkScanCLDClosure(OopClosure* c) : _oop_closure(c) { } - void do_cld(ClassLoaderData* cld) { - if (MODIFIED_ONLY) { - if (cld->has_modified_oops()) cld->oops_do(_oop_closure, CLAIM, /*clear_modified_oops*/true); - } else { - if (cld->has_modified_oops() || !WEAK) - cld->oops_do(_oop_closure, CLAIM, /*clear_modified_oops*/true); - } - } -}; - class MMTkScanObjectClosure : public BasicOopIterateClosure { - void (*_trace)(void*); - bool _follow_clds; - bool _claim_clds; + void* _trace; + CLDToOopClosure follow_cld_closure; template void do_oop_work(T* p, bool narrow) { @@ -85,27 +66,27 @@ class MMTkScanObjectClosure : public BasicOopIterateClosure { guarantee((uintptr_t(p) & (1ull << 63)) == 0, "test"); p = (T*) (uintptr_t(p) | (1ull << 63)); } - _trace((void*) p); } public: - MMTkScanObjectClosure(void* trace, bool follow_clds, bool claim_clds): _trace((void (*)(void*)) trace), _follow_clds(follow_clds), _claim_clds(claim_clds) {} + MMTkScanObjectClosure(void* trace): _trace(trace), follow_cld_closure(this, false) {} - virtual void do_oop(oop* p) { do_oop_work(p, false); } + virtual void do_oop(oop* p) { do_oop_work(p, false); } virtual void do_oop(narrowOop* p) { do_oop_work(p, true); } virtual bool do_metadata() { - return _follow_clds; + return true; } virtual void do_klass(Klass* k) { - if (!_follow_clds) return; - do_cld(k->class_loader_data()); + // follow_cld_closure.do_cld(k->class_loader_data()); + // oop op = k->klass_holder(); + // oop new_op = (oop) trace_root_object(_trace, op); + // guarantee(new_op == op, "trace_root_object returned a different value %p -> %p", op, new_op); } virtual void do_cld(ClassLoaderData* cld) { - if (!_follow_clds) return; - cld->oops_do(this, _claim_clds); + follow_cld_closure.do_cld(cld); } virtual ReferenceIterationMode reference_iteration_mode() { return DO_FIELDS; } @@ -120,14 +101,4 @@ class MMTkScanObjectClosure : public BasicOopIterateClosure { // } // }; -class CodeBlobFixRelocationClosure: public CodeBlobClosure { - public: - inline virtual void do_code_blob(CodeBlob* cb) { - nmethod* nm = cb->as_nmethod_or_null(); - if (nm != NULL) { - nm->fix_oop_relocations(); - } - } -}; - #endif // MMTK_OPENJDK_MMTK_ROOTS_CLOSURE_HPP diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index 29409f65..8aa638e7 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -197,10 +197,6 @@ static void mmtk_stop_all_mutators(void *tls, MutatorClosure closure) { nmethod::oops_do_marking_prologue(); } -static void mmtk_clear_claimed_marks() { - ClassLoaderDataGraph::clear_claimed_marks(); -} - static void mmtk_update_weak_processor(bool lxr) { // HandleMark hm(THREAD); if (lxr) { @@ -364,7 +360,7 @@ static void mmtk_scan_multiple_thread_roots(SlotsClosure closure, void* ptr, siz } static void mmtk_scan_object(void* trace, void* object, void* tls) { - MMTkScanObjectClosure cl(trace, false, false); + MMTkScanObjectClosure cl(trace); ((oop) object)->oop_iterate(&cl); } @@ -515,9 +511,6 @@ static void mmtk_enqueue_references(void** objects, size_t len) { assert(Universe::has_reference_pending_list(), "Reference pending list is empty after swap"); } -static void* mmtk_swap_reference_pending_list(void* object) { - return Universe::swap_reference_pending_list((oop) object); -} void mmtk_fix_oop_relocations(bool lxr, void* nmethods, size_t len) { if (lxr) { @@ -570,8 +563,6 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_prepare_for_roots_re_scanning, mmtk_update_weak_processor, mmtk_enqueue_references, - mmtk_swap_reference_pending_list, mmtk_fix_oop_relocations, - mmtk_clear_claimed_marks, mmtk_gc_epilogue, }; From 6ed14cd098713bbb8cf6ab5dbcafa94c0b86bb05 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 7 Apr 2026 17:28:12 +1000 Subject: [PATCH 466/494] Remove bulk thread scanning support (scan_multiple_thread_roots) Remove scan_multiple_thread_roots from the OpenJDK binding to match the corresponding removal in mmtk-core. This includes removing: - scan_multiple_thread_root implementation in scanning.rs - scan_multiple_thread_roots field from OpenJDK_Upcalls struct - mmtk_scan_multiple_thread_roots function and initializer entry in mmtkUpcalls.cpp - Function pointer declaration in mmtk.h Co-Authored-By: Claude Opus 4.6 (1M context) --- mmtk/src/lib.rs | 2 -- mmtk/src/scanning.rs | 25 ------------------------- openjdk/share/mmtk.h | 1 - openjdk/share/mmtkUpcalls.cpp | 10 ---------- 4 files changed, 38 deletions(-) diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 3c3f9d13..efdb17cd 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -107,8 +107,6 @@ pub struct OpenJDK_Upcalls { pub dump_object_string: extern "C" fn(object: ObjectReference) -> *const c_char, pub scan_roots_in_all_mutator_threads: extern "C" fn(closure: SlotsClosure), pub scan_roots_in_mutator_thread: extern "C" fn(closure: SlotsClosure, tls: VMMutatorThread), - pub scan_multiple_thread_roots: - extern "C" fn(closure: SlotsClosure, ptr: OpaquePointer, len: usize), pub scan_code_cache_roots: extern "C" fn(closure: SlotsClosure), pub scan_class_loader_data_graph_roots: extern "C" fn(closure: SlotsClosure), pub scan_oop_storage_set_roots: extern "C" fn(closure: SlotsClosure), diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 466922a1..bb4c208f 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -86,31 +86,6 @@ impl Scanning> for VMScanning { } } - fn scan_multiple_thread_root( - _tls: VMWorkerThread, - mutators: Vec, - mut factory: impl RootsWorkFactory< as mmtk::vm::VMBinding>::VMSlot>, - ) { - // let t = if cfg!(feature = "roots_breakdown") { - // Some(std::time::SystemTime::now()) - // } else { - // None - // }; - let len = mutators.len(); - let ptr = mutators.as_ptr(); - unsafe { - ((*UPCALLS).scan_multiple_thread_roots)( - to_slots_closure(&mut factory), - std::mem::transmute(ptr), - len, - ); - } - // if cfg!(feature = "roots_breakdown") { - // let ms = t.unwrap().elapsed().unwrap().as_micros() as f32 / 1000f32; - // eprintln!(" - ScanThreadRoots ({:.3}ms)", ms); - // } - } - fn scan_vm_specific_roots( _tls: VMWorkerThread, factory: impl RootsWorkFactory>, diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index 0a491bca..a3134f35 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -199,7 +199,6 @@ typedef struct { const char* (*dump_object_string) (void* object); void (*scan_roots_in_all_mutator_threads)(SlotsClosure closure); void (*scan_roots_in_mutator_thread)(SlotsClosure closure, void* tls); - void (*scan_multiple_thread_roots)(SlotsClosure closure, void* ptr, size_t len); void (*scan_code_cache_roots) (SlotsClosure closure); void (*scan_class_loader_data_graph_roots) (SlotsClosure closure); void (*scan_oop_storage_set_roots) (SlotsClosure closure); diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index 8aa638e7..47a3a4ea 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -350,15 +350,6 @@ static void mmtk_scan_roots_in_mutator_thread(SlotsClosure closure, void* tls) { thread->oops_do(&cl, &cb_cl); } -static void mmtk_scan_multiple_thread_roots(SlotsClosure closure, void* ptr, size_t len) { - ResourceMark rm; - auto mutators = (JavaThread**) ptr; - MMTkRootsClosure cl(closure); - MarkingCodeBlobClosure cb_cl(&cl, false, true); - for (size_t i = 0; i < len; i++) - mutators[i]->oops_do(&cl, &cb_cl); -} - static void mmtk_scan_object(void* trace, void* object, void* tls) { MMTkScanObjectClosure cl(trace); ((oop) object)->oop_iterate(&cl); @@ -552,7 +543,6 @@ OpenJDK_Upcalls mmtk_upcalls = { dump_object_string, mmtk_scan_roots_in_all_mutator_threads, mmtk_scan_roots_in_mutator_thread, - mmtk_scan_multiple_thread_roots, mmtk_scan_code_cache_roots, mmtk_scan_class_loader_data_graph_roots, mmtk_scan_oop_storage_set_roots, From acc3d31f083a3a535690121f8238d65488e30f6d Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 7 Apr 2026 18:15:41 +1000 Subject: [PATCH 467/494] Remove dead feature flags, instrumentation, and stale declarations - Remove ~30 unused Cargo feature flags: barrier measurement/instrumentation (barrier_measurement, yield_and_roots_timer, satb_timer, log_gc, etc.), experiment flags (field_barrier_no_*, compressed pointer modes, no_fast_alloc, same_stw_and_conc_threads, lxr_*_conc_workers), and stale flags (release_debug_assertions, discontig_space, g3_compatibilities) - Delete .cargo/config.toml that forced AVX/SSE target features - Remove DISABLE_ALLOCATION_FAST_PATH and FIELD_BARRIER_NO_* exported statics from lib.rs and their corresponding extern declarations in mmtk.h - Remove stale declarations from mmtk.h: CLASS_UNLOADING_ENABLED, ENABLE_CLASS_UNLOADING_LOGS macro, GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS_COMPRESSED, HEAP_START, HEAP_END, disable_fast_alloc() - Simplify mmtk_builder_set_conc_threads to pass through the value directly - Remove all FIELD_BARRIER_NO_* conditional guards from C++ barrier code (mmtkFieldBarrier.cpp/hpp, mmtkBarrierSet.cpp, mmtkBarrierSetC2.cpp, mmtkBarrierSetAssembler_x86.cpp), always executing the full barrier path - Remove disable_fast_alloc() checks from allocation fast-path conditions - Remove commented-out profile settings from Cargo.toml Co-Authored-By: Claude Opus 4.6 (1M context) --- mmtk/.cargo/config.toml | 5 --- mmtk/Cargo.toml | 36 ------------------- mmtk/src/api.rs | 11 +----- mmtk/src/lib.rs | 15 -------- .../cpu/x86/mmtkBarrierSetAssembler_x86.cpp | 2 +- openjdk/share/barriers/mmtkFieldBarrier.cpp | 19 ++++------ openjdk/share/barriers/mmtkFieldBarrier.hpp | 2 +- openjdk/share/mmtk.h | 21 ----------- openjdk/share/mmtkBarrierSet.cpp | 5 +-- openjdk/share/mmtkBarrierSetC2.cpp | 2 +- 10 files changed, 11 insertions(+), 107 deletions(-) delete mode 100644 mmtk/.cargo/config.toml diff --git a/mmtk/.cargo/config.toml b/mmtk/.cargo/config.toml deleted file mode 100644 index 9f12ddcd..00000000 --- a/mmtk/.cargo/config.toml +++ /dev/null @@ -1,5 +0,0 @@ -[target.x86_64-unknown-linux-gnu] -rustflags = [ - "-C", - "target-feature=+avx,+avx2,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3", -] diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 926ea398..defe2f66 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -11,16 +11,8 @@ name = "mmtk_openjdk" # be careful - LTO is only allowed for certain crate types crate-type = ["cdylib"] -[profile.dev] -panic = "abort" -codegen-units = 24 - [profile.release] lto = true -codegen-units = 1 -# codegen-units = 24 -# debug-assertions = true -# debug = true # Metadata for the OpenJDK repository [package.metadata.openjdk] @@ -54,19 +46,12 @@ built = { version = "0.7.7", features = ["git2"] } [features] default = [] -release_debug_assertions = [] mmtk_extreme_assertions = ["mmtk/extreme_assertions"] nogc_lock_free = ["mmtk/nogc_lock_free"] nogc_no_zeroing = ["mmtk/nogc_no_zeroing"] # Use the env var MMTK_VO_BIT=1 when building OpenJDK so the fastpath for setting the alloc bit will be compiled in. # See README. vo_bit = ["mmtk/vo_bit"] -no_fast_alloc = [] -roots_breakdown = [] -same_stw_and_conc_threads = [] -lxr_6_conc_workers = [] -lxr_12_conc_workers = [] -g3_compatibilities = [] # Place the mark bit in the header of objects instead of on the side. mark_bit_in_header = [] @@ -93,24 +78,3 @@ markcompact = [] jfr = [] sanity = ["mmtk/sanity"] -barrier_measurement = ["mmtk/barrier_measurement"] -yield_and_roots_timer = ["mmtk/yield_and_roots_timer"] -satb_timer = ["mmtk/satb_timer"] -instrumentation = ["mmtk/instrumentation", "no_fast_alloc"] -log_gc = ["mmtk/log_gc"] -log_stages = ["mmtk/log_stages"] -log_work_packets = ["mmtk/log_work_packets"] -discontig_space = [] - -# Compressed pointer experiments -force_narrow_oop_mode = [] -narrow_oop_mode_32bit = ["force_narrow_oop_mode"] -narrow_oop_mode_shift = ["force_narrow_oop_mode"] -narrow_oop_mode_base = ["force_narrow_oop_mode"] -narrow_oop_mode_base_and_shift = ["force_narrow_oop_mode"] - -field_barrier_no_eager_branch = [] -field_barrier_no_arraycopy = [] -field_barrier_no_arraycopy_slow = [] -field_barrier_no_c2_slow_call = [] -field_barrier_no_c2_rust_call = [] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 4fe7a73a..b6adc2ae 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -358,16 +358,7 @@ pub extern "C" fn mmtk_builder_set_threads(value: usize) { #[no_mangle] pub extern "C" fn mmtk_builder_set_conc_threads(value: usize) { let mut builder = BUILDER.lock().unwrap(); - if cfg!(feature = "same_stw_and_conc_threads") { - let threads = *builder.options.threads; - builder.options.conc_threads.set(threads); - } else if cfg!(feature = "lxr_6_conc_workers") { - builder.options.conc_threads.set(6); - } else if cfg!(feature = "lxr_12_conc_workers") { - builder.options.conc_threads.set(12); - } else { - builder.options.conc_threads.set(value); - } + builder.options.conc_threads.set(value); } /// Pass hotspot `UseTransparentHugePages` flag to mmtk diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index efdb17cd..89ebe19e 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -151,25 +151,10 @@ pub static VO_BIT_ADDRESS: uintptr_t = pub static FREE_LIST_ALLOCATOR_SIZE: uintptr_t = std::mem::size_of::>>(); -#[no_mangle] -pub static DISABLE_ALLOCATION_FAST_PATH: i32 = cfg!(feature = "no_fast_alloc") as _; - #[no_mangle] pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = std::mem::size_of::>>(); -#[no_mangle] -pub static FIELD_BARRIER_NO_EAGER_BRANCH: u8 = cfg!(feature = "field_barrier_no_eager_branch") as _; -#[no_mangle] -pub static FIELD_BARRIER_NO_ARRAYCOPY: u8 = cfg!(feature = "field_barrier_no_arraycopy") as _; -#[no_mangle] -pub static FIELD_BARRIER_NO_ARRAYCOPY_SLOW: u8 = - cfg!(feature = "field_barrier_no_arraycopy_slow") as _; -#[no_mangle] -pub static FIELD_BARRIER_NO_C2_SLOW_CALL: u8 = cfg!(feature = "field_barrier_no_c2_slow_call") as _; -#[no_mangle] -pub static FIELD_BARRIER_NO_C2_RUST_CALL: u8 = cfg!(feature = "field_barrier_no_c2_rust_call") as _; - #[no_mangle] pub static mut CONCURRENT_MARKING_ACTIVE: u8 = 0; diff --git a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp index 70332064..99f3d9d3 100644 --- a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp @@ -42,7 +42,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa assert(obj == rax, "obj must be in rax, for cmpxchg"); assert_different_registers(obj, var_size_in_bytes, t1); - if (!mmtk_enable_allocation_fastpath || disable_fast_alloc()) { + if (!mmtk_enable_allocation_fastpath) { __ jmp(slow_case); } else { // MMTk size check. If the alloc size is larger than the allowed max size for non los, diff --git a/openjdk/share/barriers/mmtkFieldBarrier.cpp b/openjdk/share/barriers/mmtkFieldBarrier.cpp index 6695f667..21d42c36 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.cpp @@ -32,8 +32,7 @@ void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, intptr_t addr = ((intptr_t) (void*) slot); const volatile uint8_t * meta_addr = (const volatile uint8_t *) (side_metadata_base_address() + (addr >> (UseCompressedOops ? 5 : 6))); uint8_t byte_val = *meta_addr; - if (!FIELD_BARRIER_NO_EAGER_BRANCH) - if (byte_val == 0) return; + if (byte_val == 0) return; intptr_t shift = (addr >> (UseCompressedOops ? 2 : 3)) & 0b111; if (((byte_val >> shift) & 1) == kUnloggedValue) { ::mmtk_object_reference_write_slow((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); @@ -94,10 +93,8 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma __ shrptr(tmp1, UseCompressedOops ? 5 : 6); __ movptr(tmp2, side_metadata_base_address()); __ movzbl(tmp2, Address(tmp2, tmp1)); - if (!FIELD_BARRIER_NO_EAGER_BRANCH) { - __ cmpl(tmp2, 0); - __ jcc(Assembler::equal, done); - } + __ cmpl(tmp2, 0); + __ jcc(Assembler::equal, done); // tmp1 = (obj >> 3) & 7 __ lea(tmp1, dst); __ shrptr(tmp1, UseCompressedOops ? 2 : 3); @@ -139,7 +136,6 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma } void MMTkFieldBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { - if (FIELD_BARRIER_NO_ARRAYCOPY) return; bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; if (dest_uninitialized) return; if (type == T_OBJECT || type == T_ARRAY) { @@ -365,18 +361,15 @@ static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* sl Node* addr = __ CastPX(__ ctrl(), slot); Node* meta_addr = __ AddP(no_base, __ ConP(side_metadata_base_address()), __ URShiftX(addr, __ ConI(UseCompressedOops ? 5 : 6))); Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); - if (!FIELD_BARRIER_NO_EAGER_BRANCH) - __ if_then(byte, BoolTest::ne, zero, unlikely); + __ if_then(byte, BoolTest::ne, zero, unlikely); Node* shift = __ URShiftX(addr, __ ConI(UseCompressedOops ? 2 : 3)); shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); __ if_then(result, BoolTest::ne, zero, unlikely); { const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeRawPtr::NOTNULL); - if (!FIELD_BARRIER_NO_C2_SLOW_CALL) - Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_object_reference_write_slow), "mmtk_barrier_call", src, slot, val, mutator); + Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_object_reference_write_slow), "mmtk_barrier_call", src, slot, val, mutator); } __ end_if(); - if (!FIELD_BARRIER_NO_EAGER_BRANCH) - __ end_if(); + __ end_if(); } else { const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), "mmtk_barrier_call", src, slot, val); diff --git a/openjdk/share/barriers/mmtkFieldBarrier.hpp b/openjdk/share/barriers/mmtkFieldBarrier.hpp index 731a09f3..d3ff953b 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.hpp @@ -21,7 +21,7 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { // Interfaces called by `MMTkBarrierSet::AccessBarrier` virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const override { - if (FIELD_BARRIER_NO_ARRAYCOPY || count == 0) return; + if (count == 0) return; ::mmtk_array_copy_pre((void*) src, (void*) dst, count, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } virtual void object_probable_write(oop new_obj) const override; diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index a3134f35..e478b860 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -5,13 +5,6 @@ #include #include -#define ENABLE_CLASS_UNLOADING_LOGS false -#if ENABLE_CLASS_UNLOADING_LOGS -#define LOG_CLS_UNLOAD(...) log_info(gc)(__VA_ARGS__) -#else -#define LOG_CLS_UNLOAD(...) -#endif // ENABLE_CLASS_UNLOADING_LOGS - #ifdef __cplusplus extern "C" { #endif @@ -27,27 +20,13 @@ typedef enum { extern const uintptr_t GLOBAL_SIDE_METADATA_BASE_ADDRESS; extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS; -extern const uintptr_t GLOBAL_SIDE_METADATA_VM_BASE_ADDRESS_COMPRESSED; extern const uintptr_t RC_TABLE_BASE_ADDRESS; extern const uintptr_t VO_BIT_ADDRESS; -extern const int DISABLE_ALLOCATION_FAST_PATH; extern const uintptr_t IMMIX_ALLOCATOR_SIZE; extern uint8_t RC_ENABLED; extern uint8_t REQUIRES_WEAK_HANDLE_BARRIER; -extern uint8_t FIELD_BARRIER_NO_EAGER_BRANCH; -extern uint8_t FIELD_BARRIER_NO_ARRAYCOPY; -extern uint8_t FIELD_BARRIER_NO_ARRAYCOPY_SLOW; -extern uint8_t FIELD_BARRIER_NO_C2_SLOW_CALL; -extern uint8_t FIELD_BARRIER_NO_C2_RUST_CALL; -extern uint8_t CLASS_UNLOADING_ENABLED; - -inline bool disable_fast_alloc() { - return DISABLE_ALLOCATION_FAST_PATH != 0; -} extern const size_t MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; extern const uintptr_t FREE_LIST_ALLOCATOR_SIZE; -extern uintptr_t HEAP_START; -extern uintptr_t HEAP_END; extern uint8_t CONCURRENT_MARKING_ACTIVE; extern const char* get_mmtk_version(); diff --git a/openjdk/share/mmtkBarrierSet.cpp b/openjdk/share/mmtkBarrierSet.cpp index 7254d33c..a7cbb475 100644 --- a/openjdk/share/mmtkBarrierSet.cpp +++ b/openjdk/share/mmtkBarrierSet.cpp @@ -103,7 +103,7 @@ MMTkBarrierSet::MMTkBarrierSet(MemRegion whole_heap): BarrierSet::FakeRtti(BarrierSet::ThirdPartyHeapBarrierSet)), _whole_heap(whole_heap), _runtime(get_selected_barrier()->create_runtime()) { - if (!mmtk_enable_allocation_fastpath || disable_fast_alloc()) { + if (!mmtk_enable_allocation_fastpath) { fprintf(stderr, "Allocation fast-path disabled\n"); } } @@ -159,17 +159,14 @@ void MMTkBarrierSetRuntime::object_reference_write_post_call(void* src, void* sl } void MMTkBarrierSetRuntime::object_reference_write_slow_call(void* src, void* slot, void* target) { - if (FIELD_BARRIER_NO_C2_RUST_CALL) return; ::mmtk_object_reference_write_slow(src, slot, target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } void MMTkBarrierSetRuntime::object_reference_array_copy_pre_call(void* src, void* dst, size_t count) { - if (FIELD_BARRIER_NO_ARRAYCOPY_SLOW) return; ::mmtk_array_copy_pre(src, dst, count, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } void MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, void* dst, size_t count) { - if (FIELD_BARRIER_NO_ARRAYCOPY_SLOW) return; ::mmtk_array_copy_post((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); } diff --git a/openjdk/share/mmtkBarrierSetC2.cpp b/openjdk/share/mmtkBarrierSetC2.cpp index 23342666..6b01401f 100644 --- a/openjdk/share/mmtkBarrierSetC2.cpp +++ b/openjdk/share/mmtkBarrierSetC2.cpp @@ -133,7 +133,7 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, } } - if (x->C->env()->dtrace_alloc_probes() || !mmtk_enable_allocation_fastpath || disable_fast_alloc() + if (x->C->env()->dtrace_alloc_probes() || !mmtk_enable_allocation_fastpath // Malloc allocator has no fastpath || (selector.tag == TAG_MALLOC || selector.tag == TAG_LARGE_OBJECT || selector.tag == TAG_FREE_LIST)) { // Force slow-path allocation From 45e43d62093bad73323589378f503f3c6cd39e50 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 7 Apr 2026 20:25:42 +1000 Subject: [PATCH 468/494] Remove array scanning optimization support from OpenJDK binding Remove the VM binding implementations for the array scanning optimization that was removed from mmtk-core. Changes: - Remove Scanning trait method implementations: obj_array_data, is_obj_array, is_val_array, get_obj_kind - Remove helper functions from object_scanning.rs: is_obj_array, is_val_array, get_obj_kind, obj_array_data - Remove ObjectKind imports Co-Authored-By: Claude Opus 4.6 (1M context) --- mmtk/src/object_scanning.rs | 27 --------------------------- mmtk/src/scanning.rs | 17 ----------------- 2 files changed, 44 deletions(-) diff --git a/mmtk/src/object_scanning.rs b/mmtk/src/object_scanning.rs index c9126f94..3df93849 100644 --- a/mmtk/src/object_scanning.rs +++ b/mmtk/src/object_scanning.rs @@ -4,7 +4,6 @@ use super::abi::*; use super::UPCALLS; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; -use mmtk::vm::ObjectKind; use mmtk::vm::SlotVisitor; use std::cell::UnsafeCell; use std::{mem, slice}; @@ -226,32 +225,6 @@ fn oop_iterate(oop: Oop, closure: &mut impl SlotVisitor< } } -pub fn is_obj_array(oop: Oop) -> bool { - oop.klass::().kind == KlassKind::ObjArray -} - -pub fn is_val_array(oop: Oop) -> bool { - oop.klass::().kind == KlassKind::TypeArray -} - -pub fn get_obj_kind(oop: Oop) -> ObjectKind { - let cls_id = oop.klass::().kind; - match cls_id { - KlassKind::TypeArray => ObjectKind::ValArray, - KlassKind::ObjArray => { - ObjectKind::ObjArray(unsafe { oop.as_array_oop().length::() as u32 }) - } - _ => ObjectKind::Scalar, - } -} - -pub fn obj_array_data(oop: Oop) -> crate::OpenJDKSlotRange { - unsafe { - let array = oop.as_array_oop(); - array.slice::(BasicType::T_OBJECT) - } -} - thread_local! { static CLOSURE: UnsafeCell<*mut u8> = const { UnsafeCell::new(std::ptr::null_mut()) }; } diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index bb4c208f..73d28fb3 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -6,7 +6,6 @@ use mmtk::memory_manager; use mmtk::scheduler::RootKind; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; -use mmtk::vm::ObjectKind; use mmtk::vm::{RootsWorkFactory, Scanning, SlotVisitor}; use mmtk::Mutator; use mmtk::MutatorContext; @@ -54,22 +53,6 @@ impl Scanning> for VMScanning { crate::object_scanning::scan_object::(object, slot_visitor, tls); } - fn obj_array_data(o: ObjectReference) -> crate::OpenJDKSlotRange { - crate::object_scanning::obj_array_data::(unsafe { std::mem::transmute(o) }) - } - - fn is_obj_array(o: ObjectReference) -> bool { - crate::object_scanning::is_obj_array::(unsafe { std::mem::transmute(o) }) - } - - fn is_val_array(o: ObjectReference) -> bool { - crate::object_scanning::is_val_array::(unsafe { std::mem::transmute(o) }) - } - - fn get_obj_kind(o: ObjectReference) -> ObjectKind { - crate::object_scanning::get_obj_kind::(unsafe { std::mem::transmute(o) }) - } - fn notify_initial_thread_scan_complete(_partial_scan: bool, _tls: VMWorkerThread) { // unimplemented!() // TODO From ef5a530f388aaf9d9fdd64a00cfd4dbf592cbd93 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 7 Apr 2026 22:49:29 +1000 Subject: [PATCH 469/494] Add panic abort configuration for development profile in Cargo.toml --- mmtk/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index defe2f66..78e0caae 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -11,6 +11,9 @@ name = "mmtk_openjdk" # be careful - LTO is only allowed for certain crate types crate-type = ["cdylib"] +[profile.dev] +panic = "abort" + [profile.release] lto = true From 095ebddbb06e49eefe8bae3c305b26e5c2790a0f Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 7 Apr 2026 23:29:17 +1000 Subject: [PATCH 470/494] Refactor object reference handling and clean up unused code in MMTk integration --- mmtk/src/api.rs | 2 +- mmtk/src/gc_work.rs | 2 +- mmtk/src/object_model.rs | 13 ------------- openjdk/share/mmtkMutator.cpp | 5 +---- openjdk/share/mmtkMutator.hpp | 4 ---- 5 files changed, 3 insertions(+), 23 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index b6adc2ae..5844964a 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -539,7 +539,7 @@ pub extern "C" fn mmtk_get_forwarded_ref( ) -> NullableObjectReference { let o: Option = object.into(); let Some(o) = o else { - return ObjectReference::NULL.into(); + return None.into(); }; match o.get_forwarded_object() { Some(o) => Some(o).into(), diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index df3c8c7a..66414cd1 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -197,7 +197,7 @@ extern "C" fn report_slots_and_renew_buffer_weak let (ptr, _, capacity) = { // TODO: Use Vec::into_raw_parts() when the method is available. use std::mem::ManuallyDrop; - let new_vec = Vec::with_capacity(F::BUFFER_SIZE); + let new_vec = Vec::with_capacity(scanning::WORK_PACKET_CAPACITY); let mut me = ManuallyDrop::new(new_vec); (me.as_mut_ptr(), me.len(), me.capacity()) }; diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index f194d8de..2e9ff88c 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -124,19 +124,6 @@ impl ObjectModel> for VMObjectModel< println!("{}", s) } - fn dump_object_s(object: ObjectReference) -> String { - use std::ffi::CStr; - let c_string = unsafe { ((*UPCALLS).dump_object_string)(std::mem::transmute(object)) }; - let c_str: &CStr = unsafe { CStr::from_ptr(c_string) }; - let s: &str = c_str.to_str().unwrap(); - s.to_string() - } - - fn get_class_pointer(object: ObjectReference) -> Address { - let oop: Oop = unsafe { std::mem::transmute(object) }; - oop.klass_ptr::() - } - fn is_object_sane(object: ObjectReference) -> bool { let oop = Oop::from(object); // The KlassKind must be one of the known variants, and cannot be InstanceStackChunk which we don't support. diff --git a/openjdk/share/mmtkMutator.cpp b/openjdk/share/mmtkMutator.cpp index 1d51ad4a..24f27bdb 100644 --- a/openjdk/share/mmtkMutator.cpp +++ b/openjdk/share/mmtkMutator.cpp @@ -15,10 +15,7 @@ MMTkMutatorContext MMTkMutatorContext::bind(::Thread* current) { printf("ERROR: Unmatched free list allocator size: rs=%zu cpp=%zu\n", FREE_LIST_ALLOCATOR_SIZE, sizeof(MMTkFreeListAllocator)); guarantee(false, "ERROR"); } - auto original_rust_mutator_pointer = (MMTkMutatorContext*) ::bind_mutator((void*) current); - MMTkMutatorContext context = *original_rust_mutator_pointer; - context.original_rust_mutator_pointer = original_rust_mutator_pointer; - return context; + return *((MMTkMutatorContext*) ::bind_mutator((void*) current)); } bool MMTkMutatorContext::is_ready_to_bind() { diff --git a/openjdk/share/mmtkMutator.hpp b/openjdk/share/mmtkMutator.hpp index b0f092d8..4fca4d17 100644 --- a/openjdk/share/mmtkMutator.hpp +++ b/openjdk/share/mmtkMutator.hpp @@ -56,9 +56,6 @@ struct ImmixAllocator { uint8_t _align[7]; uint8_t line_opt_tag; uintptr_t line_opt; - uintptr_t mutator_recycled_blocks; - uintptr_t mutator_recycled_lines; - uint8_t retry; }; struct FLBlock { @@ -114,7 +111,6 @@ struct MMTkMutatorContext { void* mutator_tls; RustDynPtr plan; MutatorConfig config; - MMTkMutatorContext* original_rust_mutator_pointer; HeapWord* alloc(size_t bytes, Allocator allocator = AllocatorDefault); From 34239e17224381b7a174b8ba4edb20f71617f566 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 8 Apr 2026 12:06:27 +1000 Subject: [PATCH 471/494] Refactor object model and scanning code: remove unused methods and simplify implementations --- mmtk/src/abi.rs | 15 +---- mmtk/src/object_model.rs | 8 +-- mmtk/src/scanning.rs | 13 ----- mmtk/src/slots.rs | 115 +-------------------------------------- 4 files changed, 6 insertions(+), 145 deletions(-) diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 5fd56ea6..d760b840 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -175,7 +175,7 @@ pub struct InstanceKlass { } #[repr(u8)] -#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[derive(Copy, Clone, Debug)] #[allow(dead_code)] pub enum ReferenceType { None, // Regular class @@ -328,17 +328,6 @@ impl OopDesc { unsafe { mem::transmute(self) } } - pub(crate) fn klass_ptr(&self) -> Address { - if COMPRESSED { - let compressed = unsafe { self.klass.narrow_klass }; - let addr = COMPRESSED_KLASS_BASE.load(Ordering::Relaxed) - + ((compressed as usize) << COMPRESSED_KLASS_SHIFT.load(Ordering::Relaxed)); - addr - } else { - unsafe { Address::from_ref(self.klass.klass) } - } - } - pub fn klass(&self) -> &'static Klass { if COMPRESSED { let compressed = unsafe { self.klass.narrow_klass }; @@ -466,7 +455,7 @@ impl ArrayOopDesc { typesize_in_bytes / BYTES_IN_WORD } } - pub fn length(&self) -> i32 { + fn length(&self) -> i32 { unsafe { (Address::from_ref(self) + Self::length_offset::()).load::() } } fn base(&self, ty: BasicType) -> Address { diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 2e9ff88c..38c59086 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -117,11 +117,9 @@ impl ObjectModel> for VMObjectModel< } fn dump_object(object: ObjectReference) { - use std::ffi::CStr; - let c_string = unsafe { ((*UPCALLS).dump_object_string)(std::mem::transmute(object)) }; - let c_str: &CStr = unsafe { CStr::from_ptr(c_string) }; - let s: &str = c_str.to_str().unwrap(); - println!("{}", s) + unsafe { + ((*UPCALLS).dump_object)(object); + } } fn is_object_sane(object: ObjectReference) -> bool { diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 73d28fb3..2f79fb9a 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -82,19 +82,6 @@ impl Scanning> for VMScanning { w.push(Box::new(ScanOopStorageSetRoots::new(factory.clone()))); w.push(Box::new(ScanWeakProcessorRoots::new(factory.clone()))); } - // if crate::singleton::() - // .get_plan() - // .requires_weak_root_scanning() - // { - // w.push(Box::new(ScanNewWeakHandleRoots::new(factory.clone())) as _); - // } - // if crate::singleton::() - // .get_plan() - // .current_gc_should_perform_class_unloading() - // { - // // w.push(Box::new(ScanWeakStringTableRoots::new(factory.clone())) as _); - // w.push(Box::new(ScanWeakCodeCacheRoots::new(factory.clone())) as _); - // } memory_manager::add_work_packets( crate::singleton::(), factory.roots_stage(), diff --git a/mmtk/src/slots.rs b/mmtk/src/slots.rs index bd19d9c2..a2f8ab3c 100644 --- a/mmtk/src/slots.rs +++ b/mmtk/src/slots.rs @@ -1,6 +1,6 @@ use std::{ ops::Range, - sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}, + sync::atomic::{AtomicBool, AtomicUsize, Ordering}, }; use super::abi::LOG_BYTES_IN_INT; @@ -209,77 +209,6 @@ impl Slot for OpenJDKSlot { fn to_address(&self) -> Address { self.untagged_address() } - - fn raw_address(&self) -> Address { - self.addr - } - - fn from_address(a: Address) -> Self { - Self { addr: a } - } - - fn compare_exchange( - &self, - old_object: Option, - new_object: Option, - success: Ordering, - failure: Ordering, - ) -> Result, Option> { - if COMPRESSED { - if self.is_compressed() { - let old_value = if let Some(o) = old_object { - Self::compress(o) - } else { - 0 - }; - let new_value = if let Some(o) = new_object { - Self::compress(o) - } else { - 0 - }; - let slot = self.untagged_address(); - unsafe { - match slot.compare_exchange::(old_value, new_value, success, failure) - { - Ok(v) => Ok(Self::decompress(v)), - Err(v) => Err(Self::decompress(v)), - } - } - } else { - let slot = self.untagged_address(); - let old_value = old_object - .map(|o| o.to_raw_address().as_usize()) - .unwrap_or(0); - let new_value = new_object - .map(|o| o.to_raw_address().as_usize()) - .unwrap_or(0); - unsafe { - match slot - .compare_exchange::(old_value, new_value, success, failure) - { - Ok(v) => Ok(ObjectReference::from_raw_address(Address::from_usize(v))), - Err(v) => Err(ObjectReference::from_raw_address(Address::from_usize(v))), - } - } - } - } else { - let old_value = old_object - .map(|o| o.to_raw_address().as_usize()) - .unwrap_or(0); - let new_value = new_object - .map(|o| o.to_raw_address().as_usize()) - .unwrap_or(0); - unsafe { - match self - .addr - .compare_exchange::(old_value, new_value, success, failure) - { - Ok(v) => Ok(ObjectReference::from_raw_address(Address::from_usize(v))), - Err(v) => Err(ObjectReference::from_raw_address(Address::from_usize(v))), - } - } - } - } } /// A range of OpenJDKSlot, usually used for arrays. @@ -299,30 +228,6 @@ impl From> for OpenJDKSlotRange { - cursor: Address, - limit: Address, - step: usize, -} - -impl Iterator for ChunkIterator { - type Item = OpenJDKSlotRange; - - fn next(&mut self) -> Option { - if self.cursor >= self.limit { - None - } else { - let start = self.cursor; - let mut end = start + self.step; - if end > self.limit { - end = self.limit; - } - self.cursor = end; - Some((start..end).into()) - } - } -} - pub struct OpenJDKSlotRangeIterator { cursor: Address, limit: Address, @@ -353,7 +258,6 @@ impl From> for Range MemorySlice for OpenJDKSlotRange { type SlotType = OpenJDKSlot; type SlotIterator = OpenJDKSlotRangeIterator; - type ChunkIterator = ChunkIterator; fn iter_slots(&self) -> Self::SlotIterator { OpenJDKSlotRangeIterator { @@ -362,14 +266,6 @@ impl MemorySlice for OpenJDKSlotRange { } } - fn chunks(&self, chunk_size: usize) -> Self::ChunkIterator { - ChunkIterator { - cursor: self.range.start.addr, - limit: self.range.end.addr, - step: chunk_size << OpenJDKSlot::::LOG_BYTES_IN_SLOT, - } - } - fn object(&self) -> Option { None } @@ -382,10 +278,6 @@ impl MemorySlice for OpenJDKSlotRange { self.range.end.addr - self.range.start.addr } - fn len(&self) -> usize { - self.bytes() >> OpenJDKSlot::::LOG_BYTES_IN_SLOT - } - fn copy(src: &Self, tgt: &Self) { debug_assert_eq!(src.bytes(), tgt.bytes()); // Raw memory copy @@ -410,9 +302,4 @@ impl MemorySlice for OpenJDKSlotRange { Range::
::copy(&src.clone().into(), &tgt.clone().into()) } } - - fn get(&self, index: usize) -> Self::SlotType { - let addr = self.range.start.addr + (index << OpenJDKSlot::::LOG_BYTES_IN_SLOT); - addr.into() - } } From d9d81984cfc62854efa3d74d71a2cacfc48bed5e Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 8 Apr 2026 15:36:49 +1000 Subject: [PATCH 472/494] Refactor MMTk barrier set and roots closure: remove unused code and simplify allocation checks --- openjdk/share/mmtkBarrierSet.cpp | 6 +----- openjdk/share/mmtkBarrierSetC2.cpp | 2 +- openjdk/share/mmtkRootsClosure.hpp | 1 - 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/openjdk/share/mmtkBarrierSet.cpp b/openjdk/share/mmtkBarrierSet.cpp index a7cbb475..7754fb88 100644 --- a/openjdk/share/mmtkBarrierSet.cpp +++ b/openjdk/share/mmtkBarrierSet.cpp @@ -102,11 +102,7 @@ MMTkBarrierSet::MMTkBarrierSet(MemRegion whole_heap): NULL, BarrierSet::FakeRtti(BarrierSet::ThirdPartyHeapBarrierSet)), _whole_heap(whole_heap), - _runtime(get_selected_barrier()->create_runtime()) { - if (!mmtk_enable_allocation_fastpath) { - fprintf(stderr, "Allocation fast-path disabled\n"); - } - } + _runtime(get_selected_barrier()->create_runtime()) {} void MMTkBarrierSet::write_ref_array_work(MemRegion mr) { guarantee(false, "NoBarrier::write_ref_arrey_work not supported"); diff --git a/openjdk/share/mmtkBarrierSetC2.cpp b/openjdk/share/mmtkBarrierSetC2.cpp index 6b01401f..0cc52247 100644 --- a/openjdk/share/mmtkBarrierSetC2.cpp +++ b/openjdk/share/mmtkBarrierSetC2.cpp @@ -133,7 +133,7 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, } } - if (x->C->env()->dtrace_alloc_probes() || !mmtk_enable_allocation_fastpath + if (!mmtk_enable_allocation_fastpath // Malloc allocator has no fastpath || (selector.tag == TAG_MALLOC || selector.tag == TAG_LARGE_OBJECT || selector.tag == TAG_FREE_LIST)) { // Force slow-path allocation diff --git a/openjdk/share/mmtkRootsClosure.hpp b/openjdk/share/mmtkRootsClosure.hpp index f46ffb94..73c55c98 100644 --- a/openjdk/share/mmtkRootsClosure.hpp +++ b/openjdk/share/mmtkRootsClosure.hpp @@ -6,7 +6,6 @@ #include "oops/oop.hpp" #include "oops/oop.inline.hpp" #include "utilities/globalDefinitions.hpp" -#include "classfile/classLoaderData.inline.hpp" class MMTkRootsClosure : public OopClosure { SlotsClosure _slots_closure; From f203c49cde179cbf999b936fca7355e54c4182c2 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Thu, 9 Apr 2026 14:04:44 +1000 Subject: [PATCH 473/494] Refactor barrier slow-paths to call Rust FFI directly, bypassing C++ wrappers All barrier types (FieldBarrier, ObjectBarrier, SATBBarrier) on all architectures (x86, aarch64, riscv) now call Rust FFI functions directly from assembler, C1, and C2 generated code, eliminating the intermediate C++ wrapper functions that only added a Thread::current() lookup. Rust FFI parameter order changes (mutator moved to last position): - mmtk_object_reference_write_pre(src, slot, target, mutator) - mmtk_object_reference_write_post(src, slot, target, mutator) - mmtk_array_copy_post(src, dst, count, mutator) Assembler changes (x86, aarch64, riscv): - UnlogBitBarrier: loads mutator from thread register, calls mmtk_object_reference_write_slow/pre/post directly - ObjectBarrier arraycopy_epilogue: calls mmtk_array_copy_post directly - SATBBarrier load_at/arraycopy_prologue: calls mmtk_load_reference and mmtk_array_copy_pre directly C1 runtime stubs (x86, aarch64, riscv): - Loads mutator as the (argc+1)-th argument from thread register - Entry points changed to Rust functions C2 (shared): - UnlogBitBarrier, SATBBarrier, FieldBarrier: all call Rust functions directly with mutator node from thread pointer Removed C++ wrapper functions from MMTkBarrierSetRuntime: - object_reference_write_pre_call - object_reference_write_post_call - object_reference_write_slow_call - object_reference_array_copy_pre_call - object_reference_array_copy_post_call - load_reference_call Co-Authored-By: Claude Opus 4.6 (1M context) --- mmtk/src/api.rs | 6 ++--- .../mmtkBarrierSetAssembler_aarch64.cpp | 13 ++++++---- .../mmtkObjectBarrierSetAssembler_aarch64.cpp | 6 ++++- .../mmtkSATBBarrierSetAssembler_aarch64.cpp | 9 ++++--- ...mtkUnlogBitBarrierSetAssembler_aarch64.cpp | 17 ++++++------- .../riscv/mmtkBarrierSetAssembler_riscv.cpp | 7 ++++-- .../mmtkObjectBarrierSetAssembler_riscv.cpp | 16 +++++++++--- .../cpu/x86/mmtkBarrierSetAssembler_x86.cpp | 16 +++++++----- .../x86/mmtkObjectBarrierSetAssembler_x86.cpp | 6 ++++- .../x86/mmtkSATBBarrierSetAssembler_x86.cpp | 11 +++++--- .../mmtkUnlogBitBarrierSetAssembler_x86.cpp | 18 ++++++------- openjdk/share/barriers/mmtkFieldBarrier.cpp | 10 +++++--- openjdk/share/barriers/mmtkObjectBarrier.cpp | 4 +-- openjdk/share/barriers/mmtkObjectBarrier.hpp | 2 +- openjdk/share/barriers/mmtkSATBBarrier.cpp | 11 +++++--- .../share/barriers/mmtkUnlogBitBarrier.cpp | 17 ++++++------- openjdk/share/mmtk.h | 6 ++--- openjdk/share/mmtkBarrierSet.cpp | 24 ------------------ openjdk/share/mmtkBarrierSet.hpp | 25 ++++--------------- 19 files changed, 109 insertions(+), 115 deletions(-) diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 5844964a..81ef4619 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -417,10 +417,10 @@ pub extern "C" fn mmtk_load_reference(o: ObjectReference, mutator: *mut libc::c_ /// Full pre barrier #[no_mangle] pub extern "C" fn mmtk_object_reference_write_pre( - mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: NullableObjectReference, + mutator: *mut libc::c_void, ) { with_mutator!(|mutator| { mutator @@ -432,10 +432,10 @@ pub extern "C" fn mmtk_object_reference_write_pre( /// Full post barrier #[no_mangle] pub extern "C" fn mmtk_object_reference_write_post( - mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: NullableObjectReference, + mutator: *mut libc::c_void, ) { with_mutator!(|mutator| { mutator @@ -486,10 +486,10 @@ pub extern "C" fn mmtk_array_copy_pre( /// Array-copy post-barrier #[no_mangle] pub extern "C" fn mmtk_array_copy_post( - mutator: *mut libc::c_void, src: Address, dst: Address, count: usize, + mutator: *mut libc::c_void, ) { with_mutator!(|mutator| { let bytes = count << log_bytes_in_slot(); diff --git a/openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.cpp b/openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.cpp index 6c939aca..3f5253f6 100644 --- a/openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.cpp +++ b/openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.cpp @@ -148,7 +148,10 @@ void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sa guarantee(false, "Too many args"); } - __ call_VM_leaf(entry_point, 3); + // Load mutator from thread-local storage as the last argument, then call Rust directly. + Register mutator_reg = (argc == 1) ? c_rarg1 : (argc == 2) ? c_rarg2 : c_rarg3; + __ lea(mutator_reg, Address(rthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); + __ call_VM_leaf(entry_point, argc + 1); __ pop_call_clobbered_registers(); __ epilogue(); @@ -188,19 +191,19 @@ void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sa // } void MMTkBarrierSetAssembler::generate_c1_load_reference_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_load_reference_runtime_stub", FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); + generate_c1_runtime_stub_general(sasm, "c1_load_reference_runtime_stub", FN_ADDR(mmtk_load_reference), 1); } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(mmtk_object_reference_write_slow), 3); } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_post_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_post_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_post_stub", FN_ADDR(mmtk_object_reference_write_slow), 3); } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_slow_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_slow_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_slow_stub", FN_ADDR(mmtk_object_reference_write_slow), 3); } #undef __ diff --git a/openjdk/cpu/aarch64/mmtkObjectBarrierSetAssembler_aarch64.cpp b/openjdk/cpu/aarch64/mmtkObjectBarrierSetAssembler_aarch64.cpp index 518441b7..ac2179bb 100644 --- a/openjdk/cpu/aarch64/mmtkObjectBarrierSetAssembler_aarch64.cpp +++ b/openjdk/cpu/aarch64/mmtkObjectBarrierSetAssembler_aarch64.cpp @@ -1,5 +1,6 @@ #include "precompiled.hpp" #include "mmtkObjectBarrier.hpp" +#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" //////////////////// Assembler //////////////////// @@ -19,9 +20,12 @@ void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec // if (is_oop && !dest_uninitialized) { if (is_oop){ __ push(saved_regs, sp); + // mmtk_array_copy_post(src, dst, count, mutator) + __ mov(c_rarg0, src); __ mov(c_rarg1, dst); __ mov(c_rarg2, count); - __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_post_call), 3); + __ lea(c_rarg3, Address(rthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); + __ call_VM_leaf(FN_ADDR(mmtk_array_copy_post), 4); __ pop(saved_regs, sp); } } diff --git a/openjdk/cpu/aarch64/mmtkSATBBarrierSetAssembler_aarch64.cpp b/openjdk/cpu/aarch64/mmtkSATBBarrierSetAssembler_aarch64.cpp index 05cc21eb..5ba0c07d 100644 --- a/openjdk/cpu/aarch64/mmtkSATBBarrierSetAssembler_aarch64.cpp +++ b/openjdk/cpu/aarch64/mmtkSATBBarrierSetAssembler_aarch64.cpp @@ -1,5 +1,6 @@ #include "precompiled.hpp" #include "mmtkSATBBarrier.hpp" +#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" //////////////////// Assembler //////////////////// @@ -30,10 +31,11 @@ void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet dec __ cbz(tmp1, done); // if (dst == 0) goto done; __ cbz(dst, done); - // Do slow-call + // Do slow-call: call Rust directly with mutator from thread register __ push_call_clobbered_registers(); __ mov(c_rarg0, dst); - __ MacroAssembler::call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); + __ lea(c_rarg1, Address(rthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); + __ MacroAssembler::call_VM_leaf(FN_ADDR(mmtk_load_reference), 2); __ pop_call_clobbered_registers(); __ bind(done); } @@ -54,7 +56,8 @@ void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decor __ mov(c_rarg0, src); __ mov(c_rarg1, dst); __ mov(c_rarg2, count); - __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_pre_call), 3); + __ lea(c_rarg3, Address(rthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); + __ call_VM_leaf(FN_ADDR(mmtk_array_copy_pre), 4); __ pop_call_clobbered_registers(); __ bind(done); } diff --git a/openjdk/cpu/aarch64/mmtkUnlogBitBarrierSetAssembler_aarch64.cpp b/openjdk/cpu/aarch64/mmtkUnlogBitBarrierSetAssembler_aarch64.cpp index 223e4f31..8ed28aee 100644 --- a/openjdk/cpu/aarch64/mmtkUnlogBitBarrierSetAssembler_aarch64.cpp +++ b/openjdk/cpu/aarch64/mmtkUnlogBitBarrierSetAssembler_aarch64.cpp @@ -2,6 +2,7 @@ #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" +#include "mmtkMutator.hpp" #include "utilities/macros.hpp" #include CPU_HEADER(mmtkUnlogBitBarrierSetAssembler) @@ -50,20 +51,16 @@ void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAs __ mov(c_rarg0, obj); // Neither the ObjectBarrier nor the SATBBarrier need to know the slot or the value. // We just set both args to nullptr. - // We may need to pass actual arguments if we support other barriers. - // - // Note: If the `compensate_val_reg` parameter in the post barrier is true, and we are using - // compressed oops, the `val` register will be holding a compressed pointer to the target object - // due to the way `BarrierSetAssembler::store_at` works. If the write barrier needs to know the - // target, we will need to decompress it before passing it to the barrier slow path. __ mov(c_rarg1, zr); __ mov(c_rarg2, zr); + // Load mutator from thread-local storage and call Rust directly. + __ lea(c_rarg3, Address(rthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); - address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call) - : pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) - : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); + address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(mmtk_object_reference_write_slow) + : pre ? FN_ADDR(mmtk_object_reference_write_pre) + : FN_ADDR(mmtk_object_reference_write_post); - __ call_VM_leaf(entry_point, 3); + __ call_VM_leaf(entry_point, 4); if (pre) { __ pop_call_clobbered_registers(); diff --git a/openjdk/cpu/riscv/mmtkBarrierSetAssembler_riscv.cpp b/openjdk/cpu/riscv/mmtkBarrierSetAssembler_riscv.cpp index 49bd58d2..cc02bd88 100644 --- a/openjdk/cpu/riscv/mmtkBarrierSetAssembler_riscv.cpp +++ b/openjdk/cpu/riscv/mmtkBarrierSetAssembler_riscv.cpp @@ -160,10 +160,13 @@ void MMTkBarrierSetAssembler::generate_c1_write_barrier_runtime_stub(StubAssembl // t2, a0-a7, t3-t6 __ push_call_clobbered_registers(); + // Load mutator from thread-local storage and call Rust directly. + Register mutator = t2; + __ la(mutator, Address(xthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); if (mmtk_enable_barrier_fastpath) { - __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), src, slot, new_val); + __ call_VM_leaf(FN_ADDR(mmtk_object_reference_write_slow), src, slot, new_val, mutator); } else { - __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), src, slot, new_val); + __ call_VM_leaf(FN_ADDR(mmtk_object_reference_write_post), src, slot, new_val, mutator); } __ pop_call_clobbered_registers(); diff --git a/openjdk/cpu/riscv/mmtkObjectBarrierSetAssembler_riscv.cpp b/openjdk/cpu/riscv/mmtkObjectBarrierSetAssembler_riscv.cpp index 9c2c761f..49d0ed91 100644 --- a/openjdk/cpu/riscv/mmtkObjectBarrierSetAssembler_riscv.cpp +++ b/openjdk/cpu/riscv/mmtkObjectBarrierSetAssembler_riscv.cpp @@ -1,5 +1,6 @@ #include "precompiled.hpp" #include "mmtkObjectBarrier.hpp" +#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" #define __ masm-> @@ -65,18 +66,20 @@ void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* // equivalently ((tmp1 & 1) == 0) go to done __ andi(tmp1, tmp1, 1); __ beqz(tmp1, done); - // setup calling convention + // setup calling convention: call Rust directly with mutator as last arg __ mv(c_rarg0, obj); __ la(c_rarg1, dst); __ mv(c_rarg2, val == noreg ? zr : val); - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); + __ la(c_rarg3, Address(xthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); + __ call_VM_leaf_base(FN_ADDR(mmtk_object_reference_write_slow), 4); __ bind(done); } else { __ mv(c_rarg0, obj); __ la(c_rarg1, dst); __ mv(c_rarg2, val == noreg ? zr : val); - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); + __ la(c_rarg3, Address(xthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); + __ call_VM_leaf_base(FN_ADDR(mmtk_object_reference_write_post), 4); } if (dst.base()->is_valid()) { __ pop_reg(dst.base()); @@ -92,7 +95,12 @@ void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec if (is_oop) { // in address generate_checkcast_copy, caller tells us to save count __ push_reg(saved_regs, sp); - __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_post_call), zr, dst, count); + // mmtk_array_copy_post(src, dst, count, mutator) + __ mv(c_rarg0, src); + __ mv(c_rarg1, dst); + __ mv(c_rarg2, count); + __ la(c_rarg3, Address(xthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); + __ call_VM_leaf_base(FN_ADDR(mmtk_array_copy_post), 4); __ pop_reg(saved_regs, sp); } } diff --git a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp index 99f3d9d3..70e896fe 100644 --- a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp @@ -167,7 +167,11 @@ void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sa // As a temporary fix, we apply field barrier to all fields in this object. __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_probable_write_pre_call), c_rarg0); } else { - __ call_VM_leaf_base(entry_point, 3); + // Load mutator from thread-local storage as the last argument, then call Rust directly. + Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); + Register mutator_reg = (argc == 1) ? c_rarg1 : (argc == 2) ? c_rarg2 : c_rarg3; + __ lea(mutator_reg, mutator); + __ call_VM_leaf_base(entry_point, argc + 1); } __ restore_live_registers(true); @@ -175,23 +179,23 @@ void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sa } void MMTkBarrierSetAssembler::generate_c1_load_reference_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_load_reference_runtime_stub", FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); + generate_c1_runtime_stub_general(sasm, "c1_load_reference_runtime_stub", FN_ADDR(mmtk_load_reference), 1); } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(mmtk_object_reference_write_slow), 3); } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub_with_patch_fix(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3, true); + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(mmtk_object_reference_write_slow), 3, true); } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_post_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_post_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_post_stub", FN_ADDR(mmtk_object_reference_write_slow), 3); } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_slow_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_slow_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_slow_stub", FN_ADDR(mmtk_object_reference_write_slow), 3); } #undef __ diff --git a/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.cpp index 474d958a..1fc20021 100644 --- a/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.cpp @@ -1,5 +1,6 @@ #include "precompiled.hpp" #include "mmtkObjectBarrier.hpp" +#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" //////////////////// Assembler //////////////////// @@ -47,10 +48,13 @@ void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec } } __ pusha(); + // mmtk_array_copy_post(src, dst, count, mutator) __ movptr(c_rarg0, src); __ movptr(c_rarg1, dst); __ movptr(c_rarg2, count); - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_post_call), 3); + Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); + __ lea(c_rarg3, mutator); + __ call_VM_leaf_base(FN_ADDR(mmtk_array_copy_post), 4); __ popa(); } } diff --git a/openjdk/cpu/x86/mmtkSATBBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkSATBBarrierSetAssembler_x86.cpp index 6dccc551..421417b9 100644 --- a/openjdk/cpu/x86/mmtkSATBBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkSATBBarrierSetAssembler_x86.cpp @@ -1,5 +1,6 @@ #include "precompiled.hpp" #include "mmtkSATBBarrier.hpp" +#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" //////////////////// Assembler //////////////////// @@ -32,10 +33,12 @@ void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet dec // if (dst == 0) goto done; __ testptr(dst, dst); __ jcc(Assembler::zero, done); - // Do slow-call + // Do slow-call: call Rust directly with mutator from thread register __ pusha(); __ mov(c_rarg0, dst); - __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); + Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); + __ lea(c_rarg1, mutator); + __ MacroAssembler::call_VM_leaf_base(FN_ADDR(mmtk_load_reference), 2); __ popa(); __ bind(done); } @@ -57,7 +60,9 @@ void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decor __ movptr(c_rarg0, src); __ movptr(c_rarg1, dst); __ movptr(c_rarg2, count); - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_pre_call), 3); + Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); + __ lea(c_rarg3, mutator); + __ call_VM_leaf_base(FN_ADDR(mmtk_array_copy_pre), 4); __ popa(); __ bind(done); } diff --git a/openjdk/cpu/x86/mmtkUnlogBitBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkUnlogBitBarrierSetAssembler_x86.cpp index c7148098..6554f88a 100644 --- a/openjdk/cpu/x86/mmtkUnlogBitBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkUnlogBitBarrierSetAssembler_x86.cpp @@ -2,6 +2,7 @@ #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" +#include "mmtkMutator.hpp" #include "utilities/macros.hpp" #include CPU_HEADER(mmtkUnlogBitBarrierSetAssembler) @@ -56,20 +57,17 @@ void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAs __ movptr(c_rarg0, obj); // Neither the ObjectBarrier nor the SATBBarrier need to know the slot or the value. // We just set both args to nullptr. - // We may need to pass actual arguments if we support other barriers. - // - // Note: If the `compensate_val_reg` parameter in the post barrier is true, and we are using - // compressed oops, the `val` register will be holding a compressed pointer to the target object - // due to the way `BarrierSetAssembler::store_at` works. If the write barrier needs to know the - // target, we will need to decompress it before passing it to the barrier slow path. __ xorptr(c_rarg1, c_rarg1); __ xorptr(c_rarg2, c_rarg2); + // Load mutator from thread-local storage and call Rust directly. + Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); + __ lea(c_rarg3, mutator); - address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call) - : pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) - : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); + address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(mmtk_object_reference_write_slow) + : pre ? FN_ADDR(mmtk_object_reference_write_pre) + : FN_ADDR(mmtk_object_reference_write_post); - __ call_VM_leaf_base(entry_point, 3); + __ call_VM_leaf_base(entry_point, 4); if (pre) { __ popa(); diff --git a/openjdk/share/barriers/mmtkFieldBarrier.cpp b/openjdk/share/barriers/mmtkFieldBarrier.cpp index 21d42c36..349d2c25 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.cpp @@ -38,7 +38,7 @@ void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, ::mmtk_object_reference_write_slow((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } } else { - ::mmtk_object_reference_write_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) slot, (void*) target); + ::mmtk_object_reference_write_pre((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } } @@ -130,7 +130,9 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma __ movptr(c_rarg2, NULL_WORD); else __ movptr(c_rarg2, val); - __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); + Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); + __ lea(c_rarg3, mutator); + __ call_VM_leaf_base(FN_ADDR(mmtk_object_reference_write_pre), 4); __ popa(); } } @@ -371,8 +373,8 @@ static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* sl } __ end_if(); __ end_if(); } else { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), "mmtk_barrier_call", src, slot, val); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeRawPtr::NOTNULL); + Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_object_reference_write_pre), "mmtk_barrier_call", src, slot, val, mutator); // Looks like this is necessary // See https://github.com/mmtk/openjdk/blob/c82e5c44adced4383162826c2c3933a83cfb139b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp#L288-L291 Node* call = __ ctrl()->in(0); diff --git a/openjdk/share/barriers/mmtkObjectBarrier.cpp b/openjdk/share/barriers/mmtkObjectBarrier.cpp index 2a4e8f4a..01cacd20 100644 --- a/openjdk/share/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/share/barriers/mmtkObjectBarrier.cpp @@ -23,10 +23,10 @@ void MMTkObjectBarrierSetRuntime::object_probable_write(oop new_obj) const { void MMTkObjectBarrierSetRuntime::object_reference_write_post(oop src, oop* slot, oop target) const { if (mmtk_enable_barrier_fastpath) { if (is_unlog_bit_set(src)) { - object_reference_write_slow_call((void*) src, (void*) slot, (void*) target); + ::mmtk_object_reference_write_slow((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } } else { - object_reference_write_post_call((void*) src, (void*) slot, (void*) target); + ::mmtk_object_reference_write_post((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } } diff --git a/openjdk/share/barriers/mmtkObjectBarrier.hpp b/openjdk/share/barriers/mmtkObjectBarrier.hpp index 14af6d8b..7e276a8c 100644 --- a/openjdk/share/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/share/barriers/mmtkObjectBarrier.hpp @@ -32,7 +32,7 @@ class MMTkObjectBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { // Interfaces called by `MMTkBarrierSet::AccessBarrier` virtual void object_reference_write_post(oop src, oop* slot, oop target) const override; virtual void object_reference_array_copy_post(oop* src, oop* dst, size_t count) const override { - object_reference_array_copy_post_call((void*) src, (void*) dst, count); + ::mmtk_array_copy_post((void*) src, (void*) dst, count, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } virtual void object_probable_write(oop new_obj) const override; }; diff --git a/openjdk/share/barriers/mmtkSATBBarrier.cpp b/openjdk/share/barriers/mmtkSATBBarrier.cpp index da139e2b..42819b5a 100644 --- a/openjdk/share/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/share/barriers/mmtkSATBBarrier.cpp @@ -1,5 +1,6 @@ #include "precompiled.hpp" #include "mmtkSATBBarrier.hpp" +#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" //////////////////// Runtime //////////////////// @@ -21,10 +22,10 @@ void MMTkSATBBarrierSetRuntime::object_probable_write(oop new_obj) const { void MMTkSATBBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, oop target) const { if (mmtk_enable_barrier_fastpath) { if (is_unlog_bit_set(src)) { - object_reference_write_slow_call((void*) src, (void*) slot, (void*) target); + ::mmtk_object_reference_write_slow((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } } else { - object_reference_write_pre_call((void*) src, (void*) slot, (void*) target); + ::mmtk_object_reference_write_pre((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); } } @@ -96,6 +97,8 @@ void MMTkSATBBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool emit_barrier) { MMTkIdealKit ideal(kit, true); Node* no_base = __ top(); + Node* tls = __ thread(); + Node* mutator = __ AddP(no_base, tls, __ ConX(in_bytes(JavaThread::third_party_heap_mutator_offset()))); float unlikely = PROB_UNLIKELY(0.999); Node* zero = __ ConI(0); Node* cm_flag = __ load(__ ctrl(), __ ConP(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); @@ -103,8 +106,8 @@ static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool em __ if_then(cm_flag, BoolTest::ne, zero, unlikely); { // No slow-call if dst is NULL __ if_then(val, BoolTest::ne, kit->null()); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); - Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", val); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeRawPtr::NOTNULL); + Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_load_reference), "mmtk_barrier_call", val, mutator); } __ end_if(); } __ end_if(); kit->sync_kit(ideal); diff --git a/openjdk/share/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/share/barriers/mmtkUnlogBitBarrier.cpp index 6feae3db..e1ba1b54 100644 --- a/openjdk/share/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/share/barriers/mmtkUnlogBitBarrier.cpp @@ -1,5 +1,6 @@ #include "precompiled.hpp" #include "mmtkUnlogBitBarrier.hpp" +#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "c1/c1_LIRAssembler.hpp" @@ -91,25 +92,23 @@ Node* MMTkUnlogBitBarrierSetC2::emit_check_unlog_bit_fast_path(MMTkIdealKit& ide } void MMTkUnlogBitBarrierSetC2::object_reference_write_pre_or_post(MMTkIdealKit& ideal, Node* src, bool pre) { + Node* no_base = __ top(); + Node* tls = __ thread(); + Node* mutator = __ AddP(no_base, tls, __ ConX(in_bytes(JavaThread::third_party_heap_mutator_offset()))); if (mmtk_enable_barrier_fastpath) { Node* result = emit_check_unlog_bit_fast_path(ideal, src); Node* zero = __ ConI(0); float unlikely = PROB_UNLIKELY(0.999); __ if_then(result, BoolTest::ne, zero, unlikely); { - Node* result = emit_check_unlog_bit_fast_path(ideal, src); - - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - address entry_point = FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeRawPtr::NOTNULL); Node* null_node = __ NullP(); - Node* x = __ make_leaf_call(tf, entry_point, "mmtk_barrier_call", src, null_node, null_node); + Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_object_reference_write_slow), "mmtk_barrier_call", src, null_node, null_node, mutator); } __ end_if(); } else { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); - address entry_point = pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) - : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeRawPtr::NOTNULL); Node* null_node = __ NullP(); - Node* x = __ make_leaf_call(tf, entry_point, "mmtk_barrier_call", src, null_node, null_node); + Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_object_reference_write_slow), "mmtk_barrier_call", src, null_node, null_node, mutator); } } diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index e478b860..9ef8656a 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -58,10 +58,10 @@ extern void post_alloc(MMTk_Mutator mutator, void* refer, extern void mmtk_load_reference(void* obj, MMTk_Mutator mutator); /// Full pre-barrier -extern void mmtk_object_reference_write_pre(MMTk_Mutator mutator, void* src, void* slot, void* target); +extern void mmtk_object_reference_write_pre(void* src, void* slot, void* target, MMTk_Mutator mutator); /// Full post-barrier -extern void mmtk_object_reference_write_post(MMTk_Mutator mutator, void* src, void* slot, void* target); +extern void mmtk_object_reference_write_post(void* src, void* slot, void* target, MMTk_Mutator mutator); /// Generic slow-path extern void mmtk_object_reference_write_slow(void* src, void* slot, void* target, MMTk_Mutator mutator); @@ -70,7 +70,7 @@ extern void mmtk_object_reference_write_slow(void* src, void* slot, void* target extern void mmtk_array_copy_pre(void* src, void* dst, size_t count, MMTk_Mutator mutator); /// Full array-copy post-barrier -extern void mmtk_array_copy_post(MMTk_Mutator mutator, void* src, void* dst, size_t count); +extern void mmtk_array_copy_post(void* src, void* dst, size_t count, MMTk_Mutator mutator); /// C2 slowpath allocation barrier extern void mmtk_object_probable_write(MMTk_Mutator mutator, void* obj); diff --git a/openjdk/share/mmtkBarrierSet.cpp b/openjdk/share/mmtkBarrierSet.cpp index 7754fb88..51a5db0e 100644 --- a/openjdk/share/mmtkBarrierSet.cpp +++ b/openjdk/share/mmtkBarrierSet.cpp @@ -146,30 +146,6 @@ bool MMTkBarrierSet::is_slow_path_call(address call) { return runtime()->is_slow_path_call(call); } -void MMTkBarrierSetRuntime::object_reference_write_pre_call(void* src, void* slot, void* target) { - ::mmtk_object_reference_write_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, target); -} - -void MMTkBarrierSetRuntime::object_reference_write_post_call(void* src, void* slot, void* target) { - ::mmtk_object_reference_write_post((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, target); -} - -void MMTkBarrierSetRuntime::object_reference_write_slow_call(void* src, void* slot, void* target) { - ::mmtk_object_reference_write_slow(src, slot, target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); -} - -void MMTkBarrierSetRuntime::object_reference_array_copy_pre_call(void* src, void* dst, size_t count) { - ::mmtk_array_copy_pre(src, dst, count, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); -} - -void MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, void* dst, size_t count) { - ::mmtk_array_copy_post((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); -} - -void MMTkBarrierSetRuntime::load_reference_call(void* ref) { - ::mmtk_load_reference(ref, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); -} - void MMTkBarrierSetRuntime::object_probable_write_pre_call(void* ref) { ::mmtk_object_probable_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); } \ No newline at end of file diff --git a/openjdk/share/mmtkBarrierSet.hpp b/openjdk/share/mmtkBarrierSet.hpp index 90e3a05c..1c930867 100644 --- a/openjdk/share/mmtkBarrierSet.hpp +++ b/openjdk/share/mmtkBarrierSet.hpp @@ -69,31 +69,16 @@ MMTkAllocatorOffsets get_tlab_top_and_end_offsets(AllocatorSelector selector); class MMTkBarrierSetRuntime: public CHeapObj { public: - /// Weak ref load barrier - static void load_reference_call(void* ref); - /// Generic pre-write barrier. Called by fast-paths. - static void object_reference_write_pre_call(void* src, void* slot, void* target); - /// Generic post-write barrier. Called by fast-paths. - static void object_reference_write_post_call(void* src, void* slot, void* target); - /// Generic slow-path. Called by fast-paths. - static void object_reference_write_slow_call(void* src, void* slot, void* target); - /// Generic arraycopy post-barrier. Called by fast-paths. - static void object_reference_array_copy_pre_call(void* src, void* dst, size_t count); - /// Generic arraycopy pre-barrier. Called by fast-paths. - static void object_reference_array_copy_post_call(void* src, void* dst, size_t count); static void object_probable_write_pre_call(void* obj); /// Check if the address is a slow-path function. virtual bool is_slow_path_call(address call) const { - return call == CAST_FROM_FN_PTR(address, object_reference_write_pre_call) - || call == CAST_FROM_FN_PTR(address, object_reference_write_post_call) - || call == CAST_FROM_FN_PTR(address, object_reference_write_slow_call) - || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call) - || call == CAST_FROM_FN_PTR(address, mmtk_array_copy_pre) + return call == CAST_FROM_FN_PTR(address, mmtk_object_reference_write_pre) + || call == CAST_FROM_FN_PTR(address, mmtk_object_reference_write_post) || call == CAST_FROM_FN_PTR(address, mmtk_object_reference_write_slow) + || call == CAST_FROM_FN_PTR(address, mmtk_array_copy_pre) + || call == CAST_FROM_FN_PTR(address, mmtk_array_copy_post) || call == CAST_FROM_FN_PTR(address, mmtk_load_reference) - || call == CAST_FROM_FN_PTR(address, object_probable_write_pre_call) - || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_post_call) - || call == CAST_FROM_FN_PTR(address, load_reference_call); + || call == CAST_FROM_FN_PTR(address, object_probable_write_pre_call); } /// Full pre-barrier From 3dc8af4dda8b2fb448b6d57b0e7b0d7b63eb4fec Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Wed, 15 Apr 2026 13:52:49 +1000 Subject: [PATCH 474/494] Refactor MMTk barrier functions to remove mutator parameter from assembly calls - Updated assembly code across various architectures (AArch64, RISC-V, x86) to eliminate the need for passing the mutator as an argument in barrier calls. - Introduced new runtime functions in MMTkBarrierSetRuntime to handle mutator retrieval internally. - Adjusted function signatures in the MMTk API to reflect the changes in argument order for load and write barriers. - Removed unnecessary includes of mmtkMutator.hpp from several files. - Ensured that all references to mutator in barrier calls are replaced with the new runtime functions. --- .github/workflows/stylecheck.yml | 10 +++--- .gitignore | 1 - mmtk/Cargo.lock | 12 +------ mmtk/Cargo.toml | 7 ++-- mmtk/src/api.rs | 12 +++---- mmtk/src/lib.rs | 6 ---- .../mmtkBarrierSetAssembler_aarch64.cpp | 13 +++---- .../mmtkObjectBarrierSetAssembler_aarch64.cpp | 6 +--- .../mmtkSATBBarrierSetAssembler_aarch64.cpp | 9 ++--- ...mtkUnlogBitBarrierSetAssembler_aarch64.cpp | 17 +++++---- .../riscv/mmtkBarrierSetAssembler_riscv.cpp | 7 ++-- .../mmtkObjectBarrierSetAssembler_riscv.cpp | 16 +++------ .../cpu/x86/mmtkBarrierSetAssembler_x86.cpp | 14 +++----- .../x86/mmtkObjectBarrierSetAssembler_x86.cpp | 6 +--- .../x86/mmtkSATBBarrierSetAssembler_x86.cpp | 11 ++---- .../mmtkUnlogBitBarrierSetAssembler_x86.cpp | 18 +++++----- openjdk/share/barriers/mmtkFieldBarrier.cpp | 36 +++++++------------ openjdk/share/barriers/mmtkFieldBarrier.hpp | 2 +- openjdk/share/barriers/mmtkObjectBarrier.cpp | 4 +-- openjdk/share/barriers/mmtkObjectBarrier.hpp | 2 +- openjdk/share/barriers/mmtkSATBBarrier.cpp | 13 +++---- openjdk/share/barriers/mmtkSATBBarrier.hpp | 2 +- .../share/barriers/mmtkUnlogBitBarrier.cpp | 17 ++++----- openjdk/share/mmtk.h | 15 ++++---- openjdk/share/mmtkBarrierSet.cpp | 24 +++++++++++++ openjdk/share/mmtkBarrierSet.hpp | 26 ++++++++++---- openjdk/share/mmtkBarrierSetC2.cpp | 1 - openjdk/share/mmtkHeap.cpp | 2 -- openjdk/share/mmtkUpcalls.cpp | 2 -- 29 files changed, 139 insertions(+), 172 deletions(-) diff --git a/.github/workflows/stylecheck.yml b/.github/workflows/stylecheck.yml index a1af1dd3..cd386419 100644 --- a/.github/workflows/stylecheck.yml +++ b/.github/workflows/stylecheck.yml @@ -13,11 +13,11 @@ jobs: style-check: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v4 - - name: Setup environments - run: ./.github/scripts/ci-setup.sh - - name: Style checks - run: ./.github/scripts/ci-style.sh + - uses: actions/checkout@v4 + - name: Setup environments + run: ./.github/scripts/ci-setup.sh + - name: Style checks + run: ./.github/scripts/ci-style.sh msrv: runs-on: ubuntu-22.04 diff --git a/.gitignore b/.gitignore index 1170cd29..a25b809e 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,5 @@ target/ /*.so /repos/mmtk-core -/repos/openjdk __pycache__ diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index aa435d7d..e23a3627 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -526,7 +526,7 @@ dependencies = [ "portable-atomic", "regex", "rustversion", - "spin 0.10.0", + "spin", "strum", "strum_macros", "sysinfo", @@ -557,7 +557,6 @@ dependencies = [ "mmtk", "once_cell", "probe", - "spin 0.9.8", ] [[package]] @@ -798,15 +797,6 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] - [[package]] name = "spin" version = "0.10.0" diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 78e0caae..8e798860 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -25,10 +25,9 @@ openjdk_version = "b557520f04b35d2a7a040d43f02efc1c4911ebb7" [dependencies] libc = "0.2" -lazy_static = "1.4.0" -spin = { version = "0.9.4", features = ["std"] } +lazy_static = "1.1" log = { version = "0.4", features = ["max_level_trace"] } -once_cell = "1.16.0" +once_cell = "1.10.0" atomic = "0.6.0" memoffset = "0.9.0" cfg-if = "1.0" @@ -79,5 +78,3 @@ genimmix = [] stickyimmix = [] markcompact = [] jfr = [] - -sanity = ["mmtk/sanity"] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 81ef4619..2cd307ae 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -410,17 +410,17 @@ pub extern "C" fn executable() -> bool { } #[no_mangle] -pub extern "C" fn mmtk_load_reference(o: ObjectReference, mutator: *mut libc::c_void) { +pub extern "C" fn mmtk_load_reference(mutator: *mut libc::c_void, o: ObjectReference) { with_mutator!(|mutator| mutator.barrier().load_weak_reference(o)) } /// Full pre barrier #[no_mangle] pub extern "C" fn mmtk_object_reference_write_pre( + mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: NullableObjectReference, - mutator: *mut libc::c_void, ) { with_mutator!(|mutator| { mutator @@ -432,10 +432,10 @@ pub extern "C" fn mmtk_object_reference_write_pre( /// Full post barrier #[no_mangle] pub extern "C" fn mmtk_object_reference_write_post( + mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: NullableObjectReference, - mutator: *mut libc::c_void, ) { with_mutator!(|mutator| { mutator @@ -447,10 +447,10 @@ pub extern "C" fn mmtk_object_reference_write_post( /// Barrier slow-path call #[no_mangle] pub extern "C" fn mmtk_object_reference_write_slow( + mutator: *mut libc::c_void, src: ObjectReference, slot: Address, target: NullableObjectReference, - mutator: *mut libc::c_void, ) { with_mutator!(|mutator| { mutator @@ -470,10 +470,10 @@ fn log_bytes_in_slot() -> usize { /// Array-copy pre-barrier #[no_mangle] pub extern "C" fn mmtk_array_copy_pre( + mutator: *mut libc::c_void, src: Address, dst: Address, count: usize, - mutator: *mut libc::c_void, ) { let bytes = count << log_bytes_in_slot(); with_mutator!(|mutator| { @@ -486,10 +486,10 @@ pub extern "C" fn mmtk_array_copy_pre( /// Array-copy post-barrier #[no_mangle] pub extern "C" fn mmtk_array_copy_post( + mutator: *mut libc::c_void, src: Address, dst: Address, count: usize, - mutator: *mut libc::c_void, ) { with_mutator!(|mutator| { let bytes = count << log_bytes_in_slot(); diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 89ebe19e..4ae12f5e 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -4,7 +4,6 @@ extern crate atomic; extern crate once_cell; #[macro_use] extern crate probe; -extern crate spin; use std::collections::HashMap; use std::ptr::null_mut; @@ -161,9 +160,6 @@ pub static mut CONCURRENT_MARKING_ACTIVE: u8 = 0; #[no_mangle] pub static mut RC_ENABLED: u8 = 0; -#[no_mangle] -pub static mut REQUIRES_WEAK_HANDLE_BARRIER: u8 = 0; - #[derive(Default)] pub struct OpenJDK; @@ -202,7 +198,6 @@ lazy_static! { .get_plan() .downcast_ref::>>() .is_some() as _; - REQUIRES_WEAK_HANDLE_BARRIER = RC_ENABLED; } *ret }; @@ -217,7 +212,6 @@ lazy_static! { .get_plan() .downcast_ref::>>() .is_some() as _; - REQUIRES_WEAK_HANDLE_BARRIER = RC_ENABLED; } *ret }; diff --git a/openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.cpp b/openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.cpp index 3f5253f6..6c939aca 100644 --- a/openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.cpp +++ b/openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.cpp @@ -148,10 +148,7 @@ void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sa guarantee(false, "Too many args"); } - // Load mutator from thread-local storage as the last argument, then call Rust directly. - Register mutator_reg = (argc == 1) ? c_rarg1 : (argc == 2) ? c_rarg2 : c_rarg3; - __ lea(mutator_reg, Address(rthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); - __ call_VM_leaf(entry_point, argc + 1); + __ call_VM_leaf(entry_point, 3); __ pop_call_clobbered_registers(); __ epilogue(); @@ -191,19 +188,19 @@ void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sa // } void MMTkBarrierSetAssembler::generate_c1_load_reference_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_load_reference_runtime_stub", FN_ADDR(mmtk_load_reference), 1); + generate_c1_runtime_stub_general(sasm, "c1_load_reference_runtime_stub", FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(mmtk_object_reference_write_slow), 3); + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_post_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_post_stub", FN_ADDR(mmtk_object_reference_write_slow), 3); + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_post_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_slow_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_slow_stub", FN_ADDR(mmtk_object_reference_write_slow), 3); + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_slow_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); } #undef __ diff --git a/openjdk/cpu/aarch64/mmtkObjectBarrierSetAssembler_aarch64.cpp b/openjdk/cpu/aarch64/mmtkObjectBarrierSetAssembler_aarch64.cpp index ac2179bb..518441b7 100644 --- a/openjdk/cpu/aarch64/mmtkObjectBarrierSetAssembler_aarch64.cpp +++ b/openjdk/cpu/aarch64/mmtkObjectBarrierSetAssembler_aarch64.cpp @@ -1,6 +1,5 @@ #include "precompiled.hpp" #include "mmtkObjectBarrier.hpp" -#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" //////////////////// Assembler //////////////////// @@ -20,12 +19,9 @@ void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec // if (is_oop && !dest_uninitialized) { if (is_oop){ __ push(saved_regs, sp); - // mmtk_array_copy_post(src, dst, count, mutator) - __ mov(c_rarg0, src); __ mov(c_rarg1, dst); __ mov(c_rarg2, count); - __ lea(c_rarg3, Address(rthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); - __ call_VM_leaf(FN_ADDR(mmtk_array_copy_post), 4); + __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_post_call), 3); __ pop(saved_regs, sp); } } diff --git a/openjdk/cpu/aarch64/mmtkSATBBarrierSetAssembler_aarch64.cpp b/openjdk/cpu/aarch64/mmtkSATBBarrierSetAssembler_aarch64.cpp index 5ba0c07d..05cc21eb 100644 --- a/openjdk/cpu/aarch64/mmtkSATBBarrierSetAssembler_aarch64.cpp +++ b/openjdk/cpu/aarch64/mmtkSATBBarrierSetAssembler_aarch64.cpp @@ -1,6 +1,5 @@ #include "precompiled.hpp" #include "mmtkSATBBarrier.hpp" -#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" //////////////////// Assembler //////////////////// @@ -31,11 +30,10 @@ void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet dec __ cbz(tmp1, done); // if (dst == 0) goto done; __ cbz(dst, done); - // Do slow-call: call Rust directly with mutator from thread register + // Do slow-call __ push_call_clobbered_registers(); __ mov(c_rarg0, dst); - __ lea(c_rarg1, Address(rthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); - __ MacroAssembler::call_VM_leaf(FN_ADDR(mmtk_load_reference), 2); + __ MacroAssembler::call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); __ pop_call_clobbered_registers(); __ bind(done); } @@ -56,8 +54,7 @@ void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decor __ mov(c_rarg0, src); __ mov(c_rarg1, dst); __ mov(c_rarg2, count); - __ lea(c_rarg3, Address(rthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); - __ call_VM_leaf(FN_ADDR(mmtk_array_copy_pre), 4); + __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_pre_call), 3); __ pop_call_clobbered_registers(); __ bind(done); } diff --git a/openjdk/cpu/aarch64/mmtkUnlogBitBarrierSetAssembler_aarch64.cpp b/openjdk/cpu/aarch64/mmtkUnlogBitBarrierSetAssembler_aarch64.cpp index 8ed28aee..223e4f31 100644 --- a/openjdk/cpu/aarch64/mmtkUnlogBitBarrierSetAssembler_aarch64.cpp +++ b/openjdk/cpu/aarch64/mmtkUnlogBitBarrierSetAssembler_aarch64.cpp @@ -2,7 +2,6 @@ #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" -#include "mmtkMutator.hpp" #include "utilities/macros.hpp" #include CPU_HEADER(mmtkUnlogBitBarrierSetAssembler) @@ -51,16 +50,20 @@ void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAs __ mov(c_rarg0, obj); // Neither the ObjectBarrier nor the SATBBarrier need to know the slot or the value. // We just set both args to nullptr. + // We may need to pass actual arguments if we support other barriers. + // + // Note: If the `compensate_val_reg` parameter in the post barrier is true, and we are using + // compressed oops, the `val` register will be holding a compressed pointer to the target object + // due to the way `BarrierSetAssembler::store_at` works. If the write barrier needs to know the + // target, we will need to decompress it before passing it to the barrier slow path. __ mov(c_rarg1, zr); __ mov(c_rarg2, zr); - // Load mutator from thread-local storage and call Rust directly. - __ lea(c_rarg3, Address(rthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); - address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(mmtk_object_reference_write_slow) - : pre ? FN_ADDR(mmtk_object_reference_write_pre) - : FN_ADDR(mmtk_object_reference_write_post); + address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call) + : pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) + : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); - __ call_VM_leaf(entry_point, 4); + __ call_VM_leaf(entry_point, 3); if (pre) { __ pop_call_clobbered_registers(); diff --git a/openjdk/cpu/riscv/mmtkBarrierSetAssembler_riscv.cpp b/openjdk/cpu/riscv/mmtkBarrierSetAssembler_riscv.cpp index cc02bd88..49bd58d2 100644 --- a/openjdk/cpu/riscv/mmtkBarrierSetAssembler_riscv.cpp +++ b/openjdk/cpu/riscv/mmtkBarrierSetAssembler_riscv.cpp @@ -160,13 +160,10 @@ void MMTkBarrierSetAssembler::generate_c1_write_barrier_runtime_stub(StubAssembl // t2, a0-a7, t3-t6 __ push_call_clobbered_registers(); - // Load mutator from thread-local storage and call Rust directly. - Register mutator = t2; - __ la(mutator, Address(xthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); if (mmtk_enable_barrier_fastpath) { - __ call_VM_leaf(FN_ADDR(mmtk_object_reference_write_slow), src, slot, new_val, mutator); + __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), src, slot, new_val); } else { - __ call_VM_leaf(FN_ADDR(mmtk_object_reference_write_post), src, slot, new_val, mutator); + __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), src, slot, new_val); } __ pop_call_clobbered_registers(); diff --git a/openjdk/cpu/riscv/mmtkObjectBarrierSetAssembler_riscv.cpp b/openjdk/cpu/riscv/mmtkObjectBarrierSetAssembler_riscv.cpp index 49d0ed91..9c2c761f 100644 --- a/openjdk/cpu/riscv/mmtkObjectBarrierSetAssembler_riscv.cpp +++ b/openjdk/cpu/riscv/mmtkObjectBarrierSetAssembler_riscv.cpp @@ -1,6 +1,5 @@ #include "precompiled.hpp" #include "mmtkObjectBarrier.hpp" -#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" #define __ masm-> @@ -66,20 +65,18 @@ void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* // equivalently ((tmp1 & 1) == 0) go to done __ andi(tmp1, tmp1, 1); __ beqz(tmp1, done); - // setup calling convention: call Rust directly with mutator as last arg + // setup calling convention __ mv(c_rarg0, obj); __ la(c_rarg1, dst); __ mv(c_rarg2, val == noreg ? zr : val); - __ la(c_rarg3, Address(xthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); - __ call_VM_leaf_base(FN_ADDR(mmtk_object_reference_write_slow), 4); + __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); __ bind(done); } else { __ mv(c_rarg0, obj); __ la(c_rarg1, dst); __ mv(c_rarg2, val == noreg ? zr : val); - __ la(c_rarg3, Address(xthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); - __ call_VM_leaf_base(FN_ADDR(mmtk_object_reference_write_post), 4); + __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); } if (dst.base()->is_valid()) { __ pop_reg(dst.base()); @@ -95,12 +92,7 @@ void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec if (is_oop) { // in address generate_checkcast_copy, caller tells us to save count __ push_reg(saved_regs, sp); - // mmtk_array_copy_post(src, dst, count, mutator) - __ mv(c_rarg0, src); - __ mv(c_rarg1, dst); - __ mv(c_rarg2, count); - __ la(c_rarg3, Address(xthread, in_bytes(JavaThread::third_party_heap_mutator_offset()))); - __ call_VM_leaf_base(FN_ADDR(mmtk_array_copy_post), 4); + __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_post_call), zr, dst, count); __ pop_reg(saved_regs, sp); } } diff --git a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp index 70e896fe..9f5e8c3d 100644 --- a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp @@ -167,11 +167,7 @@ void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sa // As a temporary fix, we apply field barrier to all fields in this object. __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_probable_write_pre_call), c_rarg0); } else { - // Load mutator from thread-local storage as the last argument, then call Rust directly. - Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); - Register mutator_reg = (argc == 1) ? c_rarg1 : (argc == 2) ? c_rarg2 : c_rarg3; - __ lea(mutator_reg, mutator); - __ call_VM_leaf_base(entry_point, argc + 1); + __ call_VM_leaf_base(entry_point, argc); } __ restore_live_registers(true); @@ -179,11 +175,11 @@ void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sa } void MMTkBarrierSetAssembler::generate_c1_load_reference_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_load_reference_runtime_stub", FN_ADDR(mmtk_load_reference), 1); + generate_c1_runtime_stub_general(sasm, "c1_load_reference_runtime_stub", FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(mmtk_object_reference_write_slow), 3); + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub_with_patch_fix(StubAssembler* sasm) { @@ -191,11 +187,11 @@ void MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stu } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_post_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_post_stub", FN_ADDR(mmtk_object_reference_write_slow), 3); + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_post_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); } void MMTkBarrierSetAssembler::generate_c1_object_reference_write_slow_runtime_stub(StubAssembler* sasm) { - generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_slow_stub", FN_ADDR(mmtk_object_reference_write_slow), 3); + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_slow_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); } #undef __ diff --git a/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.cpp index 1fc20021..474d958a 100644 --- a/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkObjectBarrierSetAssembler_x86.cpp @@ -1,6 +1,5 @@ #include "precompiled.hpp" #include "mmtkObjectBarrier.hpp" -#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" //////////////////// Assembler //////////////////// @@ -48,13 +47,10 @@ void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec } } __ pusha(); - // mmtk_array_copy_post(src, dst, count, mutator) __ movptr(c_rarg0, src); __ movptr(c_rarg1, dst); __ movptr(c_rarg2, count); - Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); - __ lea(c_rarg3, mutator); - __ call_VM_leaf_base(FN_ADDR(mmtk_array_copy_post), 4); + __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_post_call), 3); __ popa(); } } diff --git a/openjdk/cpu/x86/mmtkSATBBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkSATBBarrierSetAssembler_x86.cpp index 421417b9..6dccc551 100644 --- a/openjdk/cpu/x86/mmtkSATBBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkSATBBarrierSetAssembler_x86.cpp @@ -1,6 +1,5 @@ #include "precompiled.hpp" #include "mmtkSATBBarrier.hpp" -#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" //////////////////// Assembler //////////////////// @@ -33,12 +32,10 @@ void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet dec // if (dst == 0) goto done; __ testptr(dst, dst); __ jcc(Assembler::zero, done); - // Do slow-call: call Rust directly with mutator from thread register + // Do slow-call __ pusha(); __ mov(c_rarg0, dst); - Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); - __ lea(c_rarg1, mutator); - __ MacroAssembler::call_VM_leaf_base(FN_ADDR(mmtk_load_reference), 2); + __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); __ popa(); __ bind(done); } @@ -60,9 +57,7 @@ void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Decor __ movptr(c_rarg0, src); __ movptr(c_rarg1, dst); __ movptr(c_rarg2, count); - Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); - __ lea(c_rarg3, mutator); - __ call_VM_leaf_base(FN_ADDR(mmtk_array_copy_pre), 4); + __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_pre_call), 3); __ popa(); __ bind(done); } diff --git a/openjdk/cpu/x86/mmtkUnlogBitBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkUnlogBitBarrierSetAssembler_x86.cpp index 6554f88a..c7148098 100644 --- a/openjdk/cpu/x86/mmtkUnlogBitBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkUnlogBitBarrierSetAssembler_x86.cpp @@ -2,7 +2,6 @@ #include "c1/c1_LIRAssembler.hpp" #include "c1/c1_MacroAssembler.hpp" -#include "mmtkMutator.hpp" #include "utilities/macros.hpp" #include CPU_HEADER(mmtkUnlogBitBarrierSetAssembler) @@ -57,17 +56,20 @@ void MMTkUnlogBitBarrierSetAssembler::object_reference_write_pre_or_post(MacroAs __ movptr(c_rarg0, obj); // Neither the ObjectBarrier nor the SATBBarrier need to know the slot or the value. // We just set both args to nullptr. + // We may need to pass actual arguments if we support other barriers. + // + // Note: If the `compensate_val_reg` parameter in the post barrier is true, and we are using + // compressed oops, the `val` register will be holding a compressed pointer to the target object + // due to the way `BarrierSetAssembler::store_at` works. If the write barrier needs to know the + // target, we will need to decompress it before passing it to the barrier slow path. __ xorptr(c_rarg1, c_rarg1); __ xorptr(c_rarg2, c_rarg2); - // Load mutator from thread-local storage and call Rust directly. - Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); - __ lea(c_rarg3, mutator); - address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(mmtk_object_reference_write_slow) - : pre ? FN_ADDR(mmtk_object_reference_write_pre) - : FN_ADDR(mmtk_object_reference_write_post); + address entry_point = mmtk_enable_barrier_fastpath ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call) + : pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) + : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); - __ call_VM_leaf_base(entry_point, 4); + __ call_VM_leaf_base(entry_point, 3); if (pre) { __ popa(); diff --git a/openjdk/share/barriers/mmtkFieldBarrier.cpp b/openjdk/share/barriers/mmtkFieldBarrier.cpp index 349d2c25..e4998a0e 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.cpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.cpp @@ -17,7 +17,7 @@ static inline intptr_t side_metadata_base_address() { void MMTkFieldBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const { #if SOFT_REFERENCE_LOAD_BARRIER if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL && mmtk_get_rc((void*) value) != 0) - ::mmtk_load_reference((void*) value, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); #endif }; @@ -35,10 +35,10 @@ void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, if (byte_val == 0) return; intptr_t shift = (addr >> (UseCompressedOops ? 2 : 3)) & 0b111; if (((byte_val >> shift) & 1) == kUnloggedValue) { - ::mmtk_object_reference_write_slow((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + object_reference_write_slow_call((void*) src, (void*) slot, (void*) target); } } else { - ::mmtk_object_reference_write_pre((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + object_reference_write_pre_call((void*) src, (void*) slot, (void*) target); } } @@ -71,9 +71,7 @@ void MMTkFieldBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet de // Do slow-call __ push_call_clobbered_registers(false /* save_fpu */); __ mov(c_rarg0, dst); - Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); - __ lea(c_rarg1, mutator); - __ MacroAssembler::call_VM_leaf_base(FN_ADDR(mmtk_load_reference), 2); + __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); __ pop_call_clobbered_registers(false /* save_fpu */); __ bind(done); } @@ -116,9 +114,7 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma __ movptr(c_rarg2, NULL_WORD); else __ movptr(c_rarg2, val); - Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); - __ lea(c_rarg3, mutator); - __ call_VM_leaf_base(FN_ADDR(mmtk_object_reference_write_slow), 4); + __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); __ pop_call_clobbered_registers(false /* save_fpu */); __ bind(done); @@ -130,9 +126,7 @@ void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* ma __ movptr(c_rarg2, NULL_WORD); else __ movptr(c_rarg2, val); - Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); - __ lea(c_rarg3, mutator); - __ call_VM_leaf_base(FN_ADDR(mmtk_object_reference_write_pre), 4); + __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); __ popa(); } } @@ -142,7 +136,6 @@ void MMTkFieldBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Deco if (dest_uninitialized) return; if (type == T_OBJECT || type == T_ARRAY) { Label done; - Address mutator(r15_thread, in_bytes(JavaThread::third_party_heap_mutator_offset())); // Bailout if count is zero __ cmpptr(count, 0); __ jcc(Assembler::equal, done); @@ -152,8 +145,7 @@ void MMTkFieldBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Deco if (c_rarg0 != src) __ movptr(c_rarg0, src); if (c_rarg1 != dst) __ movptr(c_rarg1, dst); if (c_rarg2 != count) __ movptr(c_rarg2, count); - __ lea(c_rarg3, mutator); - __ call_VM_leaf_base(FN_ADDR(mmtk_array_copy_pre), 4); + __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_pre_call), 3); __ pop_call_clobbered_registers(false /* save_fpu */); __ bind(done); } @@ -355,7 +347,6 @@ void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Op static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* slot, Node* val) { Node* no_base = __ top(); Node* tls = __ thread(); - Node* mutator = __ AddP(no_base, tls, __ ConX(in_bytes(JavaThread::third_party_heap_mutator_offset()))); if (mmtk_enable_barrier_fastpath) { float unlikely = PROB_UNLIKELY(0.999); @@ -368,13 +359,13 @@ static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* sl shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); __ if_then(result, BoolTest::ne, zero, unlikely); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeRawPtr::NOTNULL); - Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_object_reference_write_slow), "mmtk_barrier_call", src, slot, val, mutator); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), "mmtk_barrier_call", src, slot, val); } __ end_if(); __ end_if(); } else { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeRawPtr::NOTNULL); - Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_object_reference_write_pre), "mmtk_barrier_call", src, slot, val, mutator); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), "mmtk_barrier_call", src, slot, val); // Looks like this is necessary // See https://github.com/mmtk/openjdk/blob/c82e5c44adced4383162826c2c3933a83cfb139b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp#L288-L291 Node* call = __ ctrl()->in(0); @@ -396,7 +387,6 @@ static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool em MMTkIdealKit ideal(kit, true); Node* no_base = __ top(); Node* tls = __ thread(); - Node* mutator = __ AddP(no_base, tls, __ ConX(in_bytes(JavaThread::third_party_heap_mutator_offset()))); float unlikely = PROB_UNLIKELY(0.999); Node* zero = __ ConI(0); Node* cm_flag = __ load(__ ctrl(), __ ConP(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); @@ -404,8 +394,8 @@ static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool em __ if_then(cm_flag, BoolTest::ne, zero, unlikely); { // No slow-call if dst is NULL __ if_then(val, BoolTest::ne, kit->null()); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeRawPtr::NOTNULL); - Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_load_reference), "mmtk_barrier_call", val, mutator); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", val); } __ end_if(); } __ end_if(); kit->sync_kit(ideal); diff --git a/openjdk/share/barriers/mmtkFieldBarrier.hpp b/openjdk/share/barriers/mmtkFieldBarrier.hpp index d3ff953b..15be8838 100644 --- a/openjdk/share/barriers/mmtkFieldBarrier.hpp +++ b/openjdk/share/barriers/mmtkFieldBarrier.hpp @@ -22,7 +22,7 @@ class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const override { if (count == 0) return; - ::mmtk_array_copy_pre((void*) src, (void*) dst, count, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) dst, count); } virtual void object_probable_write(oop new_obj) const override; virtual void load_reference(DecoratorSet decorators, oop value) const override; diff --git a/openjdk/share/barriers/mmtkObjectBarrier.cpp b/openjdk/share/barriers/mmtkObjectBarrier.cpp index 01cacd20..2a4e8f4a 100644 --- a/openjdk/share/barriers/mmtkObjectBarrier.cpp +++ b/openjdk/share/barriers/mmtkObjectBarrier.cpp @@ -23,10 +23,10 @@ void MMTkObjectBarrierSetRuntime::object_probable_write(oop new_obj) const { void MMTkObjectBarrierSetRuntime::object_reference_write_post(oop src, oop* slot, oop target) const { if (mmtk_enable_barrier_fastpath) { if (is_unlog_bit_set(src)) { - ::mmtk_object_reference_write_slow((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + object_reference_write_slow_call((void*) src, (void*) slot, (void*) target); } } else { - ::mmtk_object_reference_write_post((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + object_reference_write_post_call((void*) src, (void*) slot, (void*) target); } } diff --git a/openjdk/share/barriers/mmtkObjectBarrier.hpp b/openjdk/share/barriers/mmtkObjectBarrier.hpp index 7e276a8c..14af6d8b 100644 --- a/openjdk/share/barriers/mmtkObjectBarrier.hpp +++ b/openjdk/share/barriers/mmtkObjectBarrier.hpp @@ -32,7 +32,7 @@ class MMTkObjectBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { // Interfaces called by `MMTkBarrierSet::AccessBarrier` virtual void object_reference_write_post(oop src, oop* slot, oop target) const override; virtual void object_reference_array_copy_post(oop* src, oop* dst, size_t count) const override { - ::mmtk_array_copy_post((void*) src, (void*) dst, count, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + object_reference_array_copy_post_call((void*) src, (void*) dst, count); } virtual void object_probable_write(oop new_obj) const override; }; diff --git a/openjdk/share/barriers/mmtkSATBBarrier.cpp b/openjdk/share/barriers/mmtkSATBBarrier.cpp index 42819b5a..7f08df63 100644 --- a/openjdk/share/barriers/mmtkSATBBarrier.cpp +++ b/openjdk/share/barriers/mmtkSATBBarrier.cpp @@ -1,6 +1,5 @@ #include "precompiled.hpp" #include "mmtkSATBBarrier.hpp" -#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" //////////////////// Runtime //////////////////// @@ -8,7 +7,7 @@ void MMTkSATBBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const { if (mmtk_enable_reference_load_barrier) { if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL) - ::mmtk_load_reference((void*) value, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); } }; @@ -22,10 +21,10 @@ void MMTkSATBBarrierSetRuntime::object_probable_write(oop new_obj) const { void MMTkSATBBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, oop target) const { if (mmtk_enable_barrier_fastpath) { if (is_unlog_bit_set(src)) { - ::mmtk_object_reference_write_slow((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + object_reference_write_slow_call((void*) src, (void*) slot, (void*) target); } } else { - ::mmtk_object_reference_write_pre((void*) src, (void*) slot, (void*) target, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + object_reference_write_pre_call((void*) src, (void*) slot, (void*) target); } } @@ -97,8 +96,6 @@ void MMTkSATBBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool emit_barrier) { MMTkIdealKit ideal(kit, true); Node* no_base = __ top(); - Node* tls = __ thread(); - Node* mutator = __ AddP(no_base, tls, __ ConX(in_bytes(JavaThread::third_party_heap_mutator_offset()))); float unlikely = PROB_UNLIKELY(0.999); Node* zero = __ ConI(0); Node* cm_flag = __ load(__ ctrl(), __ ConP(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); @@ -106,8 +103,8 @@ static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool em __ if_then(cm_flag, BoolTest::ne, zero, unlikely); { // No slow-call if dst is NULL __ if_then(val, BoolTest::ne, kit->null()); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeRawPtr::NOTNULL); - Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_load_reference), "mmtk_barrier_call", val, mutator); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", val); } __ end_if(); } __ end_if(); kit->sync_kit(ideal); diff --git a/openjdk/share/barriers/mmtkSATBBarrier.hpp b/openjdk/share/barriers/mmtkSATBBarrier.hpp index 98caad4e..7869caff 100644 --- a/openjdk/share/barriers/mmtkSATBBarrier.hpp +++ b/openjdk/share/barriers/mmtkSATBBarrier.hpp @@ -30,7 +30,7 @@ class MMTkSATBBarrierSetRuntime: public MMTkUnlogBitBarrierSetRuntime { virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const override { if (count == 0) return; - ::mmtk_array_copy_pre((void*) src, (void*) dst, count, (MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) dst, count); } virtual void object_probable_write(oop new_obj) const override; virtual void load_reference(DecoratorSet decorators, oop value) const override; diff --git a/openjdk/share/barriers/mmtkUnlogBitBarrier.cpp b/openjdk/share/barriers/mmtkUnlogBitBarrier.cpp index e1ba1b54..6feae3db 100644 --- a/openjdk/share/barriers/mmtkUnlogBitBarrier.cpp +++ b/openjdk/share/barriers/mmtkUnlogBitBarrier.cpp @@ -1,6 +1,5 @@ #include "precompiled.hpp" #include "mmtkUnlogBitBarrier.hpp" -#include "mmtkMutator.hpp" #include "runtime/interfaceSupport.inline.hpp" #include "c1/c1_LIRAssembler.hpp" @@ -92,23 +91,25 @@ Node* MMTkUnlogBitBarrierSetC2::emit_check_unlog_bit_fast_path(MMTkIdealKit& ide } void MMTkUnlogBitBarrierSetC2::object_reference_write_pre_or_post(MMTkIdealKit& ideal, Node* src, bool pre) { - Node* no_base = __ top(); - Node* tls = __ thread(); - Node* mutator = __ AddP(no_base, tls, __ ConX(in_bytes(JavaThread::third_party_heap_mutator_offset()))); if (mmtk_enable_barrier_fastpath) { Node* result = emit_check_unlog_bit_fast_path(ideal, src); Node* zero = __ ConI(0); float unlikely = PROB_UNLIKELY(0.999); __ if_then(result, BoolTest::ne, zero, unlikely); { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeRawPtr::NOTNULL); + Node* result = emit_check_unlog_bit_fast_path(ideal, src); + + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + address entry_point = FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call); Node* null_node = __ NullP(); - Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_object_reference_write_slow), "mmtk_barrier_call", src, null_node, null_node, mutator); + Node* x = __ make_leaf_call(tf, entry_point, "mmtk_barrier_call", src, null_node, null_node); } __ end_if(); } else { - const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeRawPtr::NOTNULL); + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + address entry_point = pre ? FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call) + : FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call); Node* null_node = __ NullP(); - Node* x = __ make_leaf_call(tf, FN_ADDR(mmtk_object_reference_write_slow), "mmtk_barrier_call", src, null_node, null_node, mutator); + Node* x = __ make_leaf_call(tf, entry_point, "mmtk_barrier_call", src, null_node, null_node); } } diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index 9ef8656a..a1d6dcf9 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -24,7 +24,6 @@ extern const uintptr_t RC_TABLE_BASE_ADDRESS; extern const uintptr_t VO_BIT_ADDRESS; extern const uintptr_t IMMIX_ALLOCATOR_SIZE; extern uint8_t RC_ENABLED; -extern uint8_t REQUIRES_WEAK_HANDLE_BARRIER; extern const size_t MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; extern const uintptr_t FREE_LIST_ALLOCATOR_SIZE; extern uint8_t CONCURRENT_MARKING_ACTIVE; @@ -55,22 +54,22 @@ extern void post_alloc(MMTk_Mutator mutator, void* refer, size_t bytes, int allocator); /// java.lang.Reference load barrier -extern void mmtk_load_reference(void* obj, MMTk_Mutator mutator); +extern void mmtk_load_reference(MMTk_Mutator mutator, void* obj); /// Full pre-barrier -extern void mmtk_object_reference_write_pre(void* src, void* slot, void* target, MMTk_Mutator mutator); +extern void mmtk_object_reference_write_pre(MMTk_Mutator mutator, void* src, void* slot, void* target); /// Full post-barrier -extern void mmtk_object_reference_write_post(void* src, void* slot, void* target, MMTk_Mutator mutator); +extern void mmtk_object_reference_write_post(MMTk_Mutator mutator, void* src, void* slot, void* target); /// Generic slow-path -extern void mmtk_object_reference_write_slow(void* src, void* slot, void* target, MMTk_Mutator mutator); +extern void mmtk_object_reference_write_slow(MMTk_Mutator mutator, void* src, void* slot, void* target); /// Full array-copy pre-barrier -extern void mmtk_array_copy_pre(void* src, void* dst, size_t count, MMTk_Mutator mutator); +extern void mmtk_array_copy_pre(MMTk_Mutator mutator, void* src, void* dst, size_t count); /// Full array-copy post-barrier -extern void mmtk_array_copy_post(void* src, void* dst, size_t count, MMTk_Mutator mutator); +extern void mmtk_array_copy_post(MMTk_Mutator mutator, void* src, void* dst, size_t count); /// C2 slowpath allocation barrier extern void mmtk_object_probable_write(MMTk_Mutator mutator, void* obj); @@ -260,6 +259,4 @@ extern void mmtk_builder_set_transparent_hugepages(bool value); } #endif -namespace JavaClassFile {}; - #endif // MMTK_OPENJDK_MMTK_H diff --git a/openjdk/share/mmtkBarrierSet.cpp b/openjdk/share/mmtkBarrierSet.cpp index 51a5db0e..c69f5413 100644 --- a/openjdk/share/mmtkBarrierSet.cpp +++ b/openjdk/share/mmtkBarrierSet.cpp @@ -146,6 +146,30 @@ bool MMTkBarrierSet::is_slow_path_call(address call) { return runtime()->is_slow_path_call(call); } +void MMTkBarrierSetRuntime::object_reference_write_pre_call(void* src, void* slot, void* target) { + ::mmtk_object_reference_write_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, target); +} + +void MMTkBarrierSetRuntime::object_reference_write_post_call(void* src, void* slot, void* target) { + ::mmtk_object_reference_write_post((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, target); +} + +void MMTkBarrierSetRuntime::object_reference_write_slow_call(void* src, void* slot, void* target) { + ::mmtk_object_reference_write_slow((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, slot, target); +} + +void MMTkBarrierSetRuntime::object_reference_array_copy_pre_call(void* src, void* dst, size_t count) { + ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); +} + +void MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, void* dst, size_t count) { + ::mmtk_array_copy_post((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, src, dst, count); +} + +void MMTkBarrierSetRuntime::load_reference_call(void* ref) { + ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); +} + void MMTkBarrierSetRuntime::object_probable_write_pre_call(void* ref) { ::mmtk_object_probable_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); } \ No newline at end of file diff --git a/openjdk/share/mmtkBarrierSet.hpp b/openjdk/share/mmtkBarrierSet.hpp index 1c930867..67808a4d 100644 --- a/openjdk/share/mmtkBarrierSet.hpp +++ b/openjdk/share/mmtkBarrierSet.hpp @@ -69,15 +69,28 @@ MMTkAllocatorOffsets get_tlab_top_and_end_offsets(AllocatorSelector selector); class MMTkBarrierSetRuntime: public CHeapObj { public: + /// Weak ref load barrier + static void load_reference_call(void* ref); + /// Generic pre-write barrier. Called by fast-paths. + static void object_reference_write_pre_call(void* src, void* slot, void* target); + /// Generic post-write barrier. Called by fast-paths. + static void object_reference_write_post_call(void* src, void* slot, void* target); + /// Generic slow-path. Called by fast-paths. + static void object_reference_write_slow_call(void* src, void* slot, void* target); + /// Generic arraycopy post-barrier. Called by fast-paths. + static void object_reference_array_copy_pre_call(void* src, void* dst, size_t count); + /// Generic arraycopy pre-barrier. Called by fast-paths. + static void object_reference_array_copy_post_call(void* src, void* dst, size_t count); + /// static void object_probable_write_pre_call(void* obj); /// Check if the address is a slow-path function. virtual bool is_slow_path_call(address call) const { - return call == CAST_FROM_FN_PTR(address, mmtk_object_reference_write_pre) - || call == CAST_FROM_FN_PTR(address, mmtk_object_reference_write_post) - || call == CAST_FROM_FN_PTR(address, mmtk_object_reference_write_slow) - || call == CAST_FROM_FN_PTR(address, mmtk_array_copy_pre) - || call == CAST_FROM_FN_PTR(address, mmtk_array_copy_post) - || call == CAST_FROM_FN_PTR(address, mmtk_load_reference) + return call == CAST_FROM_FN_PTR(address, object_reference_write_pre_call) + || call == CAST_FROM_FN_PTR(address, object_reference_write_post_call) + || call == CAST_FROM_FN_PTR(address, object_reference_write_slow_call) + || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call) + || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_post_call) + || call == CAST_FROM_FN_PTR(address, load_reference_call) || call == CAST_FROM_FN_PTR(address, object_probable_write_pre_call); } @@ -96,7 +109,6 @@ class MMTkBarrierSetRuntime: public CHeapObj { /// So this callback is requierd for any generational collectors. virtual void object_probable_write(oop new_obj) const {}; }; - class MMTkBarrierC1; class MMTkBarrierSetC1; class MMTkBarrierC2; diff --git a/openjdk/share/mmtkBarrierSetC2.cpp b/openjdk/share/mmtkBarrierSetC2.cpp index 0cc52247..dd7a970c 100644 --- a/openjdk/share/mmtkBarrierSetC2.cpp +++ b/openjdk/share/mmtkBarrierSetC2.cpp @@ -28,7 +28,6 @@ #include "mmtkBarrierSet.hpp" #include "mmtkBarrierSetC2.hpp" #include "mmtkMutator.hpp" -#include "mmtkHeap.hpp" #include "opto/addnode.hpp" #include "opto/arraycopynode.hpp" #include "opto/callnode.hpp" diff --git a/openjdk/share/mmtkHeap.cpp b/openjdk/share/mmtkHeap.cpp index e4f184b1..802b30c8 100644 --- a/openjdk/share/mmtkHeap.cpp +++ b/openjdk/share/mmtkHeap.cpp @@ -63,8 +63,6 @@ #include "mmtkRootsClosure.hpp" #include "gc/shared/workerPolicy.hpp" -using namespace JavaClassFile; - MaybeUninit> oop_storage_set_strong_par_state; MaybeUninit> oop_storage_set_weak_par_state; diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index 47a3a4ea..e15cb722 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -62,8 +62,6 @@ #include "jfr/jfr.hpp" #endif -using namespace JavaClassFile; - // Note: This counter must be accessed using the Atomic class. static volatile size_t mmtk_start_the_world_count = 0; From e20459dda6679af8d3bb5c7093934ae10e1f9937 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 18 Apr 2026 19:33:32 +1000 Subject: [PATCH 475/494] Implement ConcurrentPlan --- mmtk/src/gc_work.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 66414cd1..5b8abd89 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -6,7 +6,7 @@ use crate::OpenJDKSlot; use crate::Slot; use crate::SlotsClosure; use crate::UPCALLS; -use mmtk::plan::immix::Pause; +use mmtk::plan::concurrent::Pause; use mmtk::scheduler::*; use mmtk::util::Address; use mmtk::vm::RootsWorkFactory; From e88bb41f408bec0724d5ce0b19119693d65ff05b Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sat, 18 Apr 2026 19:41:57 +1000 Subject: [PATCH 476/494] Remove VMCollection::set_concurrent_marking_state --- mmtk/src/collection.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index dbf155e6..a7f4fbfe 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -76,10 +76,6 @@ impl Collection> for VMCollection { } } - fn set_concurrent_marking_state(active: bool) { - unsafe { crate::CONCURRENT_MARKING_ACTIVE = if active { 1 } else { 0 } } - } - fn vm_release() { unsafe { ((*UPCALLS).gc_epilogue)(); From 198abe07b97a93e9e00dfb3f31532ba4eb5270ec Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Tue, 28 Apr 2026 17:16:19 +1000 Subject: [PATCH 477/494] Refactor work bucket stages and update root scanning logic Co-authored-by: Copilot --- mmtk/src/scanning.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 2f79fb9a..338359a6 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -82,11 +82,11 @@ impl Scanning> for VMScanning { w.push(Box::new(ScanOopStorageSetRoots::new(factory.clone()))); w.push(Box::new(ScanWeakProcessorRoots::new(factory.clone()))); } - memory_manager::add_work_packets( - crate::singleton::(), - factory.roots_stage(), - w, - ); + let mmtk = crate::singleton::(); + let stage = crate::singleton::() + .get_plan() + .root_scanning_stage(); + memory_manager::add_work_packets(mmtk, stage, w); } fn supports_return_barrier() -> bool { From b641338d76fda8ce7a48324bee849e0a569b87b0 Mon Sep 17 00:00:00 2001 From: Wenyu Zhao Date: Sun, 26 Jul 2026 04:29:37 +0000 Subject: [PATCH 478/494] Remove call to `mmtk_report_gc_start` in `mmtk_stop_all_mutators` --- openjdk/share/mmtkUpcalls.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index e15cb722..e97d6cc8 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -180,7 +180,6 @@ static void mmtk_stop_all_mutators(void *tls, MutatorClosure closure) { oop_storage_set_strong_par_state.init(); oop_storage_set_weak_par_state.init(); - mmtk_report_gc_start(); // CodeCache::gc_prologue(); #if COMPILER2_OR_JVMCI DerivedPointerTable::clear(); From e7553f4072dbf03a165cf607e73ea1cd3474613e Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 28 Jul 2026 05:49:15 +0000 Subject: [PATCH 479/494] Update mmtk-core --- mmtk/Cargo.lock | 20 ++++++++++---------- mmtk/Cargo.toml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index e23a3627..5d44a5a7 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -334,12 +334,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - [[package]] name = "idna" version = "1.1.0" @@ -500,7 +494,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.32.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=603e29bb1384f83711cfc0c7bf25d22ae38b3530#603e29bb1384f83711cfc0c7bf25d22ae38b3530" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3ce65fbf7b9124267e1fd85c39147ef8b396751a#3ce65fbf7b9124267e1fd85c39147ef8b396751a" dependencies = [ "atomic", "atomic-traits", @@ -513,20 +507,20 @@ dependencies = [ "downcast-rs", "enum-map", "env_logger", - "humantime", "is-terminal", "itertools", "lazy_static", "libc", "log", - "memoffset", "mmtk-macros", "num-traits", "num_cpus", "portable-atomic", + "probe", "regex", "rustversion", "spin", + "static_assertions", "strum", "strum_macros", "sysinfo", @@ -535,7 +529,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.32.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=603e29bb1384f83711cfc0c7bf25d22ae38b3530#603e29bb1384f83711cfc0c7bf25d22ae38b3530" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3ce65fbf7b9124267e1fd85c39147ef8b396751a#3ce65fbf7b9124267e1fd85c39147ef8b396751a" dependencies = [ "proc-macro-error", "proc-macro2", @@ -806,6 +800,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strum" version = "0.27.2" diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 8e798860..4d0ee48f 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "603e29bb1384f83711cfc0c7bf25d22ae38b3530" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "3ce65fbf7b9124267e1fd85c39147ef8b396751a" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From b888874ee7b79715385157dd8e15f4964a9f54c1 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 28 Jul 2026 06:32:42 +0000 Subject: [PATCH 480/494] Test LXR in CI --- .github/configs/normal-heap.yml | 1 + .github/scripts/ci-expected-results.yml | 38 +++++++++++++++++++ .github/scripts/ci-test-forwarding-on-side.sh | 1 + mmtk/Cargo.lock | 4 +- mmtk/Cargo.toml | 2 +- 5 files changed, 43 insertions(+), 3 deletions(-) diff --git a/.github/configs/normal-heap.yml b/.github/configs/normal-heap.yml index c98e3d60..a3f43651 100644 --- a/.github/configs/normal-heap.yml +++ b/.github/configs/normal-heap.yml @@ -11,6 +11,7 @@ configs: - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-MarkCompact" - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-Compressor" - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-ConcurrentImmix" + - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-LXR" # This will be expanded in CI when we run with the config. Keep a new line at the end. benchmarks: diff --git a/.github/scripts/ci-expected-results.yml b/.github/scripts/ci-expected-results.yml index 4d07d89d..c2cf3726 100644 --- a/.github/scripts/ci-expected-results.yml +++ b/.github/scripts/ci-expected-results.yml @@ -11,6 +11,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass batik: SemiSpace: pass GenCopy: pass @@ -21,6 +22,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass biojava: SemiSpace: pass GenCopy: pass @@ -31,6 +33,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass cassandra: SemiSpace: pass GenCopy: pass @@ -41,6 +44,7 @@ results: MarkCompact: ignore Compressor: ignore ConcurrentImmix: ignore + LXR: pass eclipse: SemiSpace: pass GenCopy: pass @@ -51,6 +55,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass fop: SemiSpace: pass GenCopy: pass @@ -61,6 +66,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass graphchi: SemiSpace: pass GenCopy: pass @@ -71,6 +77,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass h2: SemiSpace: pass GenCopy: pass @@ -81,6 +88,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass h2o: SemiSpace: pass GenCopy: pass @@ -91,6 +99,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass jme: SemiSpace: pass GenCopy: pass @@ -101,6 +110,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass jython: SemiSpace: pass GenCopy: pass @@ -111,6 +121,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass kafka: SemiSpace: pass GenCopy: pass @@ -121,6 +132,7 @@ results: MarkCompact: ignore Compressor: pass ConcurrentImmix: pass + LXR: pass luindex: SemiSpace: pass GenCopy: pass @@ -131,6 +143,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass lusearch: SemiSpace: pass GenCopy: pass @@ -141,6 +154,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass pmd: SemiSpace: pass GenCopy: pass @@ -151,6 +165,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass sunflow: SemiSpace: pass GenCopy: pass @@ -161,6 +176,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass tomcat: SemiSpace: pass GenCopy: pass @@ -171,6 +187,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass xalan: SemiSpace: pass GenCopy: pass @@ -181,6 +198,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass zxing: SemiSpace: pass GenCopy: pass @@ -191,6 +209,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass release: avrora: @@ -203,6 +222,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass batik: SemiSpace: pass GenCopy: pass @@ -213,6 +233,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass biojava: SemiSpace: pass GenCopy: pass @@ -223,6 +244,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass cassandra: SemiSpace: pass GenCopy: pass @@ -233,6 +255,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass eclipse: SemiSpace: pass GenCopy: pass @@ -243,6 +266,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass fop: SemiSpace: pass GenCopy: pass @@ -253,6 +277,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass graphchi: SemiSpace: pass GenCopy: pass @@ -263,6 +288,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass h2: SemiSpace: pass GenCopy: pass @@ -273,6 +299,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass h2o: SemiSpace: pass GenCopy: pass @@ -283,6 +310,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass jme: SemiSpace: pass GenCopy: pass @@ -293,6 +321,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass jython: SemiSpace: pass GenCopy: pass @@ -303,6 +332,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass kafka: SemiSpace: pass GenCopy: pass @@ -313,6 +343,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass luindex: SemiSpace: pass GenCopy: pass @@ -323,6 +354,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass lusearch: SemiSpace: pass GenCopy: pass @@ -333,6 +365,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass pmd: SemiSpace: pass GenCopy: pass @@ -343,6 +376,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass sunflow: SemiSpace: pass GenCopy: pass @@ -353,6 +387,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass tomcat: SemiSpace: pass GenCopy: pass @@ -363,6 +398,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass xalan: SemiSpace: ignore GenCopy: ignore @@ -373,6 +409,7 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass zxing: SemiSpace: pass GenCopy: pass @@ -383,3 +420,4 @@ results: MarkCompact: pass Compressor: pass ConcurrentImmix: pass + LXR: pass diff --git a/.github/scripts/ci-test-forwarding-on-side.sh b/.github/scripts/ci-test-forwarding-on-side.sh index a7ce4085..7cdfbdb0 100755 --- a/.github/scripts/ci-test-forwarding-on-side.sh +++ b/.github/scripts/ci-test-forwarding-on-side.sh @@ -19,3 +19,4 @@ MMTK_PLAN=Immix run_subset 4 MMTK_PLAN=GenCopy run_subset 4 MMTK_PLAN=GenImmix run_subset 4 MMTK_PLAN=StickyImmix run_subset 4 +MMTK_PLAN=LXR run_subset 4 diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 5d44a5a7..a029f3f3 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -494,7 +494,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.32.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3ce65fbf7b9124267e1fd85c39147ef8b396751a#3ce65fbf7b9124267e1fd85c39147ef8b396751a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=d5616d7f32c2869819b7a4f9efd17cac72eae986#d5616d7f32c2869819b7a4f9efd17cac72eae986" dependencies = [ "atomic", "atomic-traits", @@ -529,7 +529,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.32.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3ce65fbf7b9124267e1fd85c39147ef8b396751a#3ce65fbf7b9124267e1fd85c39147ef8b396751a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=d5616d7f32c2869819b7a4f9efd17cac72eae986#d5616d7f32c2869819b7a4f9efd17cac72eae986" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 4d0ee48f..727b2ba9 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "3ce65fbf7b9124267e1fd85c39147ef8b396751a" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "d5616d7f32c2869819b7a4f9efd17cac72eae986" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 716247aa55d18bd34b37e62d7c1ddf34c8084967 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 29 Jul 2026 15:36:52 +0800 Subject: [PATCH 481/494] Fix mmtk_enqueue_references Don't acquire the `Heap_lock`. The VM companion thread is already holding the lock, so the GC worker thread can't (and doesn't need to) acquire it again. LXR never calls `mmtk_enqueue_references`, so we can remove it for STW plans. Removed the assertion of `has_reference_pending_list`. That function requires the current thread to be either the VM thead or a thread that is holding the `Heap_lock`, but in STW GC, the VM companion thread is holding that lock, and a GC worker thread cannot do the assertion. --- openjdk/share/mmtkUpcalls.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index e97d6cc8..83367850 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -462,8 +462,6 @@ static void mmtk_enqueue_references(void** objects, size_t len) { return; } - MutexLocker x(Heap_lock); - oop first = (oop) objects[0]; // This points to the first node of the linked list. oop last = first; // This points to the last node of the linked list. @@ -496,7 +494,10 @@ static void mmtk_enqueue_references(void** objects, size_t len) { oop old_first = Universe::swap_reference_pending_list(first); HeapAccess::oop_store_at(last, java_lang_ref_Reference::discovered_offset(), old_first); - assert(Universe::has_reference_pending_list(), "Reference pending list is empty after swap"); + + // Note: At this time, `Universe::has_reference_pending_list()` should be true, but we can't assert it. + // `has_reference_pending_list` requires either the current thread is the VM thread or is holding the Heap_lock. + // But during STW GC, the Heap_lock is held by the VM companion thread. } From f77bcc8e4049f7a5d12ca2e6bdf11ad57b0429d7 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 29 Jul 2026 16:21:41 +0800 Subject: [PATCH 482/494] Bump mmtk-core and deps revision --- mmtk/Cargo.lock | 649 ++++++++++++++++++++++++++++++------------------ mmtk/Cargo.toml | 2 +- 2 files changed, 412 insertions(+), 239 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index a029f3f3..d56a1da0 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -28,37 +28,37 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys", ] [[package]] name = "anstyle-wincon" -version = "3.0.10" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys", ] [[package]] @@ -82,21 +82,27 @@ dependencies = [ [[package]] name = "atomic_refcell" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" +checksum = "21e4227379beff4205943696e6c3e0cd809bacdf3f0edd6e3dd153e2269571a4" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "built" @@ -104,43 +110,43 @@ version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" dependencies = [ - "git2", + "git2 0.20.4", ] [[package]] name = "built" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" dependencies = [ - "git2", + "git2 0.21.0", ] [[package]] name = "bytemuck" -version = "1.23.1" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.8.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fa76293b4f7bb636ab88fd78228235b5248b4d05cc589aed610f954af5d7c7a" +checksum = "f65693059b6b9c588b9f62fed1cedbf0a8b805631457ea162d68f0de186f3de5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.119", ] [[package]] name = "cc" -version = "1.2.44" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37521ac7aabe3d13122dc382493e20c9416f299d2ccd5b3a5340a2570cdeb0f3" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -156,9 +162,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "colorchoice" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "crossbeam" @@ -175,18 +181,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -194,37 +200,79 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] [[package]] name = "delegate" -version = "0.13.4" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6178a82cf56c836a3ba61a7935cdb1c49bfaa6fa4327cd5bf554a503087de26b" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 3.0.3", ] [[package]] @@ -235,9 +283,9 @@ checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" [[package]] name = "either" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "enum-map" @@ -256,14 +304,14 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.119", ] [[package]] name = "env_filter" -version = "0.1.4" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" dependencies = [ "log", "regex", @@ -271,9 +319,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.8" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" dependencies = [ "anstream", "anstyle", @@ -284,9 +332,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "form_urlencoded" @@ -299,29 +347,40 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi", - "wasip2", ] [[package]] name = "git2" -version = "0.20.2" +version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2deb07a133b1520dc1a5690e9bd08950108873d7ed5de38dcc74d3b5ebffa110" +checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" dependencies = [ - "bitflags", + "bitflags 2.13.1", "libc", "libgit2-sys", "log", "url", ] +[[package]] +name = "git2" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" +dependencies = [ + "bitflags 2.13.1", + "libc", + "libgit2-sys", + "log", +] + [[package]] name = "heck" version = "0.5.0" @@ -335,34 +394,106 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] -name = "idna" -version = "1.1.0" +name = "icu_collections" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ - "idna_adapter", - "smallvec", + "displaydoc", + "potential_utf", "utf8_iter", + "yoke", + "zerofrom", + "zerovec", ] [[package]] -name = "idna_adapter" -version = "1.1.0" +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279259b0ac81c89d11c290495fdcfa96ea3643b7df311c138b6fe8ca5237f0f8" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ - "idna_mapping", - "unicode-bidi", - "unicode-normalization", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", ] [[package]] -name = "idna_mapping" +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11c13906586a4b339310541a274dd927aff6fcbb5b8e3af90634c4b31681c792" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ - "unicode-joining-type", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -373,7 +504,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys", ] [[package]] @@ -393,33 +524,45 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.15" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ + "defmt", + "jiff-core", "jiff-static", "log", "portable-atomic", "portable-atomic-util", - "serde", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", ] [[package]] name = "jiff-static" -version = "0.2.15" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" dependencies = [ + "jiff-core", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.119", ] [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ "getrandom", "libc", @@ -433,15 +576,15 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.177" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libgit2-sys" -version = "0.18.2+1.9.1" +version = "0.18.7+1.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c42fe03df2bd3c53a3a9c7317ad91d80c81cd1fb0caec8d7cc4cd2bfa10c222" +checksum = "23c7391e4b9f4ffab1a624223cc1d7385ff9a678f490768add717de7ea2f4d89" dependencies = [ "cc", "libc", @@ -451,9 +594,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.22" +version = "1.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" dependencies = [ "cc", "libc", @@ -461,6 +604,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "lock_api" version = "0.4.14" @@ -472,15 +621,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memoffset" @@ -494,12 +643,12 @@ dependencies = [ [[package]] name = "mmtk" version = "0.32.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=d5616d7f32c2869819b7a4f9efd17cac72eae986#d5616d7f32c2869819b7a4f9efd17cac72eae986" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8f3c129768af9e7e5003e97f5d6046875688859a#8f3c129768af9e7e5003e97f5d6046875688859a" dependencies = [ "atomic", "atomic-traits", "atomic_refcell", - "built 0.8.0", + "built 0.8.1", "bytemuck", "cfg-if", "crossbeam", @@ -529,12 +678,12 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.32.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=d5616d7f32c2869819b7a4f9efd17cac72eae986#d5616d7f32c2869819b7a4f9efd17cac72eae986" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8f3c129768af9e7e5003e97f5d6046875688859a#8f3c129768af9e7e5003e97f5d6046875688859a" dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.119", ] [[package]] @@ -555,9 +704,9 @@ dependencies = [ [[package]] name = "ntapi" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" dependencies = [ "winapi", ] @@ -587,7 +736,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.13.1", ] [[package]] @@ -602,9 +751,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -620,30 +769,39 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "portable-atomic" -version = "1.11.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "portable-atomic-util" -version = "0.2.4" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "probe" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e2d2444b730c8f027344c60f9e1f1554d7a3342df9bdd425142ed119a6e5a3" +checksum = "136558b6e1ebaecc92755d0ffaf9421f519531bed30cc2ad23b22cb00965cc5e" [[package]] name = "proc-macro-error" @@ -671,33 +829,33 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.42" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] [[package]] name = "r-efi" -version = "5.3.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "regex" -version = "1.12.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -707,9 +865,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -718,9 +876,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "rustc_version" @@ -733,9 +891,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "scopeguard" @@ -745,61 +903,66 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", - "serde_derive", ] [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 3.0.3", ] [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "spin" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" dependencies = [ "lock_api", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + [[package]] name = "static_assertions" version = "1.1.0" @@ -821,7 +984,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.119", ] [[package]] @@ -836,15 +999,37 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.109" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "sysinfo" version = "0.36.1" @@ -860,52 +1045,46 @@ dependencies = [ ] [[package]] -name = "tinyvec" -version = "1.9.0" +name = "thiserror" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ - "tinyvec_macros", + "thiserror-impl", ] [[package]] -name = "tinyvec_macros" -version = "0.1.1" +name = "thiserror-impl" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] [[package]] -name = "unicode-bidi" -version = "0.3.18" +name = "tinystr" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" - -[[package]] -name = "unicode-joining-type" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8d00a78170970967fdb83f9d49b92f959ab2bb829186b113e4f4604ad98e180" - -[[package]] -name = "unicode-normalization" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" -dependencies = [ - "tinyvec", -] +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", @@ -937,15 +1116,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "wasip2" -version = "1.0.1+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" -dependencies = [ - "wit-bindgen", -] - [[package]] name = "winapi" version = "0.3.9" @@ -1022,7 +1192,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.119", ] [[package]] @@ -1033,7 +1203,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.119", ] [[package]] @@ -1076,15 +1246,6 @@ dependencies = [ "windows-link 0.1.3", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -1094,23 +1255,6 @@ dependencies = [ "windows-link 0.2.1", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link 0.2.1", - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - [[package]] name = "windows-threading" version = "0.1.0" @@ -1121,55 +1265,84 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" +name = "writeable" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] -name = "windows_i686_gnu" -version = "0.53.1" +name = "yoke" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] [[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" +name = "yoke-derive" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] [[package]] -name = "windows_i686_msvc" -version = "0.53.1" +name = "zerofrom" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] [[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" +name = "zerofrom-derive" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] [[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" +name = "zerotrie" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] [[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" +name = "zerovec" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] [[package]] -name = "wit-bindgen" -version = "0.46.0" +name = "zerovec-derive" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 727b2ba9..52ddb633 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "d5616d7f32c2869819b7a4f9efd17cac72eae986" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8f3c129768af9e7e5003e97f5d6046875688859a" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From c93672d203c8dafb024a1c40fa1ab05aaa8aac9e Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 30 Jul 2026 04:03:46 +0000 Subject: [PATCH 483/494] Dont skip post_alloc for default allocation semantics --- openjdk/share/mmtkMutator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openjdk/share/mmtkMutator.cpp b/openjdk/share/mmtkMutator.cpp index 24f27bdb..85096e76 100644 --- a/openjdk/share/mmtkMutator.cpp +++ b/openjdk/share/mmtkMutator.cpp @@ -34,7 +34,7 @@ HeapWord* MMTkMutatorContext::alloc(size_t bytes, Allocator allocator) { HeapWord* o = (HeapWord*) ::alloc((MMTk_Mutator) this, bytes, HeapWordSize, 0, allocator); // Post allocation hooks. Note that we can get a nullptr from mmtk core in the case of OOM. // Hence, only call post allocation hooks if we have a proper object. - if (o != nullptr && allocator != AllocatorDefault) { + if (o != nullptr) { ::post_alloc((MMTk_Mutator) this, o, bytes, allocator); } return o; From a3ccf46a010672c3ff3ade7ed02e785f046e6913 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 30 Jul 2026 06:07:25 +0000 Subject: [PATCH 484/494] Reinit oop storage par state for second scanning for markcompact/compressor --- openjdk/share/mmtkHeap.hpp | 5 +++++ openjdk/share/mmtkUpcalls.cpp | 2 ++ 2 files changed, 7 insertions(+) diff --git a/openjdk/share/mmtkHeap.hpp b/openjdk/share/mmtkHeap.hpp index 7c7d5ecc..444c6094 100644 --- a/openjdk/share/mmtkHeap.hpp +++ b/openjdk/share/mmtkHeap.hpp @@ -58,6 +58,11 @@ struct MaybeUninit { void init(Args... args) { new (&_data) T(args...); } + template + void reinit(Args... args) { + ((T*) &_data)->~T(); + new (&_data) T(args...); + } private: char _data[sizeof(T)]; }; diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index 83367850..222296fe 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -455,6 +455,8 @@ static void mmtk_prepare_for_roots_re_scanning() { DerivedPointerTable::update_pointers(); DerivedPointerTable::clear(); #endif + oop_storage_set_strong_par_state.reinit(); + oop_storage_set_weak_par_state.reinit(); } static void mmtk_enqueue_references(void** objects, size_t len) { From d61c340a04364f67f349e31c06f1bb5b5a5966c8 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 31 Jul 2026 05:36:34 +0000 Subject: [PATCH 485/494] Use MMTkBookKeepingCodeBlobClosure. Skip MMTkUpdateClosure in mmtk_fix_oop_relocations --- openjdk/share/mmtkUpcalls.cpp | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index 222296fe..0d25e0de 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -27,6 +27,7 @@ #include "classfile/classLoaderDataGraph.hpp" #include "classfile/stringTable.hpp" #include "code/nmethod.hpp" +#include "gc/shared/barrierSetNMethod.hpp" #include "memory/iterator.inline.hpp" #include "memory/resourceArea.hpp" #include "mmtkCollectorThread.hpp" @@ -339,11 +340,31 @@ static void mmtk_scan_roots_in_all_mutator_threads(SlotsClosure closure) { MMTkHeap::heap()->scan_roots_in_all_mutator_threads(cl); } +// A CodeBlobClosure for use during live stack walks. It performs the same nmethod +// "on stack" bookkeeping as MarkingCodeBlobClosure (mark_as_maybe_on_stack/disarm), +// but deliberately does NOT re-scan the nmethod's embedded oops: those are already +// tracked once, unconditionally, via MMTk's own code-cache-roots remembered set +// (see MMTkHeap::register_nmethod / ScanCodeCacheRoots). Using MarkingCodeBlobClosure +// here would report every on-stack nmethod's oops as GC roots twice in the same GC. +class MMTkBookKeepingCodeBlobClosure : public CodeBlobClosure { +public: + virtual void do_code_blob(CodeBlob* cb) override { + nmethod* nm = cb->as_nmethod_or_null(); + if (nm != nullptr && nm->oops_do_try_claim()) { + nm->mark_as_maybe_on_stack(); + BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); + if (bs_nm != nullptr) { + bs_nm->disarm(nm); + } + } + } +}; + static void mmtk_scan_roots_in_mutator_thread(SlotsClosure closure, void* tls) { ResourceMark rm; JavaThread* thread = (JavaThread*) tls; MMTkRootsClosure cl(closure); - MarkingCodeBlobClosure cb_cl(&cl, false, true); + MMTkBookKeepingCodeBlobClosure cb_cl; thread->oops_do(&cl, &cb_cl); } @@ -512,10 +533,12 @@ void mmtk_fix_oop_relocations(bool lxr, void* nmethods, size_t len) { nm->fix_oop_relocations(); } } else { - MMTkUpdateClosure cl; + // MMTkUpdateClosure cl; for (size_t i = 0; i < len; i++) { nmethod* nm = (nmethod*) ((nmethod**) nmethods)[i]; - nm->oops_do(&cl); + // FIXME: This caused crash for mark compact. It is likely that nmethod is already scanned, and we see to-space objects here and still try to forward it. + // I am uncertain what to do here. Just comment it out for now so we can proceed. This should be resolved before merging. + // nm->oops_do(&cl); nm->fix_oop_relocations(); } } From f378fb1a09406a5ab66e77a326fe1b3b5dcd831d Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Mon, 3 Aug 2026 01:56:55 +0000 Subject: [PATCH 486/494] Update mmtk-core: queue flip workaround for conix --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index d56a1da0..18b4b69f 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -643,7 +643,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.32.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8f3c129768af9e7e5003e97f5d6046875688859a#8f3c129768af9e7e5003e97f5d6046875688859a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e04e7ec07ea68f5dc8f744ff3aef1be9bf220b66#e04e7ec07ea68f5dc8f744ff3aef1be9bf220b66" dependencies = [ "atomic", "atomic-traits", @@ -678,7 +678,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.32.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8f3c129768af9e7e5003e97f5d6046875688859a#8f3c129768af9e7e5003e97f5d6046875688859a" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e04e7ec07ea68f5dc8f744ff3aef1be9bf220b66#e04e7ec07ea68f5dc8f744ff3aef1be9bf220b66" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 52ddb633..9e90efe6 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8f3c129768af9e7e5003e97f5d6046875688859a" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e04e7ec07ea68f5dc8f744ff3aef1be9bf220b66" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 008bf1c8f0746a7f8bd954b4b882b72869c80404 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Mon, 3 Aug 2026 03:43:58 +0000 Subject: [PATCH 487/494] Run LXR tests with no weak reference --- .github/configs/base.yml | 4 ++++ .github/configs/normal-heap.yml | 2 +- .github/scripts/ci-test-forwarding-on-side.sh | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/configs/base.yml b/.github/configs/base.yml index a9639e52..4e454d0b 100644 --- a/.github/configs/base.yml +++ b/.github/configs/base.yml @@ -80,6 +80,10 @@ modifiers: no_compressed_oops: type: JVMArg val: "-XX:-UseCompressedOops -XX:-UseCompressedClassPointers" + no_reference_types: + type: "EnvVar" + var: "MMTK_NO_REFERENCE_TYPES" + val: "true" plugins: keep_stdout_stderr: diff --git a/.github/configs/normal-heap.yml b/.github/configs/normal-heap.yml index a3f43651..cc2d493d 100644 --- a/.github/configs/normal-heap.yml +++ b/.github/configs/normal-heap.yml @@ -11,7 +11,7 @@ configs: - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-MarkCompact" - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-Compressor" - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-ConcurrentImmix" - - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-LXR" + - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|no_reference_types|mmtk_gc-LXR" # This will be expanded in CI when we run with the config. Keep a new line at the end. benchmarks: diff --git a/.github/scripts/ci-test-forwarding-on-side.sh b/.github/scripts/ci-test-forwarding-on-side.sh index 7cdfbdb0..659b7f2e 100755 --- a/.github/scripts/ci-test-forwarding-on-side.sh +++ b/.github/scripts/ci-test-forwarding-on-side.sh @@ -19,4 +19,4 @@ MMTK_PLAN=Immix run_subset 4 MMTK_PLAN=GenCopy run_subset 4 MMTK_PLAN=GenImmix run_subset 4 MMTK_PLAN=StickyImmix run_subset 4 -MMTK_PLAN=LXR run_subset 4 +MMTK_PLAN=LXR MMTK_NO_REFERENCE_TYPES=true run_subset 4 From 7b86f80120fd959bc7696eb92ec7a2769959aa3b Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Mon, 3 Aug 2026 05:15:47 +0000 Subject: [PATCH 488/494] Update mmtk-core: fix concurrent ix --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 18b4b69f..c03af327 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -643,7 +643,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.32.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e04e7ec07ea68f5dc8f744ff3aef1be9bf220b66#e04e7ec07ea68f5dc8f744ff3aef1be9bf220b66" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=4f608ccb6a2e4d4c08ca990da9ab0975186adece#4f608ccb6a2e4d4c08ca990da9ab0975186adece" dependencies = [ "atomic", "atomic-traits", @@ -678,7 +678,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.32.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=e04e7ec07ea68f5dc8f744ff3aef1be9bf220b66#e04e7ec07ea68f5dc8f744ff3aef1be9bf220b66" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=4f608ccb6a2e4d4c08ca990da9ab0975186adece#4f608ccb6a2e4d4c08ca990da9ab0975186adece" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 9e90efe6..492337a0 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "e04e7ec07ea68f5dc8f744ff3aef1be9bf220b66" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "4f608ccb6a2e4d4c08ca990da9ab0975186adece" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From dcc02e1a49b8d0edbeda754464afa7bc67212ada Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Wed, 5 Aug 2026 04:53:14 +0000 Subject: [PATCH 489/494] Remove Mutex::_no_safepoint_check_flag from JNICritical_lock --- openjdk/share/mmtkHeap.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/openjdk/share/mmtkHeap.cpp b/openjdk/share/mmtkHeap.cpp index 802b30c8..5d6aa364 100644 --- a/openjdk/share/mmtkHeap.cpp +++ b/openjdk/share/mmtkHeap.cpp @@ -352,16 +352,15 @@ bool MMTkHeap::card_mark_must_follow_store() const { //OK void MMTkHeap::collect(GCCause::Cause cause) {//later when gc is implemented in rust if (cause == GCCause::_gc_locker) { - // #ifndef PRODUCT - // auto safepoint_check_required = JNICritical_lock->_safepoint_check_required; - // JNICritical_lock->_safepoint_check_required = Monitor::_safepoint_check_sometimes; - // #endif - MutexLocker locker(JNICritical_lock, Mutex::_no_safepoint_check_flag); + // This runs on the Java mutator thread that is the last one exiting a + // JNI critical region (see GCLocker::jni_unlock), so JNICritical_lock + // must be acquired with a safepoint check, same as gcLocker.cpp does. + // Acquiring it with Mutex::_no_safepoint_check_flag here trips HotSpot's + // "This lock should always have a safepoint check for Java threads" + // assertion in Mutex::check_no_safepoint_state. + MutexLocker locker(JNICritical_lock); // Notify the VMCompanionThread to trigger another VM_MMTkSTWOperation. JNICritical_lock->notify_all(); - // #ifndef PRODUCT - // JNICritical_lock->_safepoint_check_required = safepoint_check_required; - // #endif } handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, cause != GCCause::_java_lang_system_gc); } From 48fb0879f87ac9ff8c5cb1d8713706574956bdd4 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 6 Aug 2026 20:27:40 +1200 Subject: [PATCH 490/494] Rename markcompact plans (JDK21) (#365) Co-authored-by: mmtkgc-bot --- .github/configs/base.yml | 6 +- .github/configs/normal-heap.yml | 4 +- .github/scripts/ci-expected-results.yml | 152 +++++++++--------- .github/scripts/ci-test-minimal.sh | 4 +- .../ci-test-only-normal-no-compressed-oops.sh | 4 +- .github/scripts/ci-test-only-normal.sh | 4 +- .github/scripts/ci-test-vo-bit.sh | 4 +- .github/workflows/run-dacapo-chopin-inner.yml | 2 +- mmtk/Cargo.toml | 2 +- mmtk/src/api.rs | 4 +- mmtk/src/gc_work.rs | 2 +- mmtk/src/lib.rs | 4 +- .../mmtkBarrierSetAssembler_aarch64.cpp | 4 +- .../riscv/mmtkBarrierSetAssembler_riscv.cpp | 4 +- .../cpu/x86/mmtkBarrierSetAssembler_x86.cpp | 6 +- openjdk/share/mmtk.h | 4 +- openjdk/share/mmtkBarrierSet.cpp | 8 +- openjdk/share/mmtkBarrierSetC2.cpp | 6 +- openjdk/share/mmtkMutator.hpp | 6 +- 19 files changed, 115 insertions(+), 115 deletions(-) diff --git a/.github/configs/base.yml b/.github/configs/base.yml index 4e454d0b..57ea1e81 100644 --- a/.github/configs/base.yml +++ b/.github/configs/base.yml @@ -9,11 +9,11 @@ suites: type: DaCapo release: "23.11" path: "DACAPO_PATH/dacapo-23.11-MR2-chopin.jar" - minheap: mmtk-openjdk-11-MarkCompact + minheap: mmtk-openjdk-11-Lisp2 # Min heap values are from dacapo-evaluation-git-04132797 - # TODO: Use the values provided by the released version, and use G1 instead of MarkCompact. + # TODO: Use the values provided by the released version, and use G1 instead of Lisp2. minheap_values: - mmtk-openjdk-11-MarkCompact: + mmtk-openjdk-11-Lisp2: avrora: 8 batik: 426 biojava: 197 diff --git a/.github/configs/normal-heap.yml b/.github/configs/normal-heap.yml index cc2d493d..60a81eae 100644 --- a/.github/configs/normal-heap.yml +++ b/.github/configs/normal-heap.yml @@ -8,8 +8,8 @@ configs: - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-GenImmix" - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-StickyImmix" - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-MarkSweep" - - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-MarkCompact" - - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-Compressor" + - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-Lisp2" + - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-OVC" - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-ConcurrentImmix" - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|no_reference_types|mmtk_gc-LXR" diff --git a/.github/scripts/ci-expected-results.yml b/.github/scripts/ci-expected-results.yml index c2cf3726..5aa13416 100644 --- a/.github/scripts/ci-expected-results.yml +++ b/.github/scripts/ci-expected-results.yml @@ -8,8 +8,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass batik: @@ -19,8 +19,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass biojava: @@ -30,8 +30,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass cassandra: @@ -41,8 +41,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: ignore - MarkCompact: ignore - Compressor: ignore + Lisp2: ignore + OVC: ignore ConcurrentImmix: ignore LXR: pass eclipse: @@ -52,8 +52,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: ignore - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass fop: @@ -63,8 +63,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: ignore - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass graphchi: @@ -74,8 +74,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass h2: @@ -85,8 +85,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: ignore - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass h2o: @@ -96,8 +96,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass jme: @@ -107,8 +107,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass jython: @@ -118,8 +118,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: ignore - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass kafka: @@ -129,8 +129,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: ignore - Compressor: pass + Lisp2: ignore + OVC: pass ConcurrentImmix: pass LXR: pass luindex: @@ -140,8 +140,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass lusearch: @@ -151,8 +151,8 @@ results: GenImmix: ignore StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass pmd: @@ -162,8 +162,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass sunflow: @@ -173,8 +173,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass tomcat: @@ -184,8 +184,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass xalan: @@ -195,8 +195,8 @@ results: GenImmix: pass StickyImmix: ignore MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass zxing: @@ -206,8 +206,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass @@ -219,8 +219,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass batik: @@ -230,8 +230,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass biojava: @@ -241,8 +241,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass cassandra: @@ -252,8 +252,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: ignore - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass eclipse: @@ -263,8 +263,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: ignore - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass fop: @@ -274,8 +274,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: ignore - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass graphchi: @@ -285,8 +285,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass h2: @@ -296,8 +296,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: ignore - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass h2o: @@ -307,8 +307,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: ignore - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass jme: @@ -318,8 +318,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass jython: @@ -329,8 +329,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: ignore - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass kafka: @@ -340,8 +340,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass luindex: @@ -351,8 +351,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass lusearch: @@ -362,8 +362,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass pmd: @@ -373,8 +373,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass sunflow: @@ -384,8 +384,8 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass tomcat: @@ -395,8 +395,8 @@ results: GenImmix: ignore StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass xalan: @@ -406,8 +406,8 @@ results: GenImmix: ignore StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass zxing: @@ -417,7 +417,7 @@ results: GenImmix: pass StickyImmix: pass MarkSweep: pass - MarkCompact: pass - Compressor: pass + Lisp2: pass + OVC: pass ConcurrentImmix: pass LXR: pass diff --git a/.github/scripts/ci-test-minimal.sh b/.github/scripts/ci-test-minimal.sh index e6fbfba2..fcd409db 100755 --- a/.github/scripts/ci-test-minimal.sh +++ b/.github/scripts/ci-test-minimal.sh @@ -11,8 +11,8 @@ MMTK_PLAN=Immix runbms_dacapo2006_with_heap_multiplier fop 4 MMTK_PLAN=GenImmix runbms_dacapo2006_with_heap_multiplier fop 4 MMTK_PLAN=StickyImmix runbms_dacapo2006_with_heap_multiplier fop 4 MMTK_PLAN=GenCopy runbms_dacapo2006_with_heap_multiplier fop 4 -MMTK_PLAN=MarkCompact runbms_dacapo2006_with_heap_multiplier fop 4 -MMTK_PLAN=Compressor runbms_dacapo2006_with_heap_multiplier fop 4 +MMTK_PLAN=Lisp2 runbms_dacapo2006_with_heap_multiplier fop 4 +MMTK_PLAN=OVC runbms_dacapo2006_with_heap_multiplier fop 4 MMTK_PLAN=ConcurrentImmix runbms_dacapo2006_with_heap_multiplier fop 4 -XX:-UseCompressedOops -XX:-UseCompressedClassPointers MMTK_PLAN=MarkSweep runbms_dacapo2006_with_heap_multiplier fop 8 MMTK_PLAN=NoGC runbms_dacapo2006_with_heap_size fop 1000 1000 diff --git a/.github/scripts/ci-test-only-normal-no-compressed-oops.sh b/.github/scripts/ci-test-only-normal-no-compressed-oops.sh index c5a68215..9e637e91 100755 --- a/.github/scripts/ci-test-only-normal-no-compressed-oops.sh +++ b/.github/scripts/ci-test-only-normal-no-compressed-oops.sh @@ -67,8 +67,8 @@ runbms_dacapo2006_with_heap_size antlr 1000 1000 -XX:-UseCompressedOops -XX:-Use runbms_dacapo2006_with_heap_size fop 1000 1000 -XX:-UseCompressedOops -XX:-UseCompressedClassPointers runbms_dacapo2006_with_heap_size luindex 1000 1000 -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -# --- MarkCompact --- -export MMTK_PLAN=MarkCompact +# --- Lisp2 --- +export MMTK_PLAN=Lisp2 run_all_no_compressed_oop 4 diff --git a/.github/scripts/ci-test-only-normal.sh b/.github/scripts/ci-test-only-normal.sh index be37c252..ee5caf91 100755 --- a/.github/scripts/ci-test-only-normal.sh +++ b/.github/scripts/ci-test-only-normal.sh @@ -61,8 +61,8 @@ runbms_dacapo2006_with_heap_size antlr 1000 1000 runbms_dacapo2006_with_heap_size fop 1000 1000 runbms_dacapo2006_with_heap_size luindex 1000 1000 -# --- MarkCompact --- -export MMTK_PLAN=MarkCompact +# --- Lisp2 --- +export MMTK_PLAN=Lisp2 run_all 4 diff --git a/.github/scripts/ci-test-vo-bit.sh b/.github/scripts/ci-test-vo-bit.sh index 1d7e3fce..77338bbf 100755 --- a/.github/scripts/ci-test-vo-bit.sh +++ b/.github/scripts/ci-test-vo-bit.sh @@ -43,8 +43,8 @@ export MMTK_PLAN=MarkSweep run_subset 8 -# --- Compressor --- -export MMTK_PLAN=Compressor +# --- OVC --- +export MMTK_PLAN=OVC run_subset 4 diff --git a/.github/workflows/run-dacapo-chopin-inner.yml b/.github/workflows/run-dacapo-chopin-inner.yml index 61c82810..c1ee2f44 100644 --- a/.github/workflows/run-dacapo-chopin-inner.yml +++ b/.github/workflows/run-dacapo-chopin-inner.yml @@ -136,7 +136,7 @@ jobs: popd - name: Check free space run: df -h - - name: Run ${{ env.DACAPO_VERSION }} ${{ matrix.benchmark }} on MMTk OpenJDK ${{ matrix.debug-level }} with ${{ inputs.heap-factor }}x MarkCompact minheap + - name: Run ${{ env.DACAPO_VERSION }} ${{ matrix.benchmark }} on MMTk OpenJDK ${{ matrix.debug-level }} with ${{ inputs.heap-factor }}x Lisp2 minheap run: | DACAPO_PATH=`realpath ./dacapo` sed -i "s;DACAPO_PATH;$DACAPO_PATH;g" .github/configs/base.yml diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index b068bbef..1bba356d 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -76,5 +76,5 @@ pageprotect = [] immix = [] genimmix = [] stickyimmix = [] -markcompact = [] +lisp2 = [] jfr = [] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index 1d0ffad7..6737f47b 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -101,8 +101,8 @@ pub extern "C" fn openjdk_gc_init(calls: *const OpenJDK_Upcalls) { Some(PlanSelector::GenCopy) } else if cfg!(feature = "marksweep") { Some(PlanSelector::MarkSweep) - } else if cfg!(feature = "markcompact") { - Some(PlanSelector::MarkCompact) + } else if cfg!(feature = "lisp2") { + Some(PlanSelector::Lisp2) } else if cfg!(feature = "pageprotect") { Some(PlanSelector::PageProtect) } else if cfg!(feature = "immix") { diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 5b8abd89..801ce4fd 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -166,7 +166,7 @@ impl>> // immediate operands in the machine code. This can only be done after all fields of an // nmethod have been forwarded. let stage = if mmtk.get_plan().constraints().needs_forward_after_liveness { - // For MarkCompact, we forward the children of nmethods in the transitive closure + // For Lisp2 and OVC, we forward the children of nmethods in the transitive closure // starting with SecondRoots. RefForwarding is the first safe place to call // fix_oop_relocations. WorkBucketStage::RefForwarding diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index 4059016c..a82433ef 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -228,8 +228,8 @@ fn singleton() -> &'static MMTK> { } #[no_mangle] -pub static MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES: usize = - mmtk::util::alloc::MarkCompactAllocator::>::HEADER_RESERVED_IN_BYTES; +pub static MMTK_LISP2_HEADER_RESERVED_IN_BYTES: usize = + mmtk::util::alloc::Lisp2Allocator::>::HEADER_RESERVED_IN_BYTES; lazy_static! { /// A global storage for all the cached CodeCache roots added since the last GC. diff --git a/openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.cpp b/openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.cpp index 6c939aca..3bc39b4c 100644 --- a/openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.cpp +++ b/openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.cpp @@ -60,8 +60,8 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register obj, // We need to figure out which allocator we are using by querying MMTk. AllocatorSelector selector = get_allocator_mapping(allocator); - // XXX riscv: disallow markcompact and global alloc bit for now - assert(selector.tag != TAG_MARK_COMPACT, "mark compact not supported for now"); + // XXX riscv: disallow lisp2 and global alloc bit for now + assert(selector.tag != TAG_LISP2, "mark compact not supported for now"); if (var_size_in_bytes == noreg) { // constant alloc size. If it is larger than max_non_los_bytes, we directly go to slowpath. diff --git a/openjdk/cpu/riscv/mmtkBarrierSetAssembler_riscv.cpp b/openjdk/cpu/riscv/mmtkBarrierSetAssembler_riscv.cpp index 49bd58d2..f7eaa298 100644 --- a/openjdk/cpu/riscv/mmtkBarrierSetAssembler_riscv.cpp +++ b/openjdk/cpu/riscv/mmtkBarrierSetAssembler_riscv.cpp @@ -58,8 +58,8 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register obj, // We need to figure out which allocator we are using by querying MMTk. AllocatorSelector selector = get_allocator_mapping(allocator); - // XXX riscv: disallow markcompact and global alloc bit for now - assert(selector.tag != TAG_MARK_COMPACT, "mark compact not supported for now"); + // XXX riscv: disallow lisp2 and global alloc bit for now + assert(selector.tag != TAG_LISP2, "mark compact not supported for now"); if (var_size_in_bytes == noreg) { // constant alloc size. If it is larger than max_non_los_bytes, we directly go to slowpath. diff --git a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp index fffcd238..defac781 100644 --- a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp @@ -57,7 +57,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa Allocator allocator = AllocatorDefault; // We need to figure out which allocator we are using by querying MMTk. AllocatorSelector selector = get_allocator_mapping(allocator); - if (selector.tag == TAG_MARK_COMPACT) extra_header = MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; + if (selector.tag == TAG_LISP2) extra_header = MMTK_LISP2_HEADER_RESERVED_IN_BYTES; if (var_size_in_bytes == noreg) { // constant alloc size. If it is larger than max_non_los_bytes, we directly go to slowpath. @@ -85,7 +85,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa // obj = load lab.cursor __ movptr(obj, cursor); - if (selector.tag == TAG_MARK_COMPACT) __ addptr(obj, extra_header); + if (selector.tag == TAG_LISP2) __ addptr(obj, extra_header); // end = obj + size Register end = t1; if (var_size_in_bytes == noreg) { @@ -105,7 +105,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa #ifdef MMTK_ENABLE_VO_BIT enable_vo_bit = true; #endif - if (enable_vo_bit || selector.tag == TAG_MARK_COMPACT) { + if (enable_vo_bit || selector.tag == TAG_LISP2) { Register tmp3 = rdi; Register tmp2 = rscratch1; assert_different_registers(obj, tmp2, tmp3, rcx); diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index 11cb8278..8ebcea5d 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -24,7 +24,7 @@ extern uintptr_t get_rc_table_base_address(); extern uintptr_t get_vo_bit_address(); extern const uintptr_t IMMIX_ALLOCATOR_SIZE; extern uint8_t RC_ENABLED; -extern const size_t MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; +extern const size_t MMTK_LISP2_HEADER_RESERVED_IN_BYTES; extern const uintptr_t FREE_LIST_ALLOCATOR_SIZE; extern uint8_t CONCURRENT_MARKING_ACTIVE; @@ -89,7 +89,7 @@ struct AllocatorSelector { #define TAG_LARGE_OBJECT 1 #define TAG_MALLOC 2 #define TAG_IMMIX 3 -#define TAG_MARK_COMPACT 4 +#define TAG_LISP2 4 #define TAG_FREE_LIST 5 extern AllocatorSelector get_allocator_mapping(int allocator); diff --git a/openjdk/share/mmtkBarrierSet.cpp b/openjdk/share/mmtkBarrierSet.cpp index c69f5413..e588bd88 100644 --- a/openjdk/share/mmtkBarrierSet.cpp +++ b/openjdk/share/mmtkBarrierSet.cpp @@ -60,11 +60,11 @@ MMTkAllocatorOffsets get_tlab_top_and_end_offsets(AllocatorSelector selector) { + selector.index * sizeof(BumpAllocator); tlab_top_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, cursor)); tlab_end_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, limit)); - } else if (selector.tag == TAG_MARK_COMPACT) { + } else if (selector.tag == TAG_LISP2) { int allocator_base_offset = allocators_base_offset - + in_bytes(byte_offset_of(Allocators, markcompact)) - + selector.index * sizeof(MarkCompactAllocator) - + in_bytes(byte_offset_of(MarkCompactAllocator, bump_allocator)); + + in_bytes(byte_offset_of(Allocators, lisp2)) + + selector.index * sizeof(Lisp2Allocator) + + in_bytes(byte_offset_of(Lisp2Allocator, bump_allocator)); tlab_top_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, cursor)); tlab_end_offset = allocator_base_offset + in_bytes(byte_offset_of(BumpAllocator, limit)); } else { diff --git a/openjdk/share/mmtkBarrierSetC2.cpp b/openjdk/share/mmtkBarrierSetC2.cpp index a0541cf5..708d08d4 100644 --- a/openjdk/share/mmtkBarrierSetC2.cpp +++ b/openjdk/share/mmtkBarrierSetC2.cpp @@ -87,7 +87,7 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, // We always use the default allocator. // But we need to figure out which allocator we are using by querying MMTk. AllocatorSelector selector = get_allocator_mapping(AllocatorDefault); - if (selector.tag == TAG_MARK_COMPACT) extra_header = MMTK_MARK_COMPACT_HEADER_RESERVED_IN_BYTES; + if (selector.tag == TAG_LISP2) extra_header = MMTK_LISP2_HEADER_RESERVED_IN_BYTES; // max_non_los_bytes is size_t which can be 32-bit or 64-bit, unsigned, // but const_size (see below) is a jlong which is always 64-bit signed (int64_t). @@ -227,7 +227,7 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, // See note above concerning the control input when using a TLAB Node *old_eden_top; - if (selector.tag == TAG_MARK_COMPACT) { + if (selector.tag == TAG_LISP2) { Node *offset = ConLNode::make(extra_header); x->transform_later(offset); Node *node = new LoadPNode(ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM, MemNode::unordered); @@ -289,7 +289,7 @@ void MMTkBarrierSetC2::expand_allocate(PhaseMacroExpand* x, #ifdef MMTK_ENABLE_VO_BIT enable_vo_bit = true; #endif - if (enable_vo_bit || selector.tag == TAG_MARK_COMPACT) { + if (enable_vo_bit || selector.tag == TAG_LISP2) { // set the alloc bit: // intptr_t addr = (intptr_t) (void*) fast_oop; // uint8_t* meta_addr = (uint8_t*) (vo_bit_base_address() + (addr >> 6)); diff --git a/openjdk/share/mmtkMutator.hpp b/openjdk/share/mmtkMutator.hpp index 4fca4d17..e1a1dae3 100644 --- a/openjdk/share/mmtkMutator.hpp +++ b/openjdk/share/mmtkMutator.hpp @@ -24,7 +24,7 @@ const int MAX_LARGE_OBJECT_ALLOCATORS = 2; const int MAX_MALLOC_ALLOCATORS = 1; const int MAX_IMMIX_ALLOCATORS = 2; const int MAX_FREE_LIST_ALLOCATORS = 2; -const int MAX_MARK_COMPACT_ALLOCATORS = 1; +const int MAX_LISP2_ALLOCATORS = 1; // The following types should have the same layout as the types with the same name in MMTk core (Rust) @@ -85,7 +85,7 @@ struct MallocAllocator { void* context; }; -struct MarkCompactAllocator { +struct Lisp2Allocator { struct BumpAllocator bump_allocator; }; @@ -95,7 +95,7 @@ struct Allocators { MallocAllocator malloc[MAX_MALLOC_ALLOCATORS]; ImmixAllocator immix[MAX_IMMIX_ALLOCATORS]; MMTkFreeListAllocator free_list[MAX_FREE_LIST_ALLOCATORS]; - MarkCompactAllocator markcompact[MAX_MARK_COMPACT_ALLOCATORS]; + Lisp2Allocator lisp2[MAX_LISP2_ALLOCATORS]; }; struct MutatorConfig { From 06a34a63db05be95c750bef86edaa156a7ae6bb8 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 7 Aug 2026 01:25:58 +0000 Subject: [PATCH 491/494] Update mmtk-core. Enable finalization for non-lXR plans --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- openjdk/share/mmtkHeap.cpp | 8 ++++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 2699f11a..6c76c6ee 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -643,7 +643,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.33.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8af79ad5b4cb30820a1be249a1277388a85a4b77#8af79ad5b4cb30820a1be249a1277388a85a4b77" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7bc24da9b565060468c399a6d5cd1a77fbce2a54#7bc24da9b565060468c399a6d5cd1a77fbce2a54" dependencies = [ "atomic", "atomic-traits", @@ -678,7 +678,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.33.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=8af79ad5b4cb30820a1be249a1277388a85a4b77#8af79ad5b4cb30820a1be249a1277388a85a4b77" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7bc24da9b565060468c399a6d5cd1a77fbce2a54#7bc24da9b565060468c399a6d5cd1a77fbce2a54" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 1bba356d..0f742ba5 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "8af79ad5b4cb30820a1be249a1277388a85a4b77" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7bc24da9b565060468c399a6d5cd1a77fbce2a54" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/openjdk/share/mmtkHeap.cpp b/openjdk/share/mmtkHeap.cpp index 5d6aa364..2aae5f6f 100644 --- a/openjdk/share/mmtkHeap.cpp +++ b/openjdk/share/mmtkHeap.cpp @@ -237,7 +237,9 @@ const char* MMTkHeap::version() { } void MMTkHeap::schedule_finalizer() { - // MMTkFinalizerThread::instance->schedule(); + if (!RC_ENABLED) { + MMTkFinalizerThread::instance->schedule(); + } } class MMTkIsScavengable : public BoolObjectClosure { @@ -263,7 +265,9 @@ void MMTkHeap::post_initialize() { void MMTkHeap::enable_collection() { // Initialize finalizer thread before enable_collection(). // Otherwise it is possible that we schedule finalizer (during a GC) before the finalizer thread is ready. - // MMTkFinalizerThread::initialize(); + if (!RC_ENABLED) { + MMTkFinalizerThread::initialize(); + } ::initialize_collection(0); } From 2f439b055916e8a0c69d797b17f9ec79684d3c2e Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 7 Aug 2026 04:16:29 +0000 Subject: [PATCH 492/494] Update mmtk-core --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 6c76c6ee..8e1b0d86 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -643,7 +643,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.33.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7bc24da9b565060468c399a6d5cd1a77fbce2a54#7bc24da9b565060468c399a6d5cd1a77fbce2a54" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7609f5052ed2d20af0d08ea353deb77ec1c2cd75#7609f5052ed2d20af0d08ea353deb77ec1c2cd75" dependencies = [ "atomic", "atomic-traits", @@ -678,7 +678,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.33.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7bc24da9b565060468c399a6d5cd1a77fbce2a54#7bc24da9b565060468c399a6d5cd1a77fbce2a54" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7609f5052ed2d20af0d08ea353deb77ec1c2cd75#7609f5052ed2d20af0d08ea353deb77ec1c2cd75" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 0f742ba5..bb19a81c 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7bc24da9b565060468c399a6d5cd1a77fbce2a54" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7609f5052ed2d20af0d08ea353deb77ec1c2cd75" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 27301ce392eea55ac2dfa1c32e2f16710e6c1be1 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 7 Aug 2026 14:39:39 +0800 Subject: [PATCH 493/494] Update mmtk-core --- mmtk/Cargo.lock | 10 +++++----- mmtk/Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 8e1b0d86..98c08ea2 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -133,13 +133,13 @@ dependencies = [ [[package]] name = "bytemuck_derive" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65693059b6b9c588b9f62fed1cedbf0a8b805631457ea162d68f0de186f3de5" +checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] @@ -643,7 +643,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.33.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7609f5052ed2d20af0d08ea353deb77ec1c2cd75#7609f5052ed2d20af0d08ea353deb77ec1c2cd75" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3d69aa92ee009d23901f8bc433b24ee778c5fd24#3d69aa92ee009d23901f8bc433b24ee778c5fd24" dependencies = [ "atomic", "atomic-traits", @@ -678,7 +678,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.33.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=7609f5052ed2d20af0d08ea353deb77ec1c2cd75#7609f5052ed2d20af0d08ea353deb77ec1c2cd75" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3d69aa92ee009d23901f8bc433b24ee778c5fd24#3d69aa92ee009d23901f8bc433b24ee778c5fd24" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index bb19a81c..5a14416f 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "7609f5052ed2d20af0d08ea353deb77ec1c2cd75" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "3d69aa92ee009d23901f8bc433b24ee778c5fd24" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } From 55e9fa007fedb90cb67756aa0315e72a63592160 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 7 Aug 2026 16:27:23 +0800 Subject: [PATCH 494/494] Use experimenal root-kind API We keep the existing API unchanged, while letting the OpenJDK binding use the experimental API for scanning weak and young code cache roots. --- mmtk/Cargo.lock | 4 ++-- mmtk/Cargo.toml | 2 +- mmtk/src/gc_work.rs | 6 +++--- mmtk/src/scanning.rs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 98c08ea2..60658ef4 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -643,7 +643,7 @@ dependencies = [ [[package]] name = "mmtk" version = "0.33.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3d69aa92ee009d23901f8bc433b24ee778c5fd24#3d69aa92ee009d23901f8bc433b24ee778c5fd24" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=0a8ce6ea341527b76cce1b87ede731b4dcb347a4#0a8ce6ea341527b76cce1b87ede731b4dcb347a4" dependencies = [ "atomic", "atomic-traits", @@ -678,7 +678,7 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.33.0" -source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=3d69aa92ee009d23901f8bc433b24ee778c5fd24#3d69aa92ee009d23901f8bc433b24ee778c5fd24" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=0a8ce6ea341527b76cce1b87ede731b4dcb347a4#0a8ce6ea341527b76cce1b87ede731b4dcb347a4" dependencies = [ "proc-macro-error", "proc-macro2", diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 5a14416f..2494ee5d 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -39,7 +39,7 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "3d69aa92ee009d23901f8bc433b24ee778c5fd24" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "0a8ce6ea341527b76cce1b87ede731b4dcb347a4" } # Uncomment the following to build locally # mmtk = { path = "../../mmtk-core" } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index 801ce4fd..0ff026ca 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -88,7 +88,7 @@ impl>> for root in roots { slots.push(OpenJDKSlot::::from(*root)); if slots.len() >= scanning::WORK_PACKET_CAPACITY { - self.factory.create_process_roots_work( + self.factory.create_process_roots_work_with_root_kind( std::mem::take(&mut slots), RootKind::YoungCodeCacheRoots, ); @@ -141,7 +141,7 @@ impl>> if !slots.is_empty() { self.factory - .create_process_roots_work(slots, RootKind::Strong); + .create_process_roots_work_with_root_kind(slots, RootKind::Strong); } // Use the following code to scan CodeCache directly, instead of scanning the "remembered set". // unsafe { @@ -192,7 +192,7 @@ extern "C" fn report_slots_and_renew_buffer_weak let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; let kind = RootKind::Weak; - factory.create_process_roots_work(buf, kind); + factory.create_process_roots_work_with_root_kind(buf, kind); } let (ptr, _, capacity) = { // TODO: Use Vec::into_raw_parts() when the method is available. diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 338359a6..30d2ede6 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -25,7 +25,7 @@ extern "C" fn report_slots_and_renew_buffer>( // should fix the Rust-to-C interface. let buf = unsafe { Vec::::from_raw_parts(ptr as _, length, capacity) }; let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; - factory.create_process_roots_work(buf, RootKind::Strong); + factory.create_process_roots_work_with_root_kind(buf, RootKind::Strong); } let (ptr, _, capacity) = { // TODO: Use Vec::into_raw_parts() when the method is available.