diff --git a/.github/configs/base.yml b/.github/configs/base.yml index 6cd11fa2..57ea1e81 100644 --- a/.github/configs/base.yml +++ b/.github/configs/base.yml @@ -80,6 +80,10 @@ modifiers: no_compressed_oops: type: JVMArg val: "-XX:-UseCompressedOops -XX:-UseCompressedClassPointers" + no_reference_types: + type: "EnvVar" + var: "MMTK_NO_REFERENCE_TYPES" + val: "true" plugins: keep_stdout_stderr: diff --git a/.github/configs/normal-heap.yml b/.github/configs/normal-heap.yml index 7a43ab13..60a81eae 100644 --- a/.github/configs/normal-heap.yml +++ b/.github/configs/normal-heap.yml @@ -11,6 +11,7 @@ configs: - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-Lisp2" - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-OVC" - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|mmtk_gc-ConcurrentImmix" + - "jdk21-master|dacapochopin_jdk21|ms|s|fail_on_oom|tph|preserve|no_reference_types|mmtk_gc-LXR" # This will be expanded in CI when we run with the config. Keep a new line at the end. benchmarks: diff --git a/.github/scripts/ci-expected-results.yml b/.github/scripts/ci-expected-results.yml index 1cb2364f..5aa13416 100644 --- a/.github/scripts/ci-expected-results.yml +++ b/.github/scripts/ci-expected-results.yml @@ -11,6 +11,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass batik: SemiSpace: pass GenCopy: pass @@ -21,6 +22,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass biojava: SemiSpace: pass GenCopy: pass @@ -31,6 +33,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass cassandra: SemiSpace: pass GenCopy: pass @@ -41,6 +44,7 @@ results: Lisp2: ignore OVC: ignore ConcurrentImmix: ignore + LXR: pass eclipse: SemiSpace: pass GenCopy: pass @@ -51,6 +55,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass fop: SemiSpace: pass GenCopy: pass @@ -61,6 +66,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass graphchi: SemiSpace: pass GenCopy: pass @@ -71,6 +77,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass h2: SemiSpace: pass GenCopy: pass @@ -81,6 +88,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass h2o: SemiSpace: pass GenCopy: pass @@ -91,6 +99,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass jme: SemiSpace: pass GenCopy: pass @@ -101,6 +110,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass jython: SemiSpace: pass GenCopy: pass @@ -111,6 +121,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass kafka: SemiSpace: pass GenCopy: pass @@ -121,6 +132,7 @@ results: Lisp2: ignore OVC: pass ConcurrentImmix: pass + LXR: pass luindex: SemiSpace: pass GenCopy: pass @@ -131,6 +143,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass lusearch: SemiSpace: pass GenCopy: pass @@ -141,6 +154,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass pmd: SemiSpace: pass GenCopy: pass @@ -151,6 +165,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass sunflow: SemiSpace: pass GenCopy: pass @@ -161,6 +176,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass tomcat: SemiSpace: pass GenCopy: pass @@ -171,6 +187,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass xalan: SemiSpace: pass GenCopy: pass @@ -181,6 +198,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass zxing: SemiSpace: pass GenCopy: pass @@ -191,6 +209,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass release: avrora: @@ -203,6 +222,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass batik: SemiSpace: pass GenCopy: pass @@ -213,6 +233,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass biojava: SemiSpace: pass GenCopy: pass @@ -223,6 +244,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass cassandra: SemiSpace: pass GenCopy: pass @@ -233,6 +255,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass eclipse: SemiSpace: pass GenCopy: pass @@ -243,6 +266,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass fop: SemiSpace: pass GenCopy: pass @@ -253,6 +277,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass graphchi: SemiSpace: pass GenCopy: pass @@ -263,6 +288,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass h2: SemiSpace: pass GenCopy: pass @@ -273,6 +299,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass h2o: SemiSpace: pass GenCopy: pass @@ -283,6 +310,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass jme: SemiSpace: pass GenCopy: pass @@ -293,6 +321,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass jython: SemiSpace: pass GenCopy: pass @@ -303,6 +332,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass kafka: SemiSpace: pass GenCopy: pass @@ -313,6 +343,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass luindex: SemiSpace: pass GenCopy: pass @@ -323,6 +354,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass lusearch: SemiSpace: pass GenCopy: pass @@ -333,6 +365,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass pmd: SemiSpace: pass GenCopy: pass @@ -343,6 +376,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass sunflow: SemiSpace: pass GenCopy: pass @@ -353,6 +387,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass tomcat: SemiSpace: pass GenCopy: pass @@ -363,6 +398,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass xalan: SemiSpace: ignore GenCopy: ignore @@ -373,6 +409,7 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass zxing: SemiSpace: pass GenCopy: pass @@ -383,3 +420,4 @@ results: Lisp2: pass OVC: pass ConcurrentImmix: pass + LXR: pass diff --git a/.github/scripts/ci-test-forwarding-on-side.sh b/.github/scripts/ci-test-forwarding-on-side.sh index a7ce4085..659b7f2e 100755 --- a/.github/scripts/ci-test-forwarding-on-side.sh +++ b/.github/scripts/ci-test-forwarding-on-side.sh @@ -19,3 +19,4 @@ MMTK_PLAN=Immix run_subset 4 MMTK_PLAN=GenCopy run_subset 4 MMTK_PLAN=GenImmix run_subset 4 MMTK_PLAN=StickyImmix run_subset 4 +MMTK_PLAN=LXR MMTK_NO_REFERENCE_TYPES=true run_subset 4 diff --git a/README-legacy.md b/README-legacy.md old mode 100755 new mode 100644 diff --git a/mmtk/Cargo.lock b/mmtk/Cargo.lock index 942c3d9b..60658ef4 100644 --- a/mmtk/Cargo.lock +++ b/mmtk/Cargo.lock @@ -4,18 +4,18 @@ version = 4 [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -34,9 +34,9 @@ checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -82,21 +82,27 @@ dependencies = [ [[package]] name = "atomic_refcell" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41e67cd8309bbd06cd603a9e693a784ac2e5d1e955f11286e355089fcab3047c" +checksum = "21e4227379beff4205943696e6c3e0cd809bacdf3f0edd6e3dd153e2269571a4" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "built" @@ -104,43 +110,43 @@ version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" dependencies = [ - "git2", + "git2 0.20.4", ] [[package]] name = "built" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" dependencies = [ - "git2", + "git2 0.21.0", ] [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.10.2" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.3", ] [[package]] name = "cc" -version = "1.2.57" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -175,18 +181,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.15" +version = "0.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -194,27 +200,58 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] [[package]] name = "delegate" @@ -224,18 +261,18 @@ checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.3", ] [[package]] @@ -246,9 +283,9 @@ checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" [[package]] name = "either" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "enum-map" @@ -267,14 +304,14 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "env_filter" -version = "1.0.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" dependencies = [ "log", "regex", @@ -282,9 +319,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" dependencies = [ "anstream", "anstyle", @@ -310,14 +347,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi", - "wasip2", ] [[package]] @@ -326,13 +362,25 @@ version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" dependencies = [ - "bitflags", + "bitflags 2.13.1", "libc", "libgit2-sys", "log", "url", ] +[[package]] +name = "git2" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddddbf932745a6be37109b6112d3ee09696106f848449069d3a57bba937ab82e" +dependencies = [ + "bitflags 2.13.1", + "libc", + "libgit2-sys", + "log", +] + [[package]] name = "heck" version = "0.5.0" @@ -347,12 +395,13 @@ checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -360,9 +409,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -373,9 +422,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -387,15 +436,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -407,15 +456,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -439,9 +488,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -475,10 +524,12 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" dependencies = [ + "defmt", + "jiff-core", "jiff-static", "log", "portable-atomic", @@ -486,22 +537,32 @@ dependencies = [ "serde_core", ] +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" dependencies = [ + "jiff-core", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ "getrandom", "libc", @@ -515,15 +576,15 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.183" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libgit2-sys" -version = "0.18.3+1.9.2" +version = "0.18.7+1.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +checksum = "23c7391e4b9f4ffab1a624223cc1d7385ff9a678f490768add717de7ea2f4d89" dependencies = [ "cc", "libc", @@ -533,9 +594,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.25" +version = "1.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52f4c29e2a68ac30c9087e1b772dc9f44a2b66ed44edf2266cf2be9b03dafc1" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" dependencies = [ "cc", "libc", @@ -545,9 +606,9 @@ dependencies = [ [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -560,15 +621,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memoffset" @@ -582,12 +643,12 @@ dependencies = [ [[package]] name = "mmtk" version = "0.33.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=7160a03ecd89affa14d6bf83b613ee79ed8d575f#7160a03ecd89affa14d6bf83b613ee79ed8d575f" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=0a8ce6ea341527b76cce1b87ede731b4dcb347a4#0a8ce6ea341527b76cce1b87ede731b4dcb347a4" dependencies = [ "atomic", "atomic-traits", "atomic_refcell", - "built 0.8.0", + "built 0.8.1", "bytemuck", "cfg-if", "crossbeam", @@ -617,12 +678,12 @@ dependencies = [ [[package]] name = "mmtk-macros" version = "0.33.0" -source = "git+https://github.com/mmtk/mmtk-core.git?rev=7160a03ecd89affa14d6bf83b613ee79ed8d575f#7160a03ecd89affa14d6bf83b613ee79ed8d575f" +source = "git+https://github.com/wenyuzhao/mmtk-core.git?rev=0a8ce6ea341527b76cce1b87ede731b4dcb347a4#0a8ce6ea341527b76cce1b87ede731b4dcb347a4" dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -675,7 +736,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.13.1", ] [[package]] @@ -708,30 +769,30 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "portable-atomic-util" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -768,33 +829,33 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] [[package]] name = "r-efi" -version = "5.3.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "regex" -version = "1.12.3" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -804,9 +865,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -815,9 +876,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "rustc_version" @@ -830,9 +891,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "scopeguard" @@ -842,56 +903,56 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", ] [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.3", ] [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "spin" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" dependencies = [ "lock_api", ] @@ -923,7 +984,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -938,9 +999,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -955,7 +1027,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -972,11 +1044,31 @@ dependencies = [ "windows", ] +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -1024,15 +1116,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "wasip2" -version = "1.0.2+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" -dependencies = [ - "wit-bindgen", -] - [[package]] name = "winapi" version = "0.3.9" @@ -1109,7 +1192,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -1120,7 +1203,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] [[package]] @@ -1181,23 +1264,17 @@ dependencies = [ "windows-link 0.1.3", ] -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" - [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -1206,42 +1283,42 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -1250,9 +1327,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -1261,11 +1338,11 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.119", ] diff --git a/mmtk/Cargo.toml b/mmtk/Cargo.toml index 7f69be84..2494ee5d 100644 --- a/mmtk/Cargo.toml +++ b/mmtk/Cargo.toml @@ -11,6 +11,9 @@ name = "mmtk_openjdk" # be careful - LTO is only allowed for certain crate types crate-type = ["cdylib"] +[profile.dev] +panic = "abort" + [profile.release] lto = true @@ -36,9 +39,9 @@ probe = "0.5" # - change branch # - change repo name # But other changes including adding/removing whitespaces in commented lines may break the CI. -mmtk = { git = "https://github.com/mmtk/mmtk-core.git", rev = "7160a03ecd89affa14d6bf83b613ee79ed8d575f" } +mmtk = { git = "https://github.com/wenyuzhao/mmtk-core.git", rev = "0a8ce6ea341527b76cce1b87ede731b4dcb347a4" } # Uncomment the following to build locally -# mmtk = { path = "../repos/mmtk-core" } +# mmtk = { path = "../../mmtk-core" } [build-dependencies] built = { version = "0.7.7", features = ["git2"] } diff --git a/mmtk/src/abi.rs b/mmtk/src/abi.rs index 8425619a..569dd132 100644 --- a/mmtk/src/abi.rs +++ b/mmtk/src/abi.rs @@ -114,6 +114,10 @@ impl Klass { const fn layout_helper_header_size(lh: i32) -> i32 { (lh >> Self::LH_HEADER_SIZE_SHIFT) & Self::LH_HEADER_SIZE_MASK } + + pub const fn is_instance_klass(&self) -> bool { + self.layout_helper > Self::LH_NEUTRAL_VALUE + } } #[repr(C)] diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs index b47dd91b..6737f47b 100644 --- a/mmtk/src/api.rs +++ b/mmtk/src/api.rs @@ -47,6 +47,8 @@ macro_rules! with_mutator { static NO_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("NoBarrier").unwrap()); static OBJECT_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("ObjectBarrier").unwrap()); +static FIELD_LOGGING_BARRIER: sync::Lazy = + sync::Lazy::new(|| CString::new("FieldBarrier").unwrap()); static SATB_BARRIER: sync::Lazy = sync::Lazy::new(|| CString::new("SATBBarrier").unwrap()); #[no_mangle] @@ -60,6 +62,7 @@ pub extern "C" fn mmtk_active_barrier() -> *const c_char { match singleton.get_plan().constraints().barrier { BarrierSelector::NoBarrier => NO_BARRIER.as_ptr(), BarrierSelector::ObjectBarrier => OBJECT_BARRIER.as_ptr(), + BarrierSelector::FieldBarrier => FIELD_LOGGING_BARRIER.as_ptr(), BarrierSelector::SATBBarrier => SATB_BARRIER.as_ptr(), // In case we have more barriers in mmtk-core. #[allow(unreachable_patterns)] @@ -68,6 +71,11 @@ pub extern "C" fn mmtk_active_barrier() -> *const c_char { }) } +#[no_mangle] +pub extern "C" fn mmtk_report_gc_start() { + with_singleton!(|singleton| mmtk::memory_manager::report_gc_start(singleton)); +} + /// # Safety /// Caller needs to make sure the ptr is a valid vector pointer. #[no_mangle] @@ -159,6 +167,17 @@ pub extern "C" fn flush_mutator(mutator: *mut libc::c_void) { with_mutator!(|mutator| memory_manager::flush_mutator(mutator)) } +#[no_mangle] +pub unsafe extern "C" fn release_mutator(mutator: *mut libc::c_void) { + if crate::use_compressed_oops() { + let _boxed: Box>> = + Box::from_raw(mutator as *mut Mutator>); + } else { + let _boxed: Box>> = + Box::from_raw(mutator as *mut Mutator>); + } +} + #[no_mangle] // We trust the mutator pointer is valid. #[allow(clippy::not_unsafe_ptr_arg_deref)] @@ -238,9 +257,9 @@ pub extern "C" fn total_bytes() -> usize { } #[no_mangle] -pub extern "C" fn handle_user_collection_request(tls: VMMutatorThread) { +pub extern "C" fn handle_user_collection_request(tls: VMMutatorThread, force: bool) { with_singleton!(|singleton| { - memory_manager::handle_user_collection_request(singleton, tls); + memory_manager::handle_user_collection_request(singleton, tls, force); }) } @@ -335,6 +354,13 @@ pub extern "C" fn mmtk_builder_set_threads(value: usize) { builder.options.threads.set(value); } +/// Pass hotspot `ConcGCThreads` flag to mmtk +#[no_mangle] +pub extern "C" fn mmtk_builder_set_conc_threads(value: usize) { + let mut builder = BUILDER.lock().unwrap(); + builder.options.concurrent_threads.set(value); +} + /// Pass hotspot `UseTransparentHugePages` flag to mmtk #[no_mangle] pub extern "C" fn mmtk_builder_set_transparent_hugepages(value: bool) { @@ -490,6 +516,37 @@ pub extern "C" fn get_finalized_object() -> NullableObjectReference { with_singleton!(|singleton| memory_manager::get_finalized_object(singleton).into()) } +/// Test if an object is live at the end of a GC. +/// Note: only call this method after the liveness tracing and before gc release. +#[no_mangle] +pub extern "C" fn mmtk_is_live(object: NullableObjectReference) -> usize { + let o: Option = object.into(); + let Some(object) = o else { + return 0; + }; + debug_assert!( + object.to_raw_address().is_mapped(), + "{:?} is not mapped", + object + ); + object.is_live() as _ +} + +/// If the object is non-null and forwarded, return the forwarded pointer. Otherwise, return the original pointer. +#[no_mangle] +pub extern "C" fn mmtk_get_forwarded_ref( + object: NullableObjectReference, +) -> NullableObjectReference { + let o: Option = object.into(); + let Some(o) = o else { + return None.into(); + }; + match o.get_forwarded_object() { + Some(o) => Some(o).into(), + None => object, + } +} + thread_local! { /// Cache reference slots of an nmethod while the current thread is executing /// `MMTkRegisterNMethodOopClosure`. diff --git a/mmtk/src/collection.rs b/mmtk/src/collection.rs index 4e71d610..a7f4fbfe 100644 --- a/mmtk/src/collection.rs +++ b/mmtk/src/collection.rs @@ -69,4 +69,16 @@ impl Collection> for VMCollection { ((*UPCALLS).schedule_finalizer)(); } } + + fn update_weak_processor(lxr: bool) { + unsafe { + ((*UPCALLS).update_weak_processor)(lxr); + } + } + + fn vm_release() { + unsafe { + ((*UPCALLS).gc_epilogue)(); + } + } } diff --git a/mmtk/src/gc_work.rs b/mmtk/src/gc_work.rs index ed196cd1..0ff026ca 100644 --- a/mmtk/src/gc_work.rs +++ b/mmtk/src/gc_work.rs @@ -1,13 +1,18 @@ use crate::scanning; use crate::scanning::to_slots_closure; +use crate::NewBuffer; use crate::OpenJDK; use crate::OpenJDKSlot; +use crate::Slot; +use crate::SlotsClosure; use crate::UPCALLS; +use mmtk::plan::concurrent::Pause; use mmtk::scheduler::*; use mmtk::util::Address; use mmtk::vm::RootsWorkFactory; use mmtk::vm::*; use mmtk::MMTK; +use std::marker::PhantomData; macro_rules! scan_roots_work { ($struct_name: ident, $func_name: ident) => { @@ -40,9 +45,9 @@ scan_roots_work!( scan_class_loader_data_graph_roots ); scan_roots_work!(ScanOopStorageSetRoots, scan_oop_storage_set_roots); -scan_roots_work!(ScanWeakProcessorRoots, scan_weak_processor_roots); scan_roots_work!(ScanVMThreadRoots, scan_vm_thread_roots); +#[allow(unused)] pub struct ScanCodeCacheRoots>> { factory: F, @@ -68,6 +73,11 @@ impl>> .get_plan() .generational() .is_some_and(|gen| gen.is_current_gc_nursery()); + let lxr = mmtk + .get_plan() + .downcast_ref::>>(); + let is_lxr = lxr.is_some(); + let is_rc_pause = lxr.is_some_and(|lxr| lxr.current_pause() == Some(Pause::RefCount)); let mut slots = Vec::with_capacity(scanning::WORK_PACKET_CAPACITY); @@ -78,8 +88,10 @@ impl>> for root in roots { slots.push(OpenJDKSlot::::from(*root)); if slots.len() >= scanning::WORK_PACKET_CAPACITY { - self.factory - .create_process_roots_work(std::mem::take(&mut slots)); + self.factory.create_process_roots_work_with_root_kind( + std::mem::take(&mut slots), + RootKind::YoungCodeCacheRoots, + ); } } }; @@ -95,7 +107,7 @@ impl>> let mut mature = crate::MATURE_CODE_CACHE_ROOTS.lock().unwrap(); // Only scan mature roots in full-heap collections. - if !is_current_gc_nursery { + if !is_current_gc_nursery && !(is_lxr && is_rc_pause) { for (key, roots) in mature.iter() { mature_slots += roots.len(); add_roots(roots); @@ -128,14 +140,15 @@ impl>> ); if !slots.is_empty() { - self.factory.create_process_roots_work(slots); + self.factory + .create_process_roots_work_with_root_kind(slots, RootKind::Strong); } // Use the following code to scan CodeCache directly, instead of scanning the "remembered set". // unsafe { // ((*UPCALLS).scan_code_cache_roots)(to_slots_closure(&mut self.factory)); // } - if moves_object { + if moves_object && !nmethods_to_fix.is_empty() { // Note: If the current GC doesn't move objects at all, we don't need to fix relocation. // FIXME: Even during copying GC, some GC algorithms (such as Immix) don't move every // single object. We only need to call `fix_oop_relocations` on nmethods that actually @@ -161,13 +174,70 @@ impl>> // For scavenging GCs, the mmtk-openjdk binding reports the *slots* of nmethods as // roots. They will be traced at unspecified times during the Closure stage. // SoftRefClosure is the first safe place to call fix_oop_relocations. - WorkBucketStage::SoftRefClosure + WorkBucketStage::Release }; worker.scheduler().work_buckets[stage].bulk_add(packets); } } } +extern "C" fn report_slots_and_renew_buffer_weak>( + ptr: *mut Address, + length: usize, + capacity: usize, + factory_ptr: *mut libc::c_void, +) -> NewBuffer { + if !ptr.is_null() { + let ptr = ptr as *mut S; + let buf = unsafe { Vec::::from_raw_parts(ptr, length, capacity) }; + let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; + let kind = RootKind::Weak; + factory.create_process_roots_work_with_root_kind(buf, kind); + } + let (ptr, _, capacity) = { + // TODO: Use Vec::into_raw_parts() when the method is available. + use std::mem::ManuallyDrop; + let new_vec = Vec::with_capacity(scanning::WORK_PACKET_CAPACITY); + let mut me = ManuallyDrop::new(new_vec); + (me.as_mut_ptr(), me.len(), me.capacity()) + }; + NewBuffer { ptr, capacity } +} + +fn to_slots_closure_weak>(factory: &mut F) -> SlotsClosure { + SlotsClosure { + func: report_slots_and_renew_buffer_weak::, + data: factory as *mut F as *mut libc::c_void, + } +} + +pub struct ScanWeakProcessorRoots> { + factory: F, + _p: PhantomData, +} + +impl> ScanWeakProcessorRoots { + #[allow(unused)] + pub fn new(factory: F) -> Self { + Self { + factory, + _p: PhantomData, + } + } +} + +impl> GCWork + for ScanWeakProcessorRoots +{ + fn do_work(&mut self, _worker: &mut GCWorker, _mmtk: &'static MMTK) { + unsafe { + ((*UPCALLS).scan_weak_processor_roots)(to_slots_closure_weak::( + &mut self.factory, + )); + } + } +} + struct FixRelocations { nmethods: Vec
, } @@ -182,13 +252,28 @@ impl GCWork> for FixRelocations { fn do_work( &mut self, _worker: &mut GCWorker>, - _mmtk: &'static MMTK>, + mmtk: &'static MMTK>, ) { + let is_lxr = mmtk + .get_plan() + .downcast_ref::>>() + .is_some(); + let fast_mode = is_lxr + && mmtk + .get_plan() + .downcast_ref::>>() + .is_some_and(|lxr| { + lxr.current_pause() == Some(Pause::RefCount) + || lxr.current_pause() == Some(Pause::InitialMark) + }); + let num_nmethods = self.nmethods.len(); - for nmethod in self.nmethods.iter().copied() { - unsafe { - ((*UPCALLS).fix_oop_relocations)(nmethod.to_mut_ptr()); - } + unsafe { + ((*UPCALLS).fix_oop_relocations)( + fast_mode, + self.nmethods.as_mut_ptr() as *mut _, + num_nmethods, + ); } probe!(mmtk_openjdk, fix_relocations, num_nmethods); } diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs index f47e3a37..a82433ef 100644 --- a/mmtk/src/lib.rs +++ b/mmtk/src/lib.rs @@ -1,5 +1,7 @@ #[macro_use] extern crate lazy_static; +extern crate atomic; +extern crate once_cell; #[macro_use] extern crate probe; @@ -8,6 +10,7 @@ use std::ptr::null_mut; use std::sync::Mutex; use libc::{c_char, c_void, uintptr_t}; +use mmtk::plan::lxr::LXR; use mmtk::util::alloc::AllocationError; use mmtk::util::constants::LOG_BYTES_IN_GBYTE; use mmtk::util::heap::vm_layout::VMLayout; @@ -111,8 +114,10 @@ pub struct OpenJDK_Upcalls { pub number_of_mutators: extern "C" fn() -> usize, pub schedule_finalizer: extern "C" fn(), pub prepare_for_roots_re_scanning: extern "C" fn(), + pub update_weak_processor: extern "C" fn(lxr: bool), pub enqueue_references: extern "C" fn(objects: *const ObjectReference, len: usize), - pub fix_oop_relocations: extern "C" fn(nmethod: *mut libc::c_void), + pub fix_oop_relocations: extern "C" fn(lxr: bool, nmethods: *mut libc::c_void, len: usize), + pub gc_epilogue: extern "C" fn(), } pub static mut UPCALLS: *const OpenJDK_Upcalls = null_mut(); @@ -127,6 +132,11 @@ pub extern "C" fn get_global_side_metadata_vm_base_address() -> uintptr_t { mmtk::util::metadata::side_metadata::global_side_metadata_vm_base_address().as_usize() } +#[no_mangle] +pub extern "C" fn get_rc_table_base_address() -> uintptr_t { + mmtk::util::rc::RC_TABLE.get_starting_address().as_usize() +} + #[no_mangle] pub extern "C" fn get_vo_bit_address() -> uintptr_t { mmtk::util::metadata::side_metadata::vo_bit_side_metadata_addr().as_usize() @@ -136,9 +146,16 @@ pub extern "C" fn get_vo_bit_address() -> uintptr_t { pub static FREE_LIST_ALLOCATOR_SIZE: uintptr_t = std::mem::size_of::>>(); +#[no_mangle] +pub static IMMIX_ALLOCATOR_SIZE: uintptr_t = + std::mem::size_of::>>(); + #[no_mangle] pub static mut CONCURRENT_MARKING_ACTIVE: u8 = 0; +#[no_mangle] +pub static mut RC_ENABLED: u8 = 0; + #[derive(Default)] pub struct OpenJDK; @@ -172,6 +189,12 @@ lazy_static! { let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); slots::initialize_compressed_oops_base_and_shift(); + unsafe { + RC_ENABLED = ret + .get_plan() + .downcast_ref::>>() + .is_some() as _; + } *ret }; pub static ref SINGLETON_UNCOMPRESSED: MMTK> = { @@ -180,6 +203,12 @@ lazy_static! { assert!(!MMTK_INITIALIZED.load(Ordering::Relaxed)); let ret = mmtk::memory_manager::mmtk_init(&builder); MMTK_INITIALIZED.store(true, std::sync::atomic::Ordering::SeqCst); + unsafe { + RC_ENABLED = ret + .get_plan() + .downcast_ref::>>() + .is_some() as _; + } *ret }; } diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs index 945b20b2..38c59086 100644 --- a/mmtk/src/object_model.rs +++ b/mmtk/src/object_model.rs @@ -10,6 +10,11 @@ pub struct VMObjectModel {} impl ObjectModel> for VMObjectModel { const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = vm_metadata::LOGGING_SIDE_METADATA_SPEC; + const GLOBAL_FIELD_UNLOG_BIT_SPEC: VMGlobalFieldUnlogBitSpec = if COMPRESSED { + vm_metadata::FIELD_LOGGING_SIDE_METADATA_SPEC_COMPRESSED + } else { + vm_metadata::FIELD_LOGGING_SIDE_METADATA_SPEC + }; const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec = vm_metadata::FORWARDING_POINTER_METADATA_SPEC; @@ -21,6 +26,8 @@ impl ObjectModel> for VMObjectModel< const UNIFIED_OBJECT_REFERENCE_ADDRESS: bool = true; const OBJECT_REF_OFFSET_LOWER_BOUND: isize = 0; + const COMPRESSED_PTR_ENABLED: bool = COMPRESSED; + fn copy( from: ObjectReference, copy: CopySemantics, @@ -38,6 +45,24 @@ impl ObjectModel> for VMObjectModel< to_obj } + fn try_copy( + from: ObjectReference, + copy: CopySemantics, + copy_context: &mut GCWorkerCopyContext>, + ) -> Option { + let bytes = unsafe { Oop::from(from).size::() }; + let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::(), 0, copy); + if dst.is_zero() { + return None; + } + // Copy + let src = from.to_raw_address(); + unsafe { std::ptr::copy_nonoverlapping::(src.to_ptr(), dst.to_mut_ptr(), bytes) } + let to_obj = unsafe { ObjectReference::from_raw_address_unchecked(dst) }; + copy_context.post_copy(to_obj, bytes, copy); + Some(to_obj) + } + fn copy_to(from: ObjectReference, to: ObjectReference, region: Address) -> Address { let need_copy = from != to; let bytes = unsafe { Oop::from(from).size::() }; @@ -50,7 +75,7 @@ impl ObjectModel> for VMObjectModel< unsafe { (dst + i).store((src + i).load::()) }; } } - let start = Self::ref_to_object_start(to); + let start = >>::ref_to_object_start(to); if region != Address::ZERO { fill_alignment_gap::>(region, start); } @@ -67,7 +92,7 @@ impl ObjectModel> for VMObjectModel< } fn get_size_when_copied(object: ObjectReference) -> usize { - Self::get_current_size(object) + >>::get_current_size(object) } fn get_align_when_copied(_object: ObjectReference) -> usize { diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs index 33011a55..30d2ede6 100644 --- a/mmtk/src/scanning.rs +++ b/mmtk/src/scanning.rs @@ -3,7 +3,7 @@ use crate::Slot; use crate::{NewBuffer, OpenJDKSlot, UPCALLS}; use crate::{OpenJDK, SlotsClosure}; use mmtk::memory_manager; -use mmtk::scheduler::WorkBucketStage; +use mmtk::scheduler::RootKind; use mmtk::util::opaque_pointer::*; use mmtk::util::{Address, ObjectReference}; use mmtk::vm::{RootsWorkFactory, Scanning, SlotVisitor}; @@ -25,7 +25,7 @@ extern "C" fn report_slots_and_renew_buffer>( // should fix the Rust-to-C interface. let buf = unsafe { Vec::::from_raw_parts(ptr as _, length, capacity) }; let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) }; - factory.create_process_roots_work(buf); + factory.create_process_roots_work_with_root_kind(buf, RootKind::Strong); } let (ptr, _, capacity) = { // TODO: Use Vec::into_raw_parts() when the method is available. @@ -45,10 +45,10 @@ pub(crate) fn to_slots_closure>(factory: &mut F) } impl Scanning> for VMScanning { - fn scan_object>>( + fn scan_object( tls: VMWorkerThread, object: ObjectReference, - slot_visitor: &mut SV, + slot_visitor: &mut impl SlotVisitor>, ) { crate::object_scanning::scan_object::(object, slot_visitor, tls); } @@ -73,17 +73,20 @@ impl Scanning> for VMScanning { _tls: VMWorkerThread, factory: impl RootsWorkFactory>, ) { - memory_manager::add_work_packets( - crate::singleton::(), - WorkBucketStage::Prepare, - vec![ - Box::new(ScanCodeCacheRoots::new(factory.clone())) as _, - Box::new(ScanClassLoaderDataGraphRoots::new(factory.clone())) as _, - Box::new(ScanOopStorageSetRoots::new(factory.clone())) as _, // FIXME17: Several removed roots are all put to this work packet, may cause slowdown. - Box::new(ScanWeakProcessorRoots::new(factory.clone())) as _, - Box::new(ScanVMThreadRoots::new(factory)) as _, - ], - ); + let mut w: Vec>>> = vec![ + Box::new(ScanCodeCacheRoots::new(factory.clone())), + Box::new(ScanClassLoaderDataGraphRoots::new(factory.clone())), + Box::new(ScanVMThreadRoots::new(factory.clone())), + ]; + for _ in 0..*crate::singleton::().get_options().threads { + w.push(Box::new(ScanOopStorageSetRoots::new(factory.clone()))); + w.push(Box::new(ScanWeakProcessorRoots::new(factory.clone()))); + } + let mmtk = crate::singleton::(); + let stage = crate::singleton::() + .get_plan() + .root_scanning_stage(); + memory_manager::add_work_packets(mmtk, stage, w); } fn supports_return_barrier() -> bool { diff --git a/mmtk/src/slots.rs b/mmtk/src/slots.rs index 2cd216c6..8de38bdc 100644 --- a/mmtk/src/slots.rs +++ b/mmtk/src/slots.rs @@ -205,6 +205,10 @@ impl Slot for OpenJDKSlot { unsafe { self.addr.store(object) } } } + + fn to_address(&self) -> Address { + self.untagged_address() + } } /// A range of OpenJDKSlot, usually used for arrays. diff --git a/mmtk/src/vm_metadata/constants.rs b/mmtk/src/vm_metadata/constants.rs index e9e5a6ac..630278f4 100644 --- a/mmtk/src/vm_metadata/constants.rs +++ b/mmtk/src/vm_metadata/constants.rs @@ -15,6 +15,12 @@ pub(crate) const FORWARDING_POINTER_OFFSET: isize = 0; /// 1 bit per object pub(crate) const LOGGING_SIDE_METADATA_SPEC: VMGlobalLogBitSpec = VMGlobalLogBitSpec::side_first(); +pub(crate) const FIELD_LOGGING_SIDE_METADATA_SPEC: VMGlobalFieldUnlogBitSpec = + VMGlobalFieldUnlogBitSpec::side_first(); + +pub(crate) const FIELD_LOGGING_SIDE_METADATA_SPEC_COMPRESSED: VMGlobalFieldUnlogBitSpec = + VMGlobalFieldUnlogBitSpec::side_first_compressed(); + // Global MetadataSpecs - End // PolicySpecific MetadataSpecs - Start diff --git a/openjdk/CompileThirdPartyHeap.gmk b/openjdk/CompileThirdPartyHeap.gmk index c44ba301..0c331b97 100644 --- a/openjdk/CompileThirdPartyHeap.gmk +++ b/openjdk/CompileThirdPartyHeap.gmk @@ -15,8 +15,16 @@ OPENJDK_LOCAL_VERSION=`git rev-parse HEAD` # https://doc.rust-lang.org/rustc/profile-guided-optimization.html#a-complete-cargo-workflow HOST_TRIPLE=`rustc -vV | grep host | cut -d' ' -f2` +ifdef GC_FEATURES + GC_FEATURES:=--features $(strip $(GC_FEATURES)) +endif + ifdef MMTK_PLAN - GC_FEATURES=--features $(MMTK_PLAN) + ifndef GC_FEATURES + GC_FEATURES=--features $(MMTK_PLAN) + else + GC_FEATURES:=$(strip $(GC_FEATURES))",$(MMTK_PLAN)" + endif endif ifeq ($(MMTK_VO_BIT), 1) @@ -95,13 +103,14 @@ ifeq ($(COMPILE_TYPE), cross) CARGO_TARGET = riscv64gc-unknown-linux-gnu else ifeq ($(OPENJDK_TARGET_CPU), aarch64) CARGO_TARGET = aarch64-unknown-linux-gnu - else - CARGO_TARGET = UNSUPPORTED_MMTK_OPENJDK_TARGET_CPU + else + CARGO_TARGET = UNSUPPORTED_MMTK_OPENJDK_TARGET_CPU endif CARGO_EXECUTABLE = cross CARGO_TARGET_FLAG = --target $(CARGO_TARGET) endif endif + $(LIB_MMTK): FORCE if [[ "$(OPENJDK_VERSION)" != "$(OPENJDK_LOCAL_VERSION)" ]]; then \ echo -e $(YELLOW)WARNING: Local OpenJDK version does not match version specified in mmtk/Cargo.toml$(NC); \ diff --git a/openjdk/cpu/riscv/mmtkObjectBarrierSetAssembler_riscv.cpp b/openjdk/cpu/riscv/mmtkObjectBarrierSetAssembler_riscv.cpp index 9c2c761f..79ea583b 100644 --- a/openjdk/cpu/riscv/mmtkObjectBarrierSetAssembler_riscv.cpp +++ b/openjdk/cpu/riscv/mmtkObjectBarrierSetAssembler_riscv.cpp @@ -52,7 +52,7 @@ void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* // tmp1 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6)); __ mv(tmp1, obj); __ srli(tmp1, tmp1, 6); // tmp1 = obj >> 6; - __ li(tmp2, SIDE_METADATA_BASE_ADDRESS); + __ li(tmp2, field_unlog_bits_base_address()); __ add(tmp1, tmp1, tmp2); // tmp1 = SIDE_METADATA_BASE_ADDRESS + (obj >> 6); __ lbu(tmp1, Address(tmp1, 0)); // tmp2 = (obj >> 3) & 7 diff --git a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp index 801982e0..defac781 100644 --- a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp +++ b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.cpp @@ -27,6 +27,7 @@ #include "mmtkBarrierSet.hpp" #include "mmtkBarrierSetC1.hpp" #include "mmtkMutator.hpp" +#include "mmtkHeap.hpp" #include "runtime/sharedRuntime.hpp" #include "utilities/macros.hpp" #include "c1/c1_LIRAssembler.hpp" @@ -93,8 +94,8 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa __ lea(end, Address(obj, var_size_in_bytes, Address::times_1)); } // slowpath if end < obj - __ cmpptr(end, obj); - __ jcc(Assembler::below, slow_case); + // __ cmpptr(end, obj); + // __ jcc(Assembler::below, slow_case); // slowpath if end > lab.limit __ cmpptr(end, limit); __ jcc(Assembler::above, slow_case); @@ -150,7 +151,7 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register threa #define __ sasm-> -void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sasm, const char* name, address entry_point, int argc) { +void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sasm, const char* name, address entry_point, int argc, bool do_code_patch) { __ prologue(name, false); __ save_live_registers_no_oop_map(true); @@ -161,7 +162,13 @@ void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sa guarantee(false, "Too many args"); } - __ call_VM_leaf_base(entry_point, 3); + if (do_code_patch) { + // We don't know the field offset when a code patching is required. + // As a temporary fix, we apply field barrier to all fields in this object. + __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_probable_write_pre_call), c_rarg0); + } else { + __ call_VM_leaf_base(entry_point, argc); + } __ restore_live_registers(true); __ epilogue(); @@ -175,6 +182,10 @@ void MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stu generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); } +void MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub_with_patch_fix(StubAssembler* sasm) { + generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(mmtk_object_reference_write_slow), 3, true); +} + void MMTkBarrierSetAssembler::generate_c1_object_reference_write_post_runtime_stub(StubAssembler* sasm) { generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_post_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3); } diff --git a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.hpp b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.hpp index 84d8e089..531f2314 100644 --- a/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.hpp +++ b/openjdk/cpu/x86/mmtkBarrierSetAssembler_x86.hpp @@ -39,10 +39,11 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler { // Generate runtime stubs for the "runtime code blobs" in MMTkBarrierSetC1 private: - static void generate_c1_runtime_stub_general(StubAssembler* sasm, const char* name, address func, int argc); + static void generate_c1_runtime_stub_general(StubAssembler* sasm, const char* name, address func, int argc, bool do_code_patch = false); public: static void generate_c1_load_reference_runtime_stub(StubAssembler* sasm); static void generate_c1_object_reference_write_pre_runtime_stub(StubAssembler* sasm); + static void generate_c1_object_reference_write_pre_runtime_stub_with_patch_fix(StubAssembler* sasm); static void generate_c1_object_reference_write_post_runtime_stub(StubAssembler* sasm); static void generate_c1_object_reference_write_slow_runtime_stub(StubAssembler* sasm); diff --git a/openjdk/share/barriers/mmtkFieldBarrier.cpp b/openjdk/share/barriers/mmtkFieldBarrier.cpp new file mode 100644 index 00000000..6595e5ab --- /dev/null +++ b/openjdk/share/barriers/mmtkFieldBarrier.cpp @@ -0,0 +1,537 @@ +#define private public // too lazy to change openjdk... +#define protected public +#include "mmtk.h" +#include "mmtkFieldBarrier.hpp" +#include "runtime/interfaceSupport.inline.hpp" + + +#define SOFT_REFERENCE_LOAD_BARRIER true + +constexpr int kUnloggedValue = 1; + +static inline intptr_t side_metadata_base_address() { + return UseCompressedOops ? field_unlog_bits_base_address_compressed() : field_unlog_bits_base_address(); +} + + +void MMTkFieldBarrierSetRuntime::load_reference(DecoratorSet decorators, oop value) const { +#if SOFT_REFERENCE_LOAD_BARRIER + if (CONCURRENT_MARKING_ACTIVE == 1 && value != NULL && mmtk_get_rc((void*) value) != 0) + ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) value); +#endif +}; + +void MMTkFieldBarrierSetRuntime::object_probable_write(oop new_obj) const { + if (!RC_ENABLED || mmtk_get_rc((void*) new_obj) != 0) { + ::mmtk_object_probable_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) new_obj); + } +} + +void MMTkFieldBarrierSetRuntime::object_reference_write_pre(oop src, oop* slot, oop target) const { + if (mmtk_enable_barrier_fastpath) { + intptr_t addr = ((intptr_t) (void*) slot); + const volatile uint8_t * meta_addr = (const volatile uint8_t *) (side_metadata_base_address() + (addr >> (UseCompressedOops ? 5 : 6))); + uint8_t byte_val = *meta_addr; + if (byte_val == 0) return; + intptr_t shift = (addr >> (UseCompressedOops ? 2 : 3)) & 0b111; + if (((byte_val >> shift) & 1) == kUnloggedValue) { + object_reference_write_slow_call((void*) src, (void*) slot, (void*) target); + } + } else { + object_reference_write_pre_call((void*) src, (void*) slot, (void*) target); + } +} + +#define __ masm-> + +void MMTkFieldBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) { + bool on_oop = type == T_OBJECT || type == T_ARRAY; + bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; + bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; + bool on_reference = on_weak || on_phantom; + BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); +#if SOFT_REFERENCE_LOAD_BARRIER + if (on_oop && on_reference) { + Label done; + + assert_different_registers(dst, tmp1); + + // No slow-call if SATB is not active + // intptr_t tmp1_q = CONCURRENT_MARKING_ACTIVE; + __ movptr(tmp1, intptr_t(&CONCURRENT_MARKING_ACTIVE)); + // Load with zero extension to 32 bits. + // uint32_t tmp1_l = (uint32_t)(*(unt8_t*)tmp1_q); + __ movzbl(tmp1, Address(tmp1, 0)); + // if (tmp1_l == 0) goto done; + __ testl(tmp1, tmp1); + __ jcc(Assembler::zero, done); + // if (dst == 0) goto done; + __ testptr(dst, dst); + __ jcc(Assembler::zero, done); + // Do slow-call + __ push_call_clobbered_registers(false /* save_fpu */); + __ mov(c_rarg0, dst); + __ MacroAssembler::call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1); + __ pop_call_clobbered_registers(false /* save_fpu */); + __ bind(done); + } +#endif +} + +void MMTkFieldBarrierSetAssembler::object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) const { + if (can_remove_barrier(decorators, val, /* skip_const_null */ false)) return; + if (mmtk_enable_barrier_fastpath) { + Label done; + + assert_different_registers(tmp1, tmp2, dst.base(), dst.index()); + assert_different_registers(rcx, tmp2); + + // tmp2 = load-byte (side_metadata_base_address() + (obj >> 6)); + __ lea(tmp1, dst); + __ shrptr(tmp1, UseCompressedOops ? 5 : 6); + __ movptr(tmp2, side_metadata_base_address()); + __ movzbl(tmp2, Address(tmp2, tmp1)); + __ cmpl(tmp2, 0); + __ jcc(Assembler::equal, done); + // tmp1 = (obj >> 3) & 7 + __ lea(tmp1, dst); + __ shrptr(tmp1, UseCompressedOops ? 2 : 3); + __ andptr(tmp1, 7); + // tmp2 = tmp2 >> tmp1 + __ xchgptr(tmp1, rcx); + __ shrptr(tmp2); + __ xchgptr(tmp1, rcx); + // if ((tmp2 & 1) == 1) goto slowpath; + __ andptr(tmp2, 1); + __ cmpptr(tmp2, kUnloggedValue); + __ jcc(Assembler::notEqual, done); + + // TODO: Spill fewer registers + __ push_call_clobbered_registers(false /* save_fpu */); + __ movptr(c_rarg0, dst.base()); + __ lea(c_rarg1, dst); + if (val == noreg) + __ movptr(c_rarg2, NULL_WORD); + else + __ movptr(c_rarg2, val); + __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3); + __ pop_call_clobbered_registers(false /* save_fpu */); + + __ bind(done); + } else { + __ pusha(); + __ movptr(c_rarg0, dst.base()); + __ lea(c_rarg1, dst); + if (val == noreg) + __ movptr(c_rarg2, NULL_WORD); + else + __ movptr(c_rarg2, val); + __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3); + __ popa(); + } +} + +void MMTkFieldBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) { + bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; + if (dest_uninitialized) return; + if (type == T_OBJECT || type == T_ARRAY) { + Label done; + // Bailout if count is zero + __ cmpptr(count, 0); + __ jcc(Assembler::equal, done); + __ push_call_clobbered_registers(false /* save_fpu */); + assert_different_registers(c_rarg0, dst, count); + assert_different_registers(c_rarg1, count); + if (c_rarg0 != src) __ movptr(c_rarg0, src); + if (c_rarg1 != dst) __ movptr(c_rarg1, dst); + if (c_rarg2 != count) __ movptr(c_rarg2, count); + __ call_VM_leaf_base(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_pre_call), 3); + __ pop_call_clobbered_registers(false /* save_fpu */); + __ bind(done); + } +} + + +#undef __ +#define __ ce->masm()-> + +void MMTkFieldBarrierSetAssembler::generate_c1_pre_write_barrier_stub(LIR_Assembler* ce, MMTkC1FieldBarrierStub* stub) const { + MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1(); + __ bind(*stub->entry()); + + // For pre-barriers, stub->slot may not be a resolved address. + // Manually patch the address and goes to the slow-path unconditionally. + address runtime_address; + if (stub->patch_code != lir_patch_none) { + // Patch + assert(stub->scratch->is_single_cpu(), "must be"); + assert(stub->scratch->is_register(), "Precondition."); + ce->mem2reg(stub->slot, stub->scratch, T_OBJECT, stub->patch_code, stub->info, false /*wide*/); + // Resolve address + auto masm = ce->masm(); + LIR_Address* addr = stub->slot->as_address_ptr(); + Address from_addr = ce->as_Address(addr); + __ lea(stub->scratch->as_register(), from_addr); + // Store parameter + ce->store_parameter(stub->scratch->as_pointer_register(), 1); + runtime_address = bs->object_reference_write_pre_c1_runtime_code_blob_with_patch_fix()->code_begin(); + } else { + // Store parameter + ce->store_parameter(stub->slot->as_pointer_register(), 1); + runtime_address = bs->object_reference_write_pre_c1_runtime_code_blob()->code_begin(); + } + + ce->store_parameter(stub->src->as_pointer_register(), 0); + ce->store_parameter(stub->new_val->as_pointer_register(), 2); + __ call(RuntimeAddress(runtime_address)); + __ jmp(*stub->continuation()); +} + +#undef __ + +#ifdef ASSERT +#define __ gen->lir(__FILE__, __LINE__)-> +#else +#define __ gen->lir()-> +#endif + +void MMTkFieldBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) { + DecoratorSet decorators = access.decorators(); + bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0; + bool is_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; + bool is_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; + LIRGenerator *gen = access.gen(); + + BarrierSetC1::load_at_resolved(access, result); + +#if SOFT_REFERENCE_LOAD_BARRIER + if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) { + // Register the value in the referent field with the pre-barrier + LabelObj *Lcont_anonymous; + if (is_anonymous) { + Lcont_anonymous = new LabelObj(); + generate_referent_check(access, Lcont_anonymous); + } + assert(result->is_register(), "must be"); + assert(result->type() == T_OBJECT, "must be an object"); + auto slow = new MMTkC1ReferenceLoadBarrierStub(result); + // Call slow-path only when concurrent marking is active + LIR_Opr cm_flag_addr_opr = gen->new_pointer_register(); + __ move(LIR_OprFact::longConst(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), cm_flag_addr_opr); + LIR_Address* cm_flag_addr = new LIR_Address(cm_flag_addr_opr, T_BYTE); + LIR_Opr cm_flag = gen->new_register(T_INT); + __ move(cm_flag_addr, cm_flag); + // No slow-call if SATB is not active + __ cmp(lir_cond_equal, cm_flag, LIR_OprFact::intConst(1)); + __ branch(lir_cond_equal, slow); + __ branch_destination(slow->continuation()); + if (is_anonymous) { + __ branch_destination(Lcont_anonymous->label()); + } + } +#endif +} + +void MMTkC1FieldBarrierStub::emit_code(LIR_Assembler* ce) { + MMTkFieldBarrierSetAssembler* bs = (MMTkFieldBarrierSetAssembler*) BarrierSet::barrier_set()->barrier_set_assembler(); + bs->generate_c1_pre_write_barrier_stub(ce, this); +} + +void MMTkC1FieldBarrierStub::visit(LIR_OpVisitState* visitor) { + if (info != NULL) { + visitor->do_slow_case(info); + } else { + visitor->do_slow_case(); + } + if (src != NULL) visitor->do_input(src); + if (slot != NULL) visitor->do_input(slot); + if (new_val != NULL) visitor->do_input(new_val); + if (scratch != NULL) { + assert(scratch->is_oop(), "must be"); + visitor->do_temp(scratch); + } + } + +void MMTkFieldBarrierSetC1::object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const { + LIRGenerator* gen = access.gen(); + DecoratorSet decorators = access.decorators(); + if ((decorators & IN_HEAP) == 0) return; + bool needs_patching = (decorators & C1_NEEDS_PATCHING) != 0; + if (!src->is_register()) { + LIR_Opr reg = gen->new_pointer_register(); + if (src->is_constant()) { + __ move(src, reg); + } else { + __ leal(src, reg); + } + src = reg; + } + assert(src->is_register(), "must be a register at this point"); + LabelObj *Ldone = new LabelObj(); + __ cmp(lir_cond_equal, src, LIR_OprFact::oopConst(NULL)); + __ branch(lir_cond_equal, Ldone->label()); + + if (!slot->is_register() && !needs_patching) { + LIR_Address* address = slot->as_address_ptr(); + LIR_Opr ptr = gen->new_pointer_register(); + if (!address->index()->is_valid() && address->disp() == 0) { + __ move(address->base(), ptr); + } else { + assert(address->disp() != max_jint, "lea doesn't support patched addresses!"); + __ leal(slot, ptr); + } + slot = ptr; + } else if (needs_patching && !slot->is_address()) { + assert(slot->is_register(), "must be"); + slot = LIR_OprFact::address(new LIR_Address(slot, T_OBJECT)); + } + assert(needs_patching || slot->is_register(), "must be a register at this point unless needs_patching"); + if (!new_val->is_register()) { + LIR_Opr new_val_reg = gen->new_register(T_OBJECT); + if (new_val->is_constant()) { + __ move(new_val, new_val_reg); + } else { + __ leal(new_val, new_val_reg); + } + new_val = new_val_reg; + } + assert(new_val->is_register(), "must be a register at this point"); + MMTkC1FieldBarrierStub* slow = new MMTkC1FieldBarrierStub(src, slot, new_val, access.patch_emit_info(), needs_patching ? lir_patch_normal : lir_patch_none); + if (needs_patching) slow->scratch = gen->new_register(T_OBJECT); + + if (mmtk_enable_barrier_fastpath) { + if (needs_patching) { + // FIXME: Jump to a medium-path for code patching without entering slow-path + __ jump(slow); + } else { + LIR_Opr addr = slot; + // uint8_t* meta_addr = (uint8_t*) (side_metadata_base_address() + (addr >> 6)); + LIR_Opr offset = gen->new_pointer_register(); + __ move(addr, offset); + __ unsigned_shift_right(offset, UseCompressedOops ? 5 : 6, offset); + LIR_Opr base = gen->new_pointer_register(); + __ move(LIR_OprFact::longConst(side_metadata_base_address()), base); + LIR_Address* meta_addr = new LIR_Address(base, offset, T_BYTE); + // uint8_t byte_val = *meta_addr; + LIR_Opr byte_val = gen->new_register(T_INT); + __ move(meta_addr, byte_val); + // #if MMTK_BARRIER_EAGER_BRANCH + // __ cmp(lir_cond_equal, byte_val, LIR_OprFact::intConst(0)); + // __ branch(lir_cond_equal, T_BYTE, Ldone->label()); + // #endif + // intptr_t shift = (addr >> 3) & 0b111; + LIR_Opr shift = gen->new_register(T_INT); + __ move(addr, shift); + __ unsigned_shift_right(shift, UseCompressedOops ? 2 : 3, shift); + __ logical_and(shift, LIR_OprFact::intConst(0b111), shift); + // if (((byte_val >> shift) & 1) == 1) slow; + LIR_Opr result = byte_val; + __ unsigned_shift_right(result, shift, result, LIR_OprFact::illegalOpr); + __ logical_and(result, LIR_OprFact::intConst(1), result); + __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(1)); + __ branch(lir_cond_equal, slow); + } + } else { + __ jump(slow); + } + + __ branch_destination(slow->continuation()); + __ branch_destination(Ldone->label()); +} + +#undef __ + +#define __ ideal. + + +static void insert_write_barrier_common(MMTkIdealKit& ideal, Node* src, Node* slot, Node* val) { + Node* no_base = __ top(); + Node* tls = __ thread(); + if (mmtk_enable_barrier_fastpath) { + float unlikely = PROB_UNLIKELY(0.999); + + Node* zero = __ ConI(0); + Node* addr = __ CastPX(__ ctrl(), slot); + Node* meta_addr = __ AddP(no_base, __ ConP(side_metadata_base_address()), __ URShiftX(addr, __ ConI(UseCompressedOops ? 5 : 6))); + Node* byte = __ load(__ ctrl(), meta_addr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); + __ if_then(byte, BoolTest::ne, zero, unlikely); + Node* shift = __ URShiftX(addr, __ ConI(UseCompressedOops ? 2 : 3)); + shift = __ AndI(__ ConvL2I(shift), __ ConI(7)); + Node* result = __ AndI(__ URShiftI(byte, shift), __ ConI(1)); + __ if_then(result, BoolTest::ne, zero, unlikely); { + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), "mmtk_barrier_call", src, slot, val); + } __ end_if(); + __ end_if(); + } else { + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM, TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), "mmtk_barrier_call", src, slot, val); + // Looks like this is necessary + // See https://github.com/mmtk/openjdk/blob/c82e5c44adced4383162826c2c3933a83cfb139b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp#L288-L291 + Node* call = __ ctrl()->in(0); + call->add_req(slot); + } +} + +void MMTkFieldBarrierSetC2::object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const { + if (can_remove_barrier(kit, &kit->gvn(), src, slot, val, /* skip_const_null */ false)) return; + + MMTkIdealKit ideal(kit, true); + + insert_write_barrier_common(ideal, src, slot, val); + + kit->final_sync(ideal); +} + +static void reference_load_barrier(GraphKit* kit, Node* slot, Node* val, bool emit_barrier) { + MMTkIdealKit ideal(kit, true); + Node* no_base = __ top(); + Node* tls = __ thread(); + float unlikely = PROB_UNLIKELY(0.999); + Node* zero = __ ConI(0); + Node* cm_flag = __ load(__ ctrl(), __ ConP(uintptr_t(&CONCURRENT_MARKING_ACTIVE)), TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); + // No slow-call if SATB is not active + __ if_then(cm_flag, BoolTest::ne, zero, unlikely); { + // No slow-call if dst is NULL + __ if_then(val, BoolTest::ne, kit->null()); { + const TypeFunc* tf = __ func_type(TypeOopPtr::BOTTOM); + Node* x = __ make_leaf_call(tf, FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), "mmtk_barrier_call", val); + } __ end_if(); + } __ end_if(); + kit->sync_kit(ideal); + if (emit_barrier) kit->insert_mem_bar(Op_MemBarCPUOrder); + kit->final_sync(ideal); // Final sync IdealKit and GraphKit. +} + +static void reference_load_barrier_for_unknown_load(GraphKit* kit, Node* base_oop, Node* offset, Node* slot, Node* val, bool need_mem_bar) { + // Note: This function is copied from G1BarrierSetC2::insert_pre_barrier, + // and ShenandoahBarrierSetC2::insert_pre_barrier is probably copied from G1 as well. + // It basically implements BarrierSetC1::generate_referent_check in C2 IR. + // TODO: If another barrier needs weak reference load barrier, + // consider hoisting this function to a superclass. + + // We could be accessing the referent field of a reference object. If so, when G1 + // is enabled, we need to log the value in the referent field in an SATB buffer. + // This routine performs some compile time filters and generates suitable + // runtime filters that guard the pre-barrier code. + // Also add memory barrier for non volatile load from the referent field + // to prevent commoning of loads across safepoint. + + // Some compile time checks. + + // If offset is a constant, is it java_lang_ref_Reference::_reference_offset? + const TypeX* otype = offset->find_intptr_t_type(); + if (otype != nullptr && otype->is_con() && + otype->get_con() != java_lang_ref_Reference::referent_offset()) { + // Constant offset but not the reference_offset so just return + return; + } + + // We only need to generate the runtime guards for instances. + const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr(); + if (btype != nullptr) { + if (btype->isa_aryptr()) { + // Array type so nothing to do + return; + } + + const TypeInstPtr* itype = btype->isa_instptr(); + if (itype != nullptr) { + // Can the klass of base_oop be statically determined to be + // _not_ a sub-class of Reference and _not_ Object? + ciKlass* klass = itype->instance_klass(); + if (klass->is_loaded() && + !klass->is_subtype_of(kit->env()->Reference_klass()) && + !kit->env()->Object_klass()->is_subtype_of(klass)) { + return; + } + } + } + + float likely = PROB_LIKELY( 0.999); + float unlikely = PROB_UNLIKELY(0.999); + + IdealKit ideal(kit); + + Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset()); + + __ if_then(offset, BoolTest::eq, referent_off, unlikely); { + // Update graphKit memory and control from IdealKit. + kit->sync_kit(ideal); + Node* ref_klass_con = kit->makecon(TypeKlassPtr::make(kit->env()->Reference_klass())); + Node* is_instof = kit->gen_instanceof(base_oop, ref_klass_con); + // Update IdealKit memory and control from graphKit. + __ sync_kit(kit); + Node* one = __ ConI(1); + // is_instof == 0 if base_oop == nullptr + __ if_then(is_instof, BoolTest::eq, one, unlikely); { + // Update graphKit from IdeakKit. + kit->sync_kit(ideal); + // Use the pre-barrier to record the value in the referent field + reference_load_barrier(kit, slot, val, false); + if (need_mem_bar) { + // Add memory barrier to prevent commoning reads from this field + // across safepoint since GC can change its value. + kit->insert_mem_bar(Op_MemBarCPUOrder); + } + // Update IdealKit from graphKit. + __ sync_kit(kit); + } __ end_if(); // _ref_type != ref_none + } __ end_if(); // offset == referent_offset + + // Final sync IdealKit and GraphKit. + kit->final_sync(ideal); +} + +Node* MMTkFieldBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { + DecoratorSet decorators = access.decorators(); + Node* adr = access.addr().node(); + Node* obj = access.base(); + + bool mismatched = (decorators & C2_MISMATCHED) != 0; + bool unknown = (decorators & ON_UNKNOWN_OOP_REF) != 0; + bool in_heap = (decorators & IN_HEAP) != 0; + bool in_native = (decorators & IN_NATIVE) != 0; + bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; + bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; + bool is_unordered = (decorators & MO_UNORDERED) != 0; + bool no_keepalive = (decorators & AS_NO_KEEPALIVE) != 0; + bool is_mixed = !in_heap && !in_native; + bool need_cpu_mem_bar = !is_unordered || mismatched || is_mixed; + + Node* top = Compile::current()->top(); + Node* offset = adr->is_AddP() ? adr->in(AddPNode::Offset) : top; + + // If we are reading the value of the referent field of a Reference + // object (either by using Unsafe directly or through reflection) + // then, if G1 is enabled, we need to record the referent in an + // SATB log buffer using the pre-barrier mechanism. + // Also we need to add memory barrier to prevent commoning reads + // from this field across safepoint since GC can change its value. + bool need_read_barrier = (((on_weak || on_phantom) && !no_keepalive) || + (in_heap && unknown && offset != top && obj != top)); + + if (!access.is_oop() || !need_read_barrier) { + return BarrierSetC2::load_at_resolved(access, val_type); + } + + // The other access "opt_access" is only used in arraycopy barriers. + // OpenJDK doesn't have weak arrays, so it must be "parse_access". + assert(access.is_parse_access(), "entry not supported at optimization time"); + + C2ParseAccess& parse_access = static_cast(access); + GraphKit* kit = parse_access.kit(); + Node* load = BarrierSetC2::load_at_resolved(access, val_type); + +#if SOFT_REFERENCE_LOAD_BARRIER + if (on_weak) { + reference_load_barrier(kit, adr, load, true); + } else if (unknown) { + reference_load_barrier_for_unknown_load(kit, obj, offset, adr, load, !need_cpu_mem_bar); + } +#endif + + return load; +} + +#undef __ \ No newline at end of file diff --git a/openjdk/share/barriers/mmtkFieldBarrier.hpp b/openjdk/share/barriers/mmtkFieldBarrier.hpp new file mode 100644 index 00000000..15be8838 --- /dev/null +++ b/openjdk/share/barriers/mmtkFieldBarrier.hpp @@ -0,0 +1,86 @@ +#ifndef MMTK_BARRIERS_FIELD_LOGGING_BARRIER +#define MMTK_BARRIERS_FIELD_LOGGING_BARRIER + +#include "opto/callnode.hpp" +#include "opto/idealKit.hpp" +#include "c1/c1_LIRAssembler.hpp" +#include "c1/c1_MacroAssembler.hpp" +#include "gc/shared/barrierSet.hpp" +#include "../mmtk.h" +#include "../mmtkBarrierSet.hpp" +#include "../cpu/x86/mmtkBarrierSetAssembler_x86.hpp" +#include "../mmtkBarrierSetC1.hpp" +#include "../mmtkBarrierSetC2.hpp" + +#define SIDE_METADATA_WORST_CASE_RATIO_LOG 1 +#define LOG_BYTES_IN_CHUNK 22 +#define CHUNK_MASK ((1L << LOG_BYTES_IN_CHUNK) - 1) + +class MMTkFieldBarrierSetRuntime: public MMTkBarrierSetRuntime { +public: + // Interfaces called by `MMTkBarrierSet::AccessBarrier` + virtual void object_reference_write_pre(oop src, oop* slot, oop target) const override; + virtual void object_reference_array_copy_pre(oop* src, oop* dst, size_t count) const override { + if (count == 0) return; + ::mmtk_array_copy_pre((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, (void*) src, (void*) dst, count); + } + virtual void object_probable_write(oop new_obj) const override; + virtual void load_reference(DecoratorSet decorators, oop value) const override; +}; + +struct MMTkC1FieldBarrierStub: CodeStub { + LIR_Opr src, slot, new_val; + CodeEmitInfo* info; // Code patching info + LIR_PatchCode patch_code; // Enable code patching? + LIR_Opr scratch = NULL; // Scratch register for the resolved field + + MMTkC1FieldBarrierStub(LIR_Opr src, LIR_Opr slot, LIR_Opr new_val, CodeEmitInfo* info = NULL, LIR_PatchCode patch_code = lir_patch_none): src(src), slot(slot), new_val(new_val), info(info), patch_code(patch_code) { + FrameMap* f = Compilation::current()->frame_map(); + f->update_reserved_argument_area_size(3 * BytesPerWord); + } + + virtual void emit_code(LIR_Assembler* ce) override; + + virtual void visit(LIR_OpVisitState* visitor) override; + + NOT_PRODUCT(virtual void print_name(outputStream* out) const { out->print("MMTkC1FieldBarrierStub"); }); +}; + +class MMTkFieldBarrierSetAssembler: public MMTkBarrierSetAssembler { +protected: + virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) const override; +public: + virtual void generate_c1_pre_write_barrier_stub(LIR_Assembler* ce, MMTkC1FieldBarrierStub* stub) const; + virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count) override; + virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) override; +}; + +class MMTkFieldBarrierSetC1: public MMTkBarrierSetC1 { +protected: + virtual void object_reference_write_pre(LIRAccess& access, LIR_Opr src, LIR_Opr slot, LIR_Opr new_val) const override; + + virtual void load_at_resolved(LIRAccess& access, LIR_Opr result) override; + + virtual LIR_Opr resolve_address(LIRAccess& access, bool resolve_in_register) override { + return MMTkBarrierSetC1::resolve_address_in_register(access, resolve_in_register); + } +}; + +class MMTkFieldBarrierSetC2: public MMTkBarrierSetC2 { +protected: + virtual void object_reference_write_pre(GraphKit* kit, Node* src, Node* slot, Node* val) const override; +public: + virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, bool is_clone, bool is_clone_instance, ArrayCopyPhase phase) const override { + return is_reference_type(type) && !tightly_coupled_alloc && !is_clone; + } + virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const override; +}; + +struct MMTkFieldBarrier: MMTkBarrierImpl< + MMTkFieldBarrierSetRuntime, + MMTkFieldBarrierSetAssembler, + MMTkFieldBarrierSetC1, + MMTkFieldBarrierSetC2 +> {}; + +#endif diff --git a/openjdk/share/mmtk.h b/openjdk/share/mmtk.h index cf7b44d5..8ebcea5d 100644 --- a/openjdk/share/mmtk.h +++ b/openjdk/share/mmtk.h @@ -20,7 +20,10 @@ typedef enum { extern uintptr_t get_global_side_metadata_base_address(); extern uintptr_t get_global_side_metadata_vm_base_address(); +extern uintptr_t get_rc_table_base_address(); extern uintptr_t get_vo_bit_address(); +extern const uintptr_t IMMIX_ALLOCATOR_SIZE; +extern uint8_t RC_ENABLED; extern const size_t MMTK_LISP2_HEADER_RESERVED_IN_BYTES; extern const uintptr_t FREE_LIST_ALLOCATOR_SIZE; extern uint8_t CONCURRENT_MARKING_ACTIVE; @@ -33,6 +36,9 @@ extern const char* get_mmtk_version(); extern MMTk_Mutator bind_mutator(void *tls); extern void destroy_mutator(MMTk_Mutator mutator); extern void flush_mutator(MMTk_Mutator mutator); +extern void release_mutator(MMTk_Mutator mutator); + +extern void mmtk_report_gc_start(); extern void* alloc(MMTk_Mutator mutator, size_t size, size_t align, size_t offset, int allocator); @@ -105,10 +111,14 @@ extern bool will_never_move(void* object); extern bool process(char* name, char* value); extern bool process_bulk(char* options); extern void scan_region(); -extern void handle_user_collection_request(void *tls); +extern void handle_user_collection_request(void *tls, bool force); extern void start_worker(void *tls, void* worker); +extern size_t mmtk_is_live(void* object); +extern void* mmtk_get_forwarded_ref(void* object); + + extern size_t mmtk_add_nmethod_oop(void* object); extern size_t mmtk_register_nmethod(void* nm); extern size_t mmtk_unregister_nmethod(void* nm); @@ -164,7 +174,7 @@ typedef struct { int (*static_oop_field_count_offset) (); int (*referent_offset) (); int (*discovered_offset) (); - char* (*dump_object_string) (void* object); + const char* (*dump_object_string) (void* object); void (*scan_roots_in_all_mutator_threads)(SlotsClosure closure); void (*scan_roots_in_mutator_thread)(SlotsClosure closure, void* tls); void (*scan_code_cache_roots) (SlotsClosure closure); @@ -175,8 +185,10 @@ typedef struct { size_t (*number_of_mutators)(); void (*schedule_finalizer)(); void (*prepare_for_roots_re_scanning)(); + void (*mmtk_update_weak_processor)(bool lxr); void (*enqueue_references)(void** objects, size_t len); - void (*fix_oop_relocations)(void* nmethod); + void (*fix_oop_relocations)(bool lxr, void* nmethods, size_t len); + void (*gc_epilogue)(); } OpenJDK_Upcalls; extern void openjdk_gc_init(OpenJDK_Upcalls *calls); @@ -212,8 +224,35 @@ extern void add_phantom_candidate(void* ref, void* referent); extern void mmtk_harness_begin_impl(); extern void mmtk_harness_end_impl(); +constexpr size_t log_min_obj_size = 3; + +inline uint8_t mmtk_get_rc_2bits(void* o) { + const uintptr_t index = uintptr_t((void*) o) >> log_min_obj_size; + const uint8_t byte = *((uint8_t*) (get_rc_table_base_address() + (index >> 2))); + auto v = byte >> ((index & 0b11) << 1); + return v & 0b11; +} + +inline uint8_t mmtk_get_rc_4bits(void* o) { + const uintptr_t index = uintptr_t((void*) o) >> log_min_obj_size; + const uint8_t byte = *((uint8_t*) (get_rc_table_base_address() + (index >> 1))); + auto v = byte >> ((index & 0b1) << 2); + return v & 0b1111; +} + +inline uint8_t mmtk_get_rc_8bits(void* o) { + const uintptr_t index = uintptr_t((void*) o) >> log_min_obj_size; + const uint8_t byte = *((uint8_t*) (get_rc_table_base_address() + index)); + return byte; +} + +inline uint8_t mmtk_get_rc(void* o) { + return mmtk_get_rc_2bits(o); +} + extern void mmtk_builder_read_env_var_settings(); extern void mmtk_builder_set_threads(size_t value); +extern void mmtk_builder_set_conc_threads(size_t value); extern void mmtk_builder_set_transparent_hugepages(bool value); #ifdef __cplusplus diff --git a/openjdk/share/mmtkBarrierSet.cpp b/openjdk/share/mmtkBarrierSet.cpp index 6d13e066..e588bd88 100644 --- a/openjdk/share/mmtkBarrierSet.cpp +++ b/openjdk/share/mmtkBarrierSet.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "barriers/mmtkNoBarrier.hpp" #include "barriers/mmtkObjectBarrier.hpp" +#include "barriers/mmtkFieldBarrier.hpp" #include "barriers/mmtkSATBBarrier.hpp" #include "mmtkBarrierSet.hpp" #include "utilities/macros.hpp" @@ -36,6 +37,7 @@ #ifdef COMPILER2 #include "mmtkBarrierSetC2.hpp" #endif +#include "mmtkBarrierSetAssembler_x86.hpp" bool mmtk_enable_allocation_fastpath = true; bool mmtk_enable_barrier_fastpath = true; @@ -87,6 +89,7 @@ MMTkBarrierBase* get_selected_barrier() { if (strcmp(barrier, "NoBarrier") == 0) selected_barrier = new MMTkNoBarrier(); else if (strcmp(barrier, "ObjectBarrier") == 0) selected_barrier = new MMTkObjectBarrier(); else if (strcmp(barrier, "SATBBarrier") == 0) selected_barrier = new MMTkSATBBarrier(); + else if (strcmp(barrier, "FieldBarrier") == 0) selected_barrier = new MMTkFieldBarrier(); else guarantee(false, "Unimplemented"); return selected_barrier; } @@ -166,3 +169,7 @@ void MMTkBarrierSetRuntime::object_reference_array_copy_post_call(void* src, voi void MMTkBarrierSetRuntime::load_reference_call(void* ref) { ::mmtk_load_reference((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); } + +void MMTkBarrierSetRuntime::object_probable_write_pre_call(void* ref) { + ::mmtk_object_probable_write((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, ref); +} \ No newline at end of file diff --git a/openjdk/share/mmtkBarrierSet.hpp b/openjdk/share/mmtkBarrierSet.hpp index 7778faf9..1c21bb5c 100644 --- a/openjdk/share/mmtkBarrierSet.hpp +++ b/openjdk/share/mmtkBarrierSet.hpp @@ -43,6 +43,14 @@ extern bool mmtk_enable_allocation_fastpath; extern bool mmtk_enable_barrier_fastpath; extern bool mmtk_enable_reference_load_barrier; +inline intptr_t field_unlog_bits_base_address() { + return static_cast(get_global_side_metadata_vm_base_address()); +} + +inline intptr_t field_unlog_bits_base_address_compressed() { + return static_cast(get_global_side_metadata_vm_base_address()); +} + inline intptr_t vo_bit_base_address() { return static_cast(get_vo_bit_address()); } @@ -77,6 +85,8 @@ class MMTkBarrierSetRuntime: public CHeapObj { static void object_reference_array_copy_pre_call(void* src, void* dst, size_t count); /// Generic arraycopy pre-barrier. Called by fast-paths. static void object_reference_array_copy_post_call(void* src, void* dst, size_t count); + /// + static void object_probable_write_pre_call(void* obj); /// Check if the address is a slow-path function. virtual bool is_slow_path_call(address call) const { return call == CAST_FROM_FN_PTR(address, object_reference_write_pre_call) @@ -84,7 +94,8 @@ class MMTkBarrierSetRuntime: public CHeapObj { || call == CAST_FROM_FN_PTR(address, object_reference_write_slow_call) || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_pre_call) || call == CAST_FROM_FN_PTR(address, object_reference_array_copy_post_call) - || call == CAST_FROM_FN_PTR(address, load_reference_call); + || call == CAST_FROM_FN_PTR(address, load_reference_call) + || call == CAST_FROM_FN_PTR(address, object_probable_write_pre_call); } /// Full pre-barrier @@ -102,7 +113,6 @@ class MMTkBarrierSetRuntime: public CHeapObj { /// So this callback is requierd for any generational collectors. virtual void object_probable_write(oop new_obj) const {}; }; - class MMTkBarrierC1; class MMTkBarrierSetC1; class MMTkBarrierC2; @@ -135,7 +145,7 @@ class MMTkBarrierSet : public BarrierSet { MMTkBarrierSetRuntime* _runtime; protected: - virtual void write_ref_array_work(MemRegion mr) ; + virtual void write_ref_array_work(MemRegion mr); public: MMTkBarrierSet(MemRegion whole_heap); @@ -258,7 +268,7 @@ class MMTkBarrierSet : public BarrierSet { T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); runtime()->object_reference_array_copy_pre((oop*) src, (oop*) dst, length); - bool result = Raw::oop_arraycopy(src_obj, src_offset_in_bytes, src_raw, + bool result = Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); runtime()->object_reference_array_copy_post((oop*) src, (oop*) dst, length); @@ -267,7 +277,7 @@ class MMTkBarrierSet : public BarrierSet { static void clone_in_heap(oop src, oop dst, size_t size) { // TODO: We don't need clone barriers at the moment. - Raw::clone(src, dst, size); + Raw::clone_in_heap(src, dst, size); } }; diff --git a/openjdk/share/mmtkBarrierSetC1.cpp b/openjdk/share/mmtkBarrierSetC1.cpp index 1fabee78..6ef9eeff 100644 --- a/openjdk/share/mmtkBarrierSetC1.cpp +++ b/openjdk/share/mmtkBarrierSetC1.cpp @@ -26,6 +26,7 @@ void MMTkBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) { _load_reference_c1_runtime_code_blob = do_code_blob("c1_load_reference_stub", &MMTkBarrierSetAssembler::generate_c1_load_reference_runtime_stub); _object_reference_write_pre_c1_runtime_code_blob = do_code_blob("c1_object_reference_write_pre_stub", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub); + _object_reference_write_pre_c1_runtime_code_blob_with_patch_fix = do_code_blob("c1_object_reference_write_pre_stub_with_patch_fix", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub_with_patch_fix); _object_reference_write_post_c1_runtime_code_blob = do_code_blob("c1_object_reference_write_post_stub", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_post_runtime_stub); _object_reference_write_slow_c1_runtime_code_blob = do_code_blob("c1_object_reference_write_slow_stub", &MMTkBarrierSetAssembler::generate_c1_object_reference_write_slow_runtime_stub); } diff --git a/openjdk/share/mmtkBarrierSetC1.hpp b/openjdk/share/mmtkBarrierSetC1.hpp index f75b96a6..a33701c0 100644 --- a/openjdk/share/mmtkBarrierSetC1.hpp +++ b/openjdk/share/mmtkBarrierSetC1.hpp @@ -20,6 +20,7 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { // we may consider defining new code blobs for specific barriers. CodeBlob* _load_reference_c1_runtime_code_blob; CodeBlob* _object_reference_write_pre_c1_runtime_code_blob; + CodeBlob* _object_reference_write_pre_c1_runtime_code_blob_with_patch_fix; CodeBlob* _object_reference_write_post_c1_runtime_code_blob; CodeBlob* _object_reference_write_slow_c1_runtime_code_blob; @@ -74,6 +75,7 @@ class MMTkBarrierSetC1 : public BarrierSetC1 { CodeBlob* load_reference_c1_runtime_code_blob() { return _load_reference_c1_runtime_code_blob; } CodeBlob* object_reference_write_pre_c1_runtime_code_blob() { return _object_reference_write_pre_c1_runtime_code_blob; } + CodeBlob* object_reference_write_pre_c1_runtime_code_blob_with_patch_fix() { return _object_reference_write_pre_c1_runtime_code_blob_with_patch_fix; } CodeBlob* object_reference_write_post_c1_runtime_code_blob() { return _object_reference_write_post_c1_runtime_code_blob; } CodeBlob* object_reference_write_slow_c1_runtime_code_blob() { return _object_reference_write_slow_c1_runtime_code_blob; } diff --git a/openjdk/share/mmtkHeap.cpp b/openjdk/share/mmtkHeap.cpp index c70abef5..2aae5f6f 100644 --- a/openjdk/share/mmtkHeap.cpp +++ b/openjdk/share/mmtkHeap.cpp @@ -31,6 +31,7 @@ #include "gc/shared/gcLocker.inline.hpp" #include "gc/shared/gcWhen.hpp" #include "gc/shared/oopStorageSet.inline.hpp" +#include "gc/shared/oopStorageSetParState.inline.hpp" #include "gc/shared/scavengableNMethods.hpp" #include "gc/shared/strongRootsScope.hpp" #include "gc/shared/weakProcessor.hpp" @@ -59,6 +60,12 @@ #include "services/memoryManager.hpp" #include "services/memTracker.hpp" #include "utilities/vmError.hpp" +#include "mmtkRootsClosure.hpp" +#include "gc/shared/workerPolicy.hpp" + +MaybeUninit> oop_storage_set_strong_par_state; +MaybeUninit> oop_storage_set_weak_par_state; + /* needed support from rust heap capacity @@ -195,6 +202,12 @@ jint MMTkHeap::initialize() { fprintf(stderr, "Failed to create thread"); guarantee(false, "panic"); } + + unsigned int ncpus = (unsigned int) os::initial_active_processor_count(); + _workers = new WorkerThreads("GC Thread", ncpus); + _workers->initialize_workers(); + _workers->set_active_workers(WorkerPolicy::parallel_worker_threads()); + os::start_thread(_companion_thread); // Set up the GCTaskManager // _mmtk_gc_task_manager = mmtkGCTaskManager::create(ParallelGCThreads); @@ -209,6 +222,10 @@ void MMTkHeap::set_mmtk_options(bool set_defaults) { mmtk_builder_set_threads(ParallelGCThreads); } + if (FLAG_IS_DEFAULT(ConcGCThreads) == set_defaults) { + mmtk_builder_set_conc_threads(ConcGCThreads); + } + if (FLAG_IS_DEFAULT(UseTransparentHugePages) == set_defaults) { mmtk_builder_set_transparent_hugepages(UseTransparentHugePages); } @@ -220,7 +237,9 @@ const char* MMTkHeap::version() { } void MMTkHeap::schedule_finalizer() { - MMTkFinalizerThread::instance->schedule(); + if (!RC_ENABLED) { + MMTkFinalizerThread::instance->schedule(); + } } class MMTkIsScavengable : public BoolObjectClosure { @@ -246,7 +265,9 @@ void MMTkHeap::post_initialize() { void MMTkHeap::enable_collection() { // Initialize finalizer thread before enable_collection(). // Otherwise it is possible that we schedule finalizer (during a GC) before the finalizer thread is ready. - MMTkFinalizerThread::initialize(); + if (!RC_ENABLED) { + MMTkFinalizerThread::initialize(); + } ::initialize_collection(0); } @@ -300,16 +321,6 @@ bool MMTkHeap::is_in(const void* p) const { return is_in_mmtk_spaces(const_cast(p)); } -bool MMTkHeap::is_in_reserved(const void* p) const { - //printf("calling MMTkHeap::is_in_reserved\n"); - return is_in(p); -} - -bool MMTkHeap::supports_tlab_allocation() const { - //returning false is good enough...used in universe.cpp - return false; -} - // The amount of space available for thread-local allocation buffers. size_t MMTkHeap::tlab_capacity(Thread *thr) const { //no need to further implement but we need UseTLAB=False @@ -344,17 +355,30 @@ bool MMTkHeap::card_mark_must_follow_store() const { //OK } void MMTkHeap::collect(GCCause::Cause cause) {//later when gc is implemented in rust - handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator); - // guarantee(false, "collect not supported"); + if (cause == GCCause::_gc_locker) { + // This runs on the Java mutator thread that is the last one exiting a + // JNI critical region (see GCLocker::jni_unlock), so JNICritical_lock + // must be acquired with a safepoint check, same as gcLocker.cpp does. + // Acquiring it with Mutex::_no_safepoint_check_flag here trips HotSpot's + // "This lock should always have a safepoint check for Java threads" + // assertion in Mutex::check_no_safepoint_state. + MutexLocker locker(JNICritical_lock); + // Notify the VMCompanionThread to trigger another VM_MMTkSTWOperation. + JNICritical_lock->notify_all(); + } + handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, cause != GCCause::_java_lang_system_gc); } // Perform a full collection void MMTkHeap::do_full_collection(bool clear_all_soft_refs) {//later when gc is implemented in rust - // guarantee(false, "do full collection not supported"); - - // handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator); + handle_user_collection_request((MMTk_Mutator) &Thread::current()->third_party_heap_mutator, true); } +void MMTkHeap::collect_as_vm_thread(GCCause::Cause cause) { + MMTkHeap::heap()->companion_thread()->vm_thread_requires_gc_pause(); + handle_user_collection_request(NULL, true); + MMTkHeap::heap()->companion_thread()->block_vm_thread(); +} SoftRefPolicy* MMTkHeap::soft_ref_policy() {return &_soft_ref_policy;}//OK @@ -376,37 +400,6 @@ void MMTkHeap::object_iterate(ObjectClosure* cl) { //No need to implement.Traced fprintf(stderr, "WARNING: MMTkHeap::object_iterate is not implemented, yet.\n"); } -// Similar to object_iterate() except iterates only -// over live objects. -void MMTkHeap::safe_object_iterate(ObjectClosure* cl) { //not sure..many dependencies from vm - fprintf(stderr, "WARNING: MMTkHeap::safe_object_iterate is not implemented, yet.\n"); -} - -HeapWord* MMTkHeap::block_start(const void* addr) const {//OK - guarantee(false, "block start not supported"); - return NULL; -} - -size_t MMTkHeap::block_size(const HeapWord* addr) const { //OK - guarantee(false, "block size not supported"); - return 0; -} - -bool MMTkHeap::block_is_obj(const HeapWord* addr) const { //OK - guarantee(false, "block is obj not supported"); - return false; -} - -jlong MMTkHeap::millis_since_last_gc() {//later when gc is implemented in rust - jlong ret_val = (os::javaTimeNanos() / NANOSECS_PER_MILLISEC) - _last_gc_time; - if (ret_val < 0) { - log_warning(gc)("millis_since_last_gc() would return : " JLONG_FORMAT - ". returning zero instead.", ret_val); - return 0; - } - return ret_val; -} - void MMTkHeap::prepare_for_verify() { // guarantee(false, "prepare for verify not supported"); @@ -421,12 +414,7 @@ void MMTkHeap::initialize_serviceability() {//OK } // Print heap information on the given outputStream. -void MMTkHeap::print_on(outputStream* st) const {guarantee(false, "print on not supported");} - - -// Print all GC threads (other than the VM thread) -// used by this heap. -void MMTkHeap::print_gc_threads_on(outputStream* st) const {guarantee(false, "print gc threads on not supported");} +void MMTkHeap::print_on(outputStream* st) const {} // Iterator for all GC threads (other than VM thread) void MMTkHeap::gc_threads_do(ThreadClosure* tc) const { @@ -441,7 +429,7 @@ void MMTkHeap::print_tracing_info() const { // Used to print information about locations in the hs_err file. bool MMTkHeap::print_location(outputStream* st, void* addr) const { - guarantee(false, "print location not supported"); + // guarantee(false, "print location not supported"); return false; } @@ -470,9 +458,7 @@ void MMTkHeap::register_nmethod(nmethod* nm) { // Register the nmethod mmtk_register_nmethod((void*) nm); } -// Callback for when nmethod is about to be deleted. -void MMTkHeap::flush_nmethod(nmethod* nm) { -} + void MMTkHeap::verify_nmethod(nmethod* nm) { } @@ -492,17 +478,20 @@ void MMTkHeap::scan_class_loader_data_graph_roots(OopClosure& cl) { ClassLoaderDataGraph::cld_do(&cld_cl); } void MMTkHeap::scan_oop_storage_set_roots(OopClosure& cl) { - OopStorageSet::strong_oops_do(&cl); + for (auto id : EnumRange()) { + oop_storage_set_strong_par_state->par_state(id)->oops_do(&cl); + } } void MMTkHeap::scan_weak_processor_roots(OopClosure& cl) { - // XXX zixianc: I don't understand why this is removed in - // 24b90dd889da0ea58aaa2b2311ded6f262573830 - // ResourceMark rm; - WeakProcessor::oops_do(&cl); // (really needed???) + ResourceMark rm; + for (auto id : EnumRange()) { + oop_storage_set_weak_par_state->par_state(id)->weak_oops_do(&cl); + } } void MMTkHeap::scan_vm_thread_roots(OopClosure& cl) { ResourceMark rm; - VMThread::vm_thread()->oops_do(&cl, NULL); + MarkingCodeBlobClosure cb_cl(&cl, false, true); + VMThread::vm_thread()->oops_do(&cl, &cb_cl); } void MMTkHeap::scan_roots_in_all_mutator_threads(OopClosure& cl) { @@ -544,6 +533,8 @@ HeapWord* MMTkHeap::mem_allocate_nonmove(size_t size, bool* gc_overhead_limit_wa return Thread::current()->third_party_heap_mutator.alloc(size << LogHeapWordSize, AllocatorLos); } +void MMTkHeap::register_new_weak_handle(oop* handle) {} + bool MMTkHeap::requires_barriers(stackChunkOop obj) const { ShouldNotReachHere(); return false; diff --git a/openjdk/share/mmtkHeap.hpp b/openjdk/share/mmtkHeap.hpp index 8064752f..444c6094 100644 --- a/openjdk/share/mmtkHeap.hpp +++ b/openjdk/share/mmtkHeap.hpp @@ -31,8 +31,10 @@ #include "gc/shared/gcWhen.hpp" #include "gc/shared/oopStorage.hpp" #include "gc/shared/oopStorageParState.hpp" +#include "gc/shared/oopStorageSetParState.hpp" #include "gc/shared/space.hpp" #include "gc/shared/strongRootsScope.hpp" +#include "gc/shared/workerThread.hpp" #include "gc/shared/softRefPolicy.hpp" #include "memory/iterator.hpp" #include "memory/metaspace.hpp" @@ -41,6 +43,30 @@ #include "utilities/growableArray.hpp" #include "utilities/ostream.hpp" +#define WORKER_STACK_SIZE (64 * 1024 * 1024) + +template +struct MaybeUninit { + MaybeUninit() {} + T* operator->() { + return (T*) &_data; + } + T& operator*() { + return *((T*) &_data); + } + template + void init(Args... args) { + new (&_data) T(args...); + } + template + void reinit(Args... args) { + ((T*) &_data)->~T(); + new (&_data) T(args...); + } +private: + char _data[sizeof(T)]; +}; + class GCMemoryManager; class MemoryPool; //class mmtkGCTaskManager; @@ -53,7 +79,10 @@ class MMTkHeap : public CollectedHeap { ContiguousSpace* _space; int _num_root_scan_tasks; MMTkVMCompanionThread* _companion_thread; + WorkerThreads* _workers; SoftRefPolicy _soft_ref_policy; +public: + AllocatorSelector default_allocator_selector; public: jlong _last_gc_time; @@ -64,18 +93,24 @@ class MMTkHeap : public CollectedHeap { public: MMTkHeap(); + WorkerThreads* workers() const { return _workers; } + void schedule_finalizer(); + void set_is_gc_active(bool is_gc_active) { + _is_stw_gc_active = is_gc_active; + } + inline static MMTkHeap* heap() { return _heap; } static HeapWord* allocate_from_tlab(Klass* klass, Thread* thread, size_t size); - jint initialize(); - void enable_collection(); + virtual jint initialize() override; + virtual void enable_collection() override; - virtual HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded); + virtual HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) override; HeapWord* mem_allocate_nonmove(size_t size, bool* gc_overhead_limit_was_exceeded); MMTkVMCompanionThread* companion_thread() const { @@ -83,33 +118,27 @@ class MMTkHeap : public CollectedHeap { } - Name kind() const { + virtual Name kind() const override { return CollectedHeap::ThirdPartyHeap; } - const char* name() const { + virtual const char* name() const override { return "MMTk"; } static const char* version(); - size_t capacity() const; - size_t used() const; - - bool is_maximal_no_gc() const; + virtual size_t capacity() const override; + virtual size_t used() const override; - size_t max_capacity() const; - bool is_in(const void* p) const; - bool is_in_reserved(const void* p) const; - bool supports_tlab_allocation() const; + virtual bool is_maximal_no_gc() const override; - bool supports_inline_contig_alloc() const { - return mmtk_enable_allocation_fastpath; - } + virtual size_t max_capacity() const override; + virtual bool is_in(const void* p) const override; // The amount of space available for thread-local allocation buffers. - size_t tlab_capacity(Thread *thr) const; + virtual size_t tlab_capacity(Thread *thr) const override; // The amount of used space for thread-local allocation buffers for the given thread. - size_t tlab_used(Thread *thr) const; + virtual size_t tlab_used(Thread *thr) const override; void new_collector_thread() { _n_workers += 1; @@ -127,79 +156,64 @@ class MMTkHeap : public CollectedHeap { // mark to be thus strictly sequenced after the stores. bool card_mark_must_follow_store() const; - void collect(GCCause::Cause cause); + virtual void collect(GCCause::Cause cause) override; // Perform a full collection - void do_full_collection(bool clear_all_soft_refs); + virtual void do_full_collection(bool clear_all_soft_refs) override; + virtual void collect_as_vm_thread(GCCause::Cause cause) override; - SoftRefPolicy* soft_ref_policy(); - GrowableArray memory_managers() ; - GrowableArray memory_pools(); + virtual SoftRefPolicy* soft_ref_policy() override; + + virtual GrowableArray memory_managers() override; + virtual GrowableArray memory_pools() override; // Iterate over all objects, calling "cl.do_object" on each. - void object_iterate(ObjectClosure* cl); + virtual void object_iterate(ObjectClosure* cl) override; void pin_object(JavaThread* thread, oop obj); void unpin_object(JavaThread* thread, oop obj); - // Similar to object_iterate() except iterates only - // over live objects. - void safe_object_iterate(ObjectClosure* cl) ; - - HeapWord* block_start(const void* addr) const ; - - size_t block_size(const HeapWord* addr) const ; - - bool block_is_obj(const HeapWord* addr) const; - - jlong millis_since_last_gc() ; - - void prepare_for_verify() ; + virtual void prepare_for_verify() override; + virtual void register_new_weak_handle(oop* handle) /*override*/; private: - void initialize_serviceability() ; + virtual void initialize_serviceability() override; void set_mmtk_options(bool set_defaults); public: // Print heap information on the given outputStream. - void print_on(outputStream* st) const ; - - - // Print all GC threads (other than the VM thread) - // used by this heap. - void print_gc_threads_on(outputStream* st) const; + virtual void print_on(outputStream* st) const override; // Iterator for all GC threads (other than VM thread) - void gc_threads_do(ThreadClosure* tc) const; + virtual void gc_threads_do(ThreadClosure* tc) const override; // Print any relevant tracing info that flags imply. // Default implementation does nothing. - void print_tracing_info() const ; + virtual void print_tracing_info() const override; bool print_location(outputStream* st, void* addr) const; bool requires_barriers(stackChunkOop obj) const; - void register_nmethod(nmethod* nm); - void unregister_nmethod(nmethod* nm); + virtual void register_nmethod(nmethod* nm) override; + virtual void unregister_nmethod(nmethod* nm) override; - void flush_nmethod(nmethod* nm); - void verify_nmethod(nmethod* nm); + virtual void verify_nmethod(nmethod* nm) override; // An object is scavengable if its location may move during a scavenge. // (A scavenge is a GC which is not a full GC.) inline bool is_scavengable(oop obj) { return true; } // Heap verification - void verify(VerifyOption option); + virtual void verify(VerifyOption option) override; - void post_initialize(); + virtual void post_initialize() override; void scan_roots(OopClosure& cl); diff --git a/openjdk/share/mmtkMutator.cpp b/openjdk/share/mmtkMutator.cpp index 698acd7b..85096e76 100644 --- a/openjdk/share/mmtkMutator.cpp +++ b/openjdk/share/mmtkMutator.cpp @@ -2,10 +2,15 @@ #include "precompiled.hpp" #include "mmtk.h" #include "mmtkMutator.hpp" +#include "mmtkHeap.hpp" size_t MMTkMutatorContext::max_non_los_default_alloc_bytes = 0; MMTkMutatorContext MMTkMutatorContext::bind(::Thread* current) { + if (IMMIX_ALLOCATOR_SIZE != sizeof(ImmixAllocator)) { + printf("ERROR: Unmatched immix allocator size: rs=%zu cpp=%zu\n", IMMIX_ALLOCATOR_SIZE, sizeof(ImmixAllocator)); + guarantee(false, "ERROR"); + } if (FREE_LIST_ALLOCATOR_SIZE != sizeof(MMTkFreeListAllocator)) { printf("ERROR: Unmatched free list allocator size: rs=%zu cpp=%zu\n", FREE_LIST_ALLOCATOR_SIZE, sizeof(MMTkFreeListAllocator)); guarantee(false, "ERROR"); diff --git a/openjdk/share/mmtkUpcalls.cpp b/openjdk/share/mmtkUpcalls.cpp index 86e6c3a5..0d25e0de 100644 --- a/openjdk/share/mmtkUpcalls.cpp +++ b/openjdk/share/mmtkUpcalls.cpp @@ -22,10 +22,12 @@ * */ +#include "mmtkBarrierSet.hpp" #include "precompiled.hpp" #include "classfile/classLoaderDataGraph.hpp" #include "classfile/stringTable.hpp" #include "code/nmethod.hpp" +#include "gc/shared/barrierSetNMethod.hpp" #include "memory/iterator.inline.hpp" #include "memory/resourceArea.hpp" #include "mmtkCollectorThread.hpp" @@ -39,26 +41,150 @@ #include "runtime/mutexLocker.hpp" #include "runtime/os.hpp" #include "runtime/safepoint.hpp" +#include "runtime/interfaceSupport.inline.hpp" #include "runtime/thread.hpp" #include "runtime/threads.hpp" #include "runtime/thread.inline.hpp" #include "runtime/threadSMR.hpp" #include "runtime/vmThread.hpp" +#include "gc/shared/weakProcessor.hpp" +#include "prims/resolvedMethodTable.hpp" +#include "jfr/jfr.hpp" +#include "gc/shared/oopStorage.inline.hpp" #include "runtime/osThread.hpp" #include "utilities/debug.hpp" +#include "classfile/systemDictionary.hpp" +#include "prims/jvmtiExport.hpp" +#include "runtime/jniHandles.hpp" +#include "utilities/macros.hpp" +#include "gc/shared/classUnloadingContext.hpp" +#include "code/codeCache.hpp" +#if INCLUDE_JFR +#include "jfr/jfr.hpp" +#endif // Note: This counter must be accessed using the Atomic class. static volatile size_t mmtk_start_the_world_count = 0; -static void mmtk_stop_all_mutators(void *tls, MutatorClosure closure) { - ClassLoaderDataGraph::clear_claimed_marks(); -#if COMPILER2_OR_JVMCI - DerivedPointerTable::clear(); -#endif +class MMTkIsAliveClosure : public BoolObjectClosure { +public: + inline virtual bool do_object_b(oop p) { + if (p == NULL) return false; + return mmtk_is_live((void*) p) != 0; + } +}; + +class MMTkForwardClosure : public OopClosure { + public: + inline static size_t read_forwarding_word(oop o) { + return *((size_t*) (void*) o); + } + inline static oop extract_forwarding_pointer(size_t status) { + return (oop) (void*) (status << 8 >> 8); + } + inline static bool is_forwarded(size_t status) { + return (status >> 56) != 0; + } + inline virtual void do_oop(oop* slot) { + const auto o = *slot; + if (o == NULL) return; + const auto status = read_forwarding_word(o); + if (is_forwarded(status)) { + *slot = extract_forwarding_pointer(status); + } + } + inline virtual void do_oop(narrowOop* slot) { + narrowOop heap_oop = RawAccess<>::oop_load(slot); + if (CompressedOops::is_null(heap_oop)) return; + oop o = CompressedOops::decode_not_null(heap_oop); + const auto status = read_forwarding_word(o); + if (is_forwarded(status)) { + RawAccess<>::oop_store(slot, CompressedOops::encode_not_null(extract_forwarding_pointer(status))); + } + } +}; + +class MMTkLXRFastIsAliveClosure : public BoolObjectClosure { +public: + static inline bool rc_live(oop o) { + return mmtk_get_rc((void*) o) != 0; + } + + static inline bool is_forwarded(oop o) { + return MMTkForwardClosure::is_forwarded(MMTkForwardClosure::read_forwarding_word(o)); + } + + inline virtual bool do_object_b(oop o) override { + const uintptr_t v = uintptr_t((void*) o); + // if (v >= 0x220000000000ULL || v < 0x20000000000ULL) return false; + return o != NULL && (rc_live(o) || is_forwarded(o)); + } +}; + +class MMTkLXRFastUpdateClosure : public OopClosure { + public: + inline virtual void do_oop(oop* slot) override { + const auto o = *slot; + const uintptr_t v = uintptr_t((void*) o); + if (o == NULL) { + return; + } + const auto status = MMTkForwardClosure::read_forwarding_word(o); + if (MMTkForwardClosure::is_forwarded(status)) { + *slot = MMTkForwardClosure::extract_forwarding_pointer(status); + } else if (!MMTkLXRFastIsAliveClosure::rc_live(o)) { + *slot = NULL; + } + } + inline virtual void do_oop(narrowOop* slot) override { + narrowOop heap_oop = RawAccess<>::oop_load(slot); + if (CompressedOops::is_null(heap_oop)) return; + oop o = CompressedOops::decode_not_null(heap_oop); + const uintptr_t v = uintptr_t((void*) o); + const auto status = MMTkForwardClosure::read_forwarding_word(o); + if (MMTkForwardClosure::is_forwarded(status)) { + RawAccess<>::oop_store(slot, CompressedOops::encode_not_null(MMTkForwardClosure::extract_forwarding_pointer(status))); + } else if (!MMTkLXRFastIsAliveClosure::rc_live(o)) { + RawAccess<>::oop_store(slot, CompressedOops::encode(oop(NULL))); + } + } +}; + +class MMTkUpdateClosure : public OopClosure { + public: + inline virtual void do_oop(oop* slot) override { + const auto o = *slot; + const uintptr_t v = uintptr_t((void*) o); + if (o == NULL) { + return; + } + if (mmtk_is_live((void*) o) == 0) { + *slot = NULL; + } else if (MMTkForwardClosure::is_forwarded(MMTkForwardClosure::read_forwarding_word(o))) { + *slot = MMTkForwardClosure::extract_forwarding_pointer(MMTkForwardClosure::read_forwarding_word(o)); + } + } + inline virtual void do_oop(narrowOop* slot) override { + guarantee(false, "unreachable"); + } +}; + +extern MaybeUninit> oop_storage_set_strong_par_state; +extern MaybeUninit> oop_storage_set_weak_par_state; +static void mmtk_stop_all_mutators(void *tls, MutatorClosure closure) { log_debug(gc)("Requesting the VM to suspend all mutators..."); MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_suspended, true); log_debug(gc)("Mutators stopped. Now enumerate threads for scanning..."); + MMTkHeap::heap()->set_is_gc_active(true); + + oop_storage_set_strong_par_state.init(); + oop_storage_set_weak_par_state.init(); + + // CodeCache::gc_prologue(); +#if COMPILER2_OR_JVMCI + DerivedPointerTable::clear(); +#endif JavaThreadIteratorWithHandle jtiwh; while (JavaThread *cur = jtiwh.next()) { @@ -69,19 +195,37 @@ static void mmtk_stop_all_mutators(void *tls, MutatorClosure closure) { nmethod::oops_do_marking_prologue(); } -static void mmtk_resume_mutators(void *tls) { - nmethod::oops_do_marking_epilogue(); +static void mmtk_update_weak_processor(bool lxr) { + // HandleMark hm(THREAD); + if (lxr) { + MMTkLXRFastUpdateClosure cl; + WeakProcessor::oops_do(&cl); + } else { + MMTkUpdateClosure cl; + WeakProcessor::oops_do(&cl); + } +} + +static void mmtk_gc_epilogue() { + Universe::heap()->update_capacity_and_used_at_gc(); + // CodeCache::gc_epilogue(); + // JvmtiExport::gc_epilogue(); // ClassLoaderDataGraph::purge(); #if COMPILER2_OR_JVMCI DerivedPointerTable::update_pointers(); #endif + CodeCache::arm_all_nmethods(); +} +static void mmtk_resume_mutators(void *tls) { + nmethod::oops_do_marking_epilogue(); // Note: we don't have to hold gc_lock to increment the counter. // The increment has to be done before mutators can be resumed (from `block_for_gc` or yieldpoints). // Otherwise, mutators might see an outdated start-the-world count. Atomic::inc(&mmtk_start_the_world_count); log_debug(gc)("Incremented start_the_world counter to %zu.", Atomic::load(&mmtk_start_the_world_count)); + MMTkHeap::heap()->set_is_gc_active(false); log_debug(gc)("Requesting the companion thread to resume all mutators blocking on yieldpoints..."); MMTkHeap::heap()->companion_thread()->request(MMTkVMCompanionThread::_threads_resumed, true); @@ -90,6 +234,15 @@ static void mmtk_resume_mutators(void *tls) { MutexLocker locker(MMTkHeap::heap()->gc_lock(), Mutex::_no_safepoint_check_flag); MMTkHeap::heap()->gc_lock()->notify_all(); } + + log_debug(gc)("Notifying mutators blocking on Heap_lock for reference pending list..."); + // Note: That's the ReferenceHandler thread. + { + MutexLocker x(Heap_lock, Mutex::_no_safepoint_check_flag); + if (Universe::has_reference_pending_list()) { + Heap_lock->notify_all(); + } + } } static const int GC_THREAD_KIND_WORKER = 1; @@ -187,12 +340,32 @@ static void mmtk_scan_roots_in_all_mutator_threads(SlotsClosure closure) { MMTkHeap::heap()->scan_roots_in_all_mutator_threads(cl); } +// A CodeBlobClosure for use during live stack walks. It performs the same nmethod +// "on stack" bookkeeping as MarkingCodeBlobClosure (mark_as_maybe_on_stack/disarm), +// but deliberately does NOT re-scan the nmethod's embedded oops: those are already +// tracked once, unconditionally, via MMTk's own code-cache-roots remembered set +// (see MMTkHeap::register_nmethod / ScanCodeCacheRoots). Using MarkingCodeBlobClosure +// here would report every on-stack nmethod's oops as GC roots twice in the same GC. +class MMTkBookKeepingCodeBlobClosure : public CodeBlobClosure { +public: + virtual void do_code_blob(CodeBlob* cb) override { + nmethod* nm = cb->as_nmethod_or_null(); + if (nm != nullptr && nm->oops_do_try_claim()) { + nm->mark_as_maybe_on_stack(); + BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); + if (bs_nm != nullptr) { + bs_nm->disarm(nm); + } + } + } +}; + static void mmtk_scan_roots_in_mutator_thread(SlotsClosure closure, void* tls) { ResourceMark rm; JavaThread* thread = (JavaThread*) tls; MMTkRootsClosure cl(closure); - MarkingCodeBlobClosure cb_cl(&cl, false, true); - thread->oops_do(&cl, NULL); + MMTkBookKeepingCodeBlobClosure cb_cl; + thread->oops_do(&cl, &cb_cl); } static void mmtk_scan_object(void* trace, void* object, void* tls) { @@ -269,9 +442,16 @@ static int discovered_offset() { return java_lang_ref_Reference::discovered_offset(); } -static char* dump_object_string(void* object) { +char data[1024]; + +static const char* dump_object_string(void* object) { + // HandleMark hm(THREAD); + ResourceMark rm; + if (object == NULL) return NULL; oop o = (oop) object; - return o->print_value_string(); + const char* c = o->klass()->internal_name(); + strcpy(&data[0], c); + return &data[0]; } static void mmtk_schedule_finalizer() { @@ -281,7 +461,10 @@ static void mmtk_schedule_finalizer() { static void mmtk_scan_code_cache_roots(SlotsClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_code_cache_roots(cl); } static void mmtk_scan_class_loader_data_graph_roots(SlotsClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl); } static void mmtk_scan_oop_storage_set_roots(SlotsClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_oop_storage_set_roots(cl); } -static void mmtk_scan_weak_processor_roots(SlotsClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_weak_processor_roots(cl); } +static void mmtk_scan_weak_processor_roots(SlotsClosure closure) { + MMTkRootsClosure cl(closure); + MMTkHeap::heap()->scan_weak_processor_roots(cl); +} static void mmtk_scan_vm_thread_roots(SlotsClosure closure) { MMTkRootsClosure cl(closure); MMTkHeap::heap()->scan_vm_thread_roots(cl); } static size_t mmtk_number_of_mutators() { @@ -293,6 +476,8 @@ static void mmtk_prepare_for_roots_re_scanning() { DerivedPointerTable::update_pointers(); DerivedPointerTable::clear(); #endif + oop_storage_set_strong_par_state.reinit(); + oop_storage_set_weak_par_state.reinit(); } static void mmtk_enqueue_references(void** objects, size_t len) { @@ -332,11 +517,31 @@ static void mmtk_enqueue_references(void** objects, size_t len) { oop old_first = Universe::swap_reference_pending_list(first); HeapAccess::oop_store_at(last, java_lang_ref_Reference::discovered_offset(), old_first); + + // Note: At this time, `Universe::has_reference_pending_list()` should be true, but we can't assert it. + // `has_reference_pending_list` requires either the current thread is the VM thread or is holding the Heap_lock. + // But during STW GC, the Heap_lock is held by the VM companion thread. } -void mmtk_fix_oop_relocations(void *nmptr) { - nmethod* nm = (nmethod*)nmptr; - nm->fix_oop_relocations(); + +void mmtk_fix_oop_relocations(bool lxr, void* nmethods, size_t len) { + if (lxr) { + MMTkLXRFastUpdateClosure cl; + for (size_t i = 0; i < len; i++) { + nmethod* nm = (nmethod*) ((nmethod**) nmethods)[i]; + nm->oops_do(&cl); + nm->fix_oop_relocations(); + } + } else { + // MMTkUpdateClosure cl; + for (size_t i = 0; i < len; i++) { + nmethod* nm = (nmethod*) ((nmethod**) nmethods)[i]; + // FIXME: This caused crash for mark compact. It is likely that nmethod is already scanned, and we see to-space objects here and still try to forward it. + // I am uncertain what to do here. Just comment it out for now so we can proceed. This should be resolved before merging. + // nm->oops_do(&cl); + nm->fix_oop_relocations(); + } + } } OpenJDK_Upcalls mmtk_upcalls = { @@ -369,6 +574,8 @@ OpenJDK_Upcalls mmtk_upcalls = { mmtk_number_of_mutators, mmtk_schedule_finalizer, mmtk_prepare_for_roots_re_scanning, + mmtk_update_weak_processor, mmtk_enqueue_references, mmtk_fix_oop_relocations, + mmtk_gc_epilogue, }; diff --git a/openjdk/share/mmtkVMCompanionThread.cpp b/openjdk/share/mmtkVMCompanionThread.cpp index 16171987..7c073f94 100644 --- a/openjdk/share/mmtkVMCompanionThread.cpp +++ b/openjdk/share/mmtkVMCompanionThread.cpp @@ -27,6 +27,7 @@ #include "mmtkVMCompanionThread.hpp" #include "runtime/mutex.hpp" #include "logging/log.hpp" +#include "gc/shared/gcLocker.hpp" MMTkVMCompanionThread::MMTkVMCompanionThread(): NamedThread(), @@ -46,7 +47,7 @@ void MMTkVMCompanionThread::run() { for (;;) { // Wait for suspend request log_trace(gc)("MMTkVMCompanionThread: Waiting for suspend request..."); - { + if (!_wait_for_gc_locker) { MutexLocker locker(_lock, Mutex::_no_safepoint_check_flag); assert(_reached_state == _threads_resumed, "Threads should be running at this moment."); while (_desired_state != _threads_suspended) { @@ -57,12 +58,45 @@ void MMTkVMCompanionThread::run() { // Let the VM thread stop the world. log_trace(gc)("MMTkVMCompanionThread: Letting VMThread execute VM op..."); - VM_MMTkSTWOperation op(this); - // VMThread::execute() is blocking. The companion thread will be blocked - // here waiting for the VM thread to execute op, and the VM thread will - // be blocked in do_mmtk_stw_operation() until a GC thread - // calls request(_threads_resumed). - VMThread::execute(&op); + if (_vm_thread_requires_gc_pause) { + guarantee(!_wait_for_gc_locker, "VM thread is triggering a GC when the MMTkVMCompanionThread is waiting for GC locker"); + MutexLocker locker(_lock, Mutex::_no_safepoint_check_flag); + _vm_thread_requires_gc_pause = false; + _vm_thread_suspend_for_gc = true; + _lock->notify_all(); + while (_vm_thread_suspend_for_gc) { + _lock->wait_without_safepoint_check(); + } + } else { + if (_wait_for_gc_locker) { + // When VM_MMTkSTWOperation early exits due to a jni critical region, + // _wait_for_gc_locker will be set to true before exsting safepoint. + // This main loop will continue one more iteration and reach here. + // We wait until GC locker is inactive, to avoid busy looping. + // #ifndef PRODUCT + // auto safepoint_check_required = JNICritical_lock->_safepoint_check_required; + // JNICritical_lock->_safepoint_check_required = Monitor::_safepoint_check_sometimes; + // #endif + MutexLocker locker(JNICritical_lock, Mutex::_no_safepoint_check_flag); + while (GCLocker::is_active_and_needs_gc()) { + // #ifndef PRODUCT + // JNICritical_lock->_safepoint_check_required = Monitor::_safepoint_check_sometimes; + // #endif + JNICritical_lock->wait_without_safepoint_check(); + } + // #ifndef PRODUCT + // JNICritical_lock->_safepoint_check_required = safepoint_check_required; + // #endif + // clear the flag + _wait_for_gc_locker = false; + } + VM_MMTkSTWOperation op(this); + // VMThread::execute() is blocking. The companion thread will be blocked + // here waiting for the VM thread to execute op, and the VM thread will + // be blocked in do_mmtk_stw_operation() until a GC thread + // calls request(_threads_resumed). + VMThread::execute(&op); + } } } @@ -93,6 +127,29 @@ void MMTkVMCompanionThread::request(stw_state desired_state, bool wait_until_rea } } +void MMTkVMCompanionThread::vm_thread_requires_gc_pause() { + MutexLocker locker(_lock, Mutex::_no_safepoint_check_flag); + _vm_thread_requires_gc_pause = true; +} + +void MMTkVMCompanionThread::block_vm_thread() { + { + MutexLocker locker(_lock, Mutex::_no_safepoint_check_flag); + while (!_vm_thread_suspend_for_gc) { + _lock->wait_without_safepoint_check(); + } + } + + VM_MMTkSTWOperation op(this); + VMThread::execute(&op); + + { + MutexLocker locker(_lock, Mutex::_no_safepoint_check_flag); + _vm_thread_suspend_for_gc = false; + _lock->notify_all(); + } +} + // Wait until the desired state is reached. Usually called after calling the // request method. Supposed to be called by a GC thread. void MMTkVMCompanionThread::wait_for_reached(stw_state desired_state) { diff --git a/openjdk/share/mmtkVMCompanionThread.hpp b/openjdk/share/mmtkVMCompanionThread.hpp index 5a147e17..86ce1f88 100644 --- a/openjdk/share/mmtkVMCompanionThread.hpp +++ b/openjdk/share/mmtkVMCompanionThread.hpp @@ -50,6 +50,11 @@ class MMTkVMCompanionThread: public NamedThread { Monitor* _lock; stw_state _desired_state; stw_state _reached_state; + bool _vm_thread_requires_gc_pause = false; + bool _vm_thread_suspend_for_gc = false; + bool _wait_for_gc_locker = false; + + friend class VM_MMTkSTWOperation; public: // Constructor @@ -62,6 +67,9 @@ class MMTkVMCompanionThread: public NamedThread { void request(stw_state desired_state, bool wait_until_reached); void wait_for_reached(stw_state reached_state); + void vm_thread_requires_gc_pause(); + void block_vm_thread(); + // Interface for the VM_MMTkSTWOperation void do_mmtk_stw_operation(); }; diff --git a/openjdk/share/mmtkVMOperation.cpp b/openjdk/share/mmtkVMOperation.cpp index 7ba6053f..630ebb65 100644 --- a/openjdk/share/mmtkVMOperation.cpp +++ b/openjdk/share/mmtkVMOperation.cpp @@ -26,7 +26,10 @@ #include "mmtk.h" #include "mmtkVMCompanionThread.hpp" #include "mmtkVMOperation.hpp" +#include "interpreter/oopMapCache.hpp" #include "logging/log.hpp" +#include "interpreter/oopMapCache.hpp" +#include "gc/shared/gcLocker.hpp" VM_MMTkSTWOperation::VM_MMTkSTWOperation(MMTkVMCompanionThread *companion_thread): _companion_thread(companion_thread) { @@ -38,12 +41,29 @@ bool VM_MMTkSTWOperation::doit_prologue() { } void VM_MMTkSTWOperation::doit() { + if (GCLocker::check_active_before_gc()) { + // If some threads is in JNI critical region, we don't do a GC for now, + // and end this VM operation earlier. Under such case, `GCLocker::check_active_before_gc` + // will remember there is a pending GC. After the thread exits the critical region, + // if a pending GC needs to be triggered, the java thread will + // call `MMTkHeap::collect(GCCause::_gc_locker)`. + // Since we've already have a unfinished GC request inside mmtk, + // mmtk will not trigger another GC, but simply blocking this thread. + // After all threads are successfully blocked, the previously + // triggered pending GC will proceed. + _companion_thread->_wait_for_gc_locker = true; + return; + } + // JvmtiGCMarker _jgcm; log_trace(vmthread)("Entered VM_MMTkSTWOperation::doit()."); _companion_thread->do_mmtk_stw_operation(); log_trace(vmthread)("Leaving VM_MMTkSTWOperation::doit()"); } void VM_MMTkSTWOperation::doit_epilogue() { + // Clean up old interpreter OopMap entries that were replaced + // during the GC thread root traversal. + // OopMapCache::cleanup_old_entries(); // Notify the reference processing thread if (Universe::has_reference_pending_list()) { Heap_lock->notify_all(); diff --git a/openjdk/share/thirdPartyHeapArguments.cpp b/openjdk/share/thirdPartyHeapArguments.cpp index a3acb3b2..2b4c772a 100644 --- a/openjdk/share/thirdPartyHeapArguments.cpp +++ b/openjdk/share/thirdPartyHeapArguments.cpp @@ -33,11 +33,17 @@ #include "runtime/vm_version.hpp" #include "thirdPartyHeapArguments.hpp" #include "utilities/defaultStream.hpp" +#include "runtime/abstract_vm_version.hpp" size_t ThirdPartyHeapArguments::conservative_max_heap_alignment() { return compute_heap_alignment(); } +static uint scale_concurrent_worker_threads(uint num_gc_workers) { + auto v = (num_gc_workers + 2) / 4; + return v > 1 ? v : 1; +} + void ThirdPartyHeapArguments::initialize() { GCArguments::initialize(); assert(UseThirdPartyHeap , "Error, should UseThirdPartyHeap"); @@ -48,6 +54,8 @@ void ThirdPartyHeapArguments::initialize() { assert(!FLAG_IS_DEFAULT(ParallelGCThreads), "ParallelGCThreads should not be 0."); vm_exit_during_initialization("The flag -XX:+UseUseThirdPartyHeap can not be combined with -XX:ParallelGCThreads=0", NULL); } + uint marking_thread_num = scale_concurrent_worker_threads(ParallelGCThreads); + FLAG_SET_DEFAULT(ConcGCThreads, marking_thread_num); // Note: If you add an option here that may be forwarded to an MMTk option, // make sure to add appropriate code to MMTkHeap::set_mmtk_options. }