From d2861cb445db1ed82dc324d5c38166b42b8263df Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 17 Aug 2026 22:04:43 +0200 Subject: [PATCH 01/14] test(package-deps): snapshot independent root closures Record that a package root reached under a competing virtual implementation does not receive its own link closure. Signed-off-by: Ali Caglayan --- .../test-cases/package-materialization/dune | 1 + .../independent-roots.t | 100 ++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 test/blackbox-tests/test-cases/package-materialization/independent-roots.t diff --git a/test/blackbox-tests/test-cases/package-materialization/dune b/test/blackbox-tests/test-cases/package-materialization/dune index a4dde991d06..aff5ae637dd 100644 --- a/test/blackbox-tests/test-cases/package-materialization/dune +++ b/test/blackbox-tests/test-cases/package-materialization/dune @@ -1,5 +1,6 @@ (cram (applies_to + independent-roots installed-package no-transitive-through-targets ocamlfind diff --git a/test/blackbox-tests/test-cases/package-materialization/independent-roots.t b/test/blackbox-tests/test-cases/package-materialization/independent-roots.t new file mode 100644 index 00000000000..4bc42644620 --- /dev/null +++ b/test/blackbox-tests/test-cases/package-materialization/independent-roots.t @@ -0,0 +1,100 @@ +Each library installed by an explicitly requested package is an independent +closure root. Combining the roots before resolving virtual implementations can +suppress a default implementation needed by one root. + + $ make_dune_project 3.24 + $ cat >>dune-project <<'EOF' + > (package (name roots)) + > (package (name virtual-support)) + > (package (name alternative-support)) + > EOF + + $ mkdir roots-a roots-b virtual-lib default-impl alternative-impl + $ cat >virtual-lib/dune <<'EOF' + > (library + > (name virtual_support) + > (public_name virtual-support) + > (wrapped false) + > (virtual_modules virtual_support) + > (default_implementation virtual-support.default)) + > EOF + $ echo 'val value : int' >virtual-lib/virtual_support.mli + + $ cat >default-impl/dune <<'EOF' + > (library + > (name default_impl) + > (public_name virtual-support.default) + > (implements virtual-support)) + > EOF + $ echo 'let value = 1' >default-impl/virtual_support.ml + + $ cat >alternative-impl/dune <<'EOF' + > (library + > (name alternative_impl) + > (public_name alternative-support) + > (implements virtual-support)) + > EOF + $ echo 'let value = 2' >alternative-impl/virtual_support.ml + +The first root selects the alternative implementation while the second root +uses the virtual library on its own and therefore needs the default. + + $ cat >roots-b/dune <<'EOF' + > (library + > (name roots_b) + > (public_name roots.b) + > (libraries virtual-support)) + > EOF + $ echo 'let value = Virtual_support.value' >roots-b/roots_b.ml + + $ cat >roots-a/dune <<'EOF' + > (library + > (name roots_a) + > (public_name roots.a) + > (libraries roots.b alternative-support)) + > EOF + $ echo 'let value = Roots_b.value' >roots-a/roots_a.ml + + $ cat >dune <<'EOF' + > (rule + > (target result) + > (deps (package roots)) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query -predicates byte -format "%d/%A" + > virtual-support.default)))) + > (rule + > (target alternative-result) + > (deps (package roots)) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query -predicates byte -format "%d/%A" + > alternative-support)))) + > EOF + +Both implementations and their archives must be present in the scoped layout. +The default is currently absent because `roots.b` is not closed independently. + + $ dune build result + File "dune", lines 1-7, characters 0-179: + 1 | (rule + 2 | (target result) + 3 | (deps (package roots)) + 4 | (action + 5 | (with-stdout-to %{target} + 6 | (run %{bin:ocamlfind} query -predicates byte -format "%d/%A" + 7 | virtual-support.default)))) + ocamlfind: Package `virtual-support.default' not found + [1] + + $ dune build alternative-result + File "dune", lines 8-14, characters 0-187: + 8 | (rule + 9 | (target alternative-result) + 10 | (deps (package roots)) + 11 | (action + 12 | (with-stdout-to %{target} + 13 | (run %{bin:ocamlfind} query -predicates byte -format "%d/%A" + 14 | alternative-support)))) + ocamlfind: Package `alternative-support' not found + [1] From e3b501d56675c91b120f16856a1f501498415c85 Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 17 Aug 2026 10:14:26 +0200 Subject: [PATCH 02/14] test(package-deps): snapshot the missing workspace library closure Exercise library dependencies, private libraries, stubs, redirects, namespaces, virtual implementations, PPX runtime libraries, nested Dune consumers, installed packages, and only-packages masking. Record the current missing-library failures before materializing the closure. Signed-off-by: Ali Caglayan --- .../test-cases/package-materialization/dune | 1 + .../installed-package.t | 92 +++- .../package-materialization/ocamlfind.t | 6 +- .../strict-package-deps.t | 15 +- .../transitive-closure.t | 426 +++++++++++++++++- 5 files changed, 505 insertions(+), 35 deletions(-) diff --git a/test/blackbox-tests/test-cases/package-materialization/dune b/test/blackbox-tests/test-cases/package-materialization/dune index a4dde991d06..f46de8e60d2 100644 --- a/test/blackbox-tests/test-cases/package-materialization/dune +++ b/test/blackbox-tests/test-cases/package-materialization/dune @@ -3,5 +3,6 @@ installed-package no-transitive-through-targets ocamlfind + transitive-closure virtual-without-default) (deps %{bin:ocamlfind})) diff --git a/test/blackbox-tests/test-cases/package-materialization/installed-package.t b/test/blackbox-tests/test-cases/package-materialization/installed-package.t index 575329501e0..20aecd86316 100644 --- a/test/blackbox-tests/test-cases/package-materialization/installed-package.t +++ b/test/blackbox-tests/test-cases/package-materialization/installed-package.t @@ -2,42 +2,112 @@ Test that (deps (package ...)) works with externally installed packages. Installed packages (found via findlib) go through the Installed codepath, not the layout. The layout only applies to Local (workspace) packages. -Install package "a" into a prefix: +Install packages "a" and "b" into a prefix. Library `a` depends on library +`b`, so its installed metadata records `b` as a requirement. $ mkdir a consumer prefix $ cat >a/dune-project < (lang dune 3.24) > (package (name a)) + > (package (name b)) > EOF $ cat >a/dune < (library (public_name a)) + > (library + > (public_name a) + > (libraries b)) > EOF $ cat >a/a.ml < let msg = "hello from lib a" + > let value = B.value + 1 + > EOF + + $ mkdir a/b + + $ cat >a/b/dune < (library (public_name b)) + > EOF + + $ cat >a/b/b.ml < let value = 1 > EOF $ dune build --root a @install $ dune install --root a --prefix $PWD/prefix 2>/dev/null $ test -f prefix/lib/a/META + $ test -f prefix/lib/b/META Now create a consumer project that depends on the installed package. -The consumer uses (deps (package a)) and ocamlfind to verify the -package is findable: +The consumer uses `(deps (package a))` and external OCaml tooling to verify +that both `a` and its library dependency are findable: $ cat >consumer/dune-project < (lang dune 3.24) > EOF + $ cat >consumer/main.ml < let () = print_int A.value + > EOF + $ cat >consumer/dune <<'EOF' > (rule - > (deps (package a)) - > (action (with-stdout-to out - > (run ocamlfind query a)))) + > (target main.exe) + > (deps + > main.ml + > (package a)) + > (action + > (run ocamlfind ocamlc -package a -linkpkg -o %{target} main.ml))) + > EOF + + $ OCAMLPATH=$PWD/prefix/lib dune build --root consumer main.exe + $ consumer/_build/default/main.exe + 2 + +When `--only-packages` masks a workspace library in the closure, library +resolution falls back to its installed copy. The installed library remains on +the inherited `OCAMLPATH`; it is not rematerialized as workspace support. + + $ mkdir masked masked/a-src masked/b-src + + $ cat >masked/dune-project < (lang dune 3.24) + > (package (name a)) + > (package (name b)) + > EOF + + $ cat >masked/a-src/dune < (library + > (public_name a) + > (libraries b)) + > EOF + + $ cat >masked/a-src/a.ml < let value = B.value + 10 + > EOF + + $ cat >masked/b-src/dune < (library (public_name b)) + > EOF + + $ cat >masked/b-src/b.ml < let value = 100 + > EOF + + $ cat >masked/main.ml < let () = print_int A.value + > EOF + + $ cat >masked/dune <<'EOF' + > (rule + > (target main.exe) + > (deps + > main.ml + > (package a)) + > (action + > (run %{bin:ocamlfind} ocamlc -package a -linkpkg -o %{target} main.ml))) > EOF - $ OCAMLPATH=$PWD/prefix/lib/:$OCAMLPATH dune build --root consumer out - $ cat consumer/_build/default/out - $TESTCASE_ROOT/prefix/lib/a + $ OCAMLPATH=$PWD/prefix/lib dune build --root masked --only-packages a main.exe + $ masked/_build/default/main.exe + 11 diff --git a/test/blackbox-tests/test-cases/package-materialization/ocamlfind.t b/test/blackbox-tests/test-cases/package-materialization/ocamlfind.t index ed608c8c496..beff07f746f 100644 --- a/test/blackbox-tests/test-cases/package-materialization/ocamlfind.t +++ b/test/blackbox-tests/test-cases/package-materialization/ocamlfind.t @@ -39,9 +39,9 @@ the query against the layout's OCAMLPATH. $ dune build out -Immediate-deps-only: myutil is mylib's declared opam dependency but is -NOT in the layout for (deps (package mylib)). ocamlfind fails to find -it. +Current-behavior snapshot: myutil is mylib's declared opam dependency but is +not in the immediate-only layout for (deps (package mylib)), so ocamlfind +cannot find it. $ cat >dune <<'EOF' > (rule diff --git a/test/blackbox-tests/test-cases/package-materialization/strict-package-deps.t b/test/blackbox-tests/test-cases/package-materialization/strict-package-deps.t index 4c0d2f80a7a..615cad6d174 100644 --- a/test/blackbox-tests/test-cases/package-materialization/strict-package-deps.t +++ b/test/blackbox-tests/test-cases/package-materialization/strict-package-deps.t @@ -1,6 +1,6 @@ Test that (strict_package_deps) does not affect the install layout. The -layout always uses immediate deps only. strict_package_deps controls -validation in install_rules, not layout closure. +layout currently uses immediate package dependencies only. +strict_package_deps controls validation in install_rules, not the layout. $ cat >dune-project < (lang dune 3.24) @@ -13,7 +13,9 @@ validation in install_rules, not layout closure. $ mkdir foo-src bar-src baz-src $ cat >foo-src/dune < (library (public_name foo)) + > (library + > (public_name foo) + > (libraries bar)) > EOF $ cat >foo-src/foo.ml < EOF $ cat >bar-src/dune < (library (public_name bar)) + > (library + > (public_name bar) + > (libraries baz)) > EOF $ cat >bar-src/bar.ml <dune-project < (lang dune 3.24) - > (package (name foo) (depends bar)) - > (package (name bar) (depends baz)) +The intended library closure is narrower than `foo`'s package dependencies: +package dependencies can contain unrelated executables, data, or libraries, +and are not reliably available for all kinds of packages. + + $ make_dune_project 3.24 + $ cat >>dune-project < (package (name foo) (depends package-only-dep)) + > (package (name bar)) > (package (name baz)) + > (package (name namespace)) + > (package (name package-only-dep)) + > (package (name ppx-runtime)) + > (package (name redirect-root)) + > (package (name redirect-target)) + > (package (name stubbed)) + > (package (name test-ppx)) + > (package (name virtual-root)) + > (package (name virtual-support)) > EOF - $ mkdir foo-src bar-src baz-src + $ mkdir foo-src bar-src bar-private-src bar-unrelated-src baz-src namespace-src + $ mkdir namespace-unrelated-src package-only-dep-src + $ mkdir redirect-root-src redirect-target-src virtual-root-src + $ mkdir virtual-support-src virtual-support-impl-src stubbed-src $ cat >foo-src/dune < (library (public_name foo)) + > (library + > (public_name foo) + > (libraries bar namespace.selected)) > EOF $ cat >foo-src/foo.ml < let x = 1 + > let x = Bar.y + 1 > EOF $ cat >bar-src/dune < (library (public_name bar)) + > (library + > (public_name bar) + > (libraries baz bar_private stubbed)) + > (deprecated_library_name + > (old_public_name bar.old) + > (new_public_name bar)) > EOF $ cat >bar-src/bar.ml < let y = 2 + > let y = Baz.z + Bar_private.offset + Stubbed.value () + > EOF + +The installed form of `bar` needs its package-private library too. It is part +of the library closure even though it cannot be named as a public library in +the workspace. + + $ cat >bar-private-src/dune < (library + > (name bar_private) + > (package bar)) + > EOF + + $ cat >bar-private-src/bar_private.ml < let offset = 1 + > EOF + +The package that owns `bar` also contains an unrelated library. Requiring +`bar` must not make this sibling library available. + + $ cat >bar-unrelated-src/dune < (library + > (name bar_unrelated) + > (public_name bar.unrelated)) + > EOF + + $ cat >bar-unrelated-src/bar_unrelated.ml < let unused = () > EOF $ cat >baz-src/dune < let z = 3 > EOF +Only `namespace.selected`, not a top-level `namespace` library, is in the +closure. Findlib subpackages inherit their directory but not arbitrary +top-level variables, so the filtered META drops `top_marker`. + + $ cat >namespace-src/dune < (library + > (name selected) + > (public_name namespace.selected)) + > EOF + + $ cat >namespace-src/selected.ml < let unused = () + > EOF + + $ cat >namespace-unrelated-src/dune < (library + > (name unrelated) + > (public_name namespace.unrelated)) + > EOF + + $ cat >namespace-unrelated-src/unrelated.ml < let unused = () + > EOF + + $ cat >META.namespace.template < top_marker = "drop" + > # DUNE_GEN + > EOF + + $ cat >package-only-dep-src/dune < (library + > (name package_only_dep) + > (public_name package-only-dep)) + > EOF + + $ cat >package-only-dep-src/package_only_dep.ml < let unused = () + > EOF + +Library support includes native stubs and the stublibs entries needed to load +them from bytecode. + + $ cat >stubbed-src/dune < (library + > (public_name stubbed) + > (foreign_stubs + > (language c) + > (names stubbed_stubs))) + > EOF + + $ cat >stubbed-src/stubbed.ml < external value : unit -> int = "stubbed_value" + > EOF + + $ cat >stubbed-src/stubbed_stubs.c < #include + > CAMLprim value stubbed_value(value unit) + > { + > (void) unit; + > return Val_int(4); + > } + > EOF + +A deprecated name owned by an explicitly declared package may redirect to a +library in another package. The redirect target is a root of the library +closure even when the declared package has no libraries of its own. + + $ cat >redirect-root-src/dune < (deprecated_library_name + > (old_public_name redirect-root.old) + > (new_public_name redirect-target)) + > EOF + + $ cat >redirect-target-src/dune < (library + > (name redirect_target) + > (public_name redirect-target)) + > EOF + + $ cat >redirect-target-src/redirect_target.ml < let value = 42 + > EOF + +A virtual library's default implementation must belong to the same package as +the virtual library. It is nevertheless a separate library, and is part of the +link-time library closure selected by a consumer. + + $ cat >virtual-root-src/dune < (library + > (name virtual_root) + > (public_name virtual-root) + > (libraries virtual-support)) + > EOF + + $ cat >virtual-root-src/virtual_root.ml < let value = Virtual_support.value + > EOF + + $ cat >virtual-support-src/dune < (library + > (name virtual_support) + > (public_name virtual-support) + > (wrapped false) + > (virtual_modules virtual_support) + > (default_implementation virtual-support.default)) + > EOF + + $ cat >virtual-support-src/virtual_support.mli < val value : int + > EOF + + $ cat >virtual-support-impl-src/dune < (library + > (name virtual_support_default) + > (public_name virtual-support.default) + > (implements virtual-support)) + > EOF + + $ cat >virtual-support-impl-src/virtual_support.ml < let value = 42 + > EOF + +A PPX rewriter's runtime libraries are part of the library support closure +even though they are not ordinary `requires`. + + $ make_hello_ppx_runtime_fixture + + $ cat >hello/dune < (library + > (name hello) + > (public_name ppx-runtime)) + > EOF + + $ cat >hello_ppx/dune < (library + > (name hello_ppx) + > (public_name test-ppx) + > (kind ppx_rewriter) + > (ppx_runtime_libraries ppx-runtime) + > (ppx.driver (main Hello_ppx.main))) + > EOF + + $ cat >main.ml < let () = print_int Foo.x + > EOF + $ cat >dune <<'EOF' > (rule + > (target main.exe) + > (deps + > main.ml + > (package foo)) + > (action + > (run + > %{bin:ocamlfind} + > ocamlc + > -custom + > -package + > foo + > -linkpkg + > -o + > %{target} + > main.ml))) + > (rule + > (targets main.bc stubs-result) + > (deps + > main.ml + > (package foo)) + > (action + > (progn + > (run %{bin:ocamlfind} ocamlc -package foo -linkpkg -o main.bc main.ml) + > (with-stdout-to stubs-result (run %{bin:ocamlrun} main.bc))))) + > (rule + > (target redirect) > (deps (package foo)) - > (action (with-stdout-to out (echo "ok")))) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query bar.old)))) + > (rule + > (target namespace-marker) + > (deps (package foo)) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query -format "%(top_marker)" namespace)))) + > (rule + > (target ppx-runtime-marker) + > (deps (package test-ppx)) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query ppx-runtime)))) + > (rule + > (target root-redirect) + > (deps (package redirect-root)) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query -recursive redirect-root.old)))) > EOF - $ dune build out +The package dependency does not currently supply the library closure needed +for external OCaml tooling to compile and link against `foo`. + + $ dune build main.exe && _build/default/main.exe + File "dune", lines 1-16, characters 0-179: + 1 | (rule + 2 | (target main.exe) + 3 | (deps + .... + 14 | -o + 15 | %{target} + 16 | main.ml))) + ocamlfind: Package `bar' not found - required by `foo' + [1] + $ dune build stubs-result && cat _build/default/stubs-result + File "dune", lines 17-25, characters 0-229: + 17 | (rule + 18 | (targets main.bc stubs-result) + 19 | (deps + 20 | main.ml + 21 | (package foo)) + 22 | (action + 23 | (progn + 24 | (run %{bin:ocamlfind} ocamlc -package foo -linkpkg -o main.bc main.ml) + 25 | (with-stdout-to stubs-result (run %{bin:ocamlrun} main.bc))))) + ocamlfind: Package `bar' not found - required by `foo' + [1] + $ dune build redirect + File "dune", lines 26-31, characters 0-126: + 26 | (rule + 27 | (target redirect) + 28 | (deps (package foo)) + 29 | (action + 30 | (with-stdout-to %{target} + 31 | (run %{bin:ocamlfind} query bar.old)))) + ocamlfind: Package `bar.old' not found + [1] + $ dune build namespace-marker && test -z "$(cat _build/default/namespace-marker)" + File "dune", lines 32-37, characters 0-160: + 32 | (rule + 33 | (target namespace-marker) + 34 | (deps (package foo)) + 35 | (action + 36 | (with-stdout-to %{target} + 37 | (run %{bin:ocamlfind} query -format "%(top_marker)" namespace)))) + ocamlfind: Package `namespace' not found + [1] + $ dune build ppx-runtime-marker + File "dune", lines 38-43, characters 0-145: + 38 | (rule + 39 | (target ppx-runtime-marker) + 40 | (deps (package test-ppx)) + 41 | (action + 42 | (with-stdout-to %{target} + 43 | (run %{bin:ocamlfind} query ppx-runtime)))) + ocamlfind: Package `ppx-runtime' not found + [1] + $ dune build root-redirect + File "dune", lines 44-49, characters 0-162: + 44 | (rule + 45 | (target root-redirect) + 46 | (deps (package redirect-root)) + 47 | (action + 48 | (with-stdout-to %{target} + 49 | (run %{bin:ocamlfind} query -recursive redirect-root.old)))) + ocamlfind: Package `redirect-target' not found - required by `redirect-root.old' + [1] -Only foo appears, neither bar nor baz, even though foo declares -(depends bar) and bar declares (depends baz): +The same missing closure is visible to a nested Dune invocation: the +materialized `foo` metadata names `namespace.selected`, but its package is +absent from the layout. - $ dune rules --format=json _build/default/out | jq_dune '.[] | ruleDepFilePaths' | censor | grep dune-package | sort + $ mkdir consumer + $ cat >consumer/dune-project < (lang dune 3.24) + > EOF + + $ cat >consumer/dune < (executable + > (name main) + > (libraries foo virtual-root)) + > EOF + + $ cat >consumer/main.ml < let () = print_int (Foo.x + Virtual_root.value) + > EOF + + $ cat >>dune <<'EOF' + > (rule + > (target dune-package-result) + > (deps + > (package foo) + > (package virtual-root) + > (source_tree consumer)) + > (action + > (with-stdout-to %{target} + > (chdir consumer (run %{bin:dune} exec ./main.exe))))) + > EOF + + $ dune build dune-package-result 2>&1 | censor + File "$PWD/_build/install/default/.packages/$DIGEST/lib/foo/dune-package", line 14, characters 15-33: + 14 | (requires bar namespace.selected) + ^^^^^^^^^^^^^^^^^^ + Error: Library "namespace.selected" not found. + -> required by library "foo" in + $PWD/_build/install/default/.packages/$DIGEST/lib/foo + -> required by executable main in dune:2 + -> required by _build/default/.main.eobjs/native/dune__exe__Main.cmx + -> required by _build/default/main.exe + [1] + +The current layout contains only `foo`. It contains neither the libraries in +its library closure nor `package-only-dep` from package metadata. + + $ dune rules --format=json _build/default/main.exe | + > jq_dune '.[] | ruleDepFilePaths' | + > censor | + > grep dune-package | + > sort "_build/install/default/.packages/$DIGEST/lib/foo/dune-package" + +The required libraries' compiled interfaces are consequently not tracked. + + $ dune rules --format=json _build/default/main.exe | + > jq_dune '.[] | ruleDepFilePaths' | + > censor | + > grep -E 'lib/(bar/bar|bar/__private__/bar_private/.public_cmi/bar_private|baz/baz)\.cmi' | + > sort + [1] + +No artifact belonging to the unrelated sibling is a dependency of the action. + + $ dune rules --format=json _build/default/main.exe | + > jq_dune '.[] | ruleDepFilePaths' | + > grep bar_unrelated + [1] + +The unrelated library from package `bar` is not discoverable. + + $ cat >>dune <<'EOF' + > (rule + > (target unrelated) + > (deps (package foo)) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query bar.unrelated)))) + > EOF + + $ dune build unrelated + File "dune", lines 59-64, characters 0-133: + 59 | (rule + 60 | (target unrelated) + 61 | (deps (package foo)) + 62 | (action + 63 | (with-stdout-to %{target} + 64 | (run %{bin:ocamlfind} query bar.unrelated)))) + ocamlfind: Package `bar.unrelated' not found + [1] From 0b86f0490eea2a1188a31a015b0e37869390c9e3 Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 17 Aug 2026 22:06:10 +0200 Subject: [PATCH 03/14] test(package-deps): snapshot the missing Melange closure Record that a scoped package layout retains melange_requires metadata without materializing its referenced workspace library. Signed-off-by: Ali Caglayan --- .../test-cases/package-materialization/dune | 4 + .../package-materialization/melange-closure.t | 75 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 test/blackbox-tests/test-cases/package-materialization/melange-closure.t diff --git a/test/blackbox-tests/test-cases/package-materialization/dune b/test/blackbox-tests/test-cases/package-materialization/dune index a4dde991d06..43fbead5bf0 100644 --- a/test/blackbox-tests/test-cases/package-materialization/dune +++ b/test/blackbox-tests/test-cases/package-materialization/dune @@ -5,3 +5,7 @@ ocamlfind virtual-without-default) (deps %{bin:ocamlfind})) + +(cram + (applies_to melange-closure) + (deps %{bin:melc})) diff --git a/test/blackbox-tests/test-cases/package-materialization/melange-closure.t b/test/blackbox-tests/test-cases/package-materialization/melange-closure.t new file mode 100644 index 00000000000..9add7d61cd0 --- /dev/null +++ b/test/blackbox-tests/test-cases/package-materialization/melange-closure.t @@ -0,0 +1,75 @@ +A scoped package layout retains a library's Melange-only requirements in its +`dune-package` file, so it must materialize the corresponding Melange closure. + + $ make_dune_project 3.24 + $ cat >>dune-project <<'EOF' + > (using melange 0.1) + > (package (name melange-root)) + > (package (name melange-support)) + > EOF + + $ mkdir root support consumer + $ cat >support/dune <<'EOF' + > (library + > (name melange_support) + > (public_name melange-support) + > (modes melange)) + > EOF + $ echo 'let value = 42' >support/melange_support.ml + + $ cat >root/dune <<'EOF' + > (library + > (name melange_root) + > (public_name melange-root) + > (modes melange) + > (melange.libraries melange-support)) + > EOF + $ echo 'let value = Melange_support.value' >root/melange_root.ml + + $ cat >consumer/dune-project <<'EOF' + > (lang dune 3.24) + > (using melange 0.1) + > EOF + $ cat >consumer/dune <<'EOF' + > (melange.emit + > (target out) + > (emit_stdlib false) + > (modules main) + > (libraries melange-root)) + > EOF + $ echo 'let () = Js.log Melange_root.value' >consumer/main.ml + + $ cat >dune <<'EOF' + > (rule + > (target result) + > (deps + > (package melange-root) + > (source_tree consumer)) + > (action + > (with-stdout-to %{target} + > (chdir consumer (run %{bin:dune} build @melange))))) + > EOF + +The package-only layout leaves the serialized `melange_requires` edge pointing +at a library that is absent from the layout. + + $ dune build result 2>err + [1] + $ censor required by library "melange-root" in + $PWD/_build/install/default/.packages/$DIGEST/lib/melange-root + -> required by melange target out + -> required by alias melange + File "$PWD/_build/install/default/.packages/$DIGEST/lib/melange-root/dune-package", line 15, characters 19-34: + 15 | (melange_requires melange-support) + ^^^^^^^^^^^^^^^ + Error: Library "melange-support" not found. + -> required by melange target out + -> required by library "melange-root" in + $PWD/_build/install/default/.packages/$DIGEST/lib/melange-root + -> required by _build/default/out/main.js + -> required by alias melange From 49ceb5716e5c26f3dcaa26896fb79f36a46c2037 Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 17 Aug 2026 20:28:31 +0200 Subject: [PATCH 04/14] test(package-deps): snapshot multi-hop library redirects Record that a scoped package dependency omits the intermediate metadata in a deprecated-library redirect chain. Signed-off-by: Ali Caglayan --- .../test-cases/package-materialization/dune | 1 + .../package-materialization/redirect-chain.t | 82 +++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 test/blackbox-tests/test-cases/package-materialization/redirect-chain.t diff --git a/test/blackbox-tests/test-cases/package-materialization/dune b/test/blackbox-tests/test-cases/package-materialization/dune index a4dde991d06..85aeedad8bc 100644 --- a/test/blackbox-tests/test-cases/package-materialization/dune +++ b/test/blackbox-tests/test-cases/package-materialization/dune @@ -3,5 +3,6 @@ installed-package no-transitive-through-targets ocamlfind + redirect-chain virtual-without-default) (deps %{bin:ocamlfind})) diff --git a/test/blackbox-tests/test-cases/package-materialization/redirect-chain.t b/test/blackbox-tests/test-cases/package-materialization/redirect-chain.t new file mode 100644 index 00000000000..0c9fe706d15 --- /dev/null +++ b/test/blackbox-tests/test-cases/package-materialization/redirect-chain.t @@ -0,0 +1,82 @@ +A deprecated-library redirect chain needs metadata for every intermediate name. +A scoped package dependency currently materializes only the explicitly named +package. + + $ make_dune_project 3.24 + $ cat >>dune-project < (package (name redirect-root)) + > (package (name redirect-middle)) + > (package (name redirect-target)) + > EOF + + $ mkdir root middle target middle-unrelated target-unrelated + $ cat >root/dune <<'EOF' + > (deprecated_library_name + > (old_public_name redirect-root.old) + > (new_public_name redirect-middle.old)) + > EOF + + $ cat >middle/dune <<'EOF' + > (deprecated_library_name + > (old_public_name redirect-middle.old) + > (new_public_name redirect-target)) + > EOF + + $ cat >target/dune <<'EOF' + > (library + > (name redirect_target) + > (public_name redirect-target)) + > EOF + $ echo 'let value = 42' >target/redirect_target.ml + + $ cat >middle-unrelated/dune <<'EOF' + > (library + > (name middle_unrelated) + > (public_name redirect-middle.unrelated)) + > EOF + $ echo 'let value = ()' >middle-unrelated/unrelated.ml + + $ cat >target-unrelated/dune <<'EOF' + > (library + > (name target_unrelated) + > (public_name redirect-target.unrelated)) + > EOF + $ echo 'let value = ()' >target-unrelated/unrelated.ml + + $ cat >dune <<'EOF' + > (rule + > (target recursive) + > (deps (package redirect-root)) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query -recursive redirect-root.old)))) + > (rule + > (target no-middle-unrelated) + > (deps (package redirect-root)) + > (action + > (progn + > (bash "! %{bin:ocamlfind} query redirect-middle.unrelated >/dev/null 2>&1") + > (write-file %{target} "")))) + > (rule + > (target no-target-unrelated) + > (deps (package redirect-root)) + > (action + > (progn + > (bash "! %{bin:ocamlfind} query redirect-target.unrelated >/dev/null 2>&1") + > (write-file %{target} "")))) + > EOF + + $ dune build recursive + File "dune", lines 1-6, characters 0-158: + 1 | (rule + 2 | (target recursive) + 3 | (deps (package redirect-root)) + 4 | (action + 5 | (with-stdout-to %{target} + 6 | (run %{bin:ocamlfind} query -recursive redirect-root.old)))) + ocamlfind: Package `redirect-middle.old' not found - required by `redirect-root.old' + [1] + +Unrelated siblings remain absent from the scoped layout. + + $ dune build no-middle-unrelated no-target-unrelated From 37604d01c719fb0d58f0bd62417545850bea884a Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 17 Aug 2026 22:08:37 +0200 Subject: [PATCH 05/14] test(package-deps): snapshot deprecated-package redirects Record that a support redirect owned by another package omits metadata generated for its deprecated package name. Signed-off-by: Ali Caglayan --- .../deprecated-package-redirect.t | 84 +++++++++++++++++++ .../test-cases/package-materialization/dune | 1 + 2 files changed, 85 insertions(+) create mode 100644 test/blackbox-tests/test-cases/package-materialization/deprecated-package-redirect.t diff --git a/test/blackbox-tests/test-cases/package-materialization/deprecated-package-redirect.t b/test/blackbox-tests/test-cases/package-materialization/deprecated-package-redirect.t new file mode 100644 index 00000000000..a413e501880 --- /dev/null +++ b/test/blackbox-tests/test-cases/package-materialization/deprecated-package-redirect.t @@ -0,0 +1,84 @@ +A selected redirect can belong to a deprecated package name. Its owning package +must then contribute the separate META and dune-package files generated under +that deprecated name. + + $ make_dune_project 3.24 + $ cat >>dune-project <<'EOF' + > (package (name redirect-root)) + > (package + > (name redirect-owner) + > (deprecated_package_names old-support)) + > (package (name redirect-target)) + > EOF + + $ mkdir root owner target consumer + $ cat >root/dune <<'EOF' + > (deprecated_library_name + > (old_public_name redirect-root.old) + > (new_public_name old-support.lib)) + > EOF + + $ cat >owner/dune <<'EOF' + > (deprecated_library_name + > (old_public_name old-support.lib) + > (new_public_name redirect-target)) + > EOF + + $ cat >target/dune <<'EOF' + > (library + > (name redirect_target) + > (public_name redirect-target)) + > EOF + $ echo 'let value = 42' >target/redirect_target.ml + + $ cat >consumer/dune-project <<'EOF' + > (lang dune 3.24) + > EOF + $ cat >consumer/dune <<'EOF' + > (executable + > (name main) + > (libraries redirect-root.old)) + > EOF + $ echo 'let () = print_int Redirect_target.value' >consumer/main.ml + + $ cat >dune <<'EOF' + > (rule + > (target result) + > (deps (package redirect-root)) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query -recursive redirect-root.old)))) + > (rule + > (target dune-package-result) + > (deps + > (package redirect-root) + > (source_tree consumer)) + > (action + > (with-stdout-to %{target} + > (chdir consumer (run %{bin:dune} exec ./main.exe))))) + > EOF + +The ordinary package metadata can name the intermediate redirect, but the +scoped layout currently has no metadata directory for its deprecated package. + + $ dune build result + File "dune", lines 1-6, characters 0-155: + 1 | (rule + 2 | (target result) + 3 | (deps (package redirect-root)) + 4 | (action + 5 | (with-stdout-to %{target} + 6 | (run %{bin:ocamlfind} query -recursive redirect-root.old)))) + ocamlfind: Package `old-support.lib' not found - required by `redirect-root.old' + [1] + +The nested Dune consumer likewise cannot resolve the intermediate package. + + $ dune build dune-package-result + File "dune", line 3, characters 12-29: + 3 | (libraries redirect-root.old)) + ^^^^^^^^^^^^^^^^^ + Error: Library "redirect-root.old" not found. + -> required by _build/default/.main.eobjs/native/dune__exe__Main.cmx + -> required by _build/default/main.exe + [1] diff --git a/test/blackbox-tests/test-cases/package-materialization/dune b/test/blackbox-tests/test-cases/package-materialization/dune index a4dde991d06..66c83799e7f 100644 --- a/test/blackbox-tests/test-cases/package-materialization/dune +++ b/test/blackbox-tests/test-cases/package-materialization/dune @@ -1,5 +1,6 @@ (cram (applies_to + deprecated-package-redirect installed-package no-transitive-through-targets ocamlfind From 14122c9c01879c0c564fb783f1cf92f20c140ffc Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 17 Aug 2026 21:00:39 +0200 Subject: [PATCH 06/14] test(package-deps): snapshot PPX driver replacement metadata Record that a scoped package dependency omits the library referenced by a serialized PPX driver replacement edge. Signed-off-by: Ali Caglayan --- .../ppx-subsystem-dependencies.t | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 test/blackbox-tests/test-cases/package-materialization/ppx-subsystem-dependencies.t diff --git a/test/blackbox-tests/test-cases/package-materialization/ppx-subsystem-dependencies.t b/test/blackbox-tests/test-cases/package-materialization/ppx-subsystem-dependencies.t new file mode 100644 index 00000000000..868c9bfa6c3 --- /dev/null +++ b/test/blackbox-tests/test-cases/package-materialization/ppx-subsystem-dependencies.t @@ -0,0 +1,81 @@ +PPX driver metadata serializes replacement edges that are not ordinary library +requirements. A scoped package dependency currently omits the replaced driver. + + $ make_dune_project 3.24 + $ cat >>dune-project < (package (name ppx-replacer)) + > (package (name ppx-original)) + > EOF + + $ mkdir replacer original consumer + $ cat >replacer/dune <<'EOF' + > (library + > (name replacer) + > (public_name ppx-replacer) + > (kind ppx_rewriter) + > (libraries compiler-libs.common) + > (ppx.driver + > (main "Replacer.main") + > (replaces ppx-original))) + > EOF + $ cat >replacer/replacer.ml <<'EOF' + > let main () = + > let output = ref None in + > let input = ref None in + > for i = 1 to Array.length Sys.argv - 2 do + > match Sys.argv.(i) with + > | "-o" -> output := Some Sys.argv.(i + 1) + > | "--impl" -> input := Some Sys.argv.(i + 1) + > | _ -> () + > done; + > match !input, !output with + > | Some input, Some output -> + > let ic = open_in_bin input in + > let oc = open_out_bin output in + > Fun.protect + > ~finally:(fun () -> close_in ic; close_out oc) + > (fun () -> output_string oc (really_input_string ic (in_channel_length ic))) + > | _ -> exit 2 + > EOF + + $ cat >original/dune <<'EOF' + > (library + > (name original) + > (public_name ppx-original) + > (kind ppx_rewriter) + > (libraries compiler-libs.common) + > (modules ()) + > (ppx.driver (main "(fun () -> Array.iter print_endline Sys.argv)"))) + > EOF + + $ cat >consumer/dune-project <<'EOF' + > (lang dune 3.24) + > EOF + $ cat >consumer/dune <<'EOF' + > (library + > (name user) + > (preprocess (pps ppx-replacer))) + > EOF + $ echo 'let value = ()' >consumer/user.ml + + $ cat >dune <<'EOF' + > (rule + > (target result) + > (deps + > (package ppx-replacer) + > (source_tree consumer)) + > (action + > (with-stdout-to %{target} + > (chdir consumer (run %{bin:dune} build user.cma))))) + > EOF + + $ dune build result 2>err + [1] + $ censor required by _build/default/user.pp.ml + -> required by _build/default/.user.objs/byte/user.cmo + -> required by _build/default/user.cma From a1d307518bfa15143a2acface52f511aa3946cc4 Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 17 Aug 2026 20:30:21 +0200 Subject: [PATCH 07/14] test(package-deps): snapshot serialized subsystem dependencies Record that scoped package metadata can reference an inline-test runner library outside the traversed OCaml requires closure. Signed-off-by: Ali Caglayan --- .../subsystem-dependencies.t | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 test/blackbox-tests/test-cases/package-materialization/subsystem-dependencies.t diff --git a/test/blackbox-tests/test-cases/package-materialization/subsystem-dependencies.t b/test/blackbox-tests/test-cases/package-materialization/subsystem-dependencies.t new file mode 100644 index 00000000000..1171230fdf1 --- /dev/null +++ b/test/blackbox-tests/test-cases/package-materialization/subsystem-dependencies.t @@ -0,0 +1,63 @@ +Installed dune-package metadata can refer to libraries through public subsystem +fields that are not ordinary `requires`. A scoped package dependency currently +omits those referenced libraries. + + $ make_dune_project 3.24 + $ cat >>dune-project < (package (name subsystem-root)) + > (package (name subsystem-runner)) + > EOF + + $ mkdir backend runner consumer + $ cat >backend/dune <<'EOF' + > (library + > (name backend) + > (public_name subsystem-root.backend) + > (modules ()) + > (inline_tests.backend + > (runner_libraries subsystem-runner) + > (generate_runner (echo "let () = print_endline Runner.message")))) + > EOF + + $ cat >runner/dune <<'EOF' + > (library + > (name runner) + > (public_name subsystem-runner)) + > EOF + $ cat >runner/runner.ml <<'EOF' + > let message = "subsystem dependency" + > EOF + + $ cat >consumer/dune-project <<'EOF' + > (lang dune 3.24) + > EOF + $ cat >consumer/dune <<'EOF' + > (library + > (name tested) + > (inline_tests (backend subsystem-root.backend))) + > EOF + $ echo 'let value = ()' >consumer/tested.ml + + $ cat >dune <<'EOF' + > (rule + > (target result) + > (deps + > (package subsystem-root) + > (source_tree consumer)) + > (action + > (with-stdout-to %{target} + > (chdir consumer (run %{bin:dune} runtest))))) + > EOF + + $ dune build result 2>err + [1] + $ censor required by + _build/default/.tested.inline-tests/.t.eobjs/native/dune__exe__Main.cmx + -> required by _build/default/.tested.inline-tests/inline-test-runner.exe + -> required by alias runtest-tested in dune:3 + -> required by alias runtest in dune:1 From 81f40b742341896208edba6becb50127e2825915 Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Sun, 23 Aug 2026 16:57:12 +0200 Subject: [PATCH 08/14] refactor(package-deps): resolve library support with install rules Move scoped library-closure resolution behind the install-layout resolver so metadata-specific closure edges can be added without introducing dependencies from dependency evaluation back to rule-generation modules. Signed-off-by: Ali Caglayan --- src/dune_rules/dep_conf_eval.ml | 134 +----------------------------- src/dune_rules/install_layout.ml | 15 +++- src/dune_rules/install_layout.mli | 13 +-- src/dune_rules/install_rules.ml | 125 ++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 141 deletions(-) diff --git a/src/dune_rules/dep_conf_eval.ml b/src/dune_rules/dep_conf_eval.ml index 87465553894..520abd1586f 100644 --- a/src/dune_rules/dep_conf_eval.ml +++ b/src/dune_rules/dep_conf_eval.ml @@ -221,130 +221,6 @@ let package loc pkg_name (context : Build_context.t) ~dune_version = } ;; -let library_closure lib = - let open Memo.O in - Memo.parallel_map [ Compilation_mode.Ocaml; Melange ] ~f:(fun for_ -> - let* compile_closure = - Lib.closure [ lib ] ~linking:false ~for_ |> Resolve.Memo.read_memo - and* link_closure = - Lib.partial_link_closure [ lib ] ~for_ |> Resolve.Memo.read_memo - in - Memo.return (List.rev_append compile_closure link_closure)) - >>| List.concat - >>| Lib.Set.of_list - >>| Lib.Set.to_list -;; - -let workspace_redirects context = - let open Memo.O in - let* workspace_packages = Dune_load.packages () in - Package.Name.Map.keys workspace_packages - |> Memo.parallel_map ~f:(fun package -> - let+ { Scope.DB.Lib_entry.Set.deprecated_library_names; _ } = - Scope.DB.lib_entries_of_package context package - in - List.map - deprecated_library_names - ~f:(fun { Library_redirect.old_name; new_public_name; _ } -> - Public_lib.name (fst old_name), (package, snd new_public_name))) - >>| List.concat - >>| Lib_name.Map.of_list_reduce ~f:(fun redirect _ -> redirect) -;; - -let library_support_closure context packages = - let open Memo.O in - let* public_libs = Scope.DB.public_libs context - and* redirects_by_name = workspace_redirects context in - let collect_redirects name = - let rec loop name seen redirects = - if Lib_name.Set.mem seen name - then redirects - else ( - let seen = Lib_name.Set.add seen name in - match Lib_name.Map.find redirects_by_name name with - | None -> redirects - | Some (package, target) -> - let redirects = - if Package.Name.Set.mem packages package - then redirects - else - Install_layout.Redirect.Set.add - redirects - (Install_layout.Redirect.make ~package ~name) - in - loop target seen redirects) - in - loop name Lib_name.Set.empty Install_layout.Redirect.Set.empty - in - let* roots, redirects = - Package.Name.Set.to_list packages - |> Memo.parallel_map ~f:(fun package -> - let* { Scope.DB.Lib_entry.Set.libraries; deprecated_library_names } = - Scope.DB.lib_entries_of_package context package - in - let+ redirect_targets = - Memo.parallel_map deprecated_library_names ~f:(fun { new_public_name; _ } -> - let+ target = - Lib.DB.resolve public_libs new_public_name |> Resolve.Memo.read_memo - in - target, collect_redirects (snd new_public_name)) - in - ( List.rev_append - (List.rev_map libraries ~f:Lib.Local.to_lib) - (List.map redirect_targets ~f:fst) - , List.fold_left - redirect_targets - ~init:Install_layout.Redirect.Set.empty - ~f:(fun redirects (_, selected) -> - Install_layout.Redirect.Set.union redirects selected) )) - >>| List.split - >>| fun (roots, redirects) -> - ( List.concat roots - , List.fold_left - redirects - ~init:Install_layout.Redirect.Set.empty - ~f:Install_layout.Redirect.Set.union ) - in - let extra_dependencies libraries = - Memo.parallel_map libraries ~f:(fun lib -> - if Lib.is_local lib - then - Memo.parallel_map [ Compilation_mode.Ocaml; Melange ] ~f:(fun for_ -> - Lib.ppx_runtime_deps lib ~for_ |> Resolve.Memo.read_memo) - >>| List.concat - else Memo.return []) - >>| List.concat - in - let rec loop todo expanded inspected libraries = - match todo with - | [] -> Memo.return libraries - | lib :: todo -> - if Lib.Set.mem expanded lib - then loop todo expanded inspected libraries - else - let* closure = library_closure lib in - let expanded = Lib.Set.add expanded lib in - let uninspected = - List.filter closure ~f:(fun lib -> not (Lib.Set.mem inspected lib)) - in - let inspected = List.fold_left uninspected ~init:inspected ~f:Lib.Set.add in - let libraries = List.fold_left closure ~init:libraries ~f:Lib.Set.add in - let* extra_dependencies = extra_dependencies uninspected in - loop (List.rev_append extra_dependencies todo) expanded inspected libraries - in - let+ closure = loop roots Lib.Set.empty Lib.Set.empty Lib.Set.empty in - let libraries = - Lib.Set.fold closure ~init:Install_layout.Library.Set.empty ~f:(fun lib libraries -> - match Lib.is_local lib, Lib_info.package (Lib.info lib) with - | true, Some package when not (Package.Name.Set.mem packages package) -> - Install_layout.Library.Set.add - libraries - (Install_layout.Library.make ~package ~name:(Lib.name lib)) - | _ -> libraries) - in - libraries, redirects -;; - let rec dep expander : Dep_conf.t -> _ = function | Include s -> (* TODO this is wrong. we shouldn't allow bindings here if we are in an @@ -463,15 +339,7 @@ and combined_package_deps_builder expander pkgs = let* env = if Package.Name.Set.is_empty local_package_names then Action_builder.return Env.empty - else - let* support_libraries, support_redirects = - Action_builder.of_memo (library_support_closure context.name local_package_names) - in - Install_layout.env - context.name - local_package_names - support_libraries - support_redirects + else Install_layout.env context.name local_package_names in let dune_version = Expander.project expander |> Dune_project.dune_version in let+ () = diff --git a/src/dune_rules/install_layout.ml b/src/dune_rules/install_layout.ml index f1b55c12840..1a84cda7e36 100644 --- a/src/dune_rules/install_layout.ml +++ b/src/dune_rules/install_layout.ml @@ -38,6 +38,11 @@ end module Redirect = Library +type support = + { libraries : Library.Set.t + ; redirects : Redirect.Set.t + } + type request = { packages : Package.Name.Set.t ; libraries : Library.Set.t @@ -127,6 +132,7 @@ end type resolvers = { package_entries : Context_name.t -> Package.Name.t -> Install.Entry.Sourced.Unexpanded.t list Memo.t + ; library_support : Context_name.t -> Package.Name.Set.t -> support Memo.t ; library_entries : Context_name.t -> Library.Set.t -> Redirect.Set.t -> library_entries Memo.t } @@ -159,7 +165,7 @@ let compute_entries context_name root { packages; libraries; redirects } = ; "redirects", Dyn.list Redirect.to_dyn overlapping_redirects ]; let open Memo.O in - let { package_entries; library_entries } = Fdecl.get resolvers_fdecl in + let { package_entries; library_entries; _ } = Fdecl.get resolvers_fdecl in let resolve_entry (pkg, (s : Install.Entry.Sourced.Unexpanded.t)) = let install_paths = let roots = Install.Roots.opam_from_prefix Path.root ~relative:Path.relative in @@ -279,7 +285,12 @@ let env_for_request context_name request = Install.Roots.add_to_env roots Env.empty ;; -let env context_name packages libraries redirects = +let env context_name packages = + let open Action_builder.O in + let { library_support; _ } = Fdecl.get resolvers_fdecl in + let* { libraries; redirects } = + Action_builder.of_memo (library_support context_name packages) + in env_for_request context_name { packages; libraries; redirects } ;; diff --git a/src/dune_rules/install_layout.mli b/src/dune_rules/install_layout.mli index 3ac0db00d84..2001730341f 100644 --- a/src/dune_rules/install_layout.mli +++ b/src/dune_rules/install_layout.mli @@ -15,6 +15,11 @@ end module Redirect = Library +type support = + { libraries : Library.Set.t + ; redirects : Redirect.Set.t + } + type generated_entry = { package : Package.Name.t ; section : Section.t @@ -30,6 +35,7 @@ type library_entries = type resolvers = { package_entries : Context_name.t -> Package.Name.t -> Install.Entry.Sourced.Unexpanded.t list Memo.t + ; library_support : Context_name.t -> Package.Name.Set.t -> support Memo.t ; library_entries : Context_name.t -> Library.Set.t -> Redirect.Set.t -> library_entries Memo.t } @@ -42,12 +48,7 @@ val set_resolvers : resolvers -> unit entry the layout produces. Only the selected support libraries' install entries and metadata are included; their owning packages' other entries are not. *) -val env - : Context_name.t - -> Package.Name.Set.t - -> Library.Set.t - -> Redirect.Set.t - -> Env.t Action_builder.t +val env : Context_name.t -> Package.Name.Set.t -> Env.t Action_builder.t (** Engine dispatch for [_build/install//.packages/]. Called from [Gen_rules]; the layout dir is owned by this module. Resolves to: diff --git a/src/dune_rules/install_rules.ml b/src/dune_rules/install_rules.ml index c1238f9c907..a3f0cf68047 100644 --- a/src/dune_rules/install_rules.ml +++ b/src/dune_rules/install_rules.ml @@ -1369,6 +1369,130 @@ let install_entries sctx package = Package.Name.Map.Multi.find packages package ;; +let library_closure lib = + let open Memo.O in + Memo.parallel_map [ Compilation_mode.Ocaml; Melange ] ~f:(fun for_ -> + let* compile_closure = + Lib.closure [ lib ] ~linking:false ~for_ |> Resolve.Memo.read_memo + and* link_closure = + Lib.partial_link_closure [ lib ] ~for_ |> Resolve.Memo.read_memo + in + Memo.return (List.rev_append compile_closure link_closure)) + >>| List.concat + >>| Lib.Set.of_list + >>| Lib.Set.to_list +;; + +let workspace_redirects context = + let open Memo.O in + let* workspace_packages = Dune_load.packages () in + Package.Name.Map.keys workspace_packages + |> Memo.parallel_map ~f:(fun package -> + let+ { Scope.DB.Lib_entry.Set.deprecated_library_names; _ } = + Scope.DB.lib_entries_of_package context package + in + List.map + deprecated_library_names + ~f:(fun { Library_redirect.old_name; new_public_name; _ } -> + Public_lib.name (fst old_name), (package, snd new_public_name))) + >>| List.concat + >>| Lib_name.Map.of_list_reduce ~f:(fun redirect _ -> redirect) +;; + +let library_support_closure context packages = + let open Memo.O in + let* public_libs = Scope.DB.public_libs context + and* redirects_by_name = workspace_redirects context in + let collect_redirects name = + let rec loop name seen redirects = + if Lib_name.Set.mem seen name + then redirects + else ( + let seen = Lib_name.Set.add seen name in + match Lib_name.Map.find redirects_by_name name with + | None -> redirects + | Some (package, target) -> + let redirects = + if Package.Name.Set.mem packages package + then redirects + else + Install_layout.Redirect.Set.add + redirects + (Install_layout.Redirect.make ~package ~name) + in + loop target seen redirects) + in + loop name Lib_name.Set.empty Install_layout.Redirect.Set.empty + in + let* roots, redirects = + Package.Name.Set.to_list packages + |> Memo.parallel_map ~f:(fun package -> + let* { Scope.DB.Lib_entry.Set.libraries; deprecated_library_names } = + Scope.DB.lib_entries_of_package context package + in + let+ redirect_targets = + Memo.parallel_map deprecated_library_names ~f:(fun { new_public_name; _ } -> + let+ target = + Lib.DB.resolve public_libs new_public_name |> Resolve.Memo.read_memo + in + target, collect_redirects (snd new_public_name)) + in + ( List.rev_append + (List.rev_map libraries ~f:Lib.Local.to_lib) + (List.map redirect_targets ~f:fst) + , List.fold_left + redirect_targets + ~init:Install_layout.Redirect.Set.empty + ~f:(fun redirects (_, selected) -> + Install_layout.Redirect.Set.union redirects selected) )) + >>| List.split + >>| fun (roots, redirects) -> + ( List.concat roots + , List.fold_left + redirects + ~init:Install_layout.Redirect.Set.empty + ~f:Install_layout.Redirect.Set.union ) + in + let extra_dependencies libraries = + Memo.parallel_map libraries ~f:(fun lib -> + if Lib.is_local lib + then + Memo.parallel_map [ Compilation_mode.Ocaml; Melange ] ~f:(fun for_ -> + Lib.ppx_runtime_deps lib ~for_ |> Resolve.Memo.read_memo) + >>| List.concat + else Memo.return []) + >>| List.concat + in + let rec loop todo expanded inspected libraries = + match todo with + | [] -> Memo.return libraries + | lib :: todo -> + if Lib.Set.mem expanded lib + then loop todo expanded inspected libraries + else + let* closure = library_closure lib in + let expanded = Lib.Set.add expanded lib in + let uninspected = + List.filter closure ~f:(fun lib -> not (Lib.Set.mem inspected lib)) + in + let inspected = List.fold_left uninspected ~init:inspected ~f:Lib.Set.add in + let libraries = List.fold_left closure ~init:libraries ~f:Lib.Set.add in + let* extra_dependencies = extra_dependencies uninspected in + loop (List.rev_append extra_dependencies todo) expanded inspected libraries + in + let+ closure = loop roots Lib.Set.empty Lib.Set.empty Lib.Set.empty in + let libraries = + Lib.Set.fold closure ~init:Install_layout.Library.Set.empty ~f:(fun lib libraries -> + match Lib.is_local lib, Lib_info.package (Lib.info lib) with + | true, Some package when not (Package.Name.Set.mem packages package) -> + Install_layout.Library.Set.add + libraries + (Install_layout.Library.make ~package ~name:(Lib.name lib)) + | _ -> libraries) + in + { Install_layout.libraries; redirects } +;; + let library_install_entries sctx libraries redirects = let* entries_by_library = Stanzas_to_entries.library_entries sctx in let selected_libraries = @@ -1461,6 +1585,7 @@ let () = let open Memo.O in let* sctx = Super_context.find_exn context_name in install_entries sctx package) + ; library_support = library_support_closure ; library_entries = (fun context_name libraries redirects -> let open Memo.O in From af80bc1052f5175deeddbed72bbc91fbfd5664b0 Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 17 Aug 2026 10:06:34 +0200 Subject: [PATCH 09/14] test(meta): snapshot malformed rendered template acceptance A META file template can parse before expansion and become malformed when a later generation marker is replaced. Snapshot that behavior and the presence of multiple markers. Signed-off-by: Ali Caglayan --- .../test-cases/meta-file/rendered-template.t | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 test/blackbox-tests/test-cases/meta-file/rendered-template.t diff --git a/test/blackbox-tests/test-cases/meta-file/rendered-template.t b/test/blackbox-tests/test-cases/meta-file/rendered-template.t new file mode 100644 index 00000000000..6f80f765b1a --- /dev/null +++ b/test/blackbox-tests/test-cases/meta-file/rendered-template.t @@ -0,0 +1,34 @@ +A META file template may parse before expansion but become invalid when generated +entries replace a later marker. This is currently accepted. + + $ make_dune_project_with_package 2.7 rendered + + $ cat >rendered.ml < let foo () = () + > EOF + + $ cat >dune < (library + > (public_name rendered) + > (synopsis "generated marker")) + > EOF + + $ cat >META.rendered.template <<'EOF' + > # DUNE_GEN + > package "broken" + > # DUNE_GEN + > ( + > ) + > EOF + + $ dune build @install + +Both markers were expanded, and the malformed literal was installed between +them. + + $ grep -c '^description = "generated marker"$' _build/default/META.rendered + 2 + $ grep '^package "broken"$' _build/default/META.rendered + package "broken" + $ awk '/^package "broken"$/ { getline; print; exit }' _build/default/META.rendered + description = "generated marker" From 8410f288a50162b8f59d586f4c1b2affdb27cadb Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 17 Aug 2026 10:06:34 +0200 Subject: [PATCH 10/14] test(meta): snapshot malformed promoted template acceptance A promoted META file template can differ from a stale source counterpart. Snapshot acceptance of the generated build contents when promotion is disabled. Signed-off-by: Ali Caglayan --- .../test-cases/meta-file/promoted-template.t | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 test/blackbox-tests/test-cases/meta-file/promoted-template.t diff --git a/test/blackbox-tests/test-cases/meta-file/promoted-template.t b/test/blackbox-tests/test-cases/meta-file/promoted-template.t new file mode 100644 index 00000000000..83c38c17c23 --- /dev/null +++ b/test/blackbox-tests/test-cases/meta-file/promoted-template.t @@ -0,0 +1,32 @@ +A promoted META file template may have a source counterpart that differs from +the rule output. The generated contents are currently accepted. + + $ make_dune_project_with_package 2.7 promoted + + $ cat >promoted.ml < let foo () = () + > EOF + + $ cat >META.promoted.template < # DUNE_GEN + > EOF + + $ cat >dune <<'EOF' + > (library + > (public_name promoted)) + > + > (rule + > (target META.promoted.template) + > (mode promote) + > (action + > (write-file %{target} "package \"broken\" @"))) + > EOF + + $ dune build @install --disable-promotion + +The source remains valid, while the generated template and final META contain +the malformed rule output. + + $ printf '# DUNE_GEN\n' | cmp - META.promoted.template + $ printf 'package "broken" @' | cmp - _build/default/META.promoted.template + $ printf 'package "broken" @' | cmp - _build/default/META.promoted From bb044a0298e49c05c6bbd582b4ea5fcdb9d90c81 Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 17 Aug 2026 20:24:57 +0200 Subject: [PATCH 11/14] test(package-deps): snapshot nested namespace filtering Record that a package dependency does not yet materialize a nested library namespace, including the intermediate template metadata that must be filtered once support metadata is generated. Signed-off-by: Ali Caglayan --- .../test-cases/package-materialization/dune | 1 + .../nested-namespace.t | 60 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 test/blackbox-tests/test-cases/package-materialization/nested-namespace.t diff --git a/test/blackbox-tests/test-cases/package-materialization/dune b/test/blackbox-tests/test-cases/package-materialization/dune index a4dde991d06..7d249107a64 100644 --- a/test/blackbox-tests/test-cases/package-materialization/dune +++ b/test/blackbox-tests/test-cases/package-materialization/dune @@ -1,6 +1,7 @@ (cram (applies_to installed-package + nested-namespace no-transitive-through-targets ocamlfind virtual-without-default) diff --git a/test/blackbox-tests/test-cases/package-materialization/nested-namespace.t b/test/blackbox-tests/test-cases/package-materialization/nested-namespace.t new file mode 100644 index 00000000000..d399b507f9d --- /dev/null +++ b/test/blackbox-tests/test-cases/package-materialization/nested-namespace.t @@ -0,0 +1,60 @@ +A package dependency currently omits a required library in a nested namespace. +Once the library is materialized, metadata from structural namespace nodes must +not leak into the filtered META file. + + $ make_dune_project 3.24 + $ cat >>dune-project < (package (name ns-root)) + > (package (name ns-support)) + > EOF + + $ mkdir root support + $ cat >root/dune <<'EOF' + > (library + > (name root) + > (public_name ns-root) + > (libraries ns-support.middle.selected)) + > EOF + $ echo 'let value = ()' >root/root.ml + + $ cat >support/dune <<'EOF' + > (library + > (name selected) + > (public_name ns-support.middle.selected)) + > EOF + $ echo 'let value = ()' >support/selected.ml + + $ cat >META.ns-support.template <<'EOF' + > package "middle" ( + > directory = "middle" + > intermediate_marker = "drop" + > package "selected" ( + > directory = "selected" + > selected_marker = "keep" + > ) + > ) + > EOF + + $ cat >dune <<'EOF' + > (rule + > (target marker) + > (deps (package ns-root)) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query + > -format "%(intermediate_marker)|%(selected_marker)|%d/selected.cmi" + > ns-support.middle.selected)))) + > EOF + + $ dune build marker + File "dune", lines 1-8, characters 0-223: + 1 | (rule + 2 | (target marker) + 3 | (deps (package ns-root)) + 4 | (action + 5 | (with-stdout-to %{target} + 6 | (run %{bin:ocamlfind} query + 7 | -format "%(intermediate_marker)|%(selected_marker)|%d/selected.cmi" + 8 | ns-support.middle.selected)))) + ocamlfind: Package `ns-support.middle.selected' not found + [1] From 3bcc722fe95764ff25d1ae2a71a0c5b337b900e5 Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 17 Aug 2026 22:09:34 +0200 Subject: [PATCH 12/14] test(package-deps): snapshot template artifact overrides Record that a scoped package layout preserves a template-defined archive that points to an artifact absent from the layout. Signed-off-by: Ali Caglayan --- .../test-cases/package-materialization/dune | 1 + .../template-artifacts.t | 39 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 test/blackbox-tests/test-cases/package-materialization/template-artifacts.t diff --git a/test/blackbox-tests/test-cases/package-materialization/dune b/test/blackbox-tests/test-cases/package-materialization/dune index a4dde991d06..df9f41ca40b 100644 --- a/test/blackbox-tests/test-cases/package-materialization/dune +++ b/test/blackbox-tests/test-cases/package-materialization/dune @@ -3,5 +3,6 @@ installed-package no-transitive-through-targets ocamlfind + template-artifacts virtual-without-default) (deps %{bin:ocamlfind})) diff --git a/test/blackbox-tests/test-cases/package-materialization/template-artifacts.t b/test/blackbox-tests/test-cases/package-materialization/template-artifacts.t new file mode 100644 index 00000000000..1b920f73312 --- /dev/null +++ b/test/blackbox-tests/test-cases/package-materialization/template-artifacts.t @@ -0,0 +1,39 @@ +A META file template can override path-bearing variables without changing the +library graph. Retaining such a rule while materializing only Dune-selected +artifacts produces dangling support metadata. + + $ make_dune_project 3.24 + $ cat >>dune-project <<'EOF' + > (package (name artifact-support)) + > EOF + + $ mkdir support + $ cat >support/dune <<'EOF' + > (library + > (name artifact_support) + > (public_name artifact-support)) + > EOF + $ echo 'let value = 42' >support/artifact_support.ml + + $ cat >META.artifact-support.template <<'EOF' + > # DUNE_GEN + > archive(byte,custom) = "missing.cma" + > EOF + + $ cat >dune <<'EOF' + > (rule + > (target result) + > (deps (package artifact-support)) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query -predicates byte,custom -format "%d/%A" artifact-support)))) + > EOF + +The installed META currently retains the conditional archive override even +though that artifact is not part of the package layout. + + $ dune build result + $ cat _build/default/result | censor + $PWD/_build/install/default/.packages/$DIGEST/lib/artifact-support/missing.cma + $ test -e "$(cat _build/default/result)" + [1] From a3b3f2d0fe77dfa35f7b8a99a87d6e57273253bc Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 17 Aug 2026 20:27:35 +0200 Subject: [PATCH 13/14] test(package-deps): snapshot template-added library requirements Record that scoped support metadata can advertise a dependency added by a META file template without materializing that dependency. Signed-off-by: Ali Caglayan --- .../test-cases/package-materialization/dune | 1 + .../template-dependencies.t | 132 ++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 test/blackbox-tests/test-cases/package-materialization/template-dependencies.t diff --git a/test/blackbox-tests/test-cases/package-materialization/dune b/test/blackbox-tests/test-cases/package-materialization/dune index a4dde991d06..b526c29b74d 100644 --- a/test/blackbox-tests/test-cases/package-materialization/dune +++ b/test/blackbox-tests/test-cases/package-materialization/dune @@ -3,5 +3,6 @@ installed-package no-transitive-through-targets ocamlfind + template-dependencies virtual-without-default) (deps %{bin:ocamlfind})) diff --git a/test/blackbox-tests/test-cases/package-materialization/template-dependencies.t b/test/blackbox-tests/test-cases/package-materialization/template-dependencies.t new file mode 100644 index 00000000000..f04e506acf9 --- /dev/null +++ b/test/blackbox-tests/test-cases/package-materialization/template-dependencies.t @@ -0,0 +1,132 @@ +A META file template may add a library requirement that is absent from Dune's +internal library graph. The scoped layout must not emit that requirement +without also handling its dependency. + + $ make_dune_project 3.24 + $ cat >>dune-project < (package (name template-root)) + > (package (name template-middle)) + > (package (name template-extra)) + > (package (name template-explicit)) + > EOF + + $ mkdir root middle extra unrelated explicit + $ cat >root/dune <<'EOF' + > (library + > (name root) + > (public_name template-root) + > (libraries template-middle.selected)) + > EOF + $ echo 'let value = ()' >root/root.ml + + $ cat >middle/dune <<'EOF' + > (library + > (name selected) + > (public_name template-middle.selected)) + > EOF + $ echo 'let value = ()' >middle/selected.ml + + $ cat >extra/dune <<'EOF' + > (library + > (name selected) + > (public_name template-extra.selected)) + > EOF + $ echo 'let value = ()' >extra/selected.ml + + $ cat >unrelated/dune <<'EOF' + > (library + > (name unrelated) + > (public_name template-extra.unrelated)) + > EOF + $ echo 'let value = ()' >unrelated/unrelated.ml + + $ cat >explicit/dune <<'EOF' + > (library + > (name explicit) + > (public_name template-explicit)) + > EOF + $ echo 'let value = ()' >explicit/explicit.ml + + $ cat >META.template-explicit.template <<'EOF' + > requires += "template-extra.selected" + > # DUNE_GEN + > EOF + + $ cat >META.template-middle.template <<'EOF' + > package "selected" ( + > directory = "selected" + > requires = "template-extra.selected" + > ) + > EOF + +Ordinary installation continues to accept and install the custom template. + + $ dune build @install + +A scoped package dependency currently omits even the ordinary Dune library +closure. After that closure is added, the template-only edge must not be left +dangling. + + $ cat >dune <<'EOF' + > (rule + > (target recursive) + > (deps (package template-root)) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query -recursive template-root)))) + > (rule + > (target explicit-result) + > (deps (package template-explicit)) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query -recursive template-explicit)))) + > (rule + > (target recursive-with-middle) + > (deps + > (package template-root) + > (package template-middle)) + > (action + > (with-stdout-to %{target} + > (run %{bin:ocamlfind} query -recursive template-root)))) + > EOF + +Making the ordinary middle dependency explicit exposes the template-only edge +rather than failing on the missing library closure first. + + $ dune build recursive-with-middle + File "dune", lines 13-20, characters 0-196: + 13 | (rule + 14 | (target recursive-with-middle) + 15 | (deps + 16 | (package template-root) + 17 | (package template-middle)) + 18 | (action + 19 | (with-stdout-to %{target} + 20 | (run %{bin:ocamlfind} query -recursive template-root)))) + ocamlfind: Package `template-extra.selected' not found - required by `template-middle.selected' + [1] + + $ dune build recursive + File "dune", lines 1-6, characters 0-154: + 1 | (rule + 2 | (target recursive) + 3 | (deps (package template-root)) + 4 | (action + 5 | (with-stdout-to %{target} + 6 | (run %{bin:ocamlfind} query -recursive template-root)))) + ocamlfind: Package `template-middle.selected' not found - required by `template-root' + [1] + +The same problem occurs when the dependency-changing template belongs to the +explicitly requested package. + + $ dune build explicit-result + File "dune", lines 7-12, characters 0-168: + 7 | (rule + 8 | (target explicit-result) + 9 | (deps (package template-explicit)) + 10 | (action + 11 | (with-stdout-to %{target} + 12 | (run %{bin:ocamlfind} query -recursive template-explicit)))) + ocamlfind: Package `template-extra.selected' not found - required by `template-explicit' + [1] From d1c547d42495679cd8df13b91300fe425f80e35a Mon Sep 17 00:00:00 2001 From: Ali Caglayan Date: Mon, 17 Aug 2026 22:43:46 +0200 Subject: [PATCH 14/14] test(package-deps): snapshot instrumentation metadata dependencies Record that a scoped package dependency omits the PPX library and separately owned redirect chain named by serialized instrumentation backend metadata. Signed-off-by: Ali Caglayan --- .../instrumentation-dependencies.t | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 test/blackbox-tests/test-cases/package-materialization/instrumentation-dependencies.t diff --git a/test/blackbox-tests/test-cases/package-materialization/instrumentation-dependencies.t b/test/blackbox-tests/test-cases/package-materialization/instrumentation-dependencies.t new file mode 100644 index 00000000000..081a67590d9 --- /dev/null +++ b/test/blackbox-tests/test-cases/package-materialization/instrumentation-dependencies.t @@ -0,0 +1,98 @@ +Instrumentation backend metadata names a PPX library that is not an ordinary +library requirement. The scoped layout must track the referenced library and +its metadata. + + $ make_dune_project 3.24 + $ cat >>dune-project <<'EOF' + > (package (name instrumentation-root)) + > (package (name instrumentation-old)) + > (package (name instrumentation-middle)) + > (package (name instrumentation-ppx)) + > EOF + + $ mkdir backend old middle ppx consumer + $ cat >backend/dune <<'EOF' + > (library + > (name direct) + > (public_name instrumentation-root.direct) + > (modules ()) + > (instrumentation.backend + > (ppx instrumentation-ppx))) + > (library + > (name redirected) + > (public_name instrumentation-root.redirected) + > (modules ()) + > (instrumentation.backend + > (ppx instrumentation-old.backend))) + > EOF + + $ cat >old/dune <<'EOF' + > (deprecated_library_name + > (old_public_name instrumentation-old.backend) + > (new_public_name instrumentation-middle.backend)) + > EOF + + $ cat >middle/dune <<'EOF' + > (deprecated_library_name + > (old_public_name instrumentation-middle.backend) + > (new_public_name instrumentation-ppx)) + > EOF + + $ cat >ppx/dune <<'EOF' + > (library + > (name ppx) + > (public_name instrumentation-ppx) + > (kind ppx_rewriter) + > (libraries ppxlib)) + > EOF + $ cat >ppx/ppx.ml <<'EOF' + > let () = Ppxlib.Driver.register_transformation "instrumentation-noop" + > EOF + + $ cat >consumer/dune-project <<'EOF' + > (lang dune 3.24) + > EOF + $ cat >consumer/dune <<'EOF' + > (executable + > (name main) + > (instrumentation (backend instrumentation-root.redirected))) + > EOF + $ echo 'let () = ()' >consumer/main.ml + + $ cat >dune <<'EOF' + > (rule + > (target marker) + > (deps (package instrumentation-root)) + > (action (write-file %{target} "ok"))) + > (rule + > (target nested-result) + > (deps + > (package instrumentation-root) + > (source_tree consumer)) + > (action + > (progn + > (chdir consumer + > (run %{bin:dune} build --instrument-with instrumentation-root.redirected main.exe)) + > (write-file %{target} "ok")))) + > EOF + +The PPX package's metadata is not currently a dependency of the action. + + $ dune rules --format=json _build/default/marker | + > jq_dune '.[] | ruleDepFilePaths' | + > censor | + > grep 'lib/instrumentation-.*/dune-package' | + > sort + "_build/install/default/.packages/$DIGEST/lib/instrumentation-root/dune-package" + +The nested Dune consumer cannot follow the separately owned redirect chain +named by the serialized instrumentation metadata. + + $ dune build nested-result >err 2>&1; status=$?; test ! -s err || censor required by _build/default/.main.eobjs/native/dune__exe__Main.cmx + -> required by _build/default/main.exe + [1]