fix(package-deps): reject unsafe support META templates

doc/changes/changed/14373.md

@@ -1,5 +1,5 @@
-- `(deps (package ...))` now exposes only the directly declared packages to
-  the action's environment (`OCAMLPATH`, `PATH`, etc.). Previously, other
-  packages in the workspace could be discoverable via the shared install
-  staging area. Actions that relied on undeclared packages being visible
-  must declare them explicitly. (#14373, @Alizter)
+- `(deps (package ...))` now exposes the declared packages and the workspace
+  libraries in their OCaml library closure to the action's environment.
+  Previously, every package in the workspace could be discoverable via the
+  shared install staging area. Non-library entries and unrelated libraries
+  remain available only from explicitly declared packages. (#14373, @Alizter)

doc/changes/fixed/15511.md

@@ -0,0 +1,10 @@
+- Make `(deps (package ...))` include the workspace libraries in the transitive
+  OCaml and Melange closures of the libraries it installs, including
+  complete deprecated-name redirect chains and their package metadata, PPX
+  runtime dependencies, serialized inline-test, PPX-driver, and instrumentation
+  references, and virtual-library default implementations. This lets OCaml
+  tools resolve the same library closure before and after installation without
+  exposing the libraries' complete owning packages or following the broader,
+  and not always available, package dependency graph. Scoped META templates
+  that would introduce unmaterialized dependencies or artifacts are rejected.
+  (#15511, @Alizter)

doc/changes/fixed/15994.md

@@ -0,0 +1,2 @@
+- Reject malformed META file templates at their source or generated location,
+  and preserve backslashes in generated values. (#15994, @Alizter)

doc/concepts/dependency-spec.rst

@@ -40,9 +40,14 @@ Dependencies in ``dune`` files can be specified using one of the following:
   from the workspace.
 - ``(package <pkg>)`` builds the files installed by ``<package>`` and adds
   them to the action's environment: bin entries on ``PATH``, libraries on
-  ``OCAMLPATH``, stublibs on ``CAML_LD_LIBRARY_PATH``, and so on. Only the
-  named package is added; transitive package dependencies must be listed
-  separately.
+  ``OCAMLPATH``, stublibs on ``CAML_LD_LIBRARY_PATH``, and so on. For a
+  workspace package, Dune also adds the workspace libraries in the transitive
+  OCaml and Melange closures of the libraries it installs. Only those
+  libraries' artifacts and metadata are added, not the other contents of their
+  owning packages.
+  This makes the same library closure findable before and after installation.
+  Dune does not follow dependencies from package metadata; non-library package
+  dependencies must be listed separately.
 - ``(env_var <var>)`` depends on the value of the environment variable ``<var>``.
   If this variable becomes set, becomes unset, or changes value, the target
   will be rebuilt.

doc/dev/install-layouts.md

@@ -1,23 +1,25 @@
-# Install Layouts for Package Sets
+# Install Layouts for Package and Library Sets
 
 ## Overview
 
 `(deps (package ...))` materializes a scoped install layout under
-`_build/install/<context>/.packages/<digest>/` containing only the declared
-package dependencies. This replaces the old alias-based mechanism where `(deps
-(package foo))` depended on the `.foo-files` install alias, which populated the
+`_build/install/<context>/.packages/<digest>/` containing the declared packages
+and the workspace library artifacts needed by their OCaml and Melange library
+closures.
+This replaces the old alias-based mechanism where `(deps (package foo))`
+depended on the `.foo-files` install alias, which populated the
 `_build/install/` staging area shared by all packages.
 
 ## Motivation
 
 The `_build/install/` staging area shared by all packages causes several
 problems:
 
-1. Actions can silently depend on packages they did not declare via the shared
-   environment variables (OCAMLPATH, PATH, etc.). Whether an action succeeds
-   can depend on what other packages happened to be built, making builds
-   non-deterministic. `(strict_package_deps)` validates that dependencies are
-   declared but does not prevent undeclared packages from being visible at
+1. Actions can silently depend on arbitrary packages they did not declare via
+   the shared environment variables (OCAMLPATH, PATH, etc.). Whether an action
+   succeeds can depend on what other packages happened to be built, making
+   builds non-deterministic. `(strict_package_deps)` validates that dependencies
+   are declared but does not prevent unrelated packages from being visible at
    runtime.
 
 2. The shared staging area can cause rule collisions and dependency cycles.
@@ -54,28 +56,105 @@ problems:
   non-lock-dir contexts; with a lock-dir active, external packages can
   only resolve as `Build` (if they are in `dune.lock`) or as not found.
 
-### Immediate deps only
-
-The layout includes only the immediate packages listed in `(deps (package
-...))`. No transitive expansion is performed. Actions should declare what
-they need explicitly.
-
-This is a deliberate design choice:
-
-1. Transitive closure cannot traverse lock-dir packages (they are not
-   workspace packages), so it gives incomplete results in mixed
-   workspace/lock-dir setups. Immediate deps avoid this inconsistency.
-
-2. Workspace package compilation is handled by dune internally via `Lib.DB`,
-   not via OCAMLPATH. The only consumer of OCAMLPATH in the layout is
-   user-written rule actions, where explicit deps are appropriate.
-
-3. Immediate deps keep layout *contents* tractable for the "in-and-out" problem
-   ([#8652]). This is orthogonal to whether per-package layouts exist at all.
-   When a lock-dir package depends on a workspace package, that workspace
-   package's layout can be provided to the lock-dir package's build env without
-   first computing a transitive closure that would have to cross back into other
-   lock-dir packages.
+### Action package dependencies versus package metadata
+
+Two different dependencies are involved here:
+
+- `(deps (package foo))` is an action dependency. It asks Dune to build or
+  locate the installable contents of `foo`, make them available to the action,
+  and track those contents as inputs.
+
+- `(package (name foo) (depends ...))` records package metadata. It describes
+  what must be available when `foo` is built or installed, but it does not say
+  which artifacts an arbitrary action intends to use.
+
+The install layout never computes the transitive closure of the second kind.
+For example, if package `foo` has a package dependency on `bar`, then
+`(deps (package foo))` does not expose `bar`'s executable, data files, or
+unrelated libraries. An action using those must also declare
+`(package bar)`. The only implicit expansion is the narrower OCaml library
+closure described below.
+
+### Library closure, not package closure
+
+The layout starts with the packages listed in `(deps (package ...))`. For each
+workspace package, Dune also finds the libraries it installs, computes their
+transitive OCaml and Melange library closures, and adds only the install entries
+belonging to those workspace libraries. It does not add the libraries' complete owning
+packages or traverse dependencies in package metadata.
+
+This distinction preserves two important properties:
+
+1. Installed and workspace packages have the same library semantics. The
+   `META` or `dune-package` entry for an installed library records the other
+   libraries it requires, and OCaml tools recursively resolve those libraries.
+   A scoped workspace layout must therefore make the same libraries findable.
+   Otherwise, metadata in the layout can refer to libraries outside it and a
+   command that works after installation can fail in the workspace.
+
+2. Package closure cannot be defined consistently. Dune knows package
+   dependencies for workspace and lock-directory packages, but `dune-package`
+   does not record them and findlib packages do not provide a reliable package
+   dependency graph. Following package dependencies only when that information
+   happens to be available would make `(package ...)` depend on where a package
+   came from.
+
+Package dependencies are also broader than the requirement imposed by OCaml
+library metadata: they can bring in tools, data, test dependencies, and
+unrelated libraries. Such dependencies must still be listed explicitly when an
+action uses them.
+
+Dune starts from both the libraries and deprecated-library redirect targets
+installed by each declared package. It then traverses each root library's
+resolved `requires`, PPX runtime dependencies, and any virtual library default
+implementation independently rather than trying to link all the package's
+libraries together. A package may provide independent libraries or competing
+implementations of a virtual library, so its libraries need not form one valid
+link-time closure. Dune requires a public virtual library and its default
+implementation to belong to the same package, but they remain separate
+libraries and both are selected when the default is needed. Resolution errors
+are reported rather than silently producing an incomplete layout. Every
+intermediate name in a deprecated-library redirect chain is retained, including
+metadata emitted under deprecated package names.
+
+Library metadata is normally grouped by package. For each owning package in
+the support closure, Dune therefore generates a filtered `META` and
+`dune-package` containing only the selected libraries and relevant deprecated
+library redirects. The layout similarly contains only those libraries'
+interfaces, archives, runtime files, headers, and stubs. Library references
+serialized by inline-test backends, PPX drivers, and instrumentation backends
+are also included. Unrelated
+sibling libraries and non-library entries such as binaries, data, and
+documentation are excluded.
+
+At namespace nodes that do not themselves contain a selected library, the
+filtered `META` retains only `directory`, which is inherited by findlib
+subpackages. Other template variables at such a node describe that package
+node rather than its selected descendants and are dropped.
+
+A META file template can replace dependency or artifact rules independently of
+Dune's library graph and selected install entries. Such templates remain valid
+for ordinary installation, but a scoped package layout rejects changes to
+dependency and artifact-bearing variables in the selected package hierarchy.
+Rules in unselected support subpackages are filtered first, while explicitly
+requested packages are checked without filtering.
+
+Package sites and their directories are also excluded. The filtered
+`dune-package` consequently uses an empty sites map instead of advertising
+paths that the support layout does not materialize.
+
+The consuming action depends on every path materialized in the layout. A
+support library therefore adds dependencies on its selected interfaces,
+archives, stubs, headers, runtime files, and filtered metadata—not merely an
+`OCAMLPATH` entry. The symlink or generated-file rule for each layout path then
+tracks the corresponding workspace artifact or metadata input. Changes in a
+transitively required workspace library consequently invalidate the action.
+
+Installed libraries in the closure are already visible through the context
+environment; only workspace library artifacts need adding to the scoped
+layout. Because rule actions are opaque, a support library on `OCAMLPATH` is
+also directly queryable with `ocamlfind`. The distinction is one of dependency
+selection and layout contents, not an access-control boundary.
 
 ### Environment variables
 
@@ -100,12 +179,13 @@ entirely.
 
 The `env` lazy in `Super_context.create` (`src/dune_rules/super_context.ml`) is
 just `Context.installed_env` plus `Site_env.add_packages_env`, with no staging
-paths attached. Actions only see declared dependencies via layout env vars
-consed in `extend_action`. This is what makes the strict-deps property hold for
-workspace build outputs: no workspace-built artifact is reachable via
-dune-managed env vars unless declared. The user's inherited shell env (PATH,
-OCAMLPATH, findlib config) is still visible via `Context.installed_env`, as
-documented in "Environment variables" above.
+paths attached. Actions only see the named packages and the selected artifacts
+of their library closure via layout env vars consed in `extend_action`. This is
+what makes the strict-deps property hold for workspace build outputs: no
+unrelated workspace-built artifact is reachable via dune-managed env vars.
+The user's inherited shell env (PATH, OCAMLPATH, findlib config) is still
+visible via `Context.installed_env`, as documented in "Environment variables"
+above.
 
 ### `dune exec` and the staging area
 
@@ -138,26 +218,22 @@ staging cons above, this means:
   the staging dir is populated. The dune-site / sites cram tests
   consequently run `dune build @install` before `dune exec`.
 - Cram tests that exercise dune-site libraries (`(libraries dune-site
-  dune-site.plugins)`) must declare both `(package dune-site)` and
-  `(package dune-private-libs)` in their cram-level `dune` setup.
-  `dune-site` re-exports `dune-private-libs.dune-section`, and the layout
-  does not auto-expand transitive package deps (see "Immediate deps only"
-  above). The same pattern applies to other re-exporting libraries.
-  `(package stdune)`, for example, requires `(package dyn) (package
-  ordering) (package pp) (package top-closure) (package csexp) (package
-  fs-io)`. This is the explicit-deps tradeoff in its most visible form:
-  the immediate-deps-only design surfaces a library's transitive
-  dependencies at the call site rather than implicitly pulling them in.
+  dune-site.plugins)`) can declare `(package dune-site)` in their cram-level
+  `dune` setup. `dune-site` re-exports `dune-private-libs.dune-section`, so the
+  library closure also adds that library's install artifacts and filtered
+  metadata. It does not add the other contents of `dune-private-libs`. The same
+  rule applies to other re-exporting libraries. Package dependencies that are
+  not represented in the library graph must still be declared explicitly.
 
 The current staging cons is sufficient for the existing test surface and
 matches pre-install-layouts behaviour.
 
 ### Package set structure and `_root` section collisions
 
-The layout merges all packages' install entries into a single directory tree.
-For scoped sections (`lib`, `share`, `doc`, `etc`), each package installs
-under its own subdirectory (`lib/<pkgname>/`), so collisions are impossible
-by construction. The unordered set is the correct data structure.
+The layout merges the explicit packages' install entries and the selected
+support libraries' entries into a single directory tree. For scoped sections
+(`lib`, `share`, `doc`, `etc`), entries install under their owning package's
+subdirectory (`lib/<pkgname>/`), so collisions are impossible by construction.
 
 Collisions can only occur in `_root` sections (`lib_root`, `share_root`,
 `libexec_root`), which install directly to the section root without package
@@ -189,12 +265,15 @@ the core layout mechanism.
 ### Layout key (digest derivation)
 
 The `<digest>` component of `_build/install/<context>/.packages/<digest>/` is
-the hex `Digest.repr` of the sorted package-name list of the set (see
-`Install_layout.Key.encode` in `src/dune_rules/install_layout.ml`). The sort
-makes the digest order-independent. A reverse table maps each digest back to
-its original set so `gen_rules` can decode the layout dir's name when the
-engine asks for rules. Hash collisions are detected at insertion time and raise
-a `Code_error`.
+the hex `Digest.repr` of the sorted explicit-package names and support-library
+identities. A support-library identity contains both its owning package and
+library name, so package-private names remain unambiguous across workspace
+scopes. See `Install_layout.Key.encode` in
+`src/dune_rules/install_layout.ml`. The sorting makes the digest
+order-independent. A reverse table maps each digest back to its original
+request so `gen_rules` can decode the layout dir's name when the engine asks
+for rules. Hash collisions are detected at insertion time and raise a
+`Code_error`.
 
 ### `For_rocq_only` escape hatch
 

src/dune_findlib/meta.ml

@@ -52,6 +52,31 @@ let rec filter_variable t ~f =
   }
 ;;
 
+let rec filter_by_package_hierarchy t ~selected =
+  let selected_has_rules =
+    List.exists selected.entries ~f:(function
+      | Rule { var = "directory"; _ } | Comment _ | Package _ -> false
+      | Rule _ -> true)
+  in
+  let selected_packages =
+    List.filter_map selected.entries ~f:(function
+      | Package package -> Some package
+      | Comment _ | Rule _ -> None)
+  in
+  let entries =
+    List.filter_map t.entries ~f:(function
+      | Comment _ -> None
+      | Rule ({ var = "directory"; _ } as rule) -> Some (Rule rule)
+      | Rule _ as rule -> Option.some_if selected_has_rules rule
+      | Package package ->
+        List.find selected_packages ~f:(fun selected ->
+          Option.equal Lib_name.equal selected.name package.name)
+        |> Option.map ~f:(fun selected ->
+          Package (filter_by_package_hierarchy package ~selected)))
+  in
+  { t with entries }
+;;
+
 module Simplified = struct
   module Rules = struct
     type t =
@@ -320,16 +345,16 @@ let pp_predicate p =
      | Neg p -> "-" ^ p)
 ;;
 
+let escape_quoted_string s = String.escape_only '"' (String.escape_only '\\' s)
+
 let pp_print_text s =
   let open Pp.O in
-  Pp.verbatim "\"" ++ Pp.hvbox (Pp.text (String.escape_only '"' s)) ++ Pp.verbatim "\""
+  Pp.verbatim "\"" ++ Pp.hvbox (Pp.text (escape_quoted_string s)) ++ Pp.verbatim "\""
 ;;
 
 let pp_print_string s =
   let open Pp.O in
-  Pp.verbatim "\""
-  ++ Pp.hvbox (Pp.verbatim (String.escape_only '"' s))
-  ++ Pp.verbatim "\""
+  Pp.verbatim "\"" ++ Pp.hvbox (Pp.verbatim (escape_quoted_string s)) ++ Pp.verbatim "\""
 ;;
 
 let pp_quoted_value var =

src/dune_findlib/meta.mli

@@ -29,6 +29,12 @@ and predicate =
 
 val to_dyn : t -> Dyn.t
 val filter_variable : t -> f:(string -> bool) -> t
+
+(** Filter [t] to the package hierarchy present in [selected]. At hierarchy
+    nodes with no selected rules, retain only [directory], which is inherited
+    by findlib subpackages. Comments are discarded. *)
+val filter_by_package_hierarchy : t -> selected:t -> t
+
 val parse_entries : Lexing.lexbuf -> entry list
 
 (** Add version fields to all package in [t] that don't have and have at least