Skip to content

pack: resolve every component from the devShell closure, not the store #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cavearr merged 1 commit into from
Sep 15, 2026
Merged

pack: resolve every component from the devShell closure, not the store #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 49 additions & 53 deletions pack/components.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,11 @@
is_elf,
is_python_script,
is_shell_script,
nix_locate,
python_ctypes,
python_shebang_add,
resolve_needed,
resolve_needed_in,
resolve_python_package,
write_access,
)

Expand Down Expand Up @@ -344,7 +346,7 @@ def run_phase3_yosys():
copy_python()

# -- Copy the specific python packages
copy_python_dep("click", "8.1.7")
copy_python_dep("click")


def run_phase3_nextpnr_xilinx():
Expand Down Expand Up @@ -399,20 +401,46 @@ def run_phase3_fasm():
print(ansi.DEFAULT, end='')

# --- Copy fasm and its dependencies
copy_python_dep("fasm", "")
copy_python_dep("textx", "4.0.1")
copy_python_dep("fasm")
copy_python_dep("textx")

# -- Native libraries loaded at RUNTIME via ctypes/dlopen (they are not
# -- LC_LOAD_DYLIB/DT_NEEDED dependencies of the executables), so they
# -- must be copied explicitly. On Linux: .so (antlr/libuuid/libffi).
# -- On macOS: only the libffi .dylib (for _ctypes) and libantlr (for
# -- the fast parser's libparse_fasm.dylib); libuuid is provided by
# -- must be copied explicitly. Each one is resolved from the object of
# -- the devShell closure that actually links it -- never by globbing
# -- /nix/store, whose first match is a property of the build HOST's
# -- store, not of this flake (see the libuuid note below). On Linux:
# -- .so (antlr/libuuid/libffi). On macOS: .dylib; libuuid comes from
# -- libSystem.
dst = Path.cwd() / "dist" / "lib"
if not IS_DARWIN:
# -- libantlr4
antlr_dir = nix_locate("antl")
lib_dir = antlr_dir / "lib"

# -- libffi: the copy the _ctypes extension of the shipped python3.12
# -- loads (looked up via @rpath -> dist/lib on macOS).
src = resolve_needed(python_ctypes(), r"libffi\..*")
msg = copy_file(src, dst)
print(msg)

# -- libantlr4-runtime: the copy the fasm fast parser links. The
# -- parser's native objects live inside the fasm package
# -- (fasm/parser/); which of them carries the dependency differs by
# -- platform (the cython extension dlopens libparse_fasm), so the
# -- lookup scans them all.
parser_dir = resolve_python_package("fasm") / "parser"
objects = sorted([*parser_dir.glob("*.so"), *parser_dir.glob("*.dylib")])
if not objects:
raise SystemExit(f"❌ antlr: ningun objeto nativo en {parser_dir}")
antlr_lib = resolve_needed_in(objects, r"libantlr4-runtime\..*")

if IS_DARWIN:
# -- macOS: the LC_LOAD_DYLIB path already names the real file
# -- (looked up by libparse_fasm.dylib via @rpath -> dist/lib).
msg = copy_file(antlr_lib, dst)
print(msg)
else:
# -- Linux: the loader looks the library up by soname in dist/lib,
# -- so every libantlr4-runtime.so.* of the resolved directory goes
# -- in (the soname link and the real file), as before.
lib_dir = antlr_lib.parent
pattern = "libantlr4-runtime.so.*"
files = sorted(lib_dir.glob(pattern))
if not files:
Expand All @@ -436,28 +464,9 @@ def run_phase3_fasm():
# -- server 2026-09-09; CI never saw it because a fresh runner's
# -- store holds only this flake's closure). Asking the library
# -- that needs it cannot pick a stranger.
src = resolve_needed(files[0], "libuuid.so.1")
msg = copy_file(src, dst)
print(msg)

# -- libffi.so
ffi_dir = nix_locate("libffi-3.4.6")
src = ffi_dir / "lib" / "libffi.so.8"
src = resolve_needed(files[0], r"libuuid\.so\.1")
msg = copy_file(src, dst)
print(msg)
else:
# -- libffi.*.dylib (looked up by _ctypes via @rpath -> dist/lib)
ffi_dir = nix_locate("libffi-3.4.6")
for f in (ffi_dir / "lib").glob("libffi.*.dylib"):
if not f.is_symlink():
print(copy_file(f, dst))

# -- libantlr4-runtime.*.dylib (looked up by libparse_fasm.dylib
# -- via @rpath)
antlr_dir = nix_locate("antlr-runtime-cpp")
for f in (antlr_dir / "lib").glob("libantlr4-runtime.*.dylib"):
if not f.is_symlink():
print(copy_file(f, dst))


def run_phase3_prjxray():
Expand All @@ -470,26 +479,17 @@ def run_phase3_prjxray():
# ---- Prjxray
# {prjxray}/usr/share/python3/prjxray -->
# ---> dist/lib/python3.12/site-packages/prjxray
# -- Locate the folder where the package lives
pkg_dir = nix_locate("prjxray")
src = pkg_dir / "usr" / "share" / "python3" / "prjxray"
dst = Path.cwd() / DIST / LIB / "python3.12" \
/ "site-packages" / "prjxray"

mark = ""
if dst.exists():
mark = "📌"
else:
shutil.copytree(src, dst, dirs_exist_ok=True)
mark = "✅"

print(f"➡️ Dep: {mark}prjxray")
# -- The devShell puts the prjxray derivation's usr/share/python3 on
# -- PYTHONPATH, so the interpreter resolves exactly the tree the shell
# -- linked. The store can hold several prjxray builds at once; the
# -- first glob match used to pick one of them at random.
copy_python_dep("prjxray")

# -- Python packages
copy_python_dep("pyyaml", "6.0.1", "yaml")
copy_python_dep("simplejson", "3.19.2")
copy_python_dep("intervaltree", "3.1.0")
copy_python_dep("sortedcontainers", "2.4.0")
copy_python_dep("yaml")
copy_python_dep("simplejson")
copy_python_dep("intervaltree")
copy_python_dep("sortedcontainers")

# -- File locking: best-effort instead of fatal
# --
Expand Down Expand Up @@ -584,10 +584,6 @@ def run_phase3_prjxray():
mark = "✅"
print(f"➡️ Dep: {mark}{PATCH_DIR}/util.py (locking best-effort)")

# -- DEBUG
# dir = nix_locate("nextpnr-xilinx")
# print(dir)


def process_binaries(name: str):
print()
Expand Down
171 changes: 116 additions & 55 deletions pack/relocate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
signing must come after relocation.
"""

import importlib.util
import re
import shutil
import stat
Expand Down Expand Up @@ -210,89 +211,145 @@ def copy_python():
print(f"➡️ Dep: {mark}lib/{src.name}/")


# ----------------------------------------------------------------
# -- Locate the nix path whose name contains the string 'text'
# -- Returns the full path
# ------------------------------------------------------------------
# -- Locate the python package the devShell interpreter imports
# --
# -- The devShell's PYTHONPATH points at the site-packages of every
# -- python derivation in the shell's closure, so what THIS interpreter
# -- resolves is by construction what the shell linked. The store glob
# -- this replaces answered with a property of the build HOST's store:
# -- with more than one version of a package present, its first match
# -- could be (and was) a stranger.
# --
# -- Returns the package directory (the file, for a single-file
# -- module). E.g. resolve_python_package("textx") ->
# -- /nix/store/...-python3.12-textx-4.0.1/lib/python3.12/site-packages/textx
# ------------------------------------------------------------------
def resolve_python_package(modname: str) -> Path:

spec = importlib.util.find_spec(modname)

if spec is None:
raise SystemExit(
f"❌ {modname}: no lo encuentra el intérprete "
"(¿se empaqueta fuera de `nix develop .#pack`?)")

# -- Regular/namespace package: the package directory
if spec.submodule_search_locations:
return Path(spec.submodule_search_locations[0])

# -- Single-file module
return Path(spec.origin)


# ------------------------------------------------------------------
# -- Runtime dependencies of a native object, as the loader resolves
# -- them: {dependency name: file the loader would open}.
# --
# -- E.g. nix_locate("python3.12-click-8.1.7") returns
# -- 7b7509xv9aqdrayjf1fv5ialf4gbi5wd-python3.12-click-8.1.7
# -- Packages ending in "-dev" are discarded
# -- Linux: the DT_NEEDED entries resolved through the object's RUNPATH
# -- (ldd). Darwin: the absolute LC_LOAD_DYLIB entries keyed by basename
# -- (otool -L via macpack, which already filters out the system and
# -- @rpath references). Entries the loader cannot resolve ("not
# -- found") are left out.
# ------------------------------------------------------------------
def nix_locate(text: str) -> Path:
def needed_deps(obj: Path) -> dict:

# -- Path of the nix store
nix_store = Path("/nix/store")
if IS_DARWIN:
import macpack
return {Path(dep).name: Path(dep)
for dep in macpack._otool_deps(obj)}

# -- Search pattern
pattern = f"*{text}*"
out = subprocess.run(["ldd", str(obj)],
capture_output=True, text=True, check=True).stdout

# -- The auxiliary nix outputs that do not contain the wanted files
# -- are discarded: "-dev" (headers) and "-dist" (sdist/wheel). On
# -- macOS the "-dist" output tends to appear first in the glob and
# -- used to break the copy of the python packages (it has no
# -- site-packages/<pkg>).
paths = [path for path in nix_store.glob(pattern)
if path.is_dir()
and not str(path).endswith("-dev")
and not str(path).endswith("-dist")]
deps = {}
for line in out.splitlines():
match = re.search(r'(\S+)\s+=>\s+(\S+)', line.strip())
if match and match.group(2).startswith("/"):
deps[match.group(1)] = Path(match.group(2))

# -- Return the first match
return paths[0]
return deps


# ------------------------------------------------------------------
# -- Resolve one DT_NEEDED library of an ELF object to the file the
# -- loader would actually open for it (its RUNPATH decides, so the
# -- answer is the copy this object was linked against).
# -- Resolve one runtime dependency of a native object to the file the
# -- loader would actually open for it (its RUNPATH / install names
# -- decide, so the answer is the copy this object was linked against).
# --
# -- `pattern` is a regex FULLY matched against the dependency name
# -- (the soname on Linux, the dylib basename on Darwin).
# --
# -- Use this instead of globbing /nix/store whenever the library is
# -- loaded at RUNTIME and therefore has to be copied by hand: the glob
# -- answers with a property of the build HOST's store, this answers
# -- with a property of the thing that needs the library.
# ------------------------------------------------------------------
def resolve_needed(obj: Path, soname: str) -> Path:
def resolve_needed(obj: Path, pattern: str) -> Path:

deps = subprocess.run(["ldd", str(obj)],
capture_output=True, text=True, check=True)
for name, path in needed_deps(obj).items():
if re.fullmatch(pattern, name):
return path

for line in deps.stdout.splitlines():
match = re.search(r'(\S+)\s+=>\s+(\S+)', line.strip())
if match and match.group(1) == soname:
return Path(match.group(2))
raise SystemExit(
f"❌ {pattern}: no aparece entre las dependencias de {obj.name} "
"(¿cambió como se enlaza?)")


# ------------------------------------------------------------------
# -- Like resolve_needed, but trying each object in turn: which file
# -- carries a dependency can differ by platform (the fasm parser's
# -- libantlr4-runtime is linked by libparse_fasm, next to the cython
# -- extension that dlopens it).
# ------------------------------------------------------------------
def resolve_needed_in(objects: list, pattern: str) -> Path:

for obj in objects:
for name, path in needed_deps(obj).items():
if re.fullmatch(pattern, name):
return path

names = ", ".join(obj.name for obj in objects)
raise SystemExit(
f"❌ {soname}: no aparece entre las dependencias de {obj.name} "
f"❌ {pattern}: no aparece entre las dependencias de {names} "
"(¿cambió como se enlaza?)")


# -----------------------------------------------------------------------
# -- Copy a python library from nix into the distribution
# -- The package directory is copied to dist/lib/python3.12/site-packages
# ------------------------------------------------------------------
# -- The _ctypes extension of the python3.12 the package ships (the
# -- interpreter copy_python() bundles): the libffi it loads is the
# -- libffi the bundled interpreter needs at runtime.
# --
# -- E.g. package click
# -- - Source:
# -- /nix/store/xxx-python3.12-click/lib/python3.12/site-packages/
# -- - Target:
# -- dist/lib/python3.12/site-packages
# -----------------------------------------------------------------------
def copy_python_dep(pyname: str, version: str, name: str = ""):
# -- The name filter is "_ctypes." with the dot: lib-dynload also
# -- carries _ctypes_test, which is NOT the ctypes runtime.
# ------------------------------------------------------------------
def python_ctypes() -> Path:

python = Path(str(shutil.which("python3.12")))
dynload = python.parent.parent / "lib" / "python3.12" / "lib-dynload"

if name == "":
name = pyname
objects = sorted(p for p in dynload.glob("_ctypes*.so")
if p.name.startswith("_ctypes."))
if not objects:
raise SystemExit(f"❌ _ctypes: ningun _ctypes.*.so en {dynload}")

# -- Package name (name + version)
pack_name = f"{pyname}" if version == "" else f"{pyname}-{version}"
return objects[0]

# -- Locate the folder where the package lives
pkg_dir = nix_locate(f"python3.12-{pack_name}")

# -- Source directory
site_pack = pkg_dir / "lib" / "python3.12" / "site-packages"
src = site_pack / name
# -----------------------------------------------------------------------
# -- Copy a python package into the distribution
# --
# -- The package copied is the one the devShell interpreter imports
# -- (resolve_python_package), never a /nix/store glob match. It lands in
# -- dist/lib/python3.12/site-packages
# -----------------------------------------------------------------------
def copy_python_dep(modname: str):

# -- The package/module the devShell interpreter resolves
src = resolve_python_package(modname)

# -- Target directory
dst_site_pack = Path.cwd() / DIST / LIB / "python3.12" / "site-packages"
dst = dst_site_pack / name
dst = dst_site_pack / src.name

# -- Give write permissions to the "site-packages" directory
# -- of the distribution
Expand All @@ -304,10 +361,14 @@ def copy_python_dep(pyname: str, version: str, name: str = ""):
if dst.exists():
mark = "📌"
else:
shutil.copytree(src, dst, dirs_exist_ok=True)
if src.is_dir():
shutil.copytree(src, dst, dirs_exist_ok=True)
else:
dst_site_pack.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dst)
mark = "✅"

print(f"➡️ Dep: {mark}{pack_name}")
print(f"➡️ Dep: {mark}{modname} <- {src}")


# ------------------------------------
Expand Down
Loading
Loading