enhance JuliaPackage easyblock to avoid writing to $HOME & clean up JuliaBundle

easybuild/easyblocks/generic/juliabundle.py

@@ -27,8 +27,6 @@
 
 @author: Alex Domingo (Vrije Universiteit Brussel)
 """
-import os
-
 from easybuild.easyblocks.generic.bundle import Bundle
 from easybuild.easyblocks.generic.juliapackage import EXTS_FILTER_JULIA_PACKAGES, JuliaPackage
 
@@ -76,23 +74,7 @@ def __init__(self, *args, **kwargs):
 
         self.log.info("exts_default_options: %s", self.cfg['exts_default_options'])
 
-    def prepare_step(self, *args, **kwargs):
-        """Prepare for installing bundle of Julia packages."""
-        super().prepare_step(*args, **kwargs)
-
     def install_step(self):
-        """Prepare installation environment and dd all dependencies to project environment."""
+        """Prepare installation environment and add all dependencies to project environment."""
         self.prepare_julia_env()
         self.include_pkg_dependencies()
-
-    def sanity_check_step(self, *args, **kwargs):
-        """Custom sanity check for bundle of Julia packages"""
-        custom_paths = {
-            'files': [],
-            'dirs': [os.path.join('packages', self.name)],
-        }
-        super().sanity_check_step(custom_paths=custom_paths)
-
-    def make_module_extra(self, *args, **kwargs):
-        """Custom module environment from JuliaPackage"""
-        return super().make_module_extra(*args, **kwargs)

easybuild/easyblocks/generic/juliapackage.py

@@ -31,6 +31,8 @@
 import glob
 import os
 import re
+import tempfile
+from typing import List, Tuple
 
 from easybuild.tools import LooseVersion
 
@@ -118,6 +120,18 @@ def get_julia_env(env_var):
 
         return parsed_var
 
+    def __init__(self, *args, **kwargs):
+        """Initialize JuliaPackage easyblock."""
+        super().__init__(*args, **kwargs)
+        self._tmp_depot_path = None
+
+    @property
+    def tmp_depot_path(self):
+        """Temporary path to be used as top DEPOT_PATH during module load."""
+        if not self._tmp_depot_path:
+            self._tmp_depot_path = tempfile.mkdtemp(suffix='-julia_depot')
+        return self._tmp_depot_path
+
     def julia_env_path(self, absolute=True, base=True):
         """
         Return path to installation environment file.
@@ -150,6 +164,23 @@ def set_pkg_offline(self):
                 )
                 raise EasyBuildError(errmsg, julia_version)
 
+    def determine_clean_paths(self, keep_installdir=False) -> Tuple[List[str], List[str]]:
+        """Determine cleaned DEPOT_PATH and LOAD_PATH excluding user depot paths."""
+        # Grab both DEPOT_PATH and LOAD_PATH before any changes are made
+        # given that Julia might automatically update LOAD_PATH from a change on DEPOT_PATH
+        dirty_depot = self.get_julia_env("DEPOT_PATH")
+        self.log.debug('DEPOT_PATH read from Julia environment: %s', os.pathsep.join(dirty_depot))
+        dirty_load = self.get_julia_env("LOAD_PATH")
+        self.log.debug('LOAD_PATH read from Julia environment: %s', os.pathsep.join(dirty_load))
+
+        clean_depot = [path for path in dirty_depot
+                       if not USER_DEPOT_PATTERN.search(path) and (keep_installdir or path != self.installdir)]
+
+        project_toml = self.julia_env_path(base=False)
+        clean_load = [path for path in dirty_load if not USER_DEPOT_PATTERN.search(path) and path != project_toml]
+
+        return clean_depot, clean_load
+
     def prepare_julia_env(self):
         """
         1. Remove user depot and prepend installation directory to DEPOT_PATH.
@@ -166,21 +197,11 @@ def prepare_julia_env(self):
 
         4. Enable automatic precompilation of packages after each build.
         """
-        # Grab both DEPOT_PATH and LOAD_PATH before any changes are made
-        # given that Julia might automatically update LOAD_PATH from a change on DEPOT_PATH
-        dirty_depot = self.get_julia_env("DEPOT_PATH")
-        self.log.debug('DEPOT_PATH read from Julia environment: %s', os.pathsep.join(dirty_depot))
-        dirty_load = self.get_julia_env("LOAD_PATH")
-        self.log.debug('LOAD_PATH read from Julia environment: %s', os.pathsep.join(dirty_load))
-
-        # First set DEPOT_PATH and then LOAD_PATH to avoid any automatic changes made by Julia
-        clean_depot = [path for path in dirty_depot if not USER_DEPOT_PATTERN.search(path) and path != self.installdir]
+        clean_depot, clean_load = self.determine_clean_paths()
         install_depot = os.pathsep.join([self.installdir] + clean_depot)
         self.log.debug("Preparing Julia 'DEPOT_PATH' for installation: %s", install_depot)
         env.setvar("JULIA_DEPOT_PATH", install_depot)
-
         project_toml = self.julia_env_path(base=False)
-        clean_load = [path for path in dirty_load if not USER_DEPOT_PATTERN.search(path) and path != project_toml]
         install_load = os.pathsep.join([project_toml] + clean_load)
         self.log.debug("Preparing Julia 'LOAD_PATH' for installation: %s", install_load)
         env.setvar("JULIA_LOAD_PATH", install_load)
@@ -285,7 +306,6 @@ def install_step(self):
 
     def install_extension(self):
         """Install Julia package as an extension."""
-
         if not self.src:
             errmsg = "No source found for Julia package %s, required for installation. (src: %s)"
             raise EasyBuildError(errmsg, self.name, self.src)
@@ -294,18 +314,34 @@ def install_extension(self):
         self.prepare_julia_env()
         self.install_pkg()
 
+    def load_module(self, *args, **kwargs):
+        """Set JULIA_DEPOT_PATH excluding the users depot path to avoid writing to $HOME
+
+        Required for e.g. sanity checks that run a julia command.
+        """
+        super().load_module(*args, **kwargs)
+
+        if LooseVersion(get_software_version('Julia')) >= LooseVersion('1.11'):
+            depot_path = os.environ['JULIA_DEPOT_PATH'].strip(':')  # Always set by module
+            # Append a colon at the end to exclude the users depot path (in $HOME)
+            depot_path += ':'
+        else:
+            # In older Julia versions the trailing colon doesn't prevent the user $HOME being added
+            # So use the extra logic to avoid that, see https://github.com/easybuilders/easybuild-easyblocks/pull/4102
+            depot_path = os.pathsep.join(self.determine_clean_paths(keep_installdir=True)[0])
+        # Prepend a temporary directory so the install path is not affected by sanity checks
+        env.setvar('JULIA_DEPOT_PATH', f"{self.tmp_depot_path}:{depot_path}")
+        self.set_pkg_offline()
+
     def sanity_check_step(self, *args, **kwargs):
         """Custom sanity check for JuliaPackage"""
 
-        pkg_dir = os.path.join('packages', self.name)
-
         custom_paths = {
             'files': [],
-            'dirs': [pkg_dir],
+            'dirs': [os.path.join('packages', self.name)],
         }
-        kwargs.update({'custom_paths': custom_paths})
-
-        return ExtensionEasyBlock.sanity_check_step(self, EXTS_FILTER_JULIA_PACKAGES, *args, **kwargs)
+        kwargs.setdefault('custom_paths', custom_paths)
+        return super().sanity_check_step(EXTS_FILTER_JULIA_PACKAGES, *args, **kwargs)
 
     def make_module_extra(self, *args, **kwargs):
         """
@@ -317,7 +353,7 @@ def make_module_extra(self, *args, **kwargs):
         allowing user to add custom Julia packages while having packages in this installation available.
         See issue easybuilders/easybuild-easyconfigs#17455
         """
-        mod = super().make_module_extra()
+        mod = super().make_module_extra(*args, **kwargs)
         if self.module_generator.SYNTAX:
             mod += JULIA_PATHS_SOFT_INIT[self.module_generator.SYNTAX]
         mod += self.module_generator.append_paths('JULIA_DEPOT_PATH', [''])