Support pathlib Paths in mkdir and cleanup

easybuild/tools/filetools.py

@@ -61,6 +61,7 @@
 import zlib
 from functools import partial
 from html.parser import HTMLParser
+from pathlib import Path
 import urllib.request as std_urllib
 
 from easybuild.base import fancylogger
@@ -2076,29 +2077,34 @@ def mkdir(path, parents=False, set_gid=None, sticky=None):
     :param sticky: set the sticky bit on this directory (a.k.a. the restricted deletion flag),
                    to avoid users can removing/renaming files in this directory
     """
-    if not os.path.isabs(path):
-        path = os.path.abspath(path)
+    path = Path(path)
+    if not path.is_absolute():
+        path = path.absolute()
 
     # exit early if path already exists
-    if not os.path.exists(path):
+    if not path.exists():
         if set_gid is None:
             set_gid = build_option('set_gid_bit')
         if sticky is None:
             sticky = build_option('sticky_bit')
 
         _log.info("Creating directory %s (parents: %s, set_gid: %s, sticky: %s)", path, parents, set_gid, sticky)
         # set_gid and sticky bits are only set on new directories, so we need to determine the existing parent path
-        existing_parent_path = os.path.dirname(path)
+        existing_parent_path = path.parent
         try:
             if parents:
-                # climb up until we hit an existing path or the empty string (for relative paths)
-                while existing_parent_path and not os.path.exists(existing_parent_path):
-                    existing_parent_path = os.path.dirname(existing_parent_path)
-                os.makedirs(path, exist_ok=True)
+                # climb up until we hit an existing path
+                while not existing_parent_path.exists():
+                    parent = existing_parent_path.parent
+                    # In practice impossible but to avoid infinite loops
+                    if existing_parent_path == parent:
+                        raise EasyBuildError('Did not find any existing parent path or drive')
+                    existing_parent_path = parent
+                path.mkdir(parents=True, exist_ok=True)
             else:
-                os.mkdir(path)
+                path.mkdir()
         except FileExistsError as err:
-            if os.path.exists(path):
+            if path.exists():
                 # This may happen if a parallel build creates the directory after we checked for its existence
                 _log.debug("Directory creation aborted as it seems it was already created: %s", err)
             else:
@@ -2107,8 +2113,7 @@ def mkdir(path, parents=False, set_gid=None, sticky=None):
             raise EasyBuildError("Failed to create directory %s: %s", path, err)
 
         # set group ID and sticky bits, if desired
-        new_subdir = path[len(existing_parent_path):].lstrip(os.path.sep)
-        new_path = os.path.join(existing_parent_path, new_subdir.split(os.path.sep)[0])
+        new_path = existing_parent_path / path.relative_to(existing_parent_path).parts[0]
         set_gid_sticky_bits(new_path, set_gid, sticky, recursive=True)
     else:
         _log.debug("Not creating existing path %s" % path)

test/framework/filetools.py

@@ -46,6 +46,7 @@
 import time
 import types
 from io import StringIO
+from pathlib import Path
 from test.framework.github import requires_github_access
 from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
 from unittest import TextTestRunner
@@ -836,11 +837,13 @@ def test_download_file_fallback_source_urls(self):
     def test_mkdir(self):
         """Test mkdir function."""
 
-        def check_mkdir(path, error=None, **kwargs):
+        def check_mkdir(path, error=None, expected_path=None, **kwargs):
             """Create specified directory with mkdir, and check for correctness."""
             if error is None:
+                if expected_path is None:
+                    expected_path = path
                 ft.mkdir(path, **kwargs)
-                self.assertTrue(os.path.exists(path) and os.path.isdir(path), "Directory %s exists" % path)
+                self.assertTrue(os.path.isdir(expected_path), "Directory %s exists" % expected_path)
             else:
                 self.assertErrorRegex(EasyBuildError, error, ft.mkdir, path, **kwargs)
 
@@ -861,7 +864,7 @@ def check_mkdir(path, error=None, **kwargs):
         check_mkdir(giddir, set_gid=True)
         self.assertTrue(os.stat(giddir).st_mode & stat.S_ISGID, "gid bit set %s" % giddir)
         self.assertFalse(os.stat(giddir).st_mode & stat.S_ISVTX, "no sticky bit %s" % giddir)
-        # setting stciky bit works
+        # setting sticky bit works
         stickydir = os.path.join(barfoodir, 'sticky')
         check_mkdir(stickydir, sticky=True)
         self.assertFalse(os.stat(stickydir).st_mode & stat.S_ISGID, "no gid bit %s" % stickydir)
@@ -878,6 +881,11 @@ def check_mkdir(path, error=None, **kwargs):
         # existing parent dirs are untouched, no sticky/group ID bits set
         self.assertFalse(os.stat(foodir).st_mode & (stat.S_ISGID | stat.S_ISVTX), "no gid/sticky bit %s" % foodir)
         self.assertFalse(os.stat(barfoodir).st_mode & (stat.S_ISGID | stat.S_ISVTX), "no gid/sticky bit %s" % barfoodir)
+        # Relative path works
+        ft.change_dir(foodir)
+        check_mkdir(os.path.join('relative', 'subdir'), expected_path=os.path.join(foodir, 'relative'), parents=True)
+        # pathlib paths works
+        check_mkdir(Path(self.test_prefix) / 'pathlibdir')
 
     def test_path_matches(self):
         """Test path_matches function."""