diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 7381bb1d92..b20eaac5f8 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -61,6 +61,7 @@ import zlib from functools import partial from html.parser import HTMLParser +from pathlib import Path import urllib.request as std_urllib from easybuild.base import fancylogger @@ -2076,11 +2077,12 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): :param sticky: set the sticky bit on this directory (a.k.a. the restricted deletion flag), to avoid users can removing/renaming files in this directory """ - if not os.path.isabs(path): - path = os.path.abspath(path) + path = Path(path) + if not path.is_absolute(): + path = path.absolute() # exit early if path already exists - if not os.path.exists(path): + if not path.exists(): if set_gid is None: set_gid = build_option('set_gid_bit') if sticky is None: @@ -2088,17 +2090,21 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): _log.info("Creating directory %s (parents: %s, set_gid: %s, sticky: %s)", path, parents, set_gid, sticky) # set_gid and sticky bits are only set on new directories, so we need to determine the existing parent path - existing_parent_path = os.path.dirname(path) + existing_parent_path = path.parent try: if parents: - # climb up until we hit an existing path or the empty string (for relative paths) - while existing_parent_path and not os.path.exists(existing_parent_path): - existing_parent_path = os.path.dirname(existing_parent_path) - os.makedirs(path, exist_ok=True) + # climb up until we hit an existing path + while not existing_parent_path.exists(): + parent = existing_parent_path.parent + # In practice impossible but to avoid infinite loops + if existing_parent_path == parent: + raise EasyBuildError('Did not find any existing parent path or drive') + existing_parent_path = parent + path.mkdir(parents=True, exist_ok=True) else: - os.mkdir(path) + path.mkdir() except FileExistsError as err: - if os.path.exists(path): + if path.exists(): # This may happen if a parallel build creates the directory after we checked for its existence _log.debug("Directory creation aborted as it seems it was already created: %s", err) else: @@ -2107,8 +2113,7 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): raise EasyBuildError("Failed to create directory %s: %s", path, err) # set group ID and sticky bits, if desired - new_subdir = path[len(existing_parent_path):].lstrip(os.path.sep) - new_path = os.path.join(existing_parent_path, new_subdir.split(os.path.sep)[0]) + new_path = existing_parent_path / path.relative_to(existing_parent_path).parts[0] set_gid_sticky_bits(new_path, set_gid, sticky, recursive=True) else: _log.debug("Not creating existing path %s" % path) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index e7ce260afc..60e998da28 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -46,6 +46,7 @@ import time import types from io import StringIO +from pathlib import Path from test.framework.github import requires_github_access from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner @@ -836,11 +837,13 @@ def test_download_file_fallback_source_urls(self): def test_mkdir(self): """Test mkdir function.""" - def check_mkdir(path, error=None, **kwargs): + def check_mkdir(path, error=None, expected_path=None, **kwargs): """Create specified directory with mkdir, and check for correctness.""" if error is None: + if expected_path is None: + expected_path = path ft.mkdir(path, **kwargs) - self.assertTrue(os.path.exists(path) and os.path.isdir(path), "Directory %s exists" % path) + self.assertTrue(os.path.isdir(expected_path), "Directory %s exists" % expected_path) else: self.assertErrorRegex(EasyBuildError, error, ft.mkdir, path, **kwargs) @@ -861,7 +864,7 @@ def check_mkdir(path, error=None, **kwargs): check_mkdir(giddir, set_gid=True) self.assertTrue(os.stat(giddir).st_mode & stat.S_ISGID, "gid bit set %s" % giddir) self.assertFalse(os.stat(giddir).st_mode & stat.S_ISVTX, "no sticky bit %s" % giddir) - # setting stciky bit works + # setting sticky bit works stickydir = os.path.join(barfoodir, 'sticky') check_mkdir(stickydir, sticky=True) self.assertFalse(os.stat(stickydir).st_mode & stat.S_ISGID, "no gid bit %s" % stickydir) @@ -878,6 +881,11 @@ def check_mkdir(path, error=None, **kwargs): # existing parent dirs are untouched, no sticky/group ID bits set self.assertFalse(os.stat(foodir).st_mode & (stat.S_ISGID | stat.S_ISVTX), "no gid/sticky bit %s" % foodir) self.assertFalse(os.stat(barfoodir).st_mode & (stat.S_ISGID | stat.S_ISVTX), "no gid/sticky bit %s" % barfoodir) + # Relative path works + ft.change_dir(foodir) + check_mkdir(os.path.join('relative', 'subdir'), expected_path=os.path.join(foodir, 'relative'), parents=True) + # pathlib paths works + check_mkdir(Path(self.test_prefix) / 'pathlibdir') def test_path_matches(self): """Test path_matches function."""