diff --git a/easybuild/base/exceptions.py b/easybuild/base/exceptions.py index e7adbc496c..007b14ed7f 100644 --- a/easybuild/base/exceptions.py +++ b/easybuild/base/exceptions.py @@ -34,10 +34,12 @@ import logging import os +from typing import Optional, List + from easybuild.base import fancylogger -def get_callers_logger(): +def get_callers_logger() -> Optional[logging.Logger]: """ Get logger defined in caller's environment :return: logger instance (or None if none was found) @@ -76,11 +78,11 @@ class LoggedException(Exception): # must accept an argument of type string, i.e. the log message, and an optional list of formatting arguments LOGGING_METHOD_NAME = 'error' # list of top-level package names to use to format location info; None implies not to include location info - LOC_INFO_TOP_PKG_NAMES = [] + LOC_INFO_TOP_PKG_NAMES: List[str] = [] # include location where error was raised from (enabled by default under 'python', disabled under 'python -O') INCLUDE_LOCATION = __debug__ - def __init__(self, msg, *args, **kwargs): + def __init__(self, msg: str, *args, **kwargs) -> None: """ Constructor. :param msg: exception message @@ -91,7 +93,7 @@ def __init__(self, msg, *args, **kwargs): if args: msg = msg % args - backtrace = [] + backtrace: List[str] = [] if self.LOC_INFO_TOP_PKG_NAMES is not None: # determine correct frame to fetch location information from frames_up = 1 diff --git a/easybuild/base/frozendict.py b/easybuild/base/frozendict.py index 06ad117c99..62bf6fd0e0 100644 --- a/easybuild/base/frozendict.py +++ b/easybuild/base/frozendict.py @@ -23,6 +23,7 @@ import operator from collections.abc import Mapping from functools import reduce +from typing import List from easybuild.base import fancylogger @@ -64,8 +65,7 @@ def keys(self): class FrozenDictKnownKeys(FrozenDict): """A frozen dictionary only allowing known keys.""" - # list of known keys - KNOWN_KEYS = [] + KNOWN_KEYS: List[str] = [] def __init__(self, *args, **kwargs): """Constructor, only way to define the contents.""" diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index f0db658f3b..62bd8162d5 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -45,6 +45,7 @@ from io import StringIO from optparse import Option, OptionGroup, OptionParser, OptionValueError, Values from optparse import SUPPRESS_HELP as nohelp # supported in optparse of python v2.4 +from typing import List from easybuild.base.fancylogger import getLogger, setroot, setLogLevel, getDetailsLogLevels from easybuild.base.optcomplete import autocomplete, CompleterOption @@ -905,9 +906,10 @@ class GeneralOption: INTERSPERSED = True # mix args with options CONFIGFILES_USE = True - CONFIGFILES_RAISE_MISSING = False - CONFIGFILES_INIT = [] # initial list of defaults, overwritten by go_configfiles options - CONFIGFILES_IGNORE = [] + CONFIGFILES_RAISE_MISSING: bool = False + # initial list of defaults, overwritten by go_configfiles options + CONFIGFILES_INIT: List[str] = [] + CONFIGFILES_IGNORE: List[str] = [] CONFIGFILES_MAIN_SECTION = 'MAIN' # sectionname that contains the non-grouped/non-prefixed options CONFIGFILE_CASESENSITIVE = True diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 698891a772..9380182ca8 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -48,7 +48,7 @@ import re from collections import OrderedDict from contextlib import contextmanager -from typing import Optional +from typing import Any, Dict, Optional import easybuild.tools.filetools as filetools from easybuild.base import fancylogger @@ -111,9 +111,9 @@ HAVE_AUTOPEP8 = False -_easyconfig_files_cache = {} -_easyconfigs_cache = {} -_path_indexes = {} +_easyconfig_files_cache: Dict[str, Any] = {} +_easyconfigs_cache: Dict[str, Any] = {} +_path_indexes: Dict[str, int] = {} def handle_deprecated_or_replaced_easyconfig_parameters(ec_method): diff --git a/easybuild/framework/easyconfig/format/one.py b/easybuild/framework/easyconfig/format/one.py index acd1af6f6c..e0a6f3f32c 100644 --- a/easybuild/framework/easyconfig/format/one.py +++ b/easybuild/framework/easyconfig/format/one.py @@ -37,6 +37,7 @@ import pprint import re import tempfile +from typing import List from easybuild.base import fancylogger from easybuild.framework.easyconfig.format.format import DEPENDENCY_PARAMETERS, EXCLUDED_KEYS_REPLACE_TEMPLATES @@ -100,7 +101,7 @@ class FormatOneZero(EasyConfigFormatConfigObj): PYHEADER_ALLOWED_BUILTINS = None # allow all PYHEADER_MANDATORY = ['version', 'name', 'toolchain', 'homepage', 'description'] - PYHEADER_BLACKLIST = [] + PYHEADER_BLACKLIST: List[str] = [] def __init__(self, *args, **kwargs): """FormatOneZero constructor.""" diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index 782aaa5fec..ab12035f67 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -34,6 +34,7 @@ import copy import re import sys +from typing import List from easybuild.base import fancylogger from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS @@ -152,7 +153,7 @@ class EasyConfigFormatConfigObj(EasyConfigFormat): - feed to ConfigObj """ - PYHEADER_ALLOWED_BUILTINS = [] # default no builtins + PYHEADER_ALLOWED_BUILTINS: List[str] = [] # default no builtins PYHEADER_MANDATORY = None # no defaults PYHEADER_BLACKLIST = None # no defaults diff --git a/easybuild/scripts/rpath_args.py b/easybuild/scripts/rpath_args.py index 0a15d9ede1..d63b9cce4e 100755 --- a/easybuild/scripts/rpath_args.py +++ b/easybuild/scripts/rpath_args.py @@ -37,6 +37,7 @@ import re import sys + LINKER_COMMANDS = ( # binutils 'ld', 'ld.gold', 'ld.bfd', diff --git a/easybuild/toolchains/compiler/clang.py b/easybuild/toolchains/compiler/clang.py index 73d48ddf92..3d2595293a 100644 --- a/easybuild/toolchains/compiler/clang.py +++ b/easybuild/toolchains/compiler/clang.py @@ -34,6 +34,8 @@ """ import easybuild.tools.systemtools as systemtools +from typing import List + from easybuild.tools.toolchain.compiler import Compiler @@ -104,7 +106,7 @@ class Clang(Compiler): COMPILER_CC = 'clang' COMPILER_CXX = 'clang++' - COMPILER_C_UNIQUE_OPTIONS = [] + COMPILER_C_UNIQUE_OPTIONS: List[str] = [] LIB_MULTITHREAD = ['pthread'] LIB_MATH = ['m'] diff --git a/easybuild/toolchains/compiler/cuda.py b/easybuild/toolchains/compiler/cuda.py index 2bcdf294cf..0a41555f03 100644 --- a/easybuild/toolchains/compiler/cuda.py +++ b/easybuild/toolchains/compiler/cuda.py @@ -30,6 +30,8 @@ * Kenneth Hoste (Ghent University) """ +from typing import Dict, List, Tuple + from easybuild.tools.toolchain.compiler import Compiler @@ -42,7 +44,7 @@ class Cuda(Compiler): COMPILER_CUDA_MODULE_NAME = ['CUDA'] COMPILER_CUDA_FAMILY = TC_CONSTANT_CUDA - COMPILER_CUDA_UNIQUE_OPTS = { + COMPILER_CUDA_UNIQUE_OPTS: Dict[str, Tuple[List[str], str]] = { # handle '-gencode arch=X,code=Y' nvcc options (also -arch, -code) # -arch always needs to be specified, -code is optional (defaults to -arch if missing) # -gencode is syntactic sugar for combining -arch/-code diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index dda45b5507..2daf5b08ea 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -32,6 +32,7 @@ """ import re +from typing import List import easybuild.tools.systemtools as systemtools from easybuild.tools import LooseVersion @@ -118,7 +119,7 @@ class Gcc(Compiler): COMPILER_CC = 'gcc' COMPILER_CXX = 'g++' - COMPILER_C_UNIQUE_OPTIONS = [] + COMPILER_C_UNIQUE_OPTIONS: List[str] = [] COMPILER_F77 = 'gfortran' COMPILER_F90 = 'gfortran' diff --git a/easybuild/tools/asyncprocess.py b/easybuild/tools/asyncprocess.py index 2dc2f840f4..ec20977222 100644 --- a/easybuild/tools/asyncprocess.py +++ b/easybuild/tools/asyncprocess.py @@ -73,6 +73,7 @@ import select import subprocess import time +from typing import Any, Optional, Tuple, Union PIPE = subprocess.PIPE STDOUT = subprocess.STDOUT @@ -88,27 +89,28 @@ def __init__(self, *args, **kwargs): kwargs['bufsize'] = 0 super().__init__(*args, **kwargs) - def recv(self, maxsize=None): + def recv(self, maxsize: Optional[int] = None) -> Optional[Union[str, bytes]]: return self._recv('stdout', maxsize) - def recv_err(self, maxsize=None): + def recv_err(self, maxsize: Optional[int] = None) -> Optional[Union[str, bytes]]: return self._recv('stderr', maxsize) - def send_recv(self, inp='', maxsize=None): + def send_recv(self, inp: str = '', maxsize: Optional[int] = None) -> Tuple[ + Optional[int], Optional[Union[str, bytes]], Optional[Union[str, bytes]]]: return self.send(inp), self.recv(maxsize), self.recv_err(maxsize) - def get_conn_maxsize(self, which, maxsize): + def get_conn_maxsize(self, which: str, maxsize: Optional[int]) -> Tuple[Any, int]: if maxsize is None: maxsize = 1024 elif maxsize < 1: maxsize = 1 return getattr(self, which), maxsize - def _close(self, which): + def _close(self, which: str) -> None: getattr(self, which).close() setattr(self, which, None) - def send(self, inp): + def send(self, inp: str) -> Optional[int]: if not self.stdin: return None @@ -124,7 +126,7 @@ def send(self, inp): return written - def _recv(self, which, maxsize): + def _recv(self, which: str, maxsize: Optional[int]) -> Optional[Union[str, bytes]]: conn, maxsize = self.get_conn_maxsize(which, maxsize) if conn is None: return None @@ -152,7 +154,7 @@ def _recv(self, which, maxsize): message = "Other end disconnected!" -def recv_some(p, t=.2, e=1, tr=5, stderr=0): +def recv_some(p: 'Popen', t: float = 0.2, e: int = 1, tr: int = 5, stderr: int = 0) -> bytes: if tr < 1: tr = 1 x = time.time() + t @@ -175,7 +177,7 @@ def recv_some(p, t=.2, e=1, tr=5, stderr=0): return b''.join(y) -def send_all(p, data): +def send_all(p: 'Popen', data: str) -> None: while len(data): sent = p.send(data) if sent is None: diff --git a/easybuild/tools/build_details.py b/easybuild/tools/build_details.py index e9b74cb564..c6d4414719 100644 --- a/easybuild/tools/build_details.py +++ b/easybuild/tools/build_details.py @@ -37,7 +37,7 @@ from easybuild.tools.version import EASYBLOCKS_VERSION, FRAMEWORK_VERSION -def get_build_stats(app, start_time, command_line): +def get_build_stats(app, start_time: float, command_line: str) -> OrderedDict: """ Return build statistics for this build """ diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index e6a5103c27..385099fb2f 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -42,6 +42,7 @@ from copy import copy from datetime import datetime, timezone from enum import IntEnum +from typing import Any, Optional, Tuple from easybuild.base import fancylogger from easybuild.base.exceptions import LoggedException @@ -130,7 +131,7 @@ class EasyBuildError(LoggedException): # always include location where error was raised from, even under 'python -O' INCLUDE_LOCATION = True - def __init__(self, msg, *args, exit_code=EasyBuildExit.ERROR, **kwargs): + def __init__(self, msg: str, *args, exit_code: EasyBuildExit = EasyBuildExit.ERROR, **kwargs) -> None: """Constructor: initialise EasyBuildError instance.""" if args: msg = msg % args @@ -143,12 +144,12 @@ def __str__(self): return self.msg -def raise_easybuilderror(msg, *args): +def raise_easybuilderror(msg: str, *args) -> None: """Raise EasyBuildError with given message, formatted by provided string arguments.""" raise EasyBuildError(msg, *args) -def raise_nosupport(msg, ver): +def raise_nosupport(msg: str, ver: str) -> None: """Construct error message for no longer supported behaviour, and raise an EasyBuildError.""" nosupport_msg = "NO LONGER SUPPORTED since v%s: %s; see %s for more information" raise_easybuilderror(nosupport_msg, ver, msg, DEPRECATED_DOC_URL) @@ -220,7 +221,7 @@ def nosupport(self, msg, ver): """Raise error message for no longer supported behaviour.""" raise_nosupport(msg, ver) - def error(self, msg, *args, **kwargs): + def error(self, msg: str, *args, **kwargs) -> None: """Print error message.""" ebmsg = "EasyBuild encountered an error" # Don't show caller info when error is raised from within LoggedException.__init__ @@ -232,11 +233,11 @@ def error(self, msg, *args, **kwargs): fancylogger.FancyLogger.error(self, f"{ebmsg}: {msg}", *args, **kwargs) - def devel(self, msg, *args, **kwargs): + def devel(self, msg: str, *args, **kwargs) -> None: """Print development log message""" self.log(DEVEL_LOG_LEVEL, msg, *args, **kwargs) - def exception(self, msg, *args): + def exception(self, msg: str, *args) -> None: """Print exception message and raise EasyBuildError.""" # don't raise the exception from within error ebmsg = "EasyBuild encountered an exception %s: " % self.caller_info() @@ -261,7 +262,8 @@ def exception(self, msg, *args): _init_easybuildlog = fancylogger.getLogger(fname=False) -def init_logging(logfile, logtostdout=False, silent=False, colorize=fancylogger.Colorize.AUTO, tmp_logdir=None): +def init_logging(logfile: Optional[str], logtostdout: bool = False, silent: bool = False, + colorize=fancylogger.Colorize.AUTO, tmp_logdir: Optional[str] = None) -> Tuple: """Initialize logging.""" if logtostdout: fancylogger.logToScreen(enable=True, stdout=True, colorize=colorize) @@ -286,7 +288,7 @@ def init_logging(logfile, logtostdout=False, silent=False, colorize=fancylogger. return log, logfile -def log_start(log, eb_command_line, eb_tmpdir): +def log_start(log: Any, eb_command_line: list, eb_tmpdir: str) -> None: """Log startup info.""" log.info(this_is_easybuild()) @@ -296,7 +298,7 @@ def log_start(log, eb_command_line, eb_tmpdir): log.info("Using %s as temporary directory", eb_tmpdir) -def stop_logging(logfile, logtostdout=False): +def stop_logging(logfile: Optional[str], logtostdout: bool = False) -> None: """Stop logging.""" if logtostdout: fancylogger.logToScreen(enable=False, stdout=True) @@ -304,7 +306,7 @@ def stop_logging(logfile, logtostdout=False): fancylogger.logToFile(logfile, enable=False) -def print_msg(msg, *args, **kwargs): +def print_msg(msg: str, *args, **kwargs) -> None: """ Print a message. @@ -351,7 +353,7 @@ def print_msg(msg, *args, **kwargs): sys.stdout.write(msg) -def dry_run_set_dirs(prefix, builddir, software_installdir, module_installdir): +def dry_run_set_dirs(prefix: str, builddir: str, software_installdir: str, module_installdir: str) -> None: """ Initialize for printing dry run messages. @@ -372,7 +374,7 @@ def dry_run_set_dirs(prefix, builddir, software_installdir, module_installdir): DRY_RUN_SOFTWARE_INSTALL_DIR = (re.compile(re.escape(software_installdir)), software_installdir[len(prefix):]) -def dry_run_msg(msg, *args, **kwargs): +def dry_run_msg(msg: str, *args, **kwargs) -> None: """Print dry run message.""" # replace fake build/install dir in dry run message with original value if args: @@ -389,7 +391,7 @@ def dry_run_msg(msg, *args, **kwargs): print_msg(msg, silent=silent, prefix=False) -def dry_run_warning(msg, *args, **kwargs): +def dry_run_warning(msg: str, *args, **kwargs) -> None: """Print dry run message.""" if args: msg = msg % args @@ -401,7 +403,7 @@ def dry_run_warning(msg, *args, **kwargs): dry_run_msg("\n!!!\n!!! WARNING: %s\n!!!\n" % msg, silent=silent) -def print_error(msg, *args, **kwargs): +def print_error(msg: str, *args, **kwargs) -> None: """ Print error message and exit EasyBuild """ @@ -433,7 +435,7 @@ def print_error(msg, *args, **kwargs): raise EasyBuildError(msg) # Handle legacy weirdness -def print_error_and_exit(msg, *args, exit_code=EasyBuildExit.ERROR, silent=False): +def print_error_and_exit(msg: str, *args, exit_code: EasyBuildExit = EasyBuildExit.ERROR, silent: bool = False) -> None: """ Print error message and exit EasyBuild, supports format strings @@ -449,7 +451,7 @@ def print_error_and_exit(msg, *args, exit_code=EasyBuildExit.ERROR, silent=False sys.exit(int(exit_code)) -def print_warning(msg, *args, **kwargs): +def print_warning(msg: str, *args, **kwargs) -> None: """ Print warning message. """ @@ -467,7 +469,7 @@ def print_warning(msg, *args, **kwargs): sys.stderr.write("\nWARNING: %s\n\n" % msg) -def time_str_since(start_time): +def time_str_since(start_time: datetime) -> str: """ Return string representing amount of time that has passed since specified timestamp diff --git a/easybuild/tools/bwrap.py b/easybuild/tools/bwrap.py index 8cfd8c8cd2..7240b42b7e 100644 --- a/easybuild/tools/bwrap.py +++ b/easybuild/tools/bwrap.py @@ -32,6 +32,7 @@ """ import json import os +from typing import Any from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg @@ -66,7 +67,7 @@ def get_bwrap_info(key): raise EasyBuildError(f"Unknown key specified to get bwrap info: {key}") -def set_bwrap_info(key, value): +def set_bwrap_info(key: str, value: Any) -> None: """ Set specified info w.r.t. use of bwrap """ @@ -76,7 +77,7 @@ def set_bwrap_info(key, value): raise EasyBuildError(f"Unknown key specified to set bwrap info: {key}") -def update_bwrap_info(key, value): +def update_bwrap_info(key: str, value: Any) -> None: """ Update specified info w.r.t. use of bwrap (only supports 'set' values currently) """ @@ -90,7 +91,7 @@ def update_bwrap_info(key, value): raise EasyBuildError(f"Unknown key specified to update bwrap info: {key}") -def prepare_bwrap(bwrap_installpath): +def prepare_bwrap(bwrap_installpath: str) -> None: """ Prepare for running EasyBuild with bwrap: - update _bwrap_info diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index d6769442d9..6ecd3ac5f3 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -46,6 +46,7 @@ import time from abc import ABCMeta from string import ascii_letters +from typing import Any, Dict, Optional from easybuild.base import fancylogger from easybuild.base.frozendict import FrozenDictKnownKeys @@ -216,7 +217,7 @@ def __call__(cls, *args, **kwargs): # utility function for obtaining default paths -def mk_full_default_path(name, prefix=DEFAULT_PREFIX): +def mk_full_default_path(name: str, prefix: str = DEFAULT_PREFIX) -> str: """Create full path, avoid '/' at the end.""" args = [prefix] path = DEFAULT_PATH_SUBDIRS[name] @@ -581,12 +582,12 @@ class BuildOptions(BaseBuildOptions): KNOWN_KEYS = [k for kss in [BUILD_OPTIONS_CMDLINE, BUILD_OPTIONS_OTHER] for ks in kss.values() for k in ks] -def get_pretend_installpath(): +def get_pretend_installpath() -> str: """Get the installpath when --pretend option is used""" return os.path.join(os.path.expanduser('~'), 'easybuildinstall') -def init(options, config_options_dict): +def init(options, config_options_dict: dict) -> None: """ Gather all variables and check if they're valid Variables are read in this order of preference: generaloption > legacy environment > legacy config file @@ -678,7 +679,7 @@ def init_build_options(build_options=None, cmdline_options=None): return BuildOptions(bo) -def build_option(key, **kwargs): +def build_option(key: str, **kwargs) -> Any: """Obtain value specified build option.""" # return build option value if BuildOptions has been initialised and it's a known build option @@ -699,7 +700,7 @@ def build_option(key, **kwargs): raise EasyBuildError(error_msg, exit_code=EasyBuildExit.OPTION_ERROR) -def update_build_option(key, value): +def update_build_option(key: str, value: Any) -> Any: """ Update build option with specified name to given value. @@ -715,7 +716,7 @@ def update_build_option(key, value): return orig_value -def update_build_options(key_value_dict): +def update_build_options(key_value_dict: Dict[str, Any]) -> Dict[str, Any]: """ Update build options as specified by the given dictionary (where keys are assumed to be build option names). Returns dictionary with original values for the updated build options. @@ -729,33 +730,33 @@ def update_build_options(key_value_dict): return orig_key_value_dict -def build_path(): +def build_path() -> Any: """ Return the build path """ return ConfigurationVariables()['buildpath'] -def source_paths(): +def source_paths() -> Any: """ Return the list of source paths for software """ return ConfigurationVariables()['sourcepath'] -def source_paths_data(): +def source_paths_data() -> Any: """ Return the list of source paths for data """ return ConfigurationVariables()['sourcepath_data'] -def source_path(): +def source_path() -> None: """NO LONGER SUPPORTED: use source_paths instead""" _log.nosupport("source_path() is replaced by source_paths()", '2.0') -def install_path(typ=None): +def install_path(typ: Optional[str] = None) -> str: """ Returns the install path - subdir 'software' for actual software installation (default) @@ -788,42 +789,42 @@ def install_path(typ=None): return res -def get_repository(): +def get_repository() -> Any: """ Return the repository (git, svn or file) """ return ConfigurationVariables()['repository'] -def get_repositorypath(): +def get_repositorypath() -> Any: """ Return the repository path """ return ConfigurationVariables()['repositorypath'] -def get_package_naming_scheme(): +def get_package_naming_scheme() -> Any: """ Return the package naming scheme """ return ConfigurationVariables()['package_naming_scheme'] -def package_path(): +def package_path() -> Any: """ Return the path where built packages are copied to """ return ConfigurationVariables()['packagepath'] -def container_path(): +def container_path() -> Any: """ Return the path for container recipes & images """ return ConfigurationVariables()['containerpath'] -def get_modules_tool(): +def get_modules_tool() -> Optional[Any]: """ Return modules tool (EnvironmentModules, Lmod, ...) """ @@ -831,14 +832,14 @@ def get_modules_tool(): return ConfigurationVariables().get('modules_tool', None) -def get_module_naming_scheme(): +def get_module_naming_scheme() -> Any: """ Return module naming scheme (EasyBuildMNS, HierarchicalMNS, ...) """ return ConfigurationVariables()['module_naming_scheme'] -def get_job_backend(): +def get_job_backend() -> Optional[Any]: """ Return job execution backend (PBS, GC3Pie, ...) """ @@ -846,14 +847,14 @@ def get_job_backend(): return ConfigurationVariables().get('job_backend', None) -def get_module_syntax(): +def get_module_syntax() -> Any: """ Return module syntax (Lua, Tcl) """ return ConfigurationVariables()['module_syntax'] -def get_output_style(): +def get_output_style() -> str: """Return output style to use.""" output_style = build_option('output_style', default=OUTPUT_STYLE_BASIC) @@ -873,7 +874,8 @@ def get_output_style(): return output_style -def log_file_format(return_directory=False, ec=None, date=None, timestamp=None): +def log_file_format(return_directory: bool = False, ec: Optional[Dict[str, Any]] = None, + date: Optional[str] = None, timestamp: Optional[str] = None) -> str: """ Return the format for the logfile or the directory @@ -909,7 +911,7 @@ def log_file_format(return_directory=False, ec=None, date=None, timestamp=None): return res -def log_format(ec=None): +def log_format(ec: Optional[Dict[str, Any]] = None) -> str: """ Return the logfilename format """ @@ -917,7 +919,7 @@ def log_format(ec=None): return log_file_format(return_directory=False, ec=ec) -def log_path(ec=None): +def log_path(ec: Optional[Dict[str, Any]] = None) -> str: """ Return the log path """ @@ -926,7 +928,7 @@ def log_path(ec=None): return log_file_format(return_directory=True, ec=ec, date=date, timestamp=timestamp) -def get_failed_install_build_dirs_path(ec): +def get_failed_install_build_dirs_path(ec: Dict[str, Any]) -> Optional[str]: """ Return the location where the build directory is copied to if installation failed @@ -944,7 +946,7 @@ def get_failed_install_build_dirs_path(ec): return os.path.join(base_path, f'{name}-{version}') -def get_failed_install_logs_path(ec): +def get_failed_install_logs_path(ec: Dict[str, Any]) -> Optional[str]: """ Return the location where log files are copied to if installation failed @@ -962,7 +964,7 @@ def get_failed_install_logs_path(ec): return os.path.join(base_path, f'{name}-{version}') -def get_build_log_path(): +def get_build_log_path() -> str: """ Return (temporary) directory for build log """ @@ -974,7 +976,8 @@ def get_build_log_path(): return res -def get_log_filename(name, version, add_salt=False, date=None, timestamp=None): +def get_log_filename(name: str, version: str, add_salt: bool = False, date: Optional[str] = None, + timestamp: Optional[str] = None) -> str: """ Generate a filename to be used for logging @@ -1008,7 +1011,7 @@ def get_log_filename(name, version, add_salt=False, date=None, timestamp=None): return filepath -def find_last_log(curlog): +def find_last_log(curlog: str) -> Optional[str]: """ Find location to last log file that is still available. @@ -1058,13 +1061,13 @@ def find_last_log(curlog): return res -def module_classes(): +def module_classes() -> Any: """ Return list of module classes specified in config file. """ return ConfigurationVariables()['moduleclasses'] -def read_environment(env_vars, strict=False): +def read_environment(env_vars, strict: bool = False) -> None: """NO LONGER SUPPORTED: use read_environment from easybuild.tools.environment instead""" _log.nosupport("read_environment has moved to easybuild.tools.environment", '2.0') diff --git a/easybuild/tools/entrypoints.py b/easybuild/tools/entrypoints.py index 1f7702d295..7d074eb6ce 100644 --- a/easybuild/tools/entrypoints.py +++ b/easybuild/tools/entrypoints.py @@ -30,11 +30,12 @@ """ import sys import importlib +from typing import Any, ClassVar, Dict, List, Optional, Set, TypeVar + from easybuild.tools.config import build_option from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from typing import TypeVar, List, Set, Any _T = TypeVar('_T') @@ -56,9 +57,9 @@ class EasybuildEntrypoint: - group = None + group: Optional[str] = None # Subclasses MUST override this expected_type = None - registered = {} + registered: ClassVar[Dict[str, Set["EasybuildEntrypoint"]]] = {} def __init__(self): if self.group is None: diff --git a/easybuild/tools/environment.py b/easybuild/tools/environment.py index 86e79c9830..e47136cfc3 100644 --- a/easybuild/tools/environment.py +++ b/easybuild/tools/environment.py @@ -32,6 +32,7 @@ """ import copy import os +from typing import Dict from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, dry_run_msg @@ -45,7 +46,7 @@ _log = fancylogger.getLogger('environment', fname=False) -_changes = {} +_changes: Dict[str, str] = {} def write_changes(filename): diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 366f7b87fc..19c8b78c4e 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -60,6 +60,7 @@ import time import zlib from functools import partial +from typing import Any, IO, List, Optional, Sequence, Union from html.parser import HTMLParser import urllib.request as std_urllib @@ -194,7 +195,7 @@ class ZlibChecksum: match the interface of the hashlib module """ - def __init__(self, algorithm): + def __init__(self, algorithm) -> None: self.algorithm = algorithm self.checksum = algorithm(b'') # use the same starting point as the module self.blocksize = 64 # The same as md5/sha1 @@ -208,7 +209,7 @@ def hexdigest(self): return '0x%s' % (self.checksum & 0xffffffff) -def is_readable(path): +def is_readable(path: str) -> bool: """Return whether file at specified location exists and is readable.""" try: return os.path.exists(path) and os.access(path, os.R_OK) @@ -216,7 +217,7 @@ def is_readable(path): raise EasyBuildError("Failed to check whether %s is readable: %s", path, err) -def open_file(path, mode): +def open_file(path: str, mode: str) -> IO[Any]: """Open a (usually) text file. If mode is not binary, then utf-8 encoding will be used""" # This is required for text files in Python 3, especially until Python 3.7 which implements PEP 540. # This PEP opens files in UTF-8 mode if the C locale is used, see https://www.python.org/dev/peps/pep-0540 @@ -226,7 +227,7 @@ def open_file(path, mode): return open(path, mode) -def read_file(path, log_error=True, mode='r'): +def read_file(path: str, log_error: bool = True, mode: str = 'r') -> Optional[Union[str, bytes]]: """Read contents of file at given path, in a robust way.""" txt = None try: @@ -239,8 +240,9 @@ def read_file(path, log_error=True, mode='r'): return txt -def write_file(path, data, append=False, forced=False, backup=False, always_overwrite=True, verbose=False, - show_progress=False, size=None): +def write_file(path: str, data, append: bool = False, forced: bool = False, backup: bool = False, + always_overwrite: bool = True, verbose: bool = False, show_progress: bool = False, + size: Optional[int] = None) -> None: """ Write given contents to file at given path; overwrites current file contents without backup by default! @@ -311,14 +313,14 @@ def write_file(path, data, append=False, forced=False, backup=False, always_over raise EasyBuildError("Failed to write to %s: %s", path, err) -def is_binary(contents): +def is_binary(contents) -> bool: """ Check whether given bytestring represents the contents of a binary file or not. """ return isinstance(contents, bytes) and b'\00' in bytes(contents) -def resolve_path(path): +def resolve_path(path: str) -> str: """ Return fully resolved path for given path. @@ -332,7 +334,7 @@ def resolve_path(path): return resolved_path -def symlink(source_path, symlink_path, use_abspath_source=True): +def symlink(source_path: str, symlink_path: str, use_abspath_source: bool = True) -> None: """ Create a symlink at the specified path to the given path. @@ -358,7 +360,7 @@ def symlink(source_path, symlink_path, use_abspath_source=True): raise EasyBuildError("Symlinking %s to %s failed: %s", source_path, symlink_path, err) -def remove_file(path): +def remove_file(path: str) -> None: """Remove file at specified path.""" # early exit in 'dry run' mode @@ -374,7 +376,7 @@ def remove_file(path): raise EasyBuildError("Failed to remove file %s: %s", path, err) -def empty_dir(path): +def empty_dir(path: str) -> None: """Empty directory at specified path, keeping directory itself intact.""" # early exit in 'dry run' mode if build_option('extended_dry_run'): @@ -394,7 +396,7 @@ def empty_dir(path): raise EasyBuildError(f"Failed to empty directory {path}: {err}") -def remove_dir(path): +def remove_dir(path: str) -> None: """Remove directory at specified path.""" # early exit in 'dry run' mode if build_option('extended_dry_run'): @@ -436,7 +438,7 @@ def clean_dir(path): empty_dir(path) -def remove(paths): +def remove(paths: Union[str, Sequence[str]]) -> None: """ Remove single file/directory or list of files and directories @@ -456,7 +458,7 @@ def remove(paths): raise EasyBuildError("Specified path to remove is not an existing file or directory: %s", path) -def get_cwd(must_exist=True): +def get_cwd(must_exist: bool = True) -> Optional[str]: """ Retrieve current working directory """ @@ -472,7 +474,7 @@ def get_cwd(must_exist=True): return cwd -def change_dir(path): +def change_dir(path: str) -> Optional[str]: """ Change to directory at specified location. @@ -494,8 +496,9 @@ def change_dir(path): return prev_dir -def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced=False, change_into_dir=False, - trace=True): +def extract_file(fn: str, dest: str, cmd: Optional[str] = None, extra_options: Optional[str] = None, + overwrite: bool = False, forced: bool = False, change_into_dir: bool = False, + trace: bool = True) -> str: """ Extract file at given path to specified directory :param fn: path to file to extract @@ -551,7 +554,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced return base_dir -def which(cmd, retain_all=False, check_perms=True, log_ok=True, on_error=WARN): +def which(cmd: str, retain_all: bool = False, check_perms: bool = True, log_ok: bool = True, on_error=WARN): """ Return (first) path in $PATH for specified command, or None if command is not found @@ -564,7 +567,7 @@ def which(cmd, retain_all=False, check_perms=True, log_ok=True, on_error=WARN): raise EasyBuildError("Invalid value for 'on_error': %s", on_error) if retain_all: - res = [] + res: List[str] = [] else: res = None @@ -595,7 +598,7 @@ def which(cmd, retain_all=False, check_perms=True, log_ok=True, on_error=WARN): return res -def det_common_path_prefix(paths): +def det_common_path_prefix(paths: List[str]) -> Optional[str]: """Determine common path prefix for a given list of paths.""" if not isinstance(paths, list): raise EasyBuildError("det_common_path_prefix: argument must be of type list (got %s: %s)", type(paths), paths) diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 55bfa9087b..c1725a1662 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -31,13 +31,14 @@ """ import difflib import os +from typing import Any, Dict, Callable +from importlib.util import spec_from_file_location, module_from_spec from easybuild.tools.entrypoints import EntrypointHook from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option -from importlib.util import spec_from_file_location, module_from_spec _log = fancylogger.getLogger('hooks', fname=False) @@ -122,7 +123,7 @@ # cached version of hooks, to avoid having to load them from file multiple times -_cached_hooks = {} +_cached_hooks: Dict[str, Dict[str, Callable[..., Any]]] = {} def load_source(filename, path): diff --git a/easybuild/tools/include.py b/easybuild/tools/include.py index 9c026647df..3cda193256 100644 --- a/easybuild/tools/include.py +++ b/easybuild/tools/include.py @@ -34,6 +34,7 @@ import re import sys import tempfile +from typing import Optional, Sequence from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError @@ -82,7 +83,7 @@ """ -def create_pkg(path, pkg_init_body=None): +def create_pkg(path: str, pkg_init_body: Optional[str] = None) -> None: """Write package __init__.py file at specified path.""" init_path = os.path.join(path, '__init__.py') try: @@ -102,7 +103,8 @@ def create_pkg(path, pkg_init_body=None): raise EasyBuildError("Failed to create package at %s: %s", path, err) -def set_up_eb_package(parent_path, eb_pkg_name, subpkgs=None, pkg_init_body=None): +def set_up_eb_package(parent_path: str, eb_pkg_name: str, subpkgs: Optional[Sequence[str]] = None, + pkg_init_body: Optional[str] = None) -> None: """ Set up new easybuild subnamespace in specified path. @@ -127,7 +129,7 @@ def set_up_eb_package(parent_path, eb_pkg_name, subpkgs=None, pkg_init_body=None pkgpath = os.path.dirname(pkgpath) -def verify_imports(pymods, pypkg, from_path): +def verify_imports(pymods: Sequence[str], pypkg: str, from_path: str) -> None: """Verify that import of specified modules from specified package and expected location works.""" pymod_specs = ['%s.%s' % (pypkg, pymod) for pymod in pymods] @@ -155,13 +157,13 @@ def verify_imports(pymods, pypkg, from_path): _log.debug("Import of %s from %s verified", pymod_spec, from_path) -def is_software_specific_easyblock(module): +def is_software_specific_easyblock(module: str) -> bool: """Determine whether Python module at specified location is a software-specific easyblock.""" # All software-specific easyblocks start with the prefix and derive from another class, at least EasyBlock return bool(re.search(r"^class %s[^(:]+\([^)]+\):\s*$" % EASYBLOCK_CLASS_PREFIX, read_file(module), re.M)) -def include_easyblocks(tmpdir, paths): +def include_easyblocks(tmpdir: str, paths: Sequence[str]) -> str: """Include generic and software-specific easyblocks found in specified locations.""" easyblocks_path = tempfile.mkdtemp(dir=tmpdir, prefix='included-easyblocks-') @@ -211,7 +213,7 @@ def include_easyblocks(tmpdir, paths): return easyblocks_path -def include_module_naming_schemes(tmpdir, paths): +def include_module_naming_schemes(tmpdir: str, paths: Sequence[str]) -> str: """Include module naming schemes at specified locations.""" mns_path = os.path.join(tmpdir, 'included-module-naming-schemes') @@ -243,7 +245,7 @@ def include_module_naming_schemes(tmpdir, paths): return mns_path -def include_toolchains(tmpdir, paths): +def include_toolchains(tmpdir: str, paths: Sequence[str]) -> str: """Include toolchains and toolchain components at specified locations.""" toolchains_path = os.path.join(tmpdir, 'included-toolchains') toolchain_subpkgs = ['compiler', 'fft', 'linalg', 'mpi'] diff --git a/easybuild/tools/jenkins.py b/easybuild/tools/jenkins.py index f823be518c..11501877e1 100644 --- a/easybuild/tools/jenkins.py +++ b/easybuild/tools/jenkins.py @@ -34,6 +34,7 @@ import xml.dom.minidom as xml from datetime import datetime +from typing import Any, Dict, Sequence, Tuple from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError @@ -43,7 +44,8 @@ _log = fancylogger.getLogger('jenkins', fname=False) -def write_to_xml(succes, failed, filename): +def write_to_xml(succes: Sequence[Tuple[Any, Dict[str, Any]]], + failed: Sequence[Tuple[Any, str, str, Any]], filename: str) -> None: """ Create xml output, using minimal output required according to http://stackoverflow.com/questions/4922867/junit-xml-format-specification-that-hudson-supports @@ -113,7 +115,7 @@ def create_success(name, stats): raise EasyBuildError("Failed to write out XML file %s: %s", filename, err) -def aggregate_xml_in_dirs(base_dir, output_filename): +def aggregate_xml_in_dirs(base_dir: str, output_filename: str) -> None: """ Finds all the xml files in the dirs and takes the testcase attribute out of them. These are then put in a single output file. diff --git a/easybuild/tools/job/backend.py b/easybuild/tools/job/backend.py index b3e43d777d..f2ee487b6b 100644 --- a/easybuild/tools/job/backend.py +++ b/easybuild/tools/job/backend.py @@ -33,6 +33,7 @@ from abc import ABCMeta, abstractmethod from types import SimpleNamespace +from typing import Any, Optional from easybuild.base import fancylogger from easybuild.tools.config import get_job_backend @@ -50,7 +51,7 @@ def __init__(self): @abstractmethod def _check_version(self): """Check whether version of backend complies with required version.""" - pass + return None @abstractmethod def init(self): @@ -60,10 +61,11 @@ def init(self): Jobs may be queued and only actually submitted when `complete()` is called. """ - pass + return None @abstractmethod - def make_job(self, script, name, env_vars=None, hours=None, cores=None): + def make_job(self, script: str, name: str, env_vars: Optional[dict] = None, + hours: Optional[int] = None, cores: Optional[int] = None) -> Any: """ Create and return a `Job` object with the given parameters. @@ -84,7 +86,7 @@ def queue(self, job, dependencies=frozenset()): Note that actual submission may be delayed until `complete()` is called. """ - pass + return None @abstractmethod def complete(self): @@ -97,7 +99,7 @@ def complete(self): No more job submissions should be attempted after `complete()` has been called, until a `init()` is invoked again. """ - pass + return None def avail_job_backends(check_usable=True): diff --git a/easybuild/tools/loose_version.py b/easybuild/tools/loose_version.py index af74d5167c..18579a54fd 100644 --- a/easybuild/tools/loose_version.py +++ b/easybuild/tools/loose_version.py @@ -15,6 +15,9 @@ from itertools import zip_longest +from typing import Iterable, Optional, Union + + class LooseVersion: """Version numbering for anarchists and software realists. @@ -26,7 +29,7 @@ class LooseVersion: component_re = re.compile(r'(\d+ | [a-z]+ | \.)', re.VERBOSE) - def __init__(self, vstring=None): + def __init__(self, vstring: Optional[str] = None) -> None: self._vstring = vstring if vstring: components = [x for x in self.component_re.split(vstring) @@ -50,7 +53,7 @@ def version(self): """Readonly access to the parsed version (list or None)""" return self._version - def is_prerelease(self, other, markers): + def is_prerelease(self, other: Union[str, 'LooseVersion'], markers: Iterable[str]) -> bool: """Check if this is a prerelease of other Markers is a list of strings that denote a prerelease @@ -66,13 +69,13 @@ def is_prerelease(self, other, markers): return True return False - def __str__(self): + def __str__(self) -> str: return self._vstring - def __repr__(self): + def __repr__(self) -> str: return "LooseVersion ('%s')" % str(self) - def _cmp(self, other): + def _cmp(self, other: Union[str, 'LooseVersion']) -> int: """Rich comparison method used by the operators below""" if isinstance(other, str): other = LooseVersion(other) @@ -93,20 +96,20 @@ def _cmp(self, other): return 1 return 0 - def __eq__(self, other): + def __eq__(self, other: object) -> bool: return self._cmp(other) == 0 - def __ne__(self, other): + def __ne__(self, other: object) -> bool: return self._cmp(other) != 0 - def __lt__(self, other): + def __lt__(self, other: object) -> bool: return self._cmp(other) < 0 - def __le__(self, other): + def __le__(self, other: object) -> bool: return self._cmp(other) <= 0 - def __gt__(self, other): + def __gt__(self, other: object) -> bool: return self._cmp(other) > 0 - def __ge__(self, other): + def __ge__(self, other: object) -> bool: return self._cmp(other) >= 0 diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index bba73f8684..db66b89c57 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -44,6 +44,7 @@ from contextlib import contextmanager from string import Template from textwrap import wrap +from typing import List from easybuild.base import fancylogger from easybuild.tools import LooseVersion @@ -1224,7 +1225,7 @@ class ModuleGeneratorLua(ModuleGenerator): SYNTAX = 'Lua' MODULE_FILE_EXTENSION = '.lua' MODULE_SHEBANG = '' # no 'shebang' in Lua module files - CHARS_TO_ESCAPE = [] + CHARS_TO_ESCAPE: List[str] = [] INSTALLDIR_REGEX = r'^local root\s+=\s+"(?P.*)"' LOAD_REGEX = r'^\s*(?:load|depends_on)\("(\S+)"' diff --git a/easybuild/tools/module_naming_scheme/toolchain.py b/easybuild/tools/module_naming_scheme/toolchain.py index 7e9e59295f..d3fcfe95db 100644 --- a/easybuild/tools/module_naming_scheme/toolchain.py +++ b/easybuild/tools/module_naming_scheme/toolchain.py @@ -30,6 +30,7 @@ * Kenneth Hoste (Ghent University) """ import copy +from typing import Any, Dict, Tuple from easybuild.base import fancylogger from easybuild.framework.easyconfig.easyconfig import EasyConfig, process_easyconfig, robot_find_easyconfig @@ -39,7 +40,7 @@ _log = fancylogger.getLogger('module_naming_scheme.toolchain', fname=False) -_toolchain_details_cache = {} +_toolchain_details_cache: Dict[Tuple[str, str, str], Any] = {} # different types of toolchain elements diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 41a4174650..2026595705 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -42,6 +42,7 @@ import re import shlex from enum import Enum +from typing import Any, Dict, Optional, Tuple, Type from easybuild.base import fancylogger from easybuild.tools import LooseVersion @@ -123,14 +124,14 @@ } # cache for result of module subcommands # key: tuple with $MODULEPATH and (stringified) list of extra arguments/options for module subcommand -# value: result of module subcommand -MODULE_AVAIL_CACHE = {} -MODULE_SHOW_CACHE = {} +# value: result of module subcommand (various types) +MODULE_AVAIL_CACHE: Dict[Tuple[str, str], Any] = {} +MODULE_SHOW_CACHE: Dict[Tuple[str, str], Any] = {} # cache for modules tool version # cache key: module command # value: corresponding (validated) module version -MODULE_VERSION_CACHE = {} +MODULE_VERSION_CACHE: Dict[str, str] = {} _log = fancylogger.getLogger('modules', fname=False) @@ -568,30 +569,30 @@ def set_alias_vars(self, alias, value): class ModulesTool: """An abstract interface to a tool that deals with modules.""" # name of this modules tool (used in log/warning/error messages) - NAME = None + NAME: Optional[str] = None # position and optionname TERSE_OPTION = (0, '--terse') # module command to use - COMMAND = None + COMMAND: Optional[str] = None # environment variable to determine path to module command; # used as fallback in case command is not available in $PATH - COMMAND_ENVIRONMENT = None + COMMAND_ENVIRONMENT: Optional[str] = None # run module command explicitly using this shell COMMAND_SHELL = None # option to determine the version VERSION_OPTION = '--version' # minimal required version (cannot include -beta or rc) - REQ_VERSION = None + REQ_VERSION: Optional[str] = None # minimal required version to check user's group in modulefile - REQ_VERSION_TCL_CHECK_GROUP = None + REQ_VERSION_TCL_CHECK_GROUP: Optional[str] = None # deprecated version limit (support for versions below this version is deprecated) - DEPR_VERSION = None + DEPR_VERSION: Optional[str] = None # maximum version allowed (cannot include -beta or rc) - MAX_VERSION = None + MAX_VERSION: Optional[str] = None # the regexp, should have a "version" group (multiline search) - VERSION_REGEXP = None + VERSION_REGEXP: Optional[str] = None # modules tool user cache directory - USER_CACHE_DIR = None + USER_CACHE_DIR: Optional[str] = None def __init__(self, mod_paths=None, testing=False): """ @@ -2266,7 +2267,7 @@ def mk_module_path(paths): return ':'.join(paths) -def avail_modules_tools(): +def avail_modules_tools() -> Dict[str, Type['ModulesTool']]: """ Return all known modules tools. """ diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index b602af7a91..53a0e50fde 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -46,6 +46,7 @@ import tempfile import pwd from collections import OrderedDict +from typing import Any, Dict, Tuple import easybuild.tools.environment as env from easybuild.base import fancylogger # build_log should always stay there, to ensure EasyBuildLog @@ -157,7 +158,7 @@ def cleanup_and_exit(tmpdir): sys.exit(0) -def pretty_print_opts(opts_dict): +def pretty_print_opts(opts_dict: Dict[str, Tuple[Any, Any]]) -> None: """ Pretty print options dict. @@ -194,7 +195,7 @@ def pretty_print_opts(opts_dict): print('\n'.join(lines)) -def use_color(colorize, stream=sys.stdout): +def use_color(colorize, stream: Any = sys.stdout) -> bool: """ Return ``True`` or ``False`` depending on whether ANSI color escapes are to be used when printing to `stream`. diff --git a/easybuild/tools/output.py b/easybuild/tools/output.py index d1cf1810be..58a443d830 100644 --- a/easybuild/tools/output.py +++ b/easybuild/tools/output.py @@ -35,6 +35,8 @@ from collections import OrderedDict import sys +from typing import Any, Dict, Optional + from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import OUTPUT_STYLE_RICH, build_option, get_output_style @@ -76,7 +78,7 @@ _progress_bar_cache = {} -def colorize(txt, color): +def colorize(txt: str, color: str) -> str: """ Colorize given text, with specified color. """ @@ -91,7 +93,7 @@ def colorize(txt, color): return coltxt -def escape_for_rich(txt): +def escape_for_rich(txt: str) -> str: """Make sure the text can be printed with rich if that is used""" if use_rich(): txt = rich.markup.escape(txt) @@ -105,42 +107,42 @@ class DummyRich: """ # __enter__ and __exit__ must be implemented to allow use as context manager - def __enter__(self, *args, **kwargs): + def __enter__(self, *args, **kwargs) -> None: pass - def __exit__(self, *args, **kwargs): + def __exit__(self, *args, **kwargs) -> None: pass # dummy implementations for methods supported by rich.progress.Progress class - def add_task(self, *args, **kwargs): + def add_task(self, *args, **kwargs) -> None: pass - def stop_task(self, *args, **kwargs): + def stop_task(self, *args, **kwargs) -> None: pass - def update(self, *args, **kwargs): + def update(self, *args, **kwargs) -> None: pass # internal Rich methods - def __rich_console__(self, *args, **kwargs): + def __rich_console__(self, *args, **kwargs) -> None: pass -def use_rich(): +def use_rich() -> bool: """ Return whether or not to use Rich to produce rich output. """ return get_output_style() == OUTPUT_STYLE_RICH -def show_progress_bars(): +def show_progress_bars() -> bool: """ Return whether or not to show progress bars. """ return use_rich() and build_option('show_progress_bar') and not build_option('extended_dry_run') -def rich_live_cm(): +def rich_live_cm() -> object: """ Return Live instance to use as context manager. """ @@ -165,7 +167,7 @@ def progress_bar_cache(func): Function decorator to cache created progress bars for easy retrieval. """ @functools.wraps(func) - def new_func(ignore_cache=False): + def new_func(ignore_cache: bool = False) -> object: if hasattr(func, 'cached') and not ignore_cache: progress_bar = func.cached elif use_rich() and build_option('show_progress_bar'): @@ -180,7 +182,7 @@ def new_func(ignore_cache=False): @progress_bar_cache -def status_bar(): +def status_bar() -> object: """ Get progress bar to display overall progress. """ @@ -193,7 +195,7 @@ def status_bar(): @progress_bar_cache -def easyconfig_progress_bar(): +def easyconfig_progress_bar() -> object: """ Get progress bar to display progress for installing a single easyconfig file. """ @@ -209,7 +211,7 @@ def easyconfig_progress_bar(): @progress_bar_cache -def download_all_progress_bar(): +def download_all_progress_bar() -> object: """ Get progress bar to show progress on downloading of all source files. """ @@ -224,7 +226,7 @@ def download_all_progress_bar(): @progress_bar_cache -def download_one_progress_bar(): +def download_one_progress_bar() -> object: """ Get progress bar to show progress for downloading a file of known size. """ @@ -240,7 +242,7 @@ def download_one_progress_bar(): @progress_bar_cache -def download_one_progress_bar_unknown_size(): +def download_one_progress_bar_unknown_size() -> object: """ Get progress bar to show progress for downloading a file of unknown size. """ @@ -254,7 +256,7 @@ def download_one_progress_bar_unknown_size(): @progress_bar_cache -def extensions_progress_bar(): +def extensions_progress_bar() -> object: """ Get progress bar to show progress for installing extensions. """ @@ -267,7 +269,7 @@ def extensions_progress_bar(): return progress_bar -def get_progress_bar(bar_type, ignore_cache=False, size=None): +def get_progress_bar(bar_type: str, ignore_cache: bool = False, size: Optional[int] = None) -> object: """ Get progress bar of given type. """ @@ -282,7 +284,7 @@ def get_progress_bar(bar_type, ignore_cache=False, size=None): return pbar -def start_progress_bar(bar_type, size, label=None): +def start_progress_bar(bar_type: str, size: Optional[int], label: Optional[str] = None) -> None: """ Start progress bar of given type. @@ -303,7 +305,8 @@ def start_progress_bar(bar_type, size, label=None): pbar.update(task_id, description=label) -def update_progress_bar(bar_type, label=None, progress_size=1, total=None): +def update_progress_bar(bar_type: str, label: Optional[str] = None, progress_size: int = 1, + total: Optional[int] = None) -> None: """ Update progress bar of given type (if it was started), add progress of given size. @@ -321,7 +324,7 @@ def update_progress_bar(bar_type, label=None, progress_size=1, total=None): pbar.update(task_id, total=total) -def stop_progress_bar(bar_type, visible=False): +def stop_progress_bar(bar_type: str, visible: bool = False) -> None: """ Stop progress bar of given type. """ @@ -334,7 +337,7 @@ def stop_progress_bar(bar_type, visible=False): raise EasyBuildError("Failed to stop %s progress bar, since it was never started?!", bar_type) -def print_checks(checks_data): +def print_checks(checks_data: Dict[str, Any]) -> None: """Print overview of checks that were made.""" col_titles = checks_data.pop('col_titles', ('name', 'info', 'description')) diff --git a/easybuild/tools/parallelbuild.py b/easybuild/tools/parallelbuild.py index 9363ac7505..bd523e9a09 100644 --- a/easybuild/tools/parallelbuild.py +++ b/easybuild/tools/parallelbuild.py @@ -38,6 +38,7 @@ import math import os import re +from typing import Any, Dict, List, Optional, Sequence, Union from easybuild.base import fancylogger from easybuild.framework.easyblock import get_easyblock_instance @@ -59,8 +60,10 @@ def _to_key(dep): return ActiveMNS().det_full_module_name(dep) -def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybuild-build', testing=False, - prepare_first=True, tweak_map=None, try_opts=''): +def build_easyconfigs_in_parallel(build_command: str, easyconfigs: Sequence[Dict[str, Any]], + output_dir: str = 'easybuild-build', testing: bool = False, + prepare_first: bool = True, tweak_map: Optional[Dict[str, str]] = None, + try_opts: str = '') -> Union[str, List[Any]]: """ Build easyconfigs in parallel by submitting jobs to a batch-queuing system. Return list of jobs submitted. @@ -93,7 +96,7 @@ def build_easyconfigs_in_parallel(build_command, easyconfigs, output_dir='easybu jobs = [] # keep track of which job builds which module - module_to_job = {} + module_to_job: Dict[str, Any] = {} for easyconfig in easyconfigs: # this is very important, otherwise we might have race conditions @@ -183,7 +186,8 @@ def submit_jobs(ordered_ecs, cmd_line_opts, testing=False, prepare_first=True, t tweak_map=tweak_map, try_opts=try_opts_str) -def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-build', spec=''): +def create_job(job_backend: Any, build_command: str, easyconfig: dict, + output_dir: str = 'easybuild-build', spec: str = '') -> Any: """ Creates a job to build a *single* easyconfig. @@ -228,7 +232,7 @@ def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-bui return job -def prepare_easyconfig(ec): +def prepare_easyconfig(ec: Any) -> None: """ Prepare for building specified easyconfig (fetch sources) :param ec: parsed easyconfig (EasyConfig instance) diff --git a/easybuild/tools/robot.py b/easybuild/tools/robot.py index ab744d32f5..4e6d312259 100644 --- a/easybuild/tools/robot.py +++ b/easybuild/tools/robot.py @@ -38,6 +38,7 @@ import copy import os import sys +from typing import Any, List, Optional, Sequence, Tuple from easybuild.base import fancylogger from easybuild.framework.easyconfig.easyconfig import EASYCONFIGS_ARCHIVE_DIR, ActiveMNS, process_easyconfig @@ -54,9 +55,10 @@ _log = fancylogger.getLogger('tools.robot', fname=False) -def det_robot_path(robot_paths_option, tweaked_ecs_paths, extra_ec_paths, auto_robot=False): +def det_robot_path(robot_paths_option: Sequence[str], tweaked_ecs_paths: Optional[Tuple[str, str]], + extra_ec_paths: Optional[Sequence[str]], auto_robot: bool = False) -> List[str]: """Determine robot path.""" - robot_path = robot_paths_option[:] + robot_path = list(robot_paths_option) _log.info("Using robot path(s): %s", robot_path) tweaked_ecs_path, tweaked_ecs_deps_path = None, None @@ -77,7 +79,8 @@ def det_robot_path(robot_paths_option, tweaked_ecs_paths, extra_ec_paths, auto_r return robot_path -def check_conflicts(easyconfigs, modtool, check_inter_ec_conflicts=True, return_conflicts=False): +def check_conflicts(easyconfigs: list, modtool: Any, check_inter_ec_conflicts: bool = True, + return_conflicts: bool = False) -> bool: """ Check for conflicts in dependency graphs for specified easyconfigs. @@ -105,7 +108,7 @@ def mk_key(spec): def mk_dep_keys(deps): """Create keys for given list of dependencies.""" - res = [] + res: List[Any] = [] for dep in deps: # filter out dependencies marked as external module if not dep.get('external_module', False): @@ -236,7 +239,8 @@ def check_conflict(parent, dep1, dep2): return res -def dry_run(easyconfigs, modtool, short=False, return_modules_to_install=False): +def dry_run(easyconfigs: list, modtool: Any, short: bool = False, + return_modules_to_install: bool = False) -> str: """ Compose dry run overview for supplied easyconfigs: * [ ] for unavailable @@ -312,7 +316,7 @@ def dry_run(easyconfigs, modtool, short=False, return_modules_to_install=False): return '\n'.join(lines) -def missing_deps(easyconfigs, modtool, terse=False): +def missing_deps(easyconfigs, modtool, terse: bool = False) -> str: """ Determine subset of easyconfigs for which no module is installed yet. """ @@ -336,7 +340,7 @@ def missing_deps(easyconfigs, modtool, terse=False): return '\n'.join(lines) -def raise_error_missing_deps(missing_deps, extra_msg=None): +def raise_error_missing_deps(missing_deps: Sequence[Any], extra_msg: Optional[str] = None) -> None: """Raise error to report missing dependencies.""" _log.warning("Missing dependencies (details): %s", missing_deps) @@ -352,7 +356,8 @@ def raise_error_missing_deps(missing_deps, extra_msg=None): raise EasyBuildError(error_msg, exit_code=EasyBuildExit.MISSING_DEPENDENCY) -def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_error_missing_ecs=True): +def resolve_dependencies(easyconfigs: list, modtool: Any, + retain_all_deps: bool = False, raise_error_missing_ecs: bool = True) -> list: """ Work through the list of easyconfigs to determine an optimal order :param easyconfigs: list of easyconfigs diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index dd081761c3..11a44a200f 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -48,6 +48,7 @@ import time from collections import namedtuple from datetime import datetime +from typing import Any, Optional, Sequence, Tuple, Union # import deprecated functions so they can still be imported from easybuild.tools.run, for now from easybuild._deprecated import check_async_cmd, check_log_for_errors, complete_cmd, extract_errors_from_log # noqa @@ -103,7 +104,7 @@ class RunShellCmdError(Exception): - def __init__(self, cmd_result, caller_info, *args, **kwargs): + def __init__(self, cmd_result, caller_info: Tuple[str, int, str], *args, **kwargs) -> None: """Constructor for RunShellCmdError.""" self.cmd = cmd_result.cmd self.cmd_name = os.path.basename(self.cmd.split(' ')[0]) @@ -120,7 +121,7 @@ def __init__(self, cmd_result, caller_info, *args, **kwargs): msg = f"Shell command '{self.cmd_name}' failed!" super().__init__(msg, *args, **kwargs) - def print(self): + def print(self) -> None: """ Report failed shell command for this RunShellCmdError instance """ @@ -200,7 +201,7 @@ def cache_aware_func(cmd, *args, **kwargs): return cache_aware_func -def fileprefix_from_cmd(cmd, allowed_chars=False): +def fileprefix_from_cmd(cmd: str, allowed_chars: Union[bool, str] = False) -> str: """ Simplify the cmd to only the allowed_chars we want in a filename @@ -213,7 +214,8 @@ def fileprefix_from_cmd(cmd, allowed_chars=False): return ''.join([c for c in cmd if c in allowed_chars]) -def create_cmd_scripts(cmd_str, work_dir, env, tmpdir, out_file, err_file): +def create_cmd_scripts(cmd_str: str, work_dir: str, env: Optional[dict], tmpdir: str, + out_file: str, err_file: Optional[str]) -> str: """ Create helper scripts for specified command in specified directory: - env.sh which can be sourced to define environment in which command was run; @@ -301,7 +303,7 @@ def create_cmd_scripts(cmd_str, work_dir, env, tmpdir, out_file, err_file): return cmd_fp -def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): +def _answer_question(stdout: bytes, proc, qa_patterns, qa_wait_patterns) -> bool: """ Private helper function to try and answer questions raised in interactive shell commands. """ @@ -374,10 +376,15 @@ def _answer_question(stdout, proc, qa_patterns, qa_wait_patterns): @run_shell_cmd_cache -def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, - hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, - output_file=True, stream_output=None, asynchronous=False, task_id=None, with_hooks=True, - qa_patterns=None, qa_wait_patterns=None, qa_timeout=100, log_output_on_success=True): +def run_shell_cmd(cmd: Any, fail_on_error: bool = True, split_stderr: bool = False, + stdin: Optional[Any] = None, env: Optional[dict] = None, + hidden: bool = False, in_dry_run: bool = False, verbose_dry_run: bool = False, + work_dir: Optional[str] = None, use_bash: bool = True, + output_file: bool = True, stream_output: Optional[Any] = None, + asynchronous: bool = False, task_id: Optional[Any] = None, + with_hooks: bool = True, qa_patterns: Optional[Sequence[Tuple[Any, Any]]] = None, + qa_wait_patterns: Optional[Sequence[str]] = None, qa_timeout: int = 100, + log_output_on_success: bool = True) -> RunShellCmdResult: """ Run specified (interactive) shell command, and capture output + exit code. diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 32a7960c1f..4af4fb9da6 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -31,6 +31,8 @@ * Kenneth Hoste (Ghent University) * Damian Alvarez (Forschungszentrum Juelich GmbH) """ +from typing import List + from easybuild.tools import systemtools from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option @@ -133,13 +135,13 @@ class Compiler(Toolchain): COMPILER_CC = None COMPILER_CXX = None COMPILER_C_OPTIONS = ['cstd'] - COMPILER_C_UNIQUE_OPTIONS = [] + COMPILER_C_UNIQUE_OPTIONS: List[str] = [] COMPILER_F77 = None COMPILER_F90 = None COMPILER_FC = None COMPILER_F_OPTIONS = ['i8', 'r8'] - COMPILER_F_UNIQUE_OPTIONS = [] + COMPILER_F_UNIQUE_OPTIONS: List[str] = [] LINKERS = None LINKER_TOGGLE_STATIC_DYNAMIC = None diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index 2e94f513d0..297528cfac 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -31,6 +31,8 @@ * Kenneth Hoste (Ghent University) """ +from typing import Dict, Optional + from easybuild.tools.build_log import EasyBuildError from easybuild.tools.toolchain.toolchain import Toolchain @@ -43,7 +45,7 @@ class LinAlg(Toolchain): BLAS_MODULE_NAME = None BLAS_LIB = None BLAS_LIB_MT = None - BLAS_LIB_MAP = {} + BLAS_LIB_MAP: Dict[str, Optional[str]] = {} BLAS_LIB_GROUP = False BLAS_LIB_STATIC = False BLAS_LIB_DIR = ['lib'] @@ -74,7 +76,7 @@ class LinAlg(Toolchain): SCALAPACK_REQUIRES = ['LIBBLACS', 'LIBLAPACK', 'LIBBLAS'] SCALAPACK_LIB = None SCALAPACK_LIB_MT = None - SCALAPACK_LIB_MAP = {} + SCALAPACK_LIB_MAP: Dict[str, Optional[str]] = {} SCALAPACK_LIB_GROUP = False SCALAPACK_LIB_STATIC = False SCALAPACK_LIB_DIR = ['lib'] diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index f4ce06b334..94ae4674e8 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -70,6 +70,7 @@ from easybuild.tools.toolchain.options import ToolchainOptions from easybuild.tools.toolchain.toolchainvariables import ToolchainVariables from easybuild.tools.utilities import nub, unique_ordered_extend, trace_msg +from typing import Any, Dict _log = fancylogger.getLogger('tools.toolchain', fname=False) @@ -166,7 +167,7 @@ class Toolchain: # list of class 'constants' that should be restored for every new instance of this class CLASS_CONSTANTS_TO_RESTORE = None - CLASS_CONSTANT_COPIES = {} + CLASS_CONSTANT_COPIES: Dict[str, Any] = {} @classmethod def _is_toolchain_for(cls, name): diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index e0dabe0061..587bf14532 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -38,6 +38,7 @@ import copy import re import sys +from typing import Any, Dict import easybuild.tools.toolchain from easybuild.tools.entrypoints import EntrypointToolchain @@ -49,7 +50,7 @@ TC_CONST_PREFIX = 'TC_CONSTANT_' -_initial_toolchain_instances = {} +_initial_toolchain_instances: Dict[str, Any] = {} _log = fancylogger.getLogger("toolchain.utilities") diff --git a/easybuild/tools/utilities.py b/easybuild/tools/utilities.py index 8369bc023c..89af564eff 100644 --- a/easybuild/tools/utilities.py +++ b/easybuild/tools/utilities.py @@ -36,6 +36,8 @@ import sys from string import ascii_letters, digits +from typing import Any, Callable, Dict, Iterable, List, Optional, Union + from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option @@ -47,15 +49,16 @@ INDENT_4SPACES = ' ' * 4 -def flatten(lst): +def flatten(lst: Iterable[Iterable[Any]]) -> List[Any]: """Flatten a list of lists.""" - res = [] + res: List[Any] = [] for x in lst: res.extend(x) return res -def quote_str(val, escape_newline=False, prefer_single_quotes=False, escape_backslash=False, tcl=False): +def quote_str(val, escape_newline: bool = False, prefer_single_quotes: bool = False, + escape_backslash: bool = False, tcl: bool = False): """ Obtain a new value to be used in string replacement context. @@ -105,7 +108,7 @@ def quote_py_str(val): return quote_str(val, escape_newline=True, prefer_single_quotes=True, escape_backslash=True) -def shell_quote(token): +def shell_quote(token) -> str: """ Wrap provided token in single quotes (to escape space and characters with special meaning in a shell), so it can be used in a shell command. This results in token that is not expanded/interpolated by the shell. @@ -117,7 +120,7 @@ def shell_quote(token): return "'%s'" % re.sub(r"(? str: """Remove unwanted characters from the given string and return a copy All non-letter and non-numeral characters are considered unwanted except for underscore ('_'). @@ -132,7 +135,7 @@ def capitalize_first_letter(string: str) -> str: return string[0].upper() + string[1:] -def import_available_modules(namespace): +def import_available_modules(namespace: str) -> list: """ Import all available module in the specified namespace. @@ -165,7 +168,8 @@ def import_available_modules(namespace): return modules -def only_if_module_is_available(modnames, pkgname=None, url=None): +def only_if_module_is_available(modnames: Union[str, Iterable[str]], pkgname: Optional[str] = None, + url: Optional[str] = None) -> Callable[[Callable[..., Any]], Callable[..., Any]]: """Decorator to guard functions/methods against missing required module with specified name.""" if pkgname and url is None: url = 'https://pypi.python.org/pypi/%s' % pkgname @@ -173,7 +177,7 @@ def only_if_module_is_available(modnames, pkgname=None, url=None): if isinstance(modnames, str): modnames = (modnames,) - def wrap(orig): + def wrap(orig: Callable[..., Any]) -> Callable[..., Any]: """Decorated function, raises ImportError if specified module is not available.""" try: imported = None @@ -204,13 +208,13 @@ def error(*args, **kwargs): return wrap -def trace_msg(message, silent=False): +def trace_msg(message: Any, silent: bool = False) -> None: """Print trace message.""" if build_option('trace'): print_msg(' >> ' + message, prefix=False) -def nub(list_): +def nub(list_: list) -> list: """Returns the unique items of a list of hashables, while preserving order of the original list, i.e. the first unique element encoutered is retained. @@ -225,7 +229,7 @@ def nub(list_): return list(dict.fromkeys(list_)) -def unique_ordered_extend(base, affix): +def unique_ordered_extend(base: list, affix: list) -> list: """Extend base list with elements of affix list keeping order and without duplicates""" if isinstance(affix, str): # avoid extending with strings, as iterables generate wrong result without error @@ -242,7 +246,7 @@ def unique_ordered_extend(base, affix): return nub(ext_base) # remove duplicates -def get_class_for(modulepath, class_name): +def get_class_for(modulepath: str, class_name: str) -> type: """ Get class for a given Python class name and Python module path. @@ -262,9 +266,9 @@ def get_class_for(modulepath, class_name): return klass -def get_subclasses_dict(klass, include_base_class=False): +def get_subclasses_dict(klass: type, include_base_class: bool = False) -> Dict[type, List[type]]: """Get dict with subclasses per classes, recursively from the specified base class.""" - res = {} + res: Dict[type, List[type]] = {} subclasses = klass.__subclasses__() if include_base_class: res.update({klass: subclasses}) @@ -274,12 +278,12 @@ def get_subclasses_dict(klass, include_base_class=False): return res -def get_subclasses(klass, include_base_class=False): +def get_subclasses(klass: type, include_base_class: bool = False) -> Iterable[type]: """Get list of all subclasses, recursively from the specified base class.""" return get_subclasses_dict(klass, include_base_class=include_base_class).keys() -def mk_md_table(titles, columns): +def mk_md_table(titles: Iterable[str], columns: Iterable[Iterable[str]]) -> List[str]: """ Returns a MarkDown table with given titles and columns (a nested list of string columns for each column) """ @@ -315,7 +319,7 @@ def mk_md_table(titles, columns): return table -def mk_rst_table(titles, columns): +def mk_rst_table(titles: Iterable[str], columns: Iterable[Iterable[str]]) -> List[str]: """ Returns an rst table with given titles and columns (a nested list of string columns for each column) """ @@ -354,7 +358,7 @@ def mk_rst_table(titles, columns): return table -def time2str(delta): +def time2str(delta: datetime.timedelta) -> str: """Return string representing provided datetime.timedelta value in human-readable form.""" res = None @@ -376,8 +380,8 @@ def time2str(delta): return ' '.join(res) -def natural_keys(key): +def natural_keys(key: str) -> List[Any]: """Can be used as the sort key in list.sort(key=natural_keys) to sort in natural order (i.e. respecting numbers)""" - def try_to_int(key_part): + def try_to_int(key_part: str) -> Any: return int(key_part) if key_part.isdigit() else key_part return [try_to_int(key_part) for key_part in re.split(r'(\d+)', key)] diff --git a/easybuild/tools/variables.py b/easybuild/tools/variables.py index 6f81e198de..0d7905e183 100644 --- a/easybuild/tools/variables.py +++ b/easybuild/tools/variables.py @@ -33,6 +33,7 @@ """ import copy import os +from typing import Any, Dict, Iterable, List, Optional from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError @@ -41,7 +42,7 @@ _log = fancylogger.getLogger('variables', fname=False) -def get_class(name, default_class, map_class=None): +def get_class(name: Optional[str], default_class: type, map_class: Optional[Dict[Any, Any]] = None) -> type: """Return class based on default map_class if key == str -> value = class @@ -63,7 +64,7 @@ def get_class(name, default_class, map_class=None): return klass -def join_map_class(map_classes): +def join_map_class(map_classes: Iterable[Dict[Any, Any]]) -> Dict[Any, Any]: """Join all class_maps into single class_map""" res = {} for map_class in map_classes: @@ -100,27 +101,27 @@ class StrList(list): JOIN_BEGIN_END = False - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) - def str_convert(self, x): + def str_convert(self, x) -> str: """Convert members of list to string (no prefix of begin and end)""" return ''.join([str(y) for y in [self.PREFIX, str(x), self.SUFFIX] if y is not None]) - def _str_ok(self, x): + def _str_ok(self, x) -> bool: """Test if x can be added to returned string""" test = x is not None and len(str(x)) > 0 return test - def _str_self(self): + def _str_self(self) -> List[str]: """Main part of __str__""" return [self.str_convert(x) for x in self if self._str_ok(x)] - def sanitize(self): + def sanitize(self) -> None: """Sanitize self""" - def __str__(self): + def __str__(self) -> str: """_str_self and support for BEGIN/END""" self.sanitize() xs = [self.BEGIN] + self._str_self() + [self.END] @@ -135,11 +136,11 @@ def __getattribute__(self, attr_name): else: return super().__getattribute__(attr_name) - def copy(self): + def copy(self) -> Any: """Return copy of self""" return copy.deepcopy(self) - def try_remove(self, values): + def try_remove(self, values: Iterable[Any]) -> None: """Remove without ValueError in case of missing element""" for value in values: try: @@ -206,9 +207,9 @@ def append_subdirs(self, base, subdirs=None): class ListOfLists(list): """List of lists""" DEFAULT_CLASS = StrList - PROTECTED_CLASSES = [] # classes that are not converted to DEFAULT_CLASS + PROTECTED_CLASSES: List[type] = [] # classes that are not converted to DEFAULT_CLASS # PROTECTED_INSTANCES = [AbsPathList, LibraryList] - PROTECTED_INSTANCES = [] + PROTECTED_INSTANCES: List[type] = [] PROTECT_CLASS_SELF = True # don't convert values that are same class as DEFAULT_CLASS PROTECT_INSTANCE_SELF = True # don't convert values that are instance of DEFAULT_CLASS @@ -451,10 +452,11 @@ class Variables(dict): but are in different classes """ DEFAULT_LISTCLASS = ListOfLists - MAP_LISTCLASS = {} # map between variable name and ListOfList classes (ie not the (default) class for the variable) - + # map between variable name and ListOfList classes (ie not the (default) class for the variable) + MAP_LISTCLASS: Dict[str, type] = {} DEFAULT_CLASS = StrList - MAP_CLASS = {} # predefined map to specify (default) mapping between variables and classes + # predefined map to specify (default) mapping between variables and classes + MAP_CLASS: Dict[str, type] = {} def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index c4fbae9e89..42b69b8e61 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -34,8 +34,10 @@ * Jens Timmerman (Ghent University) """ import os -from easybuild.tools import LooseVersion from socket import gethostname +from typing import Any + +from easybuild.tools import LooseVersion # note: release candidates should be versioned as a pre-release, e.g. "1.1rc1" # 1.1-rc1 would indicate a post-release, i.e., and update of 1.1, so beware! @@ -50,7 +52,7 @@ UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS' -def get_git_revision(): +def get_git_revision() -> str: """ Returns the git revision (e.g. aab4afc016b742c6d4b157427e192942d0e131fe), or UNKNOWN is getting the git revision fails @@ -91,7 +93,7 @@ def get_git_revision(): EASYBLOCKS_VERSION = UNKNOWN_EASYBLOCKS_VERSION # make sure it is smaller then anything -def this_is_easybuild(): +def this_is_easybuild() -> str: """Standard starting message""" top_version = max(FRAMEWORK_VERSION, LooseVersion(EASYBLOCKS_VERSION)) msg = "This is EasyBuild %s (framework: %s, easyblocks: %s) on host %s." @@ -105,7 +107,7 @@ def this_is_easybuild(): return msg -def different_major_versions(v1, v2): +def different_major_versions(v1: Any, v2: Any) -> bool: """Compare major versions""" # Deal with version instances being either strings or LooseVersion if isinstance(v1, str): diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000000..87cfbc1a31 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,34 @@ +[mypy] +# Core settings +python_version = 3.6 +warn_return_any = False +warn_unused_configs = True + +# Ignore missing imports - EasyBuild uses dynamic imports extensively +ignore_missing_imports = True + +# Don't check untyped calls or definitions - we're focusing on existing type hints +# and improving them, not adding them everywhere +check_untyped_defs = False +disallow_untyped_defs = False + +# Don't warn about anything related to dynamically created imports/attributes +warn_no_return = False +warn_unreachable = False +strict_optional = False + +# Show error codes to identify problems +show_error_codes = True +show_column_numbers = True + +# For subclass patterns and inheritance +warn_redundant_casts = True +warn_unused_ignores = True + +[mypy-easybuild.*] +# Apply stricter checks to the main codebase +warn_return_any = True + +[mypy-test.*] +# Less strict for tests +ignore_errors = True