Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ lint:
echo -e "\e[0Ksection_start:`date +%s`:pytest-markers\r\e[0KCheck pytest marker configuration"
make check_markers
echo -e "\e[0Ksection_end:`date +%s`:pytest-markers\r\e[0K"
echo -e "\e[0Ksection_start:`date +%s`:codespell\r\e[0KCheck for common typos with codespell"
make codespell-check
echo -e "\e[0Ksection_end:`date +%s`:codespell\r\e[0K"
echo -e "\e[0Ksection_start:`date +%s`:docs\r\e[0KBuild the docs"
make docs
echo -e "\e[0Ksection_end:`date +%s`:docs\r\e[0K"
Expand Down
6 changes: 6 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,12 @@ pyupgrade: $(PYSOURCES)
flake8: $(PYSOURCES)
flake8 --ignore=E501,W293,W291,E265,E302,E722,E126,E303,E261,E201,E202,W503,W504,W391,E128,E301,E127,E502,E129,E262,E111,E117,E306,E203,E231,E226,E741,E122,E251,E305,E701,E222,E225,E241,E305,E123,E121,E703,E704,E125,E402 $^

codespell-check: $(PYSOURCES)
@codespell $^ || (echo Probable typo found. Run \"make codespell-fix\" to accept suggested fixes, or add the word to the ignore list in setup.cfg ; exit 1)

codespell-fix: $(PYSOURCES)
codespell -w $^

preflight: mypy touched_pylint

.PHONY: help \
Expand Down
1 change: 1 addition & 0 deletions requirements-dev.txt
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,4 @@ build
check-jsonschema
strip_ansi==0.1.1
edit_distance>=1.0.9,<2
codespell
3 changes: 3 additions & 0 deletions setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,6 @@ no_warn_no_return = True

[mypy-toil.cwl.*]
strict = True

[codespell]
ignore-words-list=connexion,NotIn
8 changes: 4 additions & 4 deletions src/toil/batchSystems/abstractBatchSystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class BatchJobExitReason(enum.IntEnum):
FAILED = 2
"""Job finished, but failed."""
LOST = 3
"""Preemptable failure (job's executing host went away)."""
"""Preemptible failure (job's executing host went away)."""
KILLED = 4
"""Job killed before finishing."""
ERROR = 5
Expand Down Expand Up @@ -127,7 +127,7 @@ def supportsAutoDeployment(cls) -> bool:
If it does, the :meth:`setUserScript` can be invoked to set the resource
object representing the user script.

Note to implementors: If your implementation returns True here, it should also override
Note to implementers: If your implementation returns True here, it should also override
"""
raise NotImplementedError()

Expand Down Expand Up @@ -280,7 +280,7 @@ def setEnv(self, name: str, value: str | None = None) -> None:
variables in that inherited environment before the worker is launched.
Note that this mechanism is different to the one used by the worker
internally to set up the environment of a job. A call to this method affects
all jobs issued after this method returns. Note to implementors: This
all jobs issued after this method returns. Note to implementers: This
means that you would typically need to copy the variables before enqueuing a job.

If no value is provided it will be looked up from the current environment.
Expand Down Expand Up @@ -409,7 +409,7 @@ def setEnv(self, name: str, value: str | None = None) -> None:
method makes it possible to override specific variables in that inherited environment
before the worker is launched. Note that this mechanism is different to the one used by
the worker internally to set up the environment of a job. A call to this method affects
all jobs issued after this method returns. Note to implementors: This means that you
all jobs issued after this method returns. Note to implementers: This means that you
would typically need to copy the variables before enqueuing a job.

If no value is provided it will be looked up from the current environment.
Expand Down
2 changes: 1 addition & 1 deletion src/toil/batchSystems/abstractGridEngineBatchSystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -626,7 +626,7 @@ def setEnv(self, name, value=None):
if value and "," in value:
raise ValueError(
type(self).__name__
+ " does not support commata in environment variable values"
+ " does not support commas in environment variable values"
)
return super().setEnv(name, value)

Expand Down
4 changes: 2 additions & 2 deletions src/toil/batchSystems/kubernetes.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"""
Batch system for running Toil workflows on Kubernetes.

Ony useful with network-based job stores, like AWSJobStore.
Only useful with network-based job stores, like AWSJobStore.

Within non-privileged Kubernetes containers, additional Docker containers
cannot yet be launched. That functionality will need to wait for user-mode
Expand Down Expand Up @@ -899,7 +899,7 @@ def mount_host_path(
"workdir", self.host_path, self.worker_work_dir, create=True
)
# We also need to mount across /run/lock, where we will put
# per-node coordiantion info.
# per-node coordination info.
# Don't create this; it really should always exist.
mount_host_path("coordination", "/run/lock", "/run/lock")
else:
Expand Down
8 changes: 4 additions & 4 deletions src/toil/batchSystems/singleMachine.py
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,7 @@ def _stop_now(self, popens: Sequence[subprocess.Popen]) -> list[int]:
# It just died. Assume the pgid was its PID.
pgid = popen.pid
else:
# It is dead. Try it's PID as a PGID and hope we didn't re-use it.
# It is dead. Try it's PID as a PGID and hope we didn't reuse it.
pgid = popen.pid

if pgid != os.getpgrp():
Expand Down Expand Up @@ -439,7 +439,7 @@ def _wait_for_death(self, pgids: list[int], timeout: int = 5):
be gone.
"""
# TODO: this opens a PGID reuse risk; someone else might've reaped the
# process and its PGID may have been re-used.
# process and its PGID may have been reused.

start = datetime.datetime.now()
while (
Expand Down Expand Up @@ -759,7 +759,7 @@ def _startChild(
child_environment = dict(os.environ, **environment)

# Communicate the accelerator resources, if any, to the child process
# by modifying the environemnt
# by modifying the environment
accelerators_acquired: set[int] = (
accelerators_needed if accelerators_needed is not None else set()
)
Expand Down Expand Up @@ -844,7 +844,7 @@ def _handleChild(self, pid: int) -> None:
# It isn't reaped yet, or we have to reap all children to see if thay're done.
# Before we reap it (if possible), kill its PID as a PGID to make sure
# it isn't leaving children behind.
# TODO: This is a PGID re-use risk on Mac because the process is
# TODO: This is a PGID reuse risk on Mac because the process is
# reaped already and the PGID may have been reused.
try:
os.killpg(pid, signal.SIGKILL)
Expand Down
4 changes: 2 additions & 2 deletions src/toil/batchSystems/slurm.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ def get_partition(self, time_limit: float | None) -> str | None:
winning_partition = partition
continue
if partition.gres and not winning_partition.gres:
# Never use a partition witn GRES if you can avoid it
# Never use a partition with GRES if you can avoid it
continue
elif not partition.gres and winning_partition.gres:
# Never keep a partition with GRES if we find one without
Expand Down Expand Up @@ -1002,7 +1002,7 @@ def prepareSbatch(
i += 1

if export_all:
# We don't have any export overrides so we ened to start with
# We don't have any export overrides so we need to start with
# an ALL
export_list.append("ALL")

Expand Down
4 changes: 2 additions & 2 deletions src/toil/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -1691,8 +1691,8 @@ def import_file(
return None
if imported is None and shared_file_name is None and check_existence:
# We need to protect the caller from missing files.
# We think a file was missing, and we got None becasuse of it.
# We didn't get None instead because of usign a shared file name.
# We think a file was missing, and we got None because of it.
# We didn't get None instead because of using a shared file name.
raise FileNotFoundError(f"Could not find file {src_uri}")
return imported

Expand Down
2 changes: 1 addition & 1 deletion src/toil/cwl/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def check_cwltool_version() -> None:
# See https://github.com/pypa/setuptools/issues/3772
from setuptools.extern.packaging.version import InvalidVersion # type: ignore
except ImportError:
# It's not clear that this exception is really part fo the public API, so fake it.
# It's not clear that this exception is really part of the public API, so fake it.
class InvalidVersion(Exception): # type: ignore
pass

Expand Down
6 changes: 3 additions & 3 deletions src/toil/cwl/cwltoil.py
Original file line number Diff line number Diff line change
Expand Up @@ -883,7 +883,7 @@ def toil_make_tool(
class ProcessProtocol(Protocol, Generic[JobTypesT, CallbackTypesT]):
"""
Protocol for the type of a cwltool Process that yields jobs of the given
union type and accepts output_callbacks of the givne union type on its
union type and accepts output_callbacks of the given union type on its
run() method.
"""

Expand Down Expand Up @@ -1200,7 +1200,7 @@ def run_jobs(

# We need to know which cwltool container implementations run the container
# under the calling process's child tree. Unlisted container implementations
# are assumed to run it elsewhere through a deamon.
# are assumed to run it elsewhere through a daemon.
#
# We have to list it this way around (and not list the implementations that use
# a daemon) because some of these extend DockerCommandLineJob.
Expand Down Expand Up @@ -1468,7 +1468,7 @@ def visit(
# If we've already mapped this, map it consistently.
tgt = self._pathmap[location].target
logger.debug(
"ToilPathMapper re-using target %s for path %s",
"ToilPathMapper reusing target %s for path %s",
tgt,
location,
)
Expand Down
4 changes: 2 additions & 2 deletions src/toil/deferred.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ class DeferredFunctionManager:
be locked, and will take them over.
"""

# Define what directory the state directory should actaully be, under the base
# Define what directory the state directory should actually be, under the base
STATE_DIR_STEM = "deferred"
# Have a prefix to distinguish our deferred functions from e.g. NFS
# "silly rename" files, or other garbage that people put in our
Expand Down Expand Up @@ -198,7 +198,7 @@ def open(self):

def defer(deferredFunction):
# Just serialize deferred functions one after the other.
# If serializing later ones fails, eariler ones will still be intact.
# If serializing later ones fails, earlier ones will still be intact.
# We trust dill to protect sufficiently against partial reads later.
logger.debug("Deferring function %s" % repr(deferredFunction))
dill.dump(deferredFunction, self.stateFileOut)
Expand Down
6 changes: 3 additions & 3 deletions src/toil/fileStores/abstractFileStore.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def __init__(
prevent this file store's startCommit and the previous job's
startCommit methods from running at the same time and racing. If
they did race, it might be possible for the later job to be fully
marked as completed in the job store before the eralier job was.
marked as completed in the job store before the earlier job was.
"""
self.jobStore = jobStore
self.jobDesc = jobDesc
Expand Down Expand Up @@ -403,7 +403,7 @@ def _dumpAccessLogs(
for item in self._accessLog:
# For each access record
if len(item) == 2:
# If it has a name, dump wit the name
# If it has a name, dump with the name
file_id, dest_path = item
if os.path.exists(dest_path):
if os.path.islink(dest_path):
Expand Down Expand Up @@ -681,7 +681,7 @@ def log_to_leader(self, text: str, level: int = logging.INFO) -> None:
logged by the worker at the same level.

Does not depend on the commit system, so this is safe to use during an
ansynchronous commit, or without a commit afterward.
asynchronous commit, or without a commit afterward.

:param text: The string to log.
:param level: The logging level.
Expand Down
4 changes: 2 additions & 2 deletions src/toil/fileStores/cachingFileStore.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ class CachingFileStore(AbstractFileStore):
finish when the job's actual state after running is committed back to the
job store.

Internaly, manages caching using a database. Each node has its own
Internally, manages caching using a database. Each node has its own
database, shared between all the workers on the node. The database contains
several tables:

Expand Down Expand Up @@ -2512,7 +2512,7 @@ def shutdown(cls, shutdown_info: tuple[str, str]) -> None:
except OSError as e:
logger.warning("Caching database file %s could not be removed: %s", e)

# Delete all the files that the database refered to
# Delete all the files that the database referred to
logger.debug("Deleting cache directory %s", cache_dir)
robust_rmtree(cache_dir)

Expand Down
2 changes: 1 addition & 1 deletion src/toil/fileStores/nonCachingFileStore.py
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@ def _removeDeadJobs(
if not batchSystemShutdown:
logger.debug("Deleting the stale working directory.")
# Delete the old work directory if it still exists. Do this only during
# the life of the program and dont' do it during the batch system
# the life of the program and don't do it during the batch system
# cleanup. Leave that to the batch system cleanup code.
robust_rmtree(jobState["jobDir"])
finally:
Expand Down
Loading
Loading