Skip to content
Merged
2 changes: 1 addition & 1 deletion requirements-wdl.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
miniwdl==1.13.1
miniwdl==1.15.0
wdlparse==0.1.0
50 changes: 10 additions & 40 deletions src/toil/test/wdl/wdltoil_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@


WDL_CONFORMANCE_TEST_REPO = "https://github.com/DataBiosphere/wdl-conformance-tests.git"
WDL_CONFORMANCE_TEST_COMMIT = "12d6d8a54a11803fb529aeca18ee01cba01f1d3e"
WDL_CONFORMANCE_TEST_COMMIT = "56df9b2b3018c5bca7fd078ca4d8c539f3e5984c"
# These tests are known to require things not implemented by
# Toil and will not be run in CI.
WDL_CONFORMANCE_TESTS_UNSUPPORTED_BY_TOIL = [
Expand All @@ -49,58 +49,28 @@
# These tests (in the same order as in SPEC.md) are known to fail
WDL_11_UNIT_TESTS_UNSUPPORTED_BY_TOIL = [
"test_object", # Objects are not supported
"map_to_struct", # miniwdl cannot coerce map to struct, https://github.com/chanzuckerberg/miniwdl/issues/712
"relative_and_absolute_task", # needs root to run
"test_gpu_task", # needs gpu to run, else warning
"hisat2_task", # This needs way too many resources (and actually doesn't work?), see https://github.com/DataBiosphere/wdl-conformance-tests/blob/2d617b703a33791f75f30a9db43c3740a499cd89/README_UNIT.md?plain=1#L8
"gatk_haplotype_caller_task", # same as above
"input_ref_call", # Inputs refering into workflow body not yet implemented: see https://github.com/DataBiosphere/toil/issues/4993
"call_imported", # Same as input_ref_call since it imports it
"call_imported_task", # Same as input_ref_call since it imports it
"test_sub", # MiniWDL does not handle metacharacters properly when running regex, https://github.com/chanzuckerberg/miniwdl/issues/709
"read_bool_task", # miniwdl bug, see https://github.com/chanzuckerberg/miniwdl/issues/701
"write_json_fail", # miniwdl (and toil) bug, unserializable json is serialized, see https://github.com/chanzuckerberg/miniwdl/issues/702
"read_object_task", # object not supported
"read_objects_task", # object not supported
"write_object_task", # object not supported
"write_objects_task", # object not supported
"test_transpose", # miniwdl bug, see https://github.com/chanzuckerberg/miniwdl/issues/699
"test_as_map_fail", # miniwdl bug, evalerror, see https://github.com/chanzuckerberg/miniwdl/issues/700
"test_collect_by_key", # same as test_as_map_
]

WDL_12_UNIT_TESTS_UNSUPPORTED_BY_TOIL = WDL_11_UNIT_TESTS_UNSUPPORTED_BY_TOIL + [
"relative_paths_context", # Toil can't yet resolve File coercion at task scope relative to task file.
"relative_paths_context", # Workflow succeeds but output content does not match expected ("Expected and result do not match!")
"file_directory_equality", # String to Directory coercion not yet implemented.
"single_return_code_task", # MiniWDL 1.13.1 only knows returnCodes and not return_codes.
"all_return_codes_task", # MiniWDL 1.13.1 only knows returnCodes and not return_codes.
"test_runtime_info_task", # MiniWDL 1.13.1 can't yet expose the task global.
"placeholder_none", # 'outputs' section expected 1 results (['placeholder_none.s']), got 0 instead ([]) with exit code 1
"person_struct_task", # Doesn't work as written in the spec; see https://github.com/openwdl/wdl/issues/739
"import_structs", # Feature not yet implemented?
"environment_variable_should_echo", # Ln 14 Col 45: Unexpected token STRING1_FRAGMENT
"outputs_task", # 'outputs' section expected 2 results (['outputs.threshold', 'outputs.two_csvs']), got 3 instead (['outputs.two_csvs', 'outputs.csvs', 'outputs.threshold']) with exit code 0
"glob_task", # 'outputs' section expected 1 results (['glob.last_file_contents']), got 2 instead (['glob.last_file_contents', 'glob.outfiles']) with exit code 0
"test_hints_task", # Test is written as if the file has 3 lines, but it really has 2. See https://github.com/openwdl/wdl/issues/741
"input_hint_task", # Missing outputs in test definition: https://github.com/openwdl/wdl/issues/740
"test_allow_nested_inputs", # Ln 27 Col 3: Unexpected token HINTS
"multi_nested_inputs", # Ln 8 Col 9: Unexpected token STRING1_FRAGMENT
"allow_nested", # Ln 32 Col 9: Unexpected token STRING1_FRAGMENT
"test_find_task", # Ln 9 Col 22: No such function: find
"test_matches_task", # Ln 7 Col 29: No such function: matches
"change_extension_task", # 'outputs' section expected 2 results (['change_extension.data', 'change_extension.index']), got 3 instead (['change_extension.index', 'change_extension.data', 'change_extension.data_file']) with exit code 0
"join_paths_task", # Ln 14 Col 15: No such function: join_paths
"gen_files_task", # 'outputs' section expected 1 results (['gen_files.glob_len']), got 2 instead (['gen_files.glob_len', 'gen_files.files']) with exit code 0
"file_sizes_task", # WDL.Error.StaticTypeMismatch: Expected File? instead of Map[String,Pair[Int,File?]]
"single_return_code_task", # Task correctly exits 1 as declared via requirements.return_codes, but Toil/miniwdl still requires exit code 0 - return_codes override is not honored
"all_return_codes_task", # Task correctly exits 42 as declared via requirements.return_codes: "*", but Toil/miniwdl still requires exit code 0 - return_codes override is not honored
"test_runtime_info_task", # WDL.Error.EvalError from a KeyError; task runtime global still not fully exposed
"placeholder_none", # WDL.Error.EvalError: select_first() given empty or all-null array; prevent this or append a default value
"environment_variable_should_echo", # Now parses; Expected and result do not match!
"multi_nested_inputs", # Workflow did not fail! Fails for toil-wdl and miniwdl
"join_paths_task", # PermissionError: [Errno 13] Permission denied: '/usr/bin/sudo'
"file_sizes_task", # WDL.Error.InputError: cannot coerce Map[String,Pair[Int,File?]] to Array[File?]
"read_tsv_task", # Ln 21 Col 5: Unknown type Object
"write_tsv_task", # Ln 28 Col 16: write_tsv expects 1 argument(s)
"test_contains", # Ln 25 Col 22: No such function: contains
"chunk_array", # Ln 8 Col 17: No such function: chunk
"test_select_first", # Ln 14 Col 17: select_first expects 1 argument(s)
"test_keys", # Ln 32 Col 36: Expected Map[Any,Any] instead of Name
"test_contains_key", # Ln 18 Col 20: No such function: contains_key
"test_values", # Ln 28 Col 20: No such function: values
"test_length", # length() isn't implemented for maps and strings yet
]


Expand Down
76 changes: 72 additions & 4 deletions src/toil/wdl/wdltoil.py
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,8 @@ class WDLContext(TypedDict):

execution_dir: NotRequired[str]
"""Directory to use as the working directory for workflow code"""
source_dir: NotRequired[str]
"""Directory containing the WDL source document, for resolving WDL 1.2 source-relative File/Directory paths"""
container: NotRequired[str]
"""The type of container to use when executing a WDL task. Carries through the value of the commandline --container option."""
task_path: str
Expand Down Expand Up @@ -858,6 +860,7 @@ def fill_execution_cache(
output_bindings: WDLBindings,
file_store: AbstractFileStore,
wdl_options: WDLContext,
input_bindings: WDLBindings,
miniwdl_logger: logging.Logger | None = None,
miniwdl_config: WDL.runtime.config.Loader | None = None,
) -> WDLBindings:
Expand Down Expand Up @@ -946,7 +949,12 @@ def assign_shared_fs_path(inode: AnyINode) -> AnyINode:
)

# Save the bindings to the cache, representing all files with their shared filesystem paths.
miniwdl_cache.put(cache_key, view_shared_fs_paths(output_bindings))
miniwdl_cache.put(
cache_key,
view_shared_fs_paths(output_bindings),
inputs=view_shared_fs_paths(input_bindings),
add_paths=WDL.runtime.cache.CallCacheAddPaths(),
)
logger.debug("Saved result to cache under %s", cache_key)

# Keep using the transformed bindings so that later tasks use
Expand Down Expand Up @@ -1512,6 +1520,38 @@ def __init__(
def execution_dir(self) -> str:
return self._wdl_options.get("execution_dir", ".")

def _resolve_source_relative_path(self, filename: str) -> str:
"""
Resolve a WDL 1.2 source-relative File/Directory path to an absolute path.
"""
if os.path.isabs(filename) or is_any_url(filename):
return filename
source_dir = self._wdl_options.get("source_dir") or self.execution_dir
# The later pass over whole bindings handles virtualization.
return urljoin(source_dir, filename)

def _coerce_source_relative_path(
self, value: WDL.Value.Base, desired_type: WDL.Type.Base
) -> WDL.Value.Base:
"""
Coerce a value, resolving source-relative paths in the Files and
Directories that Toil hasn't already virtualized.
"""
value = value.coerce(desired_type)

def rewrite_path(inode: WDL.Value.File | WDL.Value.Directory) -> str:
filename: str = inode.value
if (
get_inode_virtualized_value(inode) is not None
or get_shared_fs_path(inode) is not None
):
return filename
if isinstance(inode, WDL.Value.Directory):
filename = filename.rstrip("/") + "/"
return self._resolve_source_relative_path(filename)

return WDL.Value.rewrite_paths(value, rewrite_path).coerce(desired_type)

@property
def task_path(self) -> str:
task_path: str = self._wdl_options["task_path"]
Expand Down Expand Up @@ -2415,6 +2455,8 @@ def wrapper(v: WDL.Value.Base) -> WDL.Value.File:
WDL.Env.Bindings(
WDL.Env.Binding("file", WDL.Value.File(exported_path))
),
inputs=file_input_bindings,
add_paths=WDL.runtime.cache.CallCacheAddPaths(),
)

# Apply the shared filesystem path to the virtualized file
Expand Down Expand Up @@ -2598,6 +2640,12 @@ def __init__(
),
)

def _resolve_source_relative_path(self, filename: str) -> str:
"""
Leave relative paths as they are, since they are relative to the task's working directory.
"""
return filename

def _stdout(self) -> WDL.Value.File:
"""
Get the standard output of the command that ran, as a WDL File, outside the container.
Expand Down Expand Up @@ -2816,8 +2864,9 @@ def evaluate_named_expression(
raise

if expected_type:
# Coerce to the type it should be.
value = value.coerce(expected_type)
# Coerce to the type it should be, resolving source-relative paths
# the same way StdLib functions and operators do.
value = stdlib._coerce_source_relative_path(value, expected_type)

return value

Expand Down Expand Up @@ -4023,6 +4072,9 @@ def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]:
# MiniWDL guarantees that this will be "work" under the host directory.
# MiniWDL also insists on creating it.
wdl_options["execution_dir"] = os.path.join(host_dir, "work")
# source_dir must come from pos.abspath, since Toil always loads WDL
# via URI and task.source_dir only recognizes bare local paths.
wdl_options["source_dir"] = urljoin(self._task.pos.abspath, ".")

# Set up the WDL standard library.
# We process nonexistent files in WDLTaskWrapperJob as those must be
Expand Down Expand Up @@ -4531,6 +4583,7 @@ def get_path_in_container(inode: AnyINode) -> AnyINode | None:
output_bindings,
file_store,
self._wdl_options,
unwrap(self._task_internal_bindings),
miniwdl_logger=miniwdl_logger,
miniwdl_config=miniwdl_config,
)
Expand Down Expand Up @@ -4641,6 +4694,9 @@ def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]:

if isinstance(self._node.callee, WDL.Tree.Workflow):
# This is a call of a workflow
wdl_options["source_dir"] = urljoin(
self._node.callee.pos.abspath, "."
)
subjob: WDLBaseJob = WDLWorkflowJob(
self._node.callee,
[input_bindings, passed_down_bindings],
Expand Down Expand Up @@ -5632,6 +5688,7 @@ def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]:
self._enclosing_bindings,
wdl_options=self._wdl_options,
cache_key=cache_key,
input_bindings=bindings,
local=True,
)
sink.addFollowOn(outputs_job)
Expand All @@ -5654,6 +5711,7 @@ def __init__(
enclosing_bindings: WDLBindings,
wdl_options: WDLContext,
cache_key: str | None = None,
input_bindings: Promised[WDLBindings] | None = None,
**kwargs: Any,
):
"""
Expand All @@ -5667,13 +5725,17 @@ def __init__(
:param cache_key: If set and storing into the call cache is on, will
cache the workflow execution result under the given key in a
MiniWDL-compatible way.

:param input_bindings: The workflow call's input bindings, matching
what cache_key was derived from. Required if cache_key is set.
"""
super().__init__(wdl_options=wdl_options, **kwargs)

self._bindings = bindings
self._enclosing_bindings = enclosing_bindings
self._workflow = workflow
self._cache_key = cache_key
self._input_bindings = input_bindings

@report_wdl_errors("evaluate outputs")
def run(self, file_store: AbstractFileStore) -> WDLBindings:
Expand Down Expand Up @@ -5757,8 +5819,13 @@ def run(self, file_store: AbstractFileStore) -> WDLBindings:
output_bindings = virtualize_inodes(output_bindings, standard_library)

if self._cache_key is not None:
assert self._input_bindings is not None
output_bindings = fill_execution_cache(
self._cache_key, output_bindings, file_store, self._wdl_options
self._cache_key,
output_bindings,
file_store,
self._wdl_options,
unwrap(self._input_bindings),
)

# Let Files that are not output or available outside the call go out of
Expand Down Expand Up @@ -6105,6 +6172,7 @@ def main() -> None:
# restart. For now we assume we are computing the same values.
wdl_options: WDLContext = {
"execution_dir": execution_dir,
"source_dir": urljoin(target.pos.abspath, "."),
"container": options.container,
"task_path": target.name,
"namespace": target.name,
Expand Down
Loading