Skip to content
Merged
2 changes: 1 addition & 1 deletion requirements-wdl.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
miniwdl==1.13.1
miniwdl==1.15.0
wdlparse==0.1.0
53 changes: 13 additions & 40 deletions src/toil/test/wdl/wdltoil_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@


WDL_CONFORMANCE_TEST_REPO = "https://github.com/DataBiosphere/wdl-conformance-tests.git"
WDL_CONFORMANCE_TEST_COMMIT = "12d6d8a54a11803fb529aeca18ee01cba01f1d3e"
# TODO: Revert to a pinned commit once DataBiosphere/wdl-conformance-tests#PR
# (basic_directory regex fix) merges to master.
WDL_CONFORMANCE_TEST_COMMIT = "fix-basic-directory-regex"

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It didn't work to pin 7e171c30feb8380d6f2d2808ee594227856a28f0?

# These tests are known to require things not implemented by
# Toil and will not be run in CI.
WDL_CONFORMANCE_TESTS_UNSUPPORTED_BY_TOIL = [
Expand All @@ -49,58 +51,29 @@
# These tests (in the same order as in SPEC.md) are known to fail
WDL_11_UNIT_TESTS_UNSUPPORTED_BY_TOIL = [
"test_object", # Objects are not supported
"map_to_struct", # miniwdl cannot coerce map to struct, https://github.com/chanzuckerberg/miniwdl/issues/712
"relative_and_absolute_task", # needs root to run
"test_gpu_task", # needs gpu to run, else warning
"hisat2_task", # This needs way too many resources (and actually doesn't work?), see https://github.com/DataBiosphere/wdl-conformance-tests/blob/2d617b703a33791f75f30a9db43c3740a499cd89/README_UNIT.md?plain=1#L8
"gatk_haplotype_caller_task", # same as above
"input_ref_call", # Inputs refering into workflow body not yet implemented: see https://github.com/DataBiosphere/toil/issues/4993
"call_imported", # Same as input_ref_call since it imports it
"call_imported_task", # Same as input_ref_call since it imports it
"test_sub", # MiniWDL does not handle metacharacters properly when running regex, https://github.com/chanzuckerberg/miniwdl/issues/709
"read_bool_task", # miniwdl bug, see https://github.com/chanzuckerberg/miniwdl/issues/701
"write_json_fail", # miniwdl (and toil) bug, unserializable json is serialized, see https://github.com/chanzuckerberg/miniwdl/issues/702
"read_object_task", # object not supported
"read_objects_task", # object not supported
"write_object_task", # object not supported
"write_objects_task", # object not supported
"test_transpose", # miniwdl bug, see https://github.com/chanzuckerberg/miniwdl/issues/699
"test_as_map_fail", # miniwdl bug, evalerror, see https://github.com/chanzuckerberg/miniwdl/issues/700
"test_collect_by_key", # same as test_as_map_
"test_map", # Map key not found: a File-typed Map key and a later lookup of the same path each virtualize it independently to a different toilfile: identity, since they run on different StdLib instances
]

WDL_12_UNIT_TESTS_UNSUPPORTED_BY_TOIL = WDL_11_UNIT_TESTS_UNSUPPORTED_BY_TOIL + [
"relative_paths_context", # Toil can't yet resolve File coercion at task scope relative to task file.
"relative_paths_context", # Workflow succeeds but output content does not match expected ("Expected and result do not match!")
"file_directory_equality", # String to Directory coercion not yet implemented.
"single_return_code_task", # MiniWDL 1.13.1 only knows returnCodes and not return_codes.
"all_return_codes_task", # MiniWDL 1.13.1 only knows returnCodes and not return_codes.
"test_runtime_info_task", # MiniWDL 1.13.1 can't yet expose the task global.
"placeholder_none", # 'outputs' section expected 1 results (['placeholder_none.s']), got 0 instead ([]) with exit code 1
"person_struct_task", # Doesn't work as written in the spec; see https://github.com/openwdl/wdl/issues/739
"import_structs", # Feature not yet implemented?
"environment_variable_should_echo", # Ln 14 Col 45: Unexpected token STRING1_FRAGMENT
"outputs_task", # 'outputs' section expected 2 results (['outputs.threshold', 'outputs.two_csvs']), got 3 instead (['outputs.two_csvs', 'outputs.csvs', 'outputs.threshold']) with exit code 0
"glob_task", # 'outputs' section expected 1 results (['glob.last_file_contents']), got 2 instead (['glob.last_file_contents', 'glob.outfiles']) with exit code 0
"test_hints_task", # Test is written as if the file has 3 lines, but it really has 2. See https://github.com/openwdl/wdl/issues/741
"input_hint_task", # Missing outputs in test definition: https://github.com/openwdl/wdl/issues/740
"test_allow_nested_inputs", # Ln 27 Col 3: Unexpected token HINTS
"multi_nested_inputs", # Ln 8 Col 9: Unexpected token STRING1_FRAGMENT
"allow_nested", # Ln 32 Col 9: Unexpected token STRING1_FRAGMENT
"test_find_task", # Ln 9 Col 22: No such function: find
"test_matches_task", # Ln 7 Col 29: No such function: matches
"change_extension_task", # 'outputs' section expected 2 results (['change_extension.data', 'change_extension.index']), got 3 instead (['change_extension.index', 'change_extension.data', 'change_extension.data_file']) with exit code 0
"join_paths_task", # Ln 14 Col 15: No such function: join_paths
"gen_files_task", # 'outputs' section expected 1 results (['gen_files.glob_len']), got 2 instead (['gen_files.glob_len', 'gen_files.files']) with exit code 0
"file_sizes_task", # WDL.Error.StaticTypeMismatch: Expected File? instead of Map[String,Pair[Int,File?]]
"single_return_code_task", # Task correctly exits 1 as declared via requirements.return_codes, but Toil/miniwdl still requires exit code 0 - return_codes override is not honored
"all_return_codes_task", # Task correctly exits 42 as declared via requirements.return_codes: "*", but Toil/miniwdl still requires exit code 0 - return_codes override is not honored
"test_runtime_info_task", # WDL.Error.EvalError from a KeyError; task runtime global still not fully exposed
"placeholder_none", # WDL.Error.EvalError: select_first() given empty or all-null array; prevent this or append a default value
"environment_variable_should_echo", # Now parses; Expected and result do not match!
"multi_nested_inputs", # Workflow did not fail! Fails for toil-wdl and miniwdl
"join_paths_task", # PermissionError: [Errno 13] Permission denied: '/usr/bin/sudo'
"file_sizes_task", # WDL.Error.InputError: cannot coerce Map[String,Pair[Int,File?]] to Array[File?]
"read_tsv_task", # Ln 21 Col 5: Unknown type Object
"write_tsv_task", # Ln 28 Col 16: write_tsv expects 1 argument(s)
"test_contains", # Ln 25 Col 22: No such function: contains
"chunk_array", # Ln 8 Col 17: No such function: chunk
"test_select_first", # Ln 14 Col 17: select_first expects 1 argument(s)
"test_keys", # Ln 32 Col 36: Expected Map[Any,Any] instead of Name
"test_contains_key", # Ln 18 Col 20: No such function: contains_key
"test_values", # Ln 28 Col 20: No such function: values
"test_length", # length() isn't implemented for maps and strings yet
]


Expand Down
42 changes: 40 additions & 2 deletions src/toil/wdl/wdltoil.py
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,8 @@ class WDLContext(TypedDict):

execution_dir: NotRequired[str]
"""Directory to use as the working directory for workflow code"""
source_dir: NotRequired[str]
"""Directory containing the WDL source document, for resolving WDL 1.2 source-relative File/Directory paths"""
container: NotRequired[str]
"""The type of container to use when executing a WDL task. Carries through the value of the commandline --container option."""
task_path: str
Expand Down Expand Up @@ -858,6 +860,7 @@ def fill_execution_cache(
output_bindings: WDLBindings,
file_store: AbstractFileStore,
wdl_options: WDLContext,
input_bindings: WDLBindings,
Comment on lines 859 to +863

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we now need to pass the whole input bindings to the cache, we probably should move responsibility for computing the key inside this function.

miniwdl_logger: logging.Logger | None = None,
miniwdl_config: WDL.runtime.config.Loader | None = None,
) -> WDLBindings:
Expand Down Expand Up @@ -946,7 +949,12 @@ def assign_shared_fs_path(inode: AnyINode) -> AnyINode:
)

# Save the bindings to the cache, representing all files with their shared filesystem paths.
miniwdl_cache.put(cache_key, view_shared_fs_paths(output_bindings))
miniwdl_cache.put(
cache_key,
view_shared_fs_paths(output_bindings),
inputs=view_shared_fs_paths(input_bindings),
add_paths=WDL.runtime.cache.CallCacheAddPaths(),
)
logger.debug("Saved result to cache under %s", cache_key)

# Keep using the transformed bindings so that later tasks use
Expand Down Expand Up @@ -1512,6 +1520,18 @@ def __init__(
def execution_dir(self) -> str:
return self._wdl_options.get("execution_dir", ".")

def _resolve_source_relative_path(self, filename: str) -> str:
"""
Resolve a WDL 1.2 source-relative File/Directory path to a Toil-virtualized path.
"""
if os.path.isabs(filename) or is_any_url(filename):
return filename
source_dir = self._wdl_options.get("source_dir") or self.execution_dir
# Virtualize it so it matches the same literal virtualized elsewhere,
# like a Decl's value, otherwise a Map key lookup can't find it.
# source_dir is a URI, so join with urljoin, not os.path.join.
return self._virtualize_filename(urljoin(source_dir, filename))

@property
def task_path(self) -> str:
task_path: str = self._wdl_options["task_path"]
Expand Down Expand Up @@ -2415,6 +2435,8 @@ def wrapper(v: WDL.Value.Base) -> WDL.Value.File:
WDL.Env.Bindings(
WDL.Env.Binding("file", WDL.Value.File(exported_path))
),
inputs=file_input_bindings,
add_paths=WDL.runtime.cache.CallCacheAddPaths(),
)

# Apply the shared filesystem path to the virtualized file
Expand Down Expand Up @@ -4023,6 +4045,9 @@ def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]:
# MiniWDL guarantees that this will be "work" under the host directory.
# MiniWDL also insists on creating it.
wdl_options["execution_dir"] = os.path.join(host_dir, "work")
# source_dir must come from pos.abspath, since Toil always loads WDL
# via URI and task.source_dir only recognizes bare local paths.
wdl_options["source_dir"] = urljoin(self._task.pos.abspath, ".")

# Set up the WDL standard library.
# We process nonexistent files in WDLTaskWrapperJob as those must be
Expand Down Expand Up @@ -4531,6 +4556,7 @@ def get_path_in_container(inode: AnyINode) -> AnyINode | None:
output_bindings,
file_store,
self._wdl_options,
unwrap(self._task_internal_bindings),
miniwdl_logger=miniwdl_logger,
miniwdl_config=miniwdl_config,
)
Expand Down Expand Up @@ -5632,6 +5658,7 @@ def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]:
self._enclosing_bindings,
wdl_options=self._wdl_options,
cache_key=cache_key,
input_bindings=bindings,
local=True,
)
sink.addFollowOn(outputs_job)
Expand All @@ -5654,6 +5681,7 @@ def __init__(
enclosing_bindings: WDLBindings,
wdl_options: WDLContext,
cache_key: str | None = None,
input_bindings: Promised[WDLBindings] | None = None,
**kwargs: Any,
):
"""
Expand All @@ -5667,13 +5695,17 @@ def __init__(
:param cache_key: If set and storing into the call cache is on, will
cache the workflow execution result under the given key in a
MiniWDL-compatible way.

:param input_bindings: The workflow call's input bindings, matching
what cache_key was derived from. Required if cache_key is set.
"""
super().__init__(wdl_options=wdl_options, **kwargs)

self._bindings = bindings
self._enclosing_bindings = enclosing_bindings
self._workflow = workflow
self._cache_key = cache_key
self._input_bindings = input_bindings

@report_wdl_errors("evaluate outputs")
def run(self, file_store: AbstractFileStore) -> WDLBindings:
Expand Down Expand Up @@ -5757,8 +5789,13 @@ def run(self, file_store: AbstractFileStore) -> WDLBindings:
output_bindings = virtualize_inodes(output_bindings, standard_library)

if self._cache_key is not None:
assert self._input_bindings is not None
output_bindings = fill_execution_cache(
self._cache_key, output_bindings, file_store, self._wdl_options
self._cache_key,
output_bindings,
file_store,
self._wdl_options,
unwrap(self._input_bindings),
)

# Let Files that are not output or available outside the call go out of
Expand Down Expand Up @@ -6105,6 +6142,7 @@ def main() -> None:
# restart. For now we assume we are computing the same values.
wdl_options: WDLContext = {
"execution_dir": execution_dir,
"source_dir": urljoin(target.pos.abspath, "."),
"container": options.container,
"task_path": target.name,
"namespace": target.name,
Expand Down