Repository navigation
Upgrade miniwdl to 1.15.0 and refresh WDL unit test skip lists (#5377) #5567
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
+83
−45
Merged
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit
Hold shift + click to select a range
949efc4
Upgrade miniwdl to 1.15.0 and refresh WDL unit test skip lists
annagiroti 5465b0f
Fix CallCache.put() call sites for miniwdl 1.15.0 and correct return-…
annagiroti 16bbc87
Implement WDL 1.2 source-relative File/Directory path resolution
annagiroti ea7cd60
Put test_map in the 1.1 tests because it exists there too.
adamnovak e839d89
Temporarily point WDL_CONFORMANCE_TEST_COMMIT at fix-basic-directory-…
annagiroti 5f6b091
Resolve source-relative paths when binding decls and calling sub-work…
annagiroti e3c7b23
Skip already-virtualized files when resolving source-relative paths i…
annagiroti 95f252e
Merge branch 'master' into issues/5377-wdl12-allow-nested-inputs
annagiroti File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,2 @@ | ||
| miniwdl==1.13.1 | ||
| miniwdl==1.15.0 | ||
| wdlparse==0.1.0 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -249,6 +249,8 @@ class WDLContext(TypedDict): | |
|
|
||
| execution_dir: NotRequired[str] | ||
| """Directory to use as the working directory for workflow code""" | ||
| source_dir: NotRequired[str] | ||
| """Directory containing the WDL source document, for resolving WDL 1.2 source-relative File/Directory paths""" | ||
| container: NotRequired[str] | ||
| """The type of container to use when executing a WDL task. Carries through the value of the commandline --container option.""" | ||
| task_path: str | ||
|
|
@@ -858,6 +860,7 @@ def fill_execution_cache( | |
| output_bindings: WDLBindings, | ||
| file_store: AbstractFileStore, | ||
| wdl_options: WDLContext, | ||
| input_bindings: WDLBindings, | ||
|
Comment on lines
859
to
+863
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If we now need to pass the whole input bindings to the cache, we probably should move responsibility for computing the key inside this function. |
||
| miniwdl_logger: logging.Logger | None = None, | ||
| miniwdl_config: WDL.runtime.config.Loader | None = None, | ||
| ) -> WDLBindings: | ||
|
|
@@ -946,7 +949,12 @@ def assign_shared_fs_path(inode: AnyINode) -> AnyINode: | |
| ) | ||
|
|
||
| # Save the bindings to the cache, representing all files with their shared filesystem paths. | ||
| miniwdl_cache.put(cache_key, view_shared_fs_paths(output_bindings)) | ||
| miniwdl_cache.put( | ||
| cache_key, | ||
| view_shared_fs_paths(output_bindings), | ||
| inputs=view_shared_fs_paths(input_bindings), | ||
| add_paths=WDL.runtime.cache.CallCacheAddPaths(), | ||
| ) | ||
| logger.debug("Saved result to cache under %s", cache_key) | ||
|
|
||
| # Keep using the transformed bindings so that later tasks use | ||
|
|
@@ -1512,6 +1520,18 @@ def __init__( | |
| def execution_dir(self) -> str: | ||
| return self._wdl_options.get("execution_dir", ".") | ||
|
|
||
| def _resolve_source_relative_path(self, filename: str) -> str: | ||
| """ | ||
| Resolve a WDL 1.2 source-relative File/Directory path to a Toil-virtualized path. | ||
| """ | ||
| if os.path.isabs(filename) or is_any_url(filename): | ||
| return filename | ||
| source_dir = self._wdl_options.get("source_dir") or self.execution_dir | ||
| # Virtualize it so it matches the same literal virtualized elsewhere, | ||
| # like a Decl's value, otherwise a Map key lookup can't find it. | ||
| # source_dir is a URI, so join with urljoin, not os.path.join. | ||
| return self._virtualize_filename(urljoin(source_dir, filename)) | ||
|
|
||
| @property | ||
| def task_path(self) -> str: | ||
| task_path: str = self._wdl_options["task_path"] | ||
|
|
@@ -2415,6 +2435,8 @@ def wrapper(v: WDL.Value.Base) -> WDL.Value.File: | |
| WDL.Env.Bindings( | ||
| WDL.Env.Binding("file", WDL.Value.File(exported_path)) | ||
| ), | ||
| inputs=file_input_bindings, | ||
| add_paths=WDL.runtime.cache.CallCacheAddPaths(), | ||
| ) | ||
|
|
||
| # Apply the shared filesystem path to the virtualized file | ||
|
|
@@ -4023,6 +4045,9 @@ def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]: | |
| # MiniWDL guarantees that this will be "work" under the host directory. | ||
| # MiniWDL also insists on creating it. | ||
| wdl_options["execution_dir"] = os.path.join(host_dir, "work") | ||
| # source_dir must come from pos.abspath, since Toil always loads WDL | ||
| # via URI and task.source_dir only recognizes bare local paths. | ||
| wdl_options["source_dir"] = urljoin(self._task.pos.abspath, ".") | ||
|
|
||
| # Set up the WDL standard library. | ||
| # We process nonexistent files in WDLTaskWrapperJob as those must be | ||
|
|
@@ -4531,6 +4556,7 @@ def get_path_in_container(inode: AnyINode) -> AnyINode | None: | |
| output_bindings, | ||
| file_store, | ||
| self._wdl_options, | ||
| unwrap(self._task_internal_bindings), | ||
| miniwdl_logger=miniwdl_logger, | ||
| miniwdl_config=miniwdl_config, | ||
| ) | ||
|
|
@@ -5632,6 +5658,7 @@ def run(self, file_store: AbstractFileStore) -> Promised[WDLBindings]: | |
| self._enclosing_bindings, | ||
| wdl_options=self._wdl_options, | ||
| cache_key=cache_key, | ||
| input_bindings=bindings, | ||
| local=True, | ||
| ) | ||
| sink.addFollowOn(outputs_job) | ||
|
|
@@ -5654,6 +5681,7 @@ def __init__( | |
| enclosing_bindings: WDLBindings, | ||
| wdl_options: WDLContext, | ||
| cache_key: str | None = None, | ||
| input_bindings: Promised[WDLBindings] | None = None, | ||
| **kwargs: Any, | ||
| ): | ||
| """ | ||
|
|
@@ -5667,13 +5695,17 @@ def __init__( | |
| :param cache_key: If set and storing into the call cache is on, will | ||
| cache the workflow execution result under the given key in a | ||
| MiniWDL-compatible way. | ||
|
|
||
| :param input_bindings: The workflow call's input bindings, matching | ||
| what cache_key was derived from. Required if cache_key is set. | ||
| """ | ||
| super().__init__(wdl_options=wdl_options, **kwargs) | ||
|
|
||
| self._bindings = bindings | ||
| self._enclosing_bindings = enclosing_bindings | ||
| self._workflow = workflow | ||
| self._cache_key = cache_key | ||
| self._input_bindings = input_bindings | ||
|
|
||
| @report_wdl_errors("evaluate outputs") | ||
| def run(self, file_store: AbstractFileStore) -> WDLBindings: | ||
|
|
@@ -5757,8 +5789,13 @@ def run(self, file_store: AbstractFileStore) -> WDLBindings: | |
| output_bindings = virtualize_inodes(output_bindings, standard_library) | ||
|
|
||
| if self._cache_key is not None: | ||
| assert self._input_bindings is not None | ||
| output_bindings = fill_execution_cache( | ||
| self._cache_key, output_bindings, file_store, self._wdl_options | ||
| self._cache_key, | ||
| output_bindings, | ||
| file_store, | ||
| self._wdl_options, | ||
| unwrap(self._input_bindings), | ||
| ) | ||
|
|
||
| # Let Files that are not output or available outside the call go out of | ||
|
|
@@ -6105,6 +6142,7 @@ def main() -> None: | |
| # restart. For now we assume we are computing the same values. | ||
| wdl_options: WDLContext = { | ||
| "execution_dir": execution_dir, | ||
| "source_dir": urljoin(target.pos.abspath, "."), | ||
| "container": options.container, | ||
| "task_path": target.name, | ||
| "namespace": target.name, | ||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It didn't work to pin
7e171c30feb8380d6f2d2808ee594227856a28f0?