diff --git a/doc/conf.py b/doc/conf.py index 214d821..fcef75b 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -24,6 +24,7 @@ "nbsphinx", "nbsphinx_link", "sphinx_copybutton", + "sphinx_tabs.tabs", ] templates_path = ["_templates"] exclude_patterns = ["build", "**.ipynb_checkpoints"] diff --git a/doc/howtoguides.rst b/doc/howtoguides.rst index 8285311..c4f5b91 100644 --- a/doc/howtoguides.rst +++ b/doc/howtoguides.rst @@ -15,6 +15,7 @@ Getting started howtoguides/inspect_workflows howtoguides/gui howtoguides/python + howtoguides/requirements howtoguides/engines Automation diff --git a/doc/howtoguides/requirements.rst b/doc/howtoguides/requirements.rst new file mode 100644 index 0000000..9c4c8d1 --- /dev/null +++ b/doc/howtoguides/requirements.rst @@ -0,0 +1,105 @@ +.. _requirements: + +Reproduce a workflow environment +================================ + +A :term:`workflow` stores the python environment it was created in as the ``requirements`` +field of its ``graph`` field. ``ewoks install`` recreates that environment and +``ewoks execute --env`` runs the :term:`workflow` in it. See +:ref:`this tutorial ` for a step-by-step example. + +What is stored +-------------- + +``ewoks convert`` and ``ewoks execute -o convert_destination=...`` store + +* ``python`` and ``system``: the python interpreter and the operating system. +* ``distributions``: every installed python package with its version and, when it was not + installed from the python package index, the git commit or the archive it came from. Any + :term:`package manager` can recreate the environment from this list. +* ``manager``: the :term:`package manager` that generated the requirements, with the content + of the files it needs to recreate the environment: ``requirements.txt`` for pip-venv. + +Use ``--exclude-requirements`` to store nothing. + +.. note:: + + A :term:`workflow` that stores a list of requirements instead of the structure above is + still supported: the list is parsed as a ``requirements.txt`` file. + +Select a package manager +------------------------ + +``ewoks convert`` uses the :term:`package manager` of the current python environment. It is +detected from the environment variables and files that package managers leave behind, and +from the tool that installed most of the packages. Use ``--package-manager-name`` to select +one explicitly + +.. code-block:: bash + + ewoks convert demo demo.json --test --package-manager-name pip-venv + +``ewoks install`` uses the :term:`package manager` that generated the requirements, unless it +is not installed on the machine. It installs the stored files of that :term:`package manager`. +When the requirements do not contain them, or installing them fails, it generates its own +files from the ``distributions`` list and installs those instead. Both installations are +confirmed separately unless ``--yes`` is provided + +.. code-block:: bash + + ewoks install demo.json --yes --package-manager-name pip-venv + +``--package-manager-command`` provides the command that invokes the tool, for example when it +is not on the ``PATH`` or when a faster implementation should be used + +.. code-block:: bash + + ewoks install demo.json --yes --package-manager-name pip-venv \ + --package-manager-command /path/to/python + +Choose the environment +---------------------- + +.. code-block:: bash + + # a root directory of your choice instead of the one of the package manager + ewoks install demo.json --yes --env-root /tmp/envs + + # a name of your choice instead of the workflow identifier + ewoks install demo.json --yes --env-name demo-env + + # remove the environment when it already exists + ewoks install demo.json --yes --clean + + # another python version than the one stored in the requirements + ewoks install demo.json --yes --python-version 3.12 + + # add ewoks itself when the requirements do not contain it + ewoks install demo.json --yes --with-ewoks + + # install in the current python environment instead of creating one + ewoks install demo.json --yes --in-place + +The default name is the identifier of the :term:`workflow`, which is the ``id`` field of its +``graph`` field. A :term:`workflow` without an identifier gets a name derived from its +content. + +Without ``--env-root`` the :term:`package manager` decides where the environment goes. venv +creates an environment wherever it is told to, so for pip-venv ewoks uses ``~/.ewoks/envs``. + +An environment that already exists is installed in, which adds the requirements to what is +already there. Use ``--clean`` to remove it first. Only a directory that contains a python +environment is removed. + +The environment of a :term:`workflow` is a directory: for pip-venv it is a virtual +environment. ``ewoks execute --env`` takes that directory, not the python interpreter inside +it. + +Limitations +----------- + +* ``--python-version`` is a request: pip-venv can only use the version of the python + interpreter that creates the environment. A warning is emitted when the version cannot be + provided. +* A package installed from a local directory cannot be recreated elsewhere. This is reported as + a warning when the requirements are generated. diff --git a/doc/reference/cli.rst b/doc/reference/cli.rst index ae382ca..4add2cd 100644 --- a/doc/reference/cli.rst +++ b/doc/reference/cli.rst @@ -18,9 +18,27 @@ ewoks install If no ``requirements`` field exist, **ewoks install** will try to extract requirements from the :term:`tasks ` in the :term:`workflows ` before installing them. - Unless ``--yes`` is provided, **ewoks install** will ask for confirmation before installing the packages. + **ewoks install** installs the files of the :term:`package manager` that generated the ``requirements``. + When the ``requirements`` do not contain those files, or installing them fails, the Python ``distributions`` they contain are installed instead. - By default, packages are installed in the current Python environment: if **ewoks install** is run in a virtual environment, the packages will be installed in this virtual environment. + Unless ``--yes`` is provided, **ewoks install** will ask for confirmation before each installation and before removing an existing environment. + + By default, packages are installed in a new Python environment named after the workflow identifier, which is the ``id`` field of its ``graph`` field. + The environment is created where the :term:`package manager` creates named environments (``~/.ewoks/envs`` for package managers that do not have such a directory). + Use ``--env-root`` for another directory to create the environment in and ``--env-name`` for another name than the workflow identifier. + An environment that already exists is installed in, unless ``--clean`` is provided to remove it first. + The :term:`workflow` can be executed in that environment when it contains **ewoks** itself, which is the case when the ``requirements`` contain it. + Use ``--with-ewoks`` to add **ewoks** when the ``requirements`` do not contain it, without changing any of the versions they do contain: + + .. code-block:: bash + + ewoks install myworkflow.json --yes --with-ewoks + ewoks execute --env ~/.ewoks/envs/myworkflow myworkflow.json + + Use ``--in-place`` to install in the current Python environment instead. This is not supported by all package managers. + + The package manager that generated the ``requirements`` is used to reproduce the environment when it is available. + Provide ``--package-manager-name`` (and optionally ``--package-manager-command``) to use another one: the ``requirements`` always contain the list of installed Python distributions as a fallback. ewoks convert ------------- diff --git a/doc/reference/glossary.rst b/doc/reference/glossary.rst index 9abcc6b..d506527 100644 --- a/doc/reference/glossary.rst +++ b/doc/reference/glossary.rst @@ -28,5 +28,8 @@ Glossary Execution engine An execution engine is the underlying software used to execute the :term:`workflow`. :term:`Ewoks` supports multiple execution engines: pypushflow, orange, dask and the ewoks internal excution engine. + Package manager + A package manager creates python environments and installs packages in them. :term:`Ewoks` uses package managers to store the environment in which a :term:`workflow` was created and to recreate it: pip with venv. + blissdata `Blissdata `_ is an API for accessing data from BLISS in memory. diff --git a/doc/tutorials.rst b/doc/tutorials.rst index 8287dc9..f5e7d31 100644 --- a/doc/tutorials.rst +++ b/doc/tutorials.rst @@ -8,6 +8,7 @@ Tutorials tutorials/hello_world tutorials/create_workflow tutorials/execute + tutorials/install tutorials/infrastructure tutorials/external tutorials/develop diff --git a/doc/tutorials/execute.rst b/doc/tutorials/execute.rst index 7b7bb7f..5bb5cda 100644 --- a/doc/tutorials/execute.rst +++ b/doc/tutorials/execute.rst @@ -13,6 +13,17 @@ The :ref:`python interface ` for executing :term:`workflows `. +A :term:`workflow` can also be executed in a python environment created from the +:term:`workflow` itself + +.. code-block:: bash + + ewoks install /path/to/graph.json --yes + ewoks execute --env ~/.ewoks/envs/mygraph /path/to/graph.json + +See :ref:`this tutorial ` and +:ref:`this how-to guide `. + The :code:`engine=None` argument selects the default :term:`execution engine `. Documentation on different :term:`execution engines `: * `ewoksppf `_ : execute cyclic :term:`workflows ` diff --git a/doc/tutorials/getting_started.rst b/doc/tutorials/getting_started.rst index 854319d..707dda2 100644 --- a/doc/tutorials/getting_started.rst +++ b/doc/tutorials/getting_started.rst @@ -87,6 +87,16 @@ To inspect or modify a workflow, convert it to a JSON file: ewoks convert demo demo.json --test +The JSON file also stores the python environment the :term:`workflow` needs, so it can be +recreated later: + +.. code-block:: bash + + ewoks install demo.json --yes + ewoks execute --env ~/.ewoks/envs/demo demo.json --outputs=all + +See the `Install and execute a workflow tutorial <./install.html>`_. + Learn More ========== diff --git a/doc/tutorials/install.rst b/doc/tutorials/install.rst new file mode 100644 index 0000000..aea0d2f --- /dev/null +++ b/doc/tutorials/install.rst @@ -0,0 +1,150 @@ +.. _install_tutorial: + +Install and execute a workflow +============================== + +A :term:`workflow` needs the python packages of its :term:`tasks ` to be installed. +:term:`Ewoks` can store the python environment in which a :term:`workflow` was created inside +the :term:`workflow` itself and recreate that environment later, on another machine or at +another time. + +There are two sides to this + +* the *producer* creates a :term:`workflow` and stores its requirements, +* the *re-producer* receives the :term:`workflow`, recreates the environment and executes the + :term:`workflow` in it. + +Both sides use a :term:`package manager`. The walk-through below does both sides + +.. toctree:: + :maxdepth: 1 + + install/pip_venv + +The producer and the re-producer do not need the same :term:`package manager`: the +requirements contain the installed python packages, which any :term:`package manager` can +install. See :ref:`this how-to guide ` to select a :term:`package manager`. + +Store the requirements +---------------------- + +``ewoks convert`` saves the packages installed in the current python environment as the +``requirements`` of the destination :term:`workflow`. The ``requirements`` field of the +``graph`` field looks like this + +.. code-block:: json + + { + "python": {"version": "3.12.11", "implementation": "CPython", "...": "..."}, + "system": {"system": "Linux", "machine": "x86_64", "...": "..."}, + "distributions": [ + {"name": "ewokscore", "version": "5.1.0", "installer": "pip"}, + {"name": "networkx", "version": "3.4.2", "installer": "pip"} + ], + "manager": { + "name": "pip-venv", + "version": "25.0.1", + "files": {"requirements.txt": "ewokscore==5.1.0\nnetworkx==3.4.2\n"} + } + } + +* ``distributions`` are the installed python packages. Any :term:`package manager` can + recreate the environment from this list. +* ``manager`` is the :term:`package manager` that generated the requirements together with + the files it needs to recreate the environment exactly, for example a lock file. + +Create the environment +---------------------- + +``ewoks install`` creates a python environment for the :term:`workflow`. It prints the python +interpreter of the environment it created, the command to execute the :term:`workflow` in it +and the command to remove it again + +.. code-block:: text + + Installed requirements for demo.json + Python : ewoks_envs/demo/bin/python + Execute: ewoks execute --env ewoks_envs/demo demo.json + Remove : rm -rf ewoks_envs/demo + +Without ``--yes`` you are asked to confirm after the packages have been listed. The +walk-through uses ``--env-root`` to create the environment in the working directory. Without +it the environment is created where the :term:`package manager` creates named environments. + +Execute the workflow +-------------------- + +``--env`` executes the :term:`workflow` with the python interpreter of that environment +instead of the current one. This works because the requirements contain ``ewoks`` itself: it +was installed in the environment in which the :term:`workflow` was converted. When they do +not, add ``--with-ewoks`` to ``ewoks install`` to install it without changing any of the +versions coming from the :term:`workflow`. + +Remove the environment +---------------------- + +The environment is a normal directory, so removing it is enough. This is the command that +``ewoks install`` prints when it creates the environment. + +Limitations and caveats +----------------------- + +Recreating a workflow environment does not guarantee that the workflow can be executed. +The requirements stored in a workflow describe the Python environment, but a workflow can +also depend on system software, external resources, configuration, or files that are not +captured by the requirements. + +The workflow cannot be installed +++++++++++++++++++++++++++++++++ + +Environment creation can fail when the Python packages listed in the requirements cannot +be installed in the target environment. For example: + +* A Python package requires a system package, compiler, or other native build dependency + that is not available on the target machine. +* A package contains compiled code that is not compatible with the target operating system, + CPU architecture, or Python version. +* A required package is no longer available from the configured package indexes, or requires + access to a private package repository. +* A package has dependencies that cannot be resolved together with the required package versions. +* The Python version required by a package is not available on the target machine. +* Installing a package requires network access, credentials, a license, or another resource + that is not available. +* The workflow was created on an operating system or architecture that is different from the + target system and one or more packages are platform-specific. +* A package depends on external system libraries or runtime components that are not provided + by the Python package itself. + +The stored requirements can therefore make the Python environment reproducible, +but they cannot guarantee that the environment can be recreated on every machine. + +The workflow can be installed but cannot be executed +++++++++++++++++++++++++++++++++++++++++++++++++++++ + +A successfully recreated Python environment only guarantees that the Python dependencies +can be installed. Execution can still fail because a task depends on resources or configuration +outside that environment. For example: + +* A task input points to a file that does not exist on the target machine. +* A task expects a directory, executable, configuration file, or other resource that + is not available. +* A task relies on an environment variable that is not defined in the target environment. +* A task requires access to an external service, database, network resource, or hardware device + that is unavailable. +* A task requires credentials, secrets, or configuration that are not stored in the workflow. +* A task assumes a particular working directory or filesystem layout. +* A task uses operating-system features or commands that are not available on the target system. +* A task relies on data that was available when the workflow was created but has not been + transferred with the workflow. +* A task dynamically imports or installs Python packages that are not declared in the + workflow requirements. +* A task depends on a specific version or configuration of external software that is not captured + by the Python environment. +* A task produces or consumes temporary files whose locations or permissions differ on + the target machine. +* The workflow relies on non-deterministic external state, such as the current date, remote data, + or the state of an external service. + +In these cases, `ewoks install` can successfully recreate the Python environment, while `ewoks execute` +can still fail. The requirements should therefore be considered a description of the Python environment, +rather than a complete description of everything required to execute a workflow. diff --git a/doc/tutorials/install/pip_venv.rst b/doc/tutorials/install/pip_venv.rst new file mode 100644 index 0000000..2ee3857 --- /dev/null +++ b/doc/tutorials/install/pip_venv.rst @@ -0,0 +1,133 @@ +.. _install_pip_venv: + +pip and venv +============ + +End-to-end walk-through with ``pip`` and ``venv``. Both are part of a python installation, so +there is no other :term:`package manager` to install. + +Producer side +------------- + +Create an environment, install :term:`ewoks` in it and store a :term:`workflow` with the +packages of that environment as its ``requirements``. + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + python3 -m venv ewoks_producer + source ewoks_producer/bin/activate + pip install ewoks + ewoks convert demo demo.json --test + deactivate + + .. group-tab:: macOS + + .. code-block:: bash + + python3 -m venv ewoks_producer + source ewoks_producer/bin/activate + pip install ewoks + ewoks convert demo demo.json --test + deactivate + + .. group-tab:: Windows + + .. code-block:: powershell + + py -m venv ewoks_producer + ewoks_producer\Scripts\Activate.ps1 + pip install ewoks + ewoks convert demo demo.json --test + deactivate + +Install any package that provides :term:`tasks ` instead of +``ewoks`` for a real :term:`workflow`. + +Re-producer side +---------------- + +Only ``ewoks`` itself is needed to recreate the environment of ``demo.json`` + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + python3 -m venv ewoks_reproducer + source ewoks_reproducer/bin/activate + pip install ewoks + ewoks install demo.json --yes --env-root ewoks_envs + + .. group-tab:: macOS + + .. code-block:: bash + + python3 -m venv ewoks_reproducer + source ewoks_reproducer/bin/activate + pip install ewoks + ewoks install demo.json --yes --env-root ewoks_envs + + .. group-tab:: Windows + + .. code-block:: powershell + + py -m venv ewoks_reproducer + ewoks_reproducer\Scripts\Activate.ps1 + pip install ewoks + ewoks install demo.json --yes --env-root ewoks_envs + +The environment of the :term:`workflow` is a virtual environment in ``ewoks_envs/demo``. + +Execute the workflow +-------------------- + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + ewoks execute --env ewoks_envs/demo demo.json --outputs=all + + .. group-tab:: macOS + + .. code-block:: bash + + ewoks execute --env ewoks_envs/demo demo.json --outputs=all + + .. group-tab:: Windows + + .. code-block:: powershell + + ewoks execute --env ewoks_envs\demo demo.json --outputs=all + +Clean up +-------- + +.. tabs:: + + .. group-tab:: Linux + + .. code-block:: bash + + deactivate + rm -rf ewoks_producer ewoks_reproducer ewoks_envs demo.json + + .. group-tab:: macOS + + .. code-block:: bash + + deactivate + rm -rf ewoks_producer ewoks_reproducer ewoks_envs demo.json + + .. group-tab:: Windows + + .. code-block:: powershell + + deactivate + Remove-Item -Recurse -Force ewoks_producer, ewoks_reproducer, ewoks_envs, demo.json diff --git a/pyproject.toml b/pyproject.toml index 333420c..cebf15e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,6 @@ test = [ "ipykernel", "importlib_resources", "pyqt5", - "pytest-venv", "defusedxml", ] dev = [ @@ -64,6 +63,7 @@ doc = [ "pydata-sphinx-theme", "sphinx-argparse", "sphinx-copybutton", + "sphinx-tabs", ] [tool.setuptools] diff --git a/src/ewoks/__main__.py b/src/ewoks/__main__.py index 3b97c7f..1383a56 100644 --- a/src/ewoks/__main__.py +++ b/src/ewoks/__main__.py @@ -1,8 +1,10 @@ +import subprocess import sys import traceback from argparse import ArgumentDefaultsHelpFormatter from argparse import ArgumentParser from argparse import Namespace +from pathlib import Path from pprint import pprint from subprocess import CalledProcessError from typing import Any @@ -16,12 +18,14 @@ from ewoksutils.cli_utils import cli_submit_utils from ewoksutils.cli_utils.cli_argparse import add_to_parser +from ._requirements.utils.environment import Environment from .bindings import _load_graph from .bindings import convert_graph from .bindings import execute_graph from .bindings import install_graph from .bindings import lint_graph from .bindings import show_graph +from .cli_utils import cli_arguments from .cli_utils import cli_convert_utils from .cli_utils import cli_install_utils from .cli_utils import cli_lint_utils @@ -79,7 +83,11 @@ def create_argument_parser(shell: bool = False) -> ArgumentParser: help="Check that a workflow follows the Ewoks specification", formatter_class=ArgumentDefaultsHelpFormatter, ) - add_to_parser(execute, cli_execute_utils.execute_arguments(shell=shell)) + add_to_parser( + execute, + cli_execute_utils.execute_arguments(shell=shell) + + cli_arguments.environment_arguments(), + ) add_to_parser(submit, cli_submit_utils.submit_arguments(shell=shell)) add_to_parser(cancel, cli_cancel_utils.cancel_arguments(shell=shell)) add_to_parser(convert, cli_convert_utils.convert_arguments(shell=shell)) @@ -154,8 +162,21 @@ def command_convert( for workflow, graph, destination in zip( cli_args.workflows, cli_args.graphs, cli_args.destinations ): - convert_graph(graph, destination, **cli_args.convert_options) + # The package manager is only known when the requirements are generated + manager_names: List[Optional[str]] = [] + convert_graph( + graph, + destination, + on_requirements=manager_names.append, + **cli_args.convert_options, + ) print(f"Converted {workflow} -> {destination}") + if manager_names[0] is None: + print(" Requirements: not saved") + else: + print( + f" Requirements: generated with the {manager_names[0]!r} package manager" + ) if shell: return 0 return None @@ -165,25 +186,75 @@ def command_install( cli_args: Namespace, shell: bool = False ) -> Optional[Literal[0, 1]]: cli_install_utils.parse_install_arguments(cli_args, shell=shell) - for workflow, graph in zip(cli_args.workflows, cli_args.graphs): + for workflow, source in zip(cli_args.workflows, cli_args.graphs): try: - install_graph( - graph, + environment = install_graph( + source, skip_prompt=cli_args.yes, + in_place=cli_args.in_place, + env_name=cli_args.env_name, + env_root=cli_args.env_root, + python_version=cli_args.python_version, + ensure_ewoks=cli_args.with_ewoks, + clean=cli_args.clean, package_manager_name=cli_args.package_manager_name, package_manager_command=cli_args.package_manager_command, ) - except CalledProcessError as e: - print(f"Install failed for {workflow}: {e}") + except (CalledProcessError, RuntimeError, ValueError): + traceback.print_exc() + print(f"Install failed for {workflow}") except AbortException: print(f"Install aborted for {workflow}") else: print(f"Installed requirements for {workflow}") + if not cli_args.in_place: + location = environment.location + print(f" Python : {environment.python}") + if environment.distribution_version("ewoks"): + print(f" Execute: ewoks execute --env {location} {workflow}") + else: + print( + " Execute: the environment has no ewoks to execute the " + "workflow with (use '--with-ewoks')" + ) + print(f" Remove : {_remove_command(location)}") if shell: return 0 return None +def _remove_command(location: Path) -> str: + """Shell command that removes a python environment.""" + if sys.platform == "win32": + return f'rmdir /s /q "{location}"' + return f"rm -rf {location}" + + +def command_in_environment( + location: str, argv: List[str], shell: bool = False +) -> Literal[0, 1]: + """Run the command in another python environment.""" + environment = Environment.from_location(location) + arguments = _remove_option(argv[1:], "--env") + return subprocess.call( # noqa: S603 - Arguments of this command + [environment.python, "-m", "ewoks", *arguments] + ) + + +def _remove_option(arguments: List[str], name: str) -> List[str]: + """Remove an option and its value from CLI arguments.""" + keep = [] + skip = False + for argument in arguments: + if skip: + skip = False + elif argument == name: + skip = True + elif not argument.startswith(f"{name}="): + keep.append(argument) + return keep + + def command_show(cli_args: Namespace, shell: bool = False) -> Optional[Literal[0, 1]]: cli_show_utils.parse_show_arguments(cli_args, shell=shell) for workflow, graph in zip(cli_args.workflows, cli_args.graphs): @@ -217,6 +288,9 @@ def main(argv=None, shell: bool = True) -> Union[Any, Literal[0, 1]]: argv = sys.argv cli_args = parser.parse_args(argv[1:]) + if getattr(cli_args, "env", None): + return command_in_environment(cli_args.env, argv, shell=shell) + if cli_args.command == "execute": return command_execute(cli_args, shell=shell) elif cli_args.command == "submit": diff --git a/src/ewoks/_requirements/__init__.py b/src/ewoks/_requirements/__init__.py index 6aaa98b..b58ff9c 100644 --- a/src/ewoks/_requirements/__init__.py +++ b/src/ewoks/_requirements/__init__.py @@ -1,29 +1,62 @@ """Workflow requirements.""" import logging +import re +import shutil +from pathlib import Path +from typing import Callable +from typing import Dict from typing import Optional from typing import Tuple from typing import Union from ewokscore.graph import TaskGraph +from ewokscore.hashing import uhash from .utils import parse +from .utils._supported import get_supported_managers from .utils.base_manager import BaseRequirements +from .utils.detect import get_in_place_installer +from .utils.detect import get_installer from .utils.detect import get_manager +from .utils.environment import Environment from .utils.metadata import last_resort logger = logging.getLogger(__file__) +_NO_GRAPH_ID = "notspecified" +"""Identifier of a workflow that does not have one.""" + + +def supported_managers() -> Dict[str, str]: + """Names of the supported package managers with an example command for each.""" + return { + name: manager_cls.COMMAND_EXAMPLE + for name, manager_cls in get_supported_managers().items() + } + + +def managers_supporting_in_place() -> Tuple[str, ...]: + """Names of the package managers that can install in the current environment.""" + return tuple( + name + for name, manager_cls in get_supported_managers().items() + if manager_cls.CAN_INSTALL_IN_PLACE + ) + def add_requirements( graph: TaskGraph, manager_name: Optional[str] = None, manager_command: Tuple[str, ...] = tuple(), -) -> None: - """Add requirements to a workflow definition in-place.""" +) -> str: + """Add requirements to a workflow definition in-place. Returns the name of the + package manager that generated them. + """ manager = get_manager(manager_name=manager_name, manager_command=manager_command) requirements = manager.gather_requirements() graph.graph.graph["requirements"] = requirements.model_dump() + return manager.NAME def get_requirements(graph: TaskGraph) -> BaseRequirements: @@ -45,29 +78,163 @@ def get_requirements(graph: TaskGraph) -> BaseRequirements: return requirements +def environment_name(graph: TaskGraph) -> str: + """Name of the python environment of a workflow: its identifier, or a hash of + the workflow when it does not have one. + """ + name = str(graph.graph_id) + if name == _NO_GRAPH_ID: + name = str(uhash(graph.serialize()))[:16] + return re.sub(r"[^\w.-]", "_", name)[:64] or _NO_GRAPH_ID + + def install_requirements( requirements: BaseRequirements, + env_name: Optional[str] = None, + env_root: Optional[Union[str, Path]] = None, manager_name: Optional[str] = None, - manager_command: Union[None, str, Tuple[str, ...]] = None, -) -> None: - """Install workflow requirements.""" + manager_command: Tuple[str, ...] = tuple(), + python_version: Optional[str] = None, + ensure_ewoks: bool = False, + clean: bool = False, + confirm: Optional[Callable[[str], None]] = None, +) -> Environment: + """Install workflow requirements in a python environment named `env_name`, + created inside `env_root` or inside the root directory of the package manager, + or in the current python environment when no name is provided. An environment + that already exists is installed in, or removed first when `clean`. + The files of the package manager are installed when the requirements provide + them, the installed python distributions otherwise or when that installation + failed. `ensure_ewoks` adds ewoks itself when the requirements do not contain it. + `confirm` is called with a description of an installation or a removal before + it starts. - if manager_command and not manager_name: - raise ValueError( - f"Provide 'manager_name' associated to command {manager_command}" + :raises ValueError: no environment name, the package manager cannot install in + the current environment or the environment to remove is not + a python environment + :raises RuntimeError: environment creation or installation failed + """ + if env_name is None: + if env_root: + raise ValueError("An environment root requires an environment name") + if clean: + raise ValueError("Cleaning requires an environment name") + + manager = get_installer( + requirements, manager_name=manager_name, manager_command=manager_command + ) + + if env_name is None: + if not manager.CAN_INSTALL_IN_PLACE: + if manager_name: + raise ValueError( + f"Package manager {manager_name!r} cannot install in the current " + "python environment" + ) + manager = get_in_place_installer() + location = None + remove = False + environment = Environment.current() + destination = "the current python environment" + else: + location = manager.environment_location(env_name, env_root) + if python_version is None: + python_version = requirements.python.version or None + environment = manager.environment(location) + remove = clean and location.exists() + if remove: + _assert_environment(location) + if location.exists() and not remove: + logger.warning( + "The python environment in '%s' already exists: the requirements " + "are installed in it", + location, + ) + destination = f"the existing python environment in '{location}'" + else: + destination = f"a new python environment in '{location}'" + + # The environment is created after the confirmation of the first installation + native = manager.is_native(requirements, environment) + if native: + _confirm( + confirm, + requirements.__files_info__(), + f"This will install the files of the {manager.NAME} package manager " + f"above in {destination}.", ) + else: + logger.warning( + "The requirements do not provide files that the %r package manager can " + "install. The installed python distributions are installed instead.", + manager.NAME, + ) + _confirm( + confirm, + requirements.__distributions_info__(), + f"The requirements do not provide files that the {manager.NAME} package " + "manager can install. This will install the python distributions above " + f"in {destination} instead.", + ) + + if location is not None: + if remove: + if confirm is not None: + confirm( + f"This will remove the existing python environment in '{location}'." + ) + shutil.rmtree(location) + environment = manager.create_environment(location, python_version) + + installed = False + if native: + try: + manager.install_files(requirements, environment) + installed = True + except Exception: + logger.exception( + "Continue with the installed python distributions after failure to " + "install the files of the %r package manager", + manager.NAME, + ) + _confirm( + confirm, + requirements.__distributions_info__(), + f"Installing the files of the {manager.NAME} package manager failed. " + "This will install the python distributions above in " + f"{destination} instead.", + ) + + if not installed: + manager.install_distributions(requirements, environment) + + if ensure_ewoks: + manager.ensure_ewoks(requirements, environment) + + return environment + +def _assert_environment(location: Path) -> None: + """Only a python environment is removed to create a new one. + + :raises ValueError: the location is not a python environment + """ try: - if manager_name: - raise ValueError("Ignore package manager used to generate the requirements") - else: - manager = get_manager(manager_name=requirements.manager.name) + Environment.from_location(location) except ValueError: - manager = get_manager( - manager_name=manager_name, manager_command=manager_command - ) + raise ValueError( + f"'{location}' is not a python environment: remove it yourself to " + "create an environment there" + ) from None - manager.install_requirements(requirements) + +def _confirm( + confirm: Optional[Callable[[str], None]], content: str, action: str +) -> None: + """Describe an installation and let `confirm` accept or refuse it.""" + if confirm is None: + return + confirm(f"{content}\n\n{action}") if __name__ == "__main__": @@ -75,10 +242,12 @@ def install_requirements( import time from pprint import pprint + from .utils import requirements_txt + t0 = time.perf_counter() try: - manager = get_manager(manager_name="pip") + manager = get_manager(manager_name="pip-venv") requirements = manager.gather_requirements() print() @@ -87,9 +256,13 @@ def install_requirements( finally: print("Freeze time:", time.perf_counter() - t0) - pip_freeze = requirements.manager.freeze + pip_freeze = requirements.manager.files[ + requirements_txt.REQUIREMENTS_FILENAME + ].splitlines() - dists_freeze = manager._freeze_distributions(requirements) + dists_freeze = requirements_txt.distributions_requirements( + requirements.distributions + ) dists_freeze = [s for s in dists_freeze if not s.startswith("#")] print() diff --git a/src/ewoks/_requirements/conda.py b/src/ewoks/_requirements/conda.py deleted file mode 100644 index 449a5ad..0000000 --- a/src/ewoks/_requirements/conda.py +++ /dev/null @@ -1,80 +0,0 @@ -import logging -import os -import sys -from typing import Any -from typing import Dict -from typing import Literal -from typing import Optional -from typing import Tuple - -import yaml - -from .utils.base_manager import BaseManager -from .utils.base_manager import BaseManagerInfo -from .utils.base_manager import BaseRequirements - -logger = logging.getLogger(__name__) - - -class CondaManagerInfo(BaseManagerInfo): - name: Literal["conda"] = "conda" - - -class CondaRequirements(BaseRequirements): - manager: CondaManagerInfo - environment: dict - - -class CondaManager(BaseManager): - NAME = "conda" - PRIORITY = 4 - REQUIREMENTS_MODEL = CondaRequirements - - def __init__(self, *command: str) -> None: - if not command: - command = self._get_conda_command() - super().__init__(*command) - - def version(self) -> Optional[str]: - """Returns None when this manager is not available.""" - try: - output = self._check_output("--version") - except RuntimeError: - return None - return output.strip().split(" ")[-1] - - def is_active(self) -> bool: - """Manager is explicitly active.""" - return "CONDA_PREFIX" in os.environ or os.path.exists( - os.path.join(sys.prefix, "conda-meta") - ) - - def _gather_requirements(self) -> Dict[str, Any]: - output = self._check_output("env", "export") - environment = yaml.safe_load(output) - environment.pop("name", None) - environment.pop("prefix", None) - - return {"environment": environment} - - def _install_native_requirements(self, requirements: CondaRequirements) -> bool: - text = yaml.safe_dump(requirements.environment) - with self._temporary_file(text, ".yml") as tmp_path: - self._check_call("env", "update", "-f", tmp_path) - return True - - def _install_base_requirements(self, requirements: BaseRequirements) -> bool: - raise NotImplementedError(f"{self.NAME} installation of python distributions") - - def _get_conda_command(self) -> Tuple[str, ...]: - try: - _ = self._check_output_raw("mamba", "--version") - return ("mamba",) - except Exception: # noqa S110 - pass - try: - _ = self._check_output_raw("micromamba", "--version") - return ("micromamba",) - except Exception: # noqa S110 - pass - return ("conda",) diff --git a/src/ewoks/_requirements/pip.py b/src/ewoks/_requirements/pip.py deleted file mode 100644 index f1bd48d..0000000 --- a/src/ewoks/_requirements/pip.py +++ /dev/null @@ -1,92 +0,0 @@ -import importlib.metadata -import logging -import sys -from typing import Any -from typing import Dict -from typing import List -from typing import Literal -from typing import Optional - -from .utils import pip_freeze -from .utils.base_manager import BaseManager -from .utils.base_manager import BaseManagerInfo -from .utils.base_manager import BaseRequirements - -logger = logging.getLogger(__name__) - - -class PipManagerInfo(BaseManagerInfo): - name: Literal["pip"] = "pip" - freeze: List[str] - - -class PipRequirements(BaseRequirements): - manager: PipManagerInfo - - def __info__(self) -> str: - freeze = "\n ".join(self.manager.freeze) - return f"{super().__info__()}\nRequirements:\n {freeze}" - - -class PipManager(BaseManager): - NAME = "pip" - PRIORITY = 0 - REQUIREMENTS_MODEL = PipRequirements - - def __init__(self, *command: str) -> None: - if not command: - command = sys.executable, "-m", "pip" - super().__init__(*command) - - def version(self) -> Optional[str]: - """Returns None when this manager is not available.""" - try: - return importlib.metadata.version("pip") - except importlib.metadata.PackageNotFoundError: - return None - - def is_active(self) -> bool: - """Manager is explicitly active.""" - return False - - def _gather_requirements(self) -> Dict[str, Any]: - freeze_output = self._check_output("freeze").strip().splitlines() - - return {"freeze": freeze_output} - - def _install_native_requirements(self, requirements: PipRequirements) -> bool: - freeze = requirements.manager.freeze - - if not freeze: - return False - - arguments = self._arguments(freeze) - self._check_call("install", "--no-cache-dir", *arguments) - return True - - def _install_base_requirements(self, requirements: BaseRequirements) -> bool: - freeze = self._freeze_distributions(requirements) - if not freeze: - return False - - arguments = self._arguments(freeze) - self._check_call("install", "--no-cache-dir", *arguments) - return True - - def _freeze_distributions(self, requirements: BaseRequirements) -> List[str]: - """ - Pip freeze argument from list of distributions. - """ - freeze = [] - for dist in requirements.distributions: - lines, warnings = pip_freeze.freeze_distribution(dist) - for warning in warnings: - logger.warning(warning) - freeze.extend(lines) - return freeze - - def _arguments(self, freeze: List[str]) -> List[str]: - arguments, warnings = pip_freeze.sanitize_freeze(freeze) - for warning in warnings: - logger.warning(warning) - return arguments diff --git a/src/ewoks/_requirements/pip_venv.py b/src/ewoks/_requirements/pip_venv.py new file mode 100644 index 0000000..d231210 --- /dev/null +++ b/src/ewoks/_requirements/pip_venv.py @@ -0,0 +1,147 @@ +import importlib.metadata +import sys +from pathlib import Path +from typing import Dict +from typing import List +from typing import Literal +from typing import Mapping +from typing import Optional +from typing import Sequence + +from pydantic import Field + +from .utils import process +from .utils import requirements_txt +from .utils.base_manager import BaseManager +from .utils.base_manager import BaseManagerInfo +from .utils.base_manager import BaseRequirements +from .utils.environment import Environment +from .utils.environment import create_venv +from .utils.files import temporary_files +from .utils.metadata import models + +_CONSTRAINTS_FILENAME = "constraints.txt" + + +class PipVenvManagerInfo(BaseManagerInfo): + name: Literal["pip-venv"] = Field( + default="pip-venv", + description="Environments created with `venv`, requirements installed with `pip`.", + examples=["pip-venv"], + ) + + +class PipVenvRequirements(BaseRequirements): + manager: PipVenvManagerInfo + + +class PipVenvManager(BaseManager): + """Creates environments with the `venv` module and installs in them with the + `pip` module. + + The command is the python interpreter that creates environments. For example + `PipVenvManager("/path/to/python")`. The current python interpreter is used + when no command is provided. Unlike other package managers, `venv` cannot + provide a python version other than the one of this interpreter. + """ + + NAME = "pip-venv" + PRIORITY = 0 + REQUIREMENTS_MODEL = PipVenvRequirements + COMMAND_EXAMPLE = "python" + CAN_INSTALL_IN_PLACE = True + + def __init__(self, *command: str) -> None: + if not command: + command = (sys.executable,) + super().__init__(*command) + + def version(self) -> Optional[str]: + """Returns None when this manager is not available.""" + try: + return importlib.metadata.version("pip") + except importlib.metadata.PackageNotFoundError: + return None + + def is_active(self) -> bool: + """Manager is explicitly active.""" + return False + + @classmethod + def installed_distribution(cls, distribution: models.Distribution) -> bool: + """The distribution was installed by this package manager.""" + return "pip" in cls._installer(distribution) + + def create_environment( + self, location: Path, python_version: Optional[str] = None + ) -> Environment: + """ + :raises RuntimeError: creation failed + """ + environment = self.environment(location) + create_venv(self._cmd_args[0], environment.prefix, python_version) + return environment + + def _gather_files( + self, distributions: List[models.Distribution], python_version: str + ) -> Dict[str, str]: + freeze = process.check_output(sys.executable, "-m", "pip", "freeze") + return {requirements_txt.REQUIREMENTS_FILENAME: freeze} + + def _files_from_distributions( + self, distributions: Sequence[models.Distribution], python_version: str + ) -> Dict[str, str]: + requirements = requirements_txt.distributions_requirements(distributions) + return {requirements_txt.REQUIREMENTS_FILENAME: "\n".join(requirements)} + + def _install_files( + self, files: Mapping[str, str], environment: Environment + ) -> None: + """ + :raises ValueError: no requirements to install + :raises RuntimeError: installation failed + """ + requirements = files[requirements_txt.REQUIREMENTS_FILENAME].splitlines() + self._pip_install(requirements, environment) + + def _add_ewoks( + self, requirements: BaseRequirements, environment: Environment + ) -> None: + self._pip_install( + ["ewoks"], + environment, + constraints=requirements_txt.version_constraints( + requirements.distributions + ), + ) + + def _pip_install( + self, + requirements: Sequence[str], + environment: Environment, + constraints: Optional[Sequence[str]] = None, + ) -> None: + """ + :raises ValueError: no requirements to install + :raises RuntimeError: installation failed + """ + arguments = requirements_txt.sanitize(requirements) + if not arguments: + raise ValueError("No distributions provided to install") + + files = {requirements_txt.REQUIREMENTS_FILENAME: "\n".join(arguments)} + if constraints: + files[_CONSTRAINTS_FILENAME] = "\n".join(constraints) + + with temporary_files(files) as directory: + options = ["-r", directory / requirements_txt.REQUIREMENTS_FILENAME] + if constraints: + options += ["--constraint", directory / _CONSTRAINTS_FILENAME] + process.check_call( + environment.python, + "-m", + "pip", + "install", + "--no-cache-dir", + *options, + ) diff --git a/src/ewoks/_requirements/pipenv.py b/src/ewoks/_requirements/pipenv.py deleted file mode 100644 index 7153576..0000000 --- a/src/ewoks/_requirements/pipenv.py +++ /dev/null @@ -1,74 +0,0 @@ -import importlib.metadata -import json -import os -import sys -from typing import Any -from typing import Dict -from typing import List -from typing import Literal -from typing import Optional - -from .utils.base_manager import BaseManager -from .utils.base_manager import BaseManagerInfo -from .utils.base_manager import BaseRequirements - - -class PipenvManagerInfo(BaseManagerInfo): - name: Literal["pipenv"] = "pipenv" - requirements: List[str] - - -class PipenvRequirements(BaseRequirements): - manager: PipenvManagerInfo - - def __info__(self) -> str: - requirements = "\n ".join(self.manager.requirements) - return f"{super().__info__()}\nRequirements:\n {requirements}" - - -class PipenvManager(BaseManager): - NAME = "pipenv" - PRIORITY = 3 - REQUIREMENTS_MODEL = PipenvRequirements - - def __init__(self, *command: str) -> None: - if not command: - command = sys.executable, "-m", "pipenv" - super().__init__(*command) - - def version(self) -> Optional[str]: - """Returns None when this manager is not available.""" - try: - return importlib.metadata.version("pipenv") - except importlib.metadata.PackageNotFoundError: - return None - - def is_active(self) -> bool: - """Manager is explicitly active.""" - return "PIPENV_ACTIVE" in os.environ - - def _gather_requirements(self) -> Dict[str, Any]: - output = self._check_output("lock", "--requirements") - requirements = output.strip().splitlines() - - return {"requirements": requirements} - - def _install_native_requirements(self, requirements: PipenvRequirements) -> bool: - lock_data = { - "_meta": {"hash": {"sha256": "dummy"}}, # minimal metadata - "default": { - pkg.split("==")[0]: {"version": pkg.split("==")[1]} - for pkg in requirements.requirements - }, - "develop": { - pkg.split("==")[0]: {"version": pkg.split("==")[1]} - for pkg in getattr(requirements, "dev_requirements", []) - }, - } - text = json.dumps(lock_data, indent=2) - - with self._temporary_file(text, ".lock") as tmp_path: - self._check_call("sync", "--ignore-pipfile", "-f", tmp_path) - - def _install_base_requirements(self, requirements: BaseRequirements) -> bool: - raise NotImplementedError(f"{self.NAME} installation of python distributions") diff --git a/src/ewoks/_requirements/pixi.py b/src/ewoks/_requirements/pixi.py deleted file mode 100644 index ebdb2cb..0000000 --- a/src/ewoks/_requirements/pixi.py +++ /dev/null @@ -1,62 +0,0 @@ -import os -from typing import Any -from typing import Dict -from typing import Literal -from typing import Optional - -from .utils.base_manager import BaseManager -from .utils.base_manager import BaseManagerInfo -from .utils.base_manager import BaseRequirements - - -class PixiManagerInfo(BaseManagerInfo): - name: Literal["pixi"] = "pixi" - lockfile: str - - -class PixiRequirements(BaseRequirements): - manager: PixiManagerInfo - - -class PixiManager(BaseManager): - NAME = "pixi" - PRIORITY = 5 - REQUIREMENTS_MODEL = PixiRequirements - - def __init__(self, *command: str) -> None: - if not command: - command = ("pixi",) - super().__init__(*command) - - def version(self) -> Optional[str]: - """Returns None when this manager is not available.""" - try: - output = self._check_output("--version", text=True) - return output.strip().split(" ")[-1] - except Exception: - return None - - def is_active(self) -> bool: - """Manager is explicitly active.""" - return "PIXI_PROJECT_ROOT" in os.environ - - def _gather_requirements(self) -> Dict[str, Any]: - if os.path.exists("pixi.lock"): - with open("pixi.lock", "r", encoding="utf-8") as f: - lock_content = f.read() - elif os.path.exists("pixi.toml"): - with open("pixi.toml", "r", encoding="utf-8") as f: - lock_content = f.read() - else: - raise RuntimeError("No pixi.lock or pixi.toml file found") - - return {"lockfile": lock_content} - - def _install_requirements(self, requirements: PixiRequirements) -> bool: - with self._temporary_file(requirements.lockfile, ".lock") as tmp_path: - self._check_call("install", cwd=os.path.dirname(tmp_path)) - - return True - - def _install_base_requirements(self, requirements: BaseRequirements) -> bool: - raise NotImplementedError(f"{self.NAME} installation of python distributions") diff --git a/src/ewoks/_requirements/poetry.py b/src/ewoks/_requirements/poetry.py deleted file mode 100644 index b9190d9..0000000 --- a/src/ewoks/_requirements/poetry.py +++ /dev/null @@ -1,59 +0,0 @@ -import importlib.metadata -import os -import sys -from typing import Any -from typing import Dict -from typing import List -from typing import Literal -from typing import Optional - -from .utils.base_manager import BaseManager -from .utils.base_manager import BaseManagerInfo -from .utils.base_manager import BaseRequirements - - -class PoetryManagerInfo(BaseManagerInfo): - name: Literal["poetry"] = "pip" - requirements: List[str] - - -class PoetryRequirements(BaseRequirements): - manager: PoetryManagerInfo - - -class PoetryManager(BaseManager): - NAME = "poetry" - PRIORITY = 2 - REQUIREMENTS_MODEL = PoetryRequirements - - def __init__(self, *command: str) -> None: - if not command: - command = sys.executable, "-m", "poetry" - super().__init__(*command) - - def version(self) -> Optional[str]: - """Returns None when this manager is not available.""" - try: - return importlib.metadata.version("poetry") - except importlib.metadata.PackageNotFoundError: - return None - - def is_active(self) -> bool: - """Manager is explicitly active.""" - return "POETRY_ACTIVE" in os.environ - - def _gather_requirements(self) -> Dict[str, Any]: - output = self._check_output("export", "--without-hashes") - requirements = output.strip().splitlines() - - return {"requirements": requirements} - - def _install_native_requirements(self, requirements: PoetryRequirements) -> bool: - text = "\n".join(requirements.requirements) - with self._temporary_file(text, ".txt") as tmp_path: - self._check_call("add", "--lock", "--file", tmp_path) - - return True - - def _install_base_requirements(self, requirements: BaseRequirements) -> bool: - raise NotImplementedError(f"{self.NAME} installation of python distributions") diff --git a/src/ewoks/_requirements/utils/_supported.py b/src/ewoks/_requirements/utils/_supported.py index a9bec1f..348c9e9 100644 --- a/src/ewoks/_requirements/utils/_supported.py +++ b/src/ewoks/_requirements/utils/_supported.py @@ -2,24 +2,13 @@ from typing import Dict from typing import Type -from ..pip import PipManager +from ..pip_venv import PipVenvManager from .base_manager import BaseManager -# from ..conda import CondaManager -# from ..pipenv import PipenvManager -# from ..pixi import PixiManager -# from ..poetry import PoetryManager -# from ..uv import UvManager - @lru_cache(1) def get_supported_managers() -> Dict[str, Type[BaseManager]]: managers = [ - PipManager, - # UvManager, - # PoetryManager, - # PipenvManager, - # CondaManager, - # PixiManager, + PipVenvManager, ] return {manager_cls.NAME: manager_cls for manager_cls in managers} diff --git a/src/ewoks/_requirements/utils/base_manager.py b/src/ewoks/_requirements/utils/base_manager.py index 3eccd13..f27f867 100644 --- a/src/ewoks/_requirements/utils/base_manager.py +++ b/src/ewoks/_requirements/utils/base_manager.py @@ -1,62 +1,126 @@ import logging -import os -import subprocess -import tempfile from abc import abstractmethod -from contextlib import contextmanager -from typing import Any +from pathlib import Path +from textwrap import indent from typing import Dict -from typing import Generator from typing import List +from typing import Mapping from typing import Optional +from typing import Sequence +from typing import Union +from pydantic import Field + +from . import process +from .environment import Environment from .metadata import models from .metadata.from_python import current_requirements logger = logging.getLogger(__name__) +EWOKS_ENVIRONMENTS_ROOT = Path("~", ".ewoks", "envs") +"""Root directory of named environments for package managers that do not create +them in a directory of their own.""" + class BaseManagerInfo(models.BaseModel): - name: str - version: str + name: str = Field( + description="Package manager that generated the requirements.", + examples=["pip-venv"], + ) + version: str = Field( + description="Version of the package manager.", examples=["25.0.1"] + ) + files: Dict[str, str] = Field( + default_factory=dict, + description=( + "Content of the files the package manager needs to reproduce the " + "environment: 'requirements.txt' for pip-venv. Empty when the package " + "manager could not generate them." + ), + examples=[{"requirements.txt": "ewoks==7.0.0\nnetworkx==3.4.2\n"}], + ) class BaseRequirements(models.BaseModel): - system: models.SystemInfo - python: models.PythonInfo - distributions: List[models.Distribution] - manager: BaseManagerInfo + system: models.SystemInfo = Field( + description="Operating system on which the requirements were generated." + ) + python: models.PythonInfo = Field( + description="Python interpreter for which the requirements were generated." + ) + distributions: List[models.Distribution] = Field( + description=( + "Installed python distributions. Any package manager can reproduce the " + "environment from this list, also one that does not understand the " + "files of the package manager that generated the requirements." + ), + examples=[[{"name": "networkx", "version": "3.4.2"}]], + ) + manager: BaseManagerInfo = Field( + description="Package manager that generated the requirements." + ) def __info__(self) -> str: - return ( + return f"{self.__files_info__()}\n{self.__distributions_info__()}" + + def __files_info__(self) -> str: + """The package manager with the content of the files it needs to reproduce + the environment.""" + info = ( f"Manager: {self.manager.name} ({self.manager.version}) " - f"python={self.python.version}) " + f"python={self.python.version} " f"distributions={len(self.distributions)}" ) + for filename in sorted(self.manager.files): + content = indent(self.manager.files[filename].strip(), " ") + info = f"{info}\n\n{filename}:\n{content}" + return info + + def __distributions_info__(self) -> str: + """The installed python distributions.""" + distributions = "\n ".join( + f"{dist.name}=={dist.version}" for dist in self.distributions + ) + return f"Distributions:\n {distributions}" class BaseManager: """Defines the interface all package managers must implement. - If `MyManager` is an implementation of this interface then - Ewoks workflow requirements can be obtained like this: + If `MyManager` is an implementation of this interface then the requirements + of the current python environment can be obtained like this: .. code-block:: python manager = MyManager() requirements = manager.gather_requirements() - Ewoks workflow requirements can be installed like this: + Those requirements can be reproduced in another python environment like this: .. code-block:: python manager = MyManager() - manager.install_requirements(requirements) + environment = manager.create_environment(Path("/path/to/environment")) + if manager.is_native(requirements, environment): + manager.install_files(requirements, environment) + else: + manager.install_distributions(requirements, environment) + + An implementation provides `version`, `is_active`, `create_environment`, + `_files_from_distributions`, `_install_files` and `_add_ewoks`. Package managers + that can inspect a python environment also override `_gather_files`. Package + managers that keep named environments in a directory of their own also override + `environments_root`. Package managers that are not named after the tool that + installs the distributions also override `installed_distribution`. """ NAME = NotImplemented PRIORITY = NotImplemented REQUIREMENTS_MODEL = NotImplemented + COMMAND_EXAMPLE = NotImplemented # example of an associated shell command + ENVIRONMENT_SUBDIR = "" # environment prefix relative to the location + CAN_INSTALL_IN_PLACE = False # can install in the current environment def __init__(self, *command: str) -> None: if not command: @@ -76,7 +140,19 @@ def is_active(self) -> bool: """Manager is explicitly active.""" pass - def gather_requirements(self) -> Optional[BaseRequirements]: + @classmethod + def installed_distribution(cls, distribution: models.Distribution) -> bool: + """The distribution was installed by this package manager.""" + return cls.NAME in cls._installer(distribution) + + @staticmethod + def _installer(distribution: models.Distribution) -> str: + """Tool that installed the distribution. It can contain more than the name + of the tool, for example its version. + """ + return (distribution.installer or "").lower() + + def gather_requirements(self) -> BaseRequirements: """ Return requirements associated to the current python environment. @@ -86,102 +162,171 @@ def gather_requirements(self) -> Optional[BaseRequirements]: if manager_version is None: raise RuntimeError(f"{self.NAME!r} is not available") + metadata = current_requirements() + files = self._gather_files( + metadata["distributions"], metadata["python"]["version"] + ) + + manager = dict(name=self.NAME, version=manager_version, files=files) + return self.REQUIREMENTS_MODEL(manager=manager, **metadata) + + def environment(self, location: Union[str, Path]) -> Environment: + """Environment the manager creates at this location.""" + return Environment.at_location(location, self.ENVIRONMENT_SUBDIR) + + def environment_location( + self, name: str, root: Optional[Union[str, Path]] = None + ) -> Path: + """Location of the environment with this name, inside a root directory of + the user or inside the root directory of this package manager. + """ + return Path(root or self.environments_root()).expanduser() / name + + def environments_root(self) -> Path: + """Root directory in which the package manager creates named environments. + Package managers that create an environment wherever they are told to (venv) + use a directory of ewoks. + """ + return EWOKS_ENVIRONMENTS_ROOT + + @abstractmethod + def create_environment( + self, location: Path, python_version: Optional[str] = None + ) -> Environment: + """Create an empty python environment. + + :raises RuntimeError: creation failed + """ + pass + + def is_native( + self, requirements: BaseRequirements, environment: Environment + ) -> bool: + """The files of the requirements are the files of this package manager and + the environment has the layout this package manager creates. + """ + return ( + isinstance(requirements, self.REQUIREMENTS_MODEL) + and bool(requirements.manager.files) + and self._owns(environment) + ) + + def install_files( + self, requirements: BaseRequirements, environment: Environment + ) -> None: + """Install the files of the requirements in an existing environment. The + requirements must be native (see `is_native`). + + :raises RuntimeError: installation failed + """ try: - parameters = self._gather_requirements() + self._install_files(requirements.manager.files, environment) except Exception as ex: logger.error( - "%s: failed to generate requirements (%s)", type(self).__name__, ex + "%s: failed to install the requirement files (%s)", + type(self).__name__, + ex, ) - return None - - manager = dict(name=self.NAME, version=manager_version, **parameters) - return self.REQUIREMENTS_MODEL(manager=manager, **current_requirements()) + raise - def install_requirements(self, requirements: BaseRequirements) -> None: - """ - Install requirements into the current environment. + def install_distributions( + self, requirements: BaseRequirements, environment: Environment + ) -> None: + """Install the python distributions of the requirements in an existing + environment. - :raises ValueError: no distibutions provided to install + :raises ValueError: no distributions provided to install + :raises RuntimeError: installation failed """ + if not requirements.distributions: + raise ValueError("No distributions provided to install") + # Requirements that do not specify a python version are described for the + # python version of the environment they are installed in + python_version = requirements.python.version or environment.python_version() try: - return self._install_requirements(requirements) + files = self._files_from_distributions( + requirements.distributions, python_version + ) + self._install_files(files, environment) except Exception as ex: logger.error( - "%s: failed to install requirements (%s)", type(self).__name__, ex + "%s: failed to install the python distributions (%s)", + type(self).__name__, + ex, ) raise - def _install_requirements(self, requirements: BaseRequirements) -> None: - reraise = None - - if isinstance(requirements, self.REQUIREMENTS_MODEL): - try: - if self._install_native_requirements(requirements): - return - except Exception as ex: - reraise = ex - logger.debug( - ( - "Failed installing requirements native to package %s. " - "Try installing python distributions." - ), - self.NAME, - ex, - ) - pass - - if self._install_base_requirements(requirements): - return + def ensure_ewoks( + self, requirements: BaseRequirements, environment: Environment + ) -> None: + """Install ewoks in an existing environment without changing the versions + of the requirements. - if reraise: - raise reraise - raise ValueError("No distibutions provided to install") + :raises RuntimeError: installation failed + """ + if environment.distribution_version("ewoks"): + return + logger.info("Add ewoks to %s", environment.location) + self._add_ewoks(requirements, environment) + + def _owns(self, environment: Environment) -> bool: + """The environment has the layout this package manager creates.""" + return environment.python == self.environment(environment.location).python + + def _gather_files( + self, distributions: List[models.Distribution], python_version: str + ) -> Dict[str, str]: + """Files that describe the current python environment. Package managers + that can inspect an environment override this. + """ + try: + return self._files_from_distributions(distributions, python_version) + except RuntimeError as ex: + logger.warning( + "%s cannot describe the current python environment (%s). It will be " + "reproduced from the installed python distributions.", + self.NAME, + ex, + ) + return dict() @abstractmethod - def _gather_requirements(self) -> Dict[str, Any]: - pass + def _files_from_distributions( + self, distributions: Sequence[models.Distribution], python_version: str + ) -> Dict[str, str]: + """Files the package manager needs to install these python distributions. - @abstractmethod - def _install_native_requirements(self, requirements: BaseRequirements) -> bool: + :raises RuntimeError: the files cannot be generated + """ pass @abstractmethod - def _install_base_requirements(self, requirements: BaseRequirements) -> bool: - pass + def _install_files( + self, files: Mapping[str, str], environment: Environment + ) -> None: + """Install the files of the package manager in an existing environment. - def _check_output(self, *args: str) -> str: - return self._check_output_raw(*[*self._cmd_args, *args]) + :raises RuntimeError: installation failed + """ + pass - def _check_call(self, *args: str) -> int: - return self._check_call_raw(*[*self._cmd_args, *args]) + @abstractmethod + def _add_ewoks( + self, requirements: BaseRequirements, environment: Environment + ) -> None: + """Install ewoks in an existing environment without changing the versions + of the requirements. - @staticmethod - def _check_output_raw(*args: str) -> str: - try: - return subprocess.check_output(args, text=True) # noqa: S603 - Internal manager call - except Exception as ex: - raise RuntimeError(f"Command failed: {args}") from ex + :raises RuntimeError: installation failed + """ + pass - @staticmethod - def _check_call_raw(*args: str) -> int: - try: - return subprocess.check_call(args) # noqa: S603 - Internal manager call - except Exception as ex: - raise RuntimeError(f"Command failed: {args}") from ex + def _check_output( + self, *args: Union[str, Path], extra_env: Optional[Mapping[str, str]] = None + ) -> str: + return process.check_output(*self._cmd_args, *args, extra_env=extra_env) - @contextmanager - def _temporary_file(self, text: str, suffix: str) -> Generator[str, None, None]: - tmp_path = None - try: - with tempfile.NamedTemporaryFile("w", suffix=suffix, delete=False) as tmp: - tmp.write(text) - tmp_path = tmp.name - - yield tmp_path - - finally: - if tmp_path: - try: - os.remove(tmp_path) - except OSError: - logger.debug("Could not delete temporary file: %s", tmp_path) + def _check_call( + self, *args: Union[str, Path], extra_env: Optional[Mapping[str, str]] = None + ) -> None: + process.check_call(*self._cmd_args, *args, extra_env=extra_env) diff --git a/src/ewoks/_requirements/utils/detect.py b/src/ewoks/_requirements/utils/detect.py index 3d82727..b6516e2 100644 --- a/src/ewoks/_requirements/utils/detect.py +++ b/src/ewoks/_requirements/utils/detect.py @@ -7,6 +7,7 @@ from ._supported import get_supported_managers from .base_manager import BaseManager +from .base_manager import BaseRequirements from .metadata.from_python import current_requirements logger = logging.getLogger(__name__) @@ -16,8 +17,9 @@ def get_manager( manager_name: Optional[str] = None, manager_command: Tuple[str, ...] = tuple(), ) -> BaseManager: - """ - :raise ValueError: package manager not support or not available + """Package manager that describes the current python environment. + + :raise ValueError: package manager not supported or not available :raise RuntimeError: no package manager available """ if manager_name: @@ -36,6 +38,63 @@ def get_manager( return manager +def get_installer( + requirements: BaseRequirements, + manager_name: Optional[str] = None, + manager_command: Tuple[str, ...] = tuple(), +) -> BaseManager: + """Package manager that installs requirements. The package manager that + generated the requirements is used when available. + + :raise ValueError: package manager not supported or not available + :raise RuntimeError: no package manager available + """ + if manager_name: + return _select_manager(manager_name, manager_command) + + if manager_command: + raise ValueError( + f"Provide 'manager_name' associated to command {manager_command}" + ) + + try: + return _select_manager(requirements.manager.name, tuple()) + except ValueError: + logger.debug( + "Package manager %r that generated the requirements is not available", + requirements.manager.name, + ) + + managers = _managers() + scores = {name: (manager.PRIORITY,) for name, manager in managers.items()} + manager = _first_available(managers, scores) + if manager is None: + raise RuntimeError("No known package manager installed or available") + + return manager + + +def get_in_place_installer() -> BaseManager: + """Package manager that can install in the current python environment. + + :raise RuntimeError: no package manager available + """ + managers = { + name: manager + for name, manager in _managers().items() + if manager.CAN_INSTALL_IN_PLACE + } + scores = {name: (manager.PRIORITY,) for name, manager in managers.items()} + manager = _first_available(managers, scores) + if manager is None: + raise RuntimeError( + "No package manager available that can install in the current python " + "environment" + ) + + return manager + + def _select_manager(manager_name: str, manager_command: Tuple[str, ...]) -> BaseManager: managers = get_supported_managers() @@ -50,64 +109,59 @@ def _select_manager(manager_name: str, manager_command: Tuple[str, ...]) -> Base return manager -def _detect_manager() -> Optional[BaseManager]: - # Available package managers - available_managers = { +def _managers() -> Dict[str, BaseManager]: + return { name: manager_cls() for name, manager_cls in get_supported_managers().items() } - available_managers = { - name: manager - for name, manager in available_managers.items() - if manager.version() - } - if not available_managers: - return None + + +def _first_available( + managers: Dict[str, BaseManager], scores: Dict[str, Tuple[int, ...]] +) -> Optional[BaseManager]: + """Available package manager with the highest score. Availability is checked + in score order because it may be an expensive check. + """ + for name in sorted(scores, key=lambda name: scores[name], reverse=True): + manager = managers[name] + if manager.version(): + logger.debug("Package manager %r selected\n scores = %s", name, scores) + return manager + return None + + +def _detect_manager() -> Optional[BaseManager]: + managers = _managers() # Select the active manager with the highest priority active_managers = { - name: manager - for name, manager in available_managers.items() - if manager.is_active() + name: manager for name, manager in managers.items() if manager.is_active() } if active_managers: - name = max(active_managers, key=lambda name: active_managers[name].PRIORITY) - manager = active_managers[name] - logger.debug( - "Detected active %r package manager\n available = %s\n active = %s", - name, - list(available_managers), - list(active_managers), - ) - return manager - - # Infer most likely package manager - counts = _installer_distribution_count() - if set(counts) & set(available_managers): - # Use the number of installed distributions as the score - crit = "distribution count" - scores = {name: counts.get(name, -1) for name in available_managers} - else: - # Use the package manager priority as the score - crit = "priority" scores = { - name: manager.PRIORITY for name, manager in available_managers.items() + name: (manager.PRIORITY,) for name, manager in active_managers.items() } + manager = _first_available(active_managers, scores) + if manager is not None: + return manager - name = max(scores, key=scores.get) - logger.debug( - "Package manager selection based on %s\n %s", - crit, - "\n ".join( - f"{k} = {v} {'(SELECTED)' if k == name else ''}" for k, v in scores.items() - ), - ) - return available_managers[name] + # Infer most likely package manager + counts = _manager_distribution_count() + scores = { + name: (counts.get(name, -1), manager.PRIORITY) + for name, manager in managers.items() + } + return _first_available(managers, scores) @lru_cache(1) -def _installer_distribution_count() -> Dict[str, int]: +def _manager_distribution_count() -> Dict[str, int]: + """Number of installed python distributions per package manager. Distributions + installed by a tool that is not a supported package manager are not counted. + """ + managers = get_supported_managers() counts: Counter = Counter() for dist in current_requirements()["distributions"]: - if dist.installer: - counts[dist.installer] += 1 + for name, manager_cls in managers.items(): + if manager_cls.installed_distribution(dist): + counts[name] += 1 return dict(counts) diff --git a/src/ewoks/_requirements/utils/environment.py b/src/ewoks/_requirements/utils/environment.py new file mode 100644 index 0000000..8ce113a --- /dev/null +++ b/src/ewoks/_requirements/utils/environment.py @@ -0,0 +1,137 @@ +import logging +import os +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Optional +from typing import Tuple +from typing import Union + +from . import process + +logger = logging.getLogger(__name__) + +ENVIRONMENT_SUBDIRS: Tuple[str, ...] = ( + "", # environment prefix, for example created by `python -m venv` +) + + +@dataclass(frozen=True) +class Environment: + """Python environment a package manager creates, installs into and runs.""" + + location: Path + """Directory provided by the user: an environment prefix or a project.""" + + python: Path + """Python interpreter of the environment.""" + + @classmethod + def current(cls) -> "Environment": + """Environment of the running python interpreter.""" + return cls(location=Path(sys.prefix), python=Path(sys.executable)) + + @classmethod + def at_location(cls, location: Union[str, Path], subdir: str = "") -> "Environment": + """Environment with a prefix at a fixed position inside the location.""" + location = Path(os.path.abspath(location)) + prefix = location / subdir if subdir else location + return cls(location=location, python=python_executable(prefix)) + + @classmethod + def from_location(cls, location: Union[str, Path]) -> "Environment": + """Environment found inside the location. + + :raises ValueError: no python interpreter found + """ + for subdir in ENVIRONMENT_SUBDIRS: + environment = cls.at_location(location, subdir) + if environment.python.is_file(): + return environment + raise ValueError(f"No python environment found in '{location}'") + + @property + def prefix(self) -> Path: + """Environment prefix, which is the location itself unless the package + manager creates the environment inside the location. + """ + directory = self.python.parent + if directory.name in ("bin", "Scripts"): + return directory.parent + return directory + + def exists(self) -> bool: + return self.python.is_file() + + def python_version(self) -> str: + """ + :raises RuntimeError: environment has no python interpreter + """ + return interpreter_version(self.python) + + def distribution_version(self, name: str) -> Optional[str]: + """Returns None when the distribution is not installed. + + :raises RuntimeError: environment has no python interpreter + """ + code = ( + "import importlib.metadata as m\n" + f"try: print(m.version({name!r}))\n" + "except m.PackageNotFoundError: pass" + ) + return self._python_output(code) or None + + def _python_output(self, code: str) -> str: + return process.check_output(self.python, "-c", code).strip() + + +def python_executable(prefix: Path) -> Path: + """Python interpreter of an environment prefix. The interpreter of a virtual + environment is returned when the environment does not exist yet. + """ + relative_paths = _interpreters() + for relative in relative_paths: + python = prefix / relative + if python.is_file(): + return python + return prefix / relative_paths[0] + + +def _interpreters() -> Tuple[Path, ...]: + """Interpreter locations inside an environment prefix, the first one being where + a virtual environment has it. On Windows an environment that is not virtual (a + python installation) has it in the prefix itself. + """ + if sys.platform == "win32": + return (Path("Scripts", "python.exe"), Path("python.exe")) + return (Path("bin", "python"),) + + +def interpreter_version(python: Union[str, Path]) -> str: + """Version of a python interpreter. + + :raises RuntimeError: not a python interpreter + """ + return process.check_output( + python, "-c", "import platform; print(platform.python_version())" + ).strip() + + +def create_venv( + base_python: Union[str, Path], prefix: Path, python_version: Optional[str] = None +) -> None: + """Create a virtual environment with the `venv` module, which cannot provide + another python version than the one of the interpreter that creates it. + + :raises RuntimeError: creation failed + """ + if python_version: + base_version = interpreter_version(base_python) + if python_version != base_version: + logger.warning( + "'venv' cannot provide python %s: creating the environment with " + "python %s instead", + python_version, + base_version, + ) + process.check_call(base_python, "-m", "venv", prefix) diff --git a/src/ewoks/_requirements/utils/files.py b/src/ewoks/_requirements/utils/files.py new file mode 100644 index 0000000..172a440 --- /dev/null +++ b/src/ewoks/_requirements/utils/files.py @@ -0,0 +1,34 @@ +import tempfile +from contextlib import contextmanager +from pathlib import Path +from typing import Dict +from typing import Generator +from typing import Mapping + + +def write_files(directory: Path, files: Mapping[str, str]) -> None: + """Write files, relative to a directory that is created when missing.""" + for name, content in files.items(): + filename = directory / name + filename.parent.mkdir(parents=True, exist_ok=True) + filename.write_text(content, encoding="utf-8") + + +def read_files(directory: Path, *names: str) -> Dict[str, str]: + """Read files, relative to a directory, skipping the ones that do not exist.""" + files = {} + for name in names: + try: + files[name] = (directory / name).read_text(encoding="utf-8") + except OSError: + continue + return files + + +@contextmanager +def temporary_files(files: Mapping[str, str]) -> Generator[Path, None, None]: + """Write files in a temporary directory which is yielded.""" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + write_files(path, files) + yield path diff --git a/src/ewoks/_requirements/utils/metadata/from_pip_freeze.py b/src/ewoks/_requirements/utils/metadata/from_pip_freeze.py deleted file mode 100644 index d1a77a6..0000000 --- a/src/ewoks/_requirements/utils/metadata/from_pip_freeze.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Any -from typing import Dict -from typing import List - -from . import _unknown - - -def pip_freeze_requirements(freeze: List[str]) -> Dict[str, Any]: - metadata = _unknown.unknown_requirements() - metadata["manager"] = dict(name="pip", version="", freeze=freeze) - return metadata diff --git a/src/ewoks/_requirements/utils/metadata/from_requirements_txt.py b/src/ewoks/_requirements/utils/metadata/from_requirements_txt.py new file mode 100644 index 0000000..1e9e3e9 --- /dev/null +++ b/src/ewoks/_requirements/utils/metadata/from_requirements_txt.py @@ -0,0 +1,30 @@ +from typing import Any +from typing import Dict +from typing import List + +from .. import requirements_txt +from . import _unknown + + +def requirements_txt_metadata(requirements: List[str]) -> Dict[str, Any]: + """Requirements for a legacy requirements list in `requirements.txt` format. + + The requirements are parsed into python distributions so that any package + manager can reproduce the environment, not only the one that installs a + `requirements.txt` file. + """ + metadata = dict(_unknown.unknown_requirements()) + metadata["distributions"] = [ + distribution + for distribution in map( + requirements_txt.distribution_from_requirement, + requirements_txt.sanitize(requirements), + ) + if distribution is not None + ] + metadata["manager"] = dict( + name="pip-venv", + version="", + files={requirements_txt.REQUIREMENTS_FILENAME: "\n".join(requirements)}, + ) + return metadata diff --git a/src/ewoks/_requirements/utils/metadata/last_resort.py b/src/ewoks/_requirements/utils/metadata/last_resort.py index fb650ec..99e78fe 100644 --- a/src/ewoks/_requirements/utils/metadata/last_resort.py +++ b/src/ewoks/_requirements/utils/metadata/last_resort.py @@ -5,7 +5,7 @@ from ewokscore.graph import TaskGraph -from .from_pip_freeze import pip_freeze_requirements +from .from_requirements_txt import requirements_txt_metadata logger = logging.getLogger(__name__) @@ -14,8 +14,7 @@ def last_resort_requirements(graph: TaskGraph) -> Dict[str, Any]: """Last resort when installing a workflow that does not have requirements: guess the requirements from the workflow nodes. """ - freeze: Set[str] = set() - distributions: Set[str] = set() + requirements: Set[str] = set() for node_id, node in graph.graph.nodes.items(): task_identifier = node["task_identifier"] @@ -29,14 +28,13 @@ def last_resort_requirements(graph: TaskGraph) -> Dict[str, Any]: ) continue - freeze.add(package) - distributions.add(package) + requirements.add(package) elif task_type == "notebook": logger.warning( f"Requirement extraction may be incomplete for node {node_id}: {task_type} is only partially supported." ) - freeze.add("ewokscore[notebook]") + requirements.add("ewokscore[notebook]") elif task_type == "script": logger.warning( @@ -47,8 +45,4 @@ def last_resort_requirements(graph: TaskGraph) -> Dict[str, Any]: f"Could not extract requirements for node {node_id}: unsupported task type {task_type}." ) - requirements = pip_freeze_requirements(sorted(freeze)) - requirements["distributions"] = [ - {"name": name, "version": ""} for name in sorted(distributions) - ] - return requirements + return requirements_txt_metadata(sorted(requirements)) diff --git a/src/ewoks/_requirements/utils/metadata/models.py b/src/ewoks/_requirements/utils/metadata/models.py index c446438..f1893da 100644 --- a/src/ewoks/_requirements/utils/metadata/models.py +++ b/src/ewoks/_requirements/utils/metadata/models.py @@ -6,34 +6,81 @@ class SystemInfo(BaseModel): - system: str - release: str - version: str - machine: str - processor: str + system: str = Field(description="Operating system name.", examples=["Linux"]) + release: str = Field( + description="Operating system release.", examples=["5.15.0-139-generic"] + ) + version: str = Field( + description="Operating system version.", + examples=["#149-Ubuntu SMP Fri Apr 11 19:19:52 UTC 2025"], + ) + machine: str = Field(description="Machine type.", examples=["x86_64"]) + processor: str = Field(description="Processor name.", examples=["x86_64"]) class PythonInfo(BaseModel): - version: str - implementation: str - compiler: str - build: str + version: str = Field(description="Python version.", examples=["3.12.11"]) + implementation: str = Field( + description="Python implementation.", examples=["CPython"] + ) + compiler: str = Field( + description="Compiler used to build python.", examples=["GCC 11.4.0"] + ) + build: str = Field( + description="Build number and date of the python interpreter.", + examples=["main, Jun 11 2025 10:57:12"], + ) class GitInfo(BaseModel): - commit: str - remote: Optional[str] = None - uncommitted_changes: bool = Field(default=False, description="Uncommited changes") + commit: str = Field( + description="Commit from which the distribution was installed.", + examples=["249ea97730a0134de4ecd1b7f136cc48bc0ec2de"], + ) + remote: Optional[str] = Field( + default=None, + description=( + "Repository from which the distribution was installed. Without a " + "repository the distribution cannot be installed elsewhere." + ), + examples=["https://github.com/ewoks-kit/ewokscore.git"], + ) + uncommitted_changes: bool = Field( + default=False, + description=( + "The repository had uncommitted changes, so the commit does not " + "describe the installed distribution completely." + ), + examples=[False], + ) class ArchiveInfo(BaseModel): - url: str - hashes: Dict[str, str] + url: str = Field( + description="Archive from which the distribution was installed.", + examples=["https://host/ewokscore-5.1.0-py3-none-any.whl"], + ) + hashes: Dict[str, str] = Field( + description="Hashes of the archive, by algorithm name.", + examples=[{"sha256": "5f8e9c1a"}], + ) class Distribution(BaseModel): - name: str - version: str - git: Optional[GitInfo] = None - archive: Optional[ArchiveInfo] = None - installer: Optional[str] = None + name: str = Field(description="Distribution name.", examples=["ewokscore"]) + version: str = Field( + description="Distribution version, empty when unknown.", examples=["5.1.0"] + ) + git: Optional[GitInfo] = Field( + default=None, + description="Set when the distribution was installed from a git repository.", + ) + archive: Optional[ArchiveInfo] = Field( + default=None, + description="Set when the distribution was installed from an archive.", + ) + installer: Optional[str] = Field( + default=None, + description="Tool that installed the distribution.", + examples=["pip"], + ) diff --git a/src/ewoks/_requirements/utils/parse.py b/src/ewoks/_requirements/utils/parse.py index 91e2a5d..0ef4330 100644 --- a/src/ewoks/_requirements/utils/parse.py +++ b/src/ewoks/_requirements/utils/parse.py @@ -3,13 +3,13 @@ from ._supported import get_supported_managers from .base_manager import BaseRequirements -from .metadata.from_pip_freeze import pip_freeze_requirements +from .metadata.from_requirements_txt import requirements_txt_metadata def parse_requirements(requirements: Union[dict, List[str]]) -> BaseRequirements: if isinstance(requirements, list): # Legacy 'pip freeze' list - requirements = pip_freeze_requirements(requirements) + requirements = requirements_txt_metadata(requirements) if not isinstance(requirements, dict): raise TypeError( diff --git a/src/ewoks/_requirements/utils/process.py b/src/ewoks/_requirements/utils/process.py new file mode 100644 index 0000000..8495f2c --- /dev/null +++ b/src/ewoks/_requirements/utils/process.py @@ -0,0 +1,45 @@ +import logging +import os +import subprocess +from pathlib import Path +from typing import Mapping +from typing import Optional +from typing import Union + +logger = logging.getLogger(__name__) + + +def check_output( + *args: Union[str, Path], extra_env: Optional[Mapping[str, str]] = None +) -> str: + """ + :raises RuntimeError: command failed + """ + logger.debug("Capture output of %s", args) + try: + return subprocess.check_output( # noqa: S603 - Internal manager call + args, text=True, env=_environment(extra_env) + ) + except Exception as ex: + raise RuntimeError(f"Command failed: {args}") from ex + + +def check_call( + *args: Union[str, Path], extra_env: Optional[Mapping[str, str]] = None +) -> None: + """ + :raises RuntimeError: command failed + """ + logger.debug("Execute %s", args) + try: + subprocess.check_call( # noqa: S603 - Internal manager call + args, env=_environment(extra_env) + ) + except Exception as ex: + raise RuntimeError(f"Command failed: {args}") from ex + + +def _environment(extra_env: Optional[Mapping[str, str]]) -> Optional[Mapping[str, str]]: + if not extra_env: + return None + return {**os.environ, **extra_env} diff --git a/src/ewoks/_requirements/utils/pip_freeze.py b/src/ewoks/_requirements/utils/requirements_txt.py similarity index 54% rename from src/ewoks/_requirements/utils/pip_freeze.py rename to src/ewoks/_requirements/utils/requirements_txt.py index 38697ad..3478844 100644 --- a/src/ewoks/_requirements/utils/pip_freeze.py +++ b/src/ewoks/_requirements/utils/requirements_txt.py @@ -1,6 +1,14 @@ +"""The `requirements.txt` format, as generated by `pip freeze`. Package managers +use it to describe python distributions, either to install them or to declare them +in a manifest. +""" + +import logging import os import re +from typing import Dict from typing import List +from typing import Optional from typing import Sequence from typing import Tuple from urllib.parse import ParseResult @@ -10,16 +18,155 @@ from packaging.requirements import InvalidRequirement from packaging.requirements import Requirement +from packaging.utils import NormalizedName +from packaging.utils import canonicalize_name from .metadata import models +logger = logging.getLogger(__name__) + +REQUIREMENTS_FILENAME = "requirements.txt" +"""Name under which requirements in `requirements.txt` format are stored.""" + +_GIT_URL_PREFIX = "git+" + +_URL_HASH_ALGORITHMS = ("sha512", "sha384", "sha256", "sha224", "sha1", "md5") +"""Hash algorithms accepted in the fragment of a direct reference URL, strongest +first.""" + + +def sanitize(requirements: Sequence[str]) -> List[str]: + """Requirements in `requirements.txt` format, ready to be installed.""" + return _log(*_sanitize(requirements)) + + +def distributions_requirements( + distributions: Sequence[models.Distribution], +) -> List[str]: + """Requirements in `requirements.txt` format for a list of distributions.""" + requirements = [] + warnings = [] + for dist in distributions: + dist_requirements, dist_warnings = _distribution_requirements(dist) + requirements.extend(dist_requirements) + warnings.extend(dist_warnings) + return _log(requirements, warnings) + + +def manifest_requirements( + distributions: Sequence[models.Distribution], +) -> List[str]: + """PEP 508 requirements for a list of distributions, for use in a package + manager manifest. Distributions installed from a local directory are skipped + because they cannot be resolved elsewhere. Names are unique because a manifest + cannot contain the same dependency twice. + """ + requirements: Dict[NormalizedName, str] = dict() + for requirement in sanitize(distributions_requirements(distributions)): + if requirement.startswith("-e "): + continue + try: + name = canonicalize_name(Requirement(requirement).name) + except InvalidRequirement: + logger.warning("Skip invalid requirement %r", requirement) + continue + if name in requirements: + logger.warning("Skip duplicate requirement %r", requirement) + continue + requirements[name] = requirement + return list(requirements.values()) + + +def version_constraints(distributions: Sequence[models.Distribution]) -> List[str]: + """Version constraints for a list of distributions. -def freeze_distribution( + Only versions are constrained: direct references are not allowed in a + constraints file. + """ + return [f"{dist.name}=={dist.version}" for dist in distributions if dist.version] + + +def distribution_from_requirement(requirement: str) -> Optional[models.Distribution]: + """Distribution described by a single requirement. Returns None when the + requirement cannot be parsed. + """ + try: + parsed = Requirement(requirement) + except InvalidRequirement: + logger.warning("Skip invalid requirement %r", requirement) + return None + + version = "" + for specifier in parsed.specifier: + if specifier.operator in ("==", "==="): + version = specifier.version + break + + if not parsed.url: + return models.Distribution(name=parsed.name, version=version) + + if parsed.url.startswith(_GIT_URL_PREFIX): + url, separator, commit = parsed.url[len(_GIT_URL_PREFIX) :].rpartition("@") + if separator and commit and "/" not in commit: + return models.Distribution( + name=parsed.name, + version=version, + git=models.GitInfo(commit=commit, remote=url), + ) + + # A URL without git revision is kept as it is + return models.Distribution( + name=parsed.name, version=version, archive=_archive_from_url(parsed.url) + ) + + +def _archive_url(archive: models.ArchiveInfo) -> str: + """Archive URL with one hash in its fragment, which pip verifies when it + downloads the archive. The hashes are not provided as `--hash` options because + pip then requires a hash for every requirement. + """ + for algorithm in _URL_HASH_ALGORITHMS: + value = archive.hashes.get(algorithm) + if value: + return f"{archive.url}#{algorithm}={value}" + return archive.url + + +def _archive_from_url(url: str) -> models.ArchiveInfo: + """Archive of a direct reference URL, with the hashes in its fragment (see + `_archive_url`).""" + base, separator, fragment = url.partition("#") + if not separator: + return models.ArchiveInfo(url=url, hashes=dict()) + + hashes = dict() + others = [] + for part in fragment.split("&"): + algorithm, _, value = part.partition("=") + if value and algorithm in _URL_HASH_ALGORITHMS: + hashes[algorithm] = value + elif part: + others.append(part) + + if not hashes: + return models.ArchiveInfo(url=url, hashes=hashes) + if others: + base = f"{base}#{'&'.join(others)}" + return models.ArchiveInfo(url=base, hashes=hashes) + + +def _log(requirements: List[str], warnings: List[str]) -> List[str]: + for warning in warnings: + logger.warning(warning) + return requirements + + +def _distribution_requirements( dist: models.Distribution, ) -> Tuple[List[str], List[str]]: """ - Return the pip freeze argument corresponding to the distribution with - associated warnings regarding reproducibility. + Return the requirements corresponding to the distribution with associated + warnings regarding reproducibility. """ lines = [] warnings = [] @@ -30,13 +177,7 @@ def freeze_distribution( pypi_req = dist.name if dist.archive: - archive_req = f"{dist.name} @ {dist.archive.url}" - - if dist.archive.hashes: - for algo, value in dist.archive.hashes.items(): - archive_req += f" --hash={algo}:{value}" - - lines.append(archive_req) + lines.append(f"{dist.name} @ {_archive_url(dist.archive)}") return lines, warnings if dist.git: @@ -97,8 +238,8 @@ def _normalize_git_url(url: str, preserve_ssh: bool = False) -> str: return url -def sanitize_freeze(requirements: Sequence[str]) -> Tuple[List[str], List[str]]: - """Sanitize a list of requirements coming from 'pip freeze'. +def _sanitize(requirements: Sequence[str]) -> Tuple[List[str], List[str]]: + """Sanitize a list of requirements in `requirements.txt` format. Returns a sanitized list with warnings regarding applied changes. """ diff --git a/src/ewoks/_requirements/uv.py b/src/ewoks/_requirements/uv.py deleted file mode 100644 index f5269f3..0000000 --- a/src/ewoks/_requirements/uv.py +++ /dev/null @@ -1,61 +0,0 @@ -from typing import Any -from typing import Dict -from typing import List -from typing import Literal -from typing import Optional - -from .utils.base_manager import BaseManager -from .utils.base_manager import BaseManagerInfo -from .utils.base_manager import BaseRequirements - - -class UvManagerInfo(BaseManagerInfo): - name: Literal["uv"] = "uv" - requirements: List[str] - - -class UvRequirements(BaseRequirements): - manager: UvManagerInfo - - def __info__(self) -> str: - requirements = "\n ".join(self.manager.requirements) - return f"{super().__info__()}\nRequirements:\n {requirements}" - - -class UvManager(BaseManager): - NAME = "uv" - PRIORITY = 1 - REQUIREMENTS_MODEL = UvRequirements - - def __init__(self, *command: str) -> None: - if not command: - command = ("uv",) - super().__init__(*command) - - def version(self) -> Optional[str]: - """Returns None when this manager is not available.""" - try: - output = self._check_output("--version") - return output.strip().split(" ")[-1] - except RuntimeError: - return None - - def is_active(self) -> bool: - """Manager is explicitly active.""" - pass - - def _gather_requirements(self) -> Dict[str, Any]: - output = self._check_output("pip", "freeze") - requirements = output.strip().splitlines() - - return {"requirements": requirements} - - def _install_native_requirements(self, requirements: UvRequirements) -> bool: - text = "\n".join(requirements.requirements) - with self._temporary_file(text, ".txt") as tmp_path: - self._check_call("add", "-r", tmp_path) - - return True - - def _install_base_requirements(self, requirements: BaseRequirements) -> bool: - raise NotImplementedError(f"{self.NAME} installation of python distributions") diff --git a/src/ewoks/bindings.py b/src/ewoks/bindings.py index a26ec5d..91d9f2f 100644 --- a/src/ewoks/bindings.py +++ b/src/ewoks/bindings.py @@ -7,6 +7,7 @@ from contextlib import contextmanager from pathlib import Path from typing import Any +from typing import Callable from typing import Dict from typing import Generator from typing import List @@ -25,6 +26,7 @@ from . import _engines from . import _requirements from . import graph_cache +from ._requirements.utils.environment import Environment from .errors import AbortException try: @@ -203,7 +205,11 @@ def convert_graph( save_requirements: bool = True, package_manager_name: Optional[str] = None, package_manager_command: Union[None, str, Tuple[str, ...]] = None, + on_requirements: Optional[Callable[[Optional[str]], None]] = None, ) -> Union[str, dict]: + """`on_requirements` is called with the name of the package manager that + generated the requirements, or `None` when they are not saved. + """ if load_options is None: load_options = dict() if save_options is None: @@ -216,14 +222,19 @@ def convert_graph( elif isinstance(package_manager_command, str): package_manager_command = _split_command(package_manager_command) - try: - _requirements.add_requirements( - graph, - manager_name=package_manager_name, - manager_command=package_manager_command, - ) - except Exception: - logger.exception("Continue after failure to add workflow requirements") + manager_name = _requirements.add_requirements( + graph, + manager_name=package_manager_name, + manager_command=package_manager_command, + ) + logger.info("Requirements generated with the %r package manager", manager_name) + else: + manager_name = None + logger.info("Requirements not saved") + + if on_requirements is not None: + on_requirements(manager_name) + return save_graph(graph, destination, **save_options) @@ -267,39 +278,65 @@ def _print_graph( def install_graph( source, skip_prompt: bool = False, + in_place: bool = False, + env_name: Optional[str] = None, + env_root: Optional[Union[str, Path]] = None, + python_version: Optional[str] = None, + ensure_ewoks: bool = False, + clean: bool = False, package_manager_name: Optional[str] = None, package_manager_command: Union[None, str, Tuple[str, ...]] = None, load_options: Optional[dict] = None, -) -> None: +) -> Environment: + """Install the requirements of a workflow in a python environment named + `env_name` (the workflow identifier by default), created inside `env_root` or + inside the root directory of the package manager. With `in_place` the + requirements are installed in the current python environment instead. + `ensure_ewoks` adds ewoks itself when the requirements do not contain it. `clean` + removes the environment when it already exists. Unless `skip_prompt`, every + installation and removal must be confirmed. + + :raises ValueError: `in_place` with an environment name, root or `clean` + :raises AbortException: installation not confirmed + """ + if in_place and (env_name or env_root or clean): + raise ValueError( + "Installing in the current python environment does not create an " + "environment" + ) if load_options is None: load_options = dict() graph = load_graph(source, **load_options) requirements = _requirements.get_requirements(graph) + if not in_place and env_name is None: + env_name = _requirements.environment_name(graph) + if not package_manager_command: package_manager_command = tuple() elif isinstance(package_manager_command, str): package_manager_command = _split_command(package_manager_command) - if skip_prompt: - _requirements.install_requirements( - requirements, - manager_name=package_manager_name, - manager_command=package_manager_command, - ) - return - - answer = input( - f"{requirements.__info__()}\n\nThis will install the packages above. Do you want to proceed (y/N)?" + return _requirements.install_requirements( + requirements, + env_name=env_name, + env_root=env_root, + manager_name=package_manager_name, + manager_command=package_manager_command, + python_version=python_version, + ensure_ewoks=ensure_ewoks, + clean=clean, + confirm=None if skip_prompt else _confirm_installation, ) - if answer.lower() == "y" or answer.lower() == "yes": - _requirements.install_requirements( - requirements, - manager_name=package_manager_name, - manager_command=package_manager_command, - ) - else: + + +def _confirm_installation(description: str) -> None: + """ + :raises AbortException: installation not confirmed + """ + answer = input(f"{description}\nDo you want to proceed (y/N)?") + if answer.lower() not in ("y", "yes"): raise AbortException() diff --git a/src/ewoks/cli_utils/cli_arguments.py b/src/ewoks/cli_utils/cli_arguments.py index a5664cd..59e42a8 100644 --- a/src/ewoks/cli_utils/cli_arguments.py +++ b/src/ewoks/cli_utils/cli_arguments.py @@ -3,6 +3,47 @@ from ewoksutils.cli_utils.cli_arguments import CLIArg from .._engines import get_graph_representations +from .._requirements import supported_managers + + +def package_manager_arguments(action: str) -> List[CLIArg]: + """CLI arguments to select the package manager that will `action` requirements.""" + examples = ", ".join( + f'"{command}" for {name}' for name, command in supported_managers().items() + ) + return [ + CLIArg( + "package_manager_name", + ["--package-manager-name"], + type=str.lower, + choices=list(supported_managers()), + help=f"Package manager to {action} the workflow requirements.", + ), + CLIArg( + "package_manager_command", + ["--package-manager-command"], + type=str, + help=( + f"Command that invokes the package manager which will {action} the " + f"workflow requirements. For example {examples}." + ), + ), + ] + + +def environment_arguments() -> List[CLIArg]: + """CLI arguments to select the python environment that runs the command.""" + return [ + CLIArg( + "env", + ["--env"], + type=str, + help=( + "Location of the python environment in which to run, for example " + "created by 'ewoks install'. Default: the current environment." + ), + ), + ] def ewoks_load_arguments() -> List[CLIArg]: diff --git a/src/ewoks/cli_utils/cli_convert_utils.py b/src/ewoks/cli_utils/cli_convert_utils.py index a2f5b77..a2732b4 100644 --- a/src/ewoks/cli_utils/cli_convert_utils.py +++ b/src/ewoks/cli_utils/cli_convert_utils.py @@ -8,6 +8,7 @@ from .._engines import get_graph_representations from .cli_arguments import ewoks_load_arguments +from .cli_arguments import package_manager_arguments from .cli_parse import parse_destinations @@ -49,19 +50,8 @@ def convert_arguments( action="store_true", help="Do not include the packages of the current Python environment as requirements in the destination workflow.", ), - CLIArg( - "package_manager_name", - ["--package-manager-name"], - type=str, - help='Package manager name to generate requirements. For example "pip"', - ), - CLIArg( - "package_manager_command", - ["--package-manager-command"], - type=str, - help='Package manager command to generate requirements. For example "python -m pip"', - ), ] + args_list += package_manager_arguments("generate") return args_list diff --git a/src/ewoks/cli_utils/cli_install_utils.py b/src/ewoks/cli_utils/cli_install_utils.py index ca2bfdd..63fa2de 100644 --- a/src/ewoks/cli_utils/cli_install_utils.py +++ b/src/ewoks/cli_utils/cli_install_utils.py @@ -7,6 +7,10 @@ from ewoksutils.cli_utils import cli_parse from ewoksutils.cli_utils.cli_spec import CLIArg +from .._requirements import managers_supporting_in_place +from .._requirements.utils.base_manager import EWOKS_ENVIRONMENTS_ROOT +from .cli_arguments import package_manager_arguments + logger = logging.getLogger(__name__) @@ -27,18 +31,63 @@ def install_arguments( help="Automatically accept installation prompts.", ), CLIArg( - "package_manager_name", - ["--package-manager-name"], + "env_name", + ["--env-name"], type=str, - help='Package manager name. For example "pip"', + help=( + "Name of the python environment to create. Default: the workflow " + "identifier." + ), ), CLIArg( - "package_manager_command", - ["--package-manager-command"], + "env_root", + ["--env-root"], type=str, - help='Package manager command. For example "python -m pip"', + help=( + "Directory in which the python environment is created. Default: the " + "directory in which the package manager creates environments " + f"('{EWOKS_ENVIRONMENTS_ROOT}' for package managers that do not have " + "one)." + ), + ), + CLIArg( + "clean", + ["--clean"], + action="store_true", + help=( + "Remove the python environment when it already exists instead of " + "installing in it." + ), + ), + CLIArg( + "in_place", + ["--in-place"], + action="store_true", + help=( + "Install in the current python environment instead of creating one " + f"(only {', '.join(managers_supporting_in_place())})." + ), + ), + CLIArg( + "python_version", + ["--python-version"], + type=str, + help=( + "Python version of the environment to create. Default: the python " + "version of the workflow requirements." + ), + ), + CLIArg( + "with_ewoks", + ["--with-ewoks"], + action="store_true", + help=( + "Add ewoks to the environment when the requirements do not contain " + "it, so the workflow can be executed in the environment." + ), ), ] + args_list += package_manager_arguments("install") return args_list @@ -46,3 +95,10 @@ def parse_install_arguments(cli_args: Namespace, shell: bool = False) -> None: if shell: cli_log_utils.parse_log_arguments(cli_args) cli_args.workflows, cli_args.graphs = cli_parse.parse_workflows(cli_args) + if cli_args.env_name and len(cli_args.workflows) > 1: + raise ValueError("'--env-name' requires a single workflow") + if cli_args.in_place and (cli_args.env_name or cli_args.env_root or cli_args.clean): + raise ValueError( + "'--in-place' cannot be combined with '--env-name', '--env-root' or " + "'--clean'" + ) diff --git a/src/ewoks/tests/requirements/conftest.py b/src/ewoks/tests/requirements/conftest.py new file mode 100644 index 0000000..7c92c11 --- /dev/null +++ b/src/ewoks/tests/requirements/conftest.py @@ -0,0 +1,106 @@ +import shutil +from pathlib import Path +from typing import Dict +from typing import Iterator + +import pytest + +from ..._requirements.utils.base_manager import BaseManager +from ..._requirements.utils.environment import Environment +from .managers import FAST_MANAGER_CASES +from .managers import MANAGER_CASES +from .managers import ManagerCase + +_IDS = [case.NAME for case in MANAGER_CASES] +_FAST_IDS = [case.NAME for case in FAST_MANAGER_CASES] + + +@pytest.fixture(scope="package", autouse=True) +def isolated_home(tmp_path_factory) -> Iterator[Path]: + """Home directory of the package managers, in which they cache their downloads + and store their configuration. It is inside the pytest temporary directory so a + test run does not write anywhere else. The downside is that the caches are + empty at the start of every test run. + """ + home = tmp_path_factory.mktemp("home") + with pytest.MonkeyPatch.context() as monkeypatch: + for name, path in _home_variables(home).items(): + monkeypatch.setenv(name, str(path)) + yield home + # The caches are not reused by the next test run and fill up the disk + shutil.rmtree(home, ignore_errors=True) + + +def _home_variables(home: Path) -> Dict[str, Path]: + """Environment variables that move everything a package manager writes + outside a python environment to this home directory. + """ + return { + # Home directory on Linux, macOS and Windows + "HOME": home, + "USERPROFILE": home, + "APPDATA": home, + "LOCALAPPDATA": home, + "XDG_CACHE_HOME": home / ".cache", + "XDG_CONFIG_HOME": home / ".config", + "XDG_DATA_HOME": home / ".local" / "share", + } + + +@pytest.fixture +def env_root(tmp_path) -> Iterator[Path]: + """Root directory of the environments created by a test.""" + yield tmp_path + # Environments are large and there are many tests + shutil.rmtree(tmp_path, ignore_errors=True) + + +@pytest.fixture(params=MANAGER_CASES, ids=_IDS) +def manager_case(request) -> ManagerCase: + """Repeats the test for every package manager.""" + return _manager_case(request) + + +@pytest.fixture +def manager(manager_case) -> BaseManager: + """Package manager under test.""" + return manager_case.manager() + + +@pytest.fixture +def environment(manager_case, tmp_path) -> Iterator[Environment]: + """Empty python environment created by the package manager under test.""" + yield from _environment(manager_case, tmp_path) + + +@pytest.fixture(params=FAST_MANAGER_CASES, ids=_FAST_IDS) +def fast_manager_case(request) -> ManagerCase: + """Repeats the test for every package manager that is fast. Use this when the + package manager is not the subject of the test.""" + return _manager_case(request) + + +@pytest.fixture +def fast_manager(fast_manager_case) -> BaseManager: + """Package manager under test, one that is fast.""" + return fast_manager_case.manager() + + +@pytest.fixture +def fast_environment(fast_manager_case, tmp_path) -> Iterator[Environment]: + """Empty python environment created by a package manager that is fast.""" + yield from _environment(fast_manager_case, tmp_path) + + +def _manager_case(request) -> ManagerCase: + case: ManagerCase = request.param + if case.manager().version() is None: + pytest.skip(f"{case.NAME} is not installed") + return case + + +def _environment(case: ManagerCase, tmp_path) -> Iterator[Environment]: + location = tmp_path / "environment" + yield case.manager().create_environment(location) + # Environments are large and there are many tests + shutil.rmtree(location, ignore_errors=True) diff --git a/src/ewoks/tests/requirements/managers.py b/src/ewoks/tests/requirements/managers.py new file mode 100644 index 0000000..0f16d11 --- /dev/null +++ b/src/ewoks/tests/requirements/managers.py @@ -0,0 +1,71 @@ +"""Package manager specific knowledge needed by the package manager tests. + +Add a `ManagerCase` to `MANAGER_CASES` to include a package manager in the +tests that apply to all package managers. The tests use the `manager_case`, +`manager` and `environment` fixtures (see `conftest.py`). +""" + +from typing import Dict +from typing import List +from typing import Sequence +from typing import Tuple +from typing import Type + +from ..._requirements.pip_venv import PipVenvManager +from ..._requirements.utils.base_manager import BaseManager +from ..._requirements.utils.metadata import models +from ..._requirements.utils.requirements_txt import REQUIREMENTS_FILENAME + + +class ManagerCase: + NAME: str = NotImplemented + + MANAGER_CLS: Type[BaseManager] = NotImplemented + + INSTALLER: str = NotImplemented + """`INSTALLER` metadata of a distribution installed by the package manager.""" + + SLOW: bool = False + """Creating an environment takes tens of seconds.""" + + def command(self) -> Tuple[str, ...]: + """Command that invokes the package manager (the default when empty).""" + return tuple() + + def manager(self) -> BaseManager: + return self.MANAGER_CLS(*self.command()) + + def cli_command(self) -> str: + """Value for the `--package-manager-command` CLI argument.""" + return " ".join(self.command()) + + def native_files( + self, distributions: Sequence[models.Distribution], python_version: str + ) -> Dict[str, str]: + """Files the package manager generates to reproduce an environment with + these distributions. Empty when they cannot be generated. + """ + raise NotImplementedError + + +class PipVenvCase(ManagerCase): + NAME = "pip-venv" + MANAGER_CLS = PipVenvManager + INSTALLER = "pip" + + def native_files( + self, distributions: Sequence[models.Distribution], python_version: str + ) -> Dict[str, str]: + requirements = "\n".join( + f"{dist.name}=={dist.version}" for dist in distributions + ) + return {REQUIREMENTS_FILENAME: requirements} + + +MANAGER_CASES: List[ManagerCase] = [ + PipVenvCase(), +] + +FAST_MANAGER_CASES: List[ManagerCase] = [ + case for case in MANAGER_CASES if not case.SLOW +] diff --git a/src/ewoks/tests/requirements/pip/__init__.py b/src/ewoks/tests/requirements/pip/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/ewoks/tests/requirements/pip/test_freeze.py b/src/ewoks/tests/requirements/pip/test_freeze.py deleted file mode 100644 index 61b0b7f..0000000 --- a/src/ewoks/tests/requirements/pip/test_freeze.py +++ /dev/null @@ -1,95 +0,0 @@ -import pytest - -from ...._requirements.utils.pip_freeze import sanitize_freeze - - -def test_normal_requirement(): - req = ["ewoks==1.1.0"] - sanitized, warnings = sanitize_freeze(req) - assert sanitized == ["ewoks==1.1.0"] - assert warnings == [] - - -def test_editable_ssh_vcs_url_normalized(): - ssh_project_name = "querypool" - ssh_project_url = "gitlab.esrf.fr/dau/querypool.git" - ssh_project_commit = "ab6acc7e140ed33eb896b1336a5a5aac6b60cc0f" - req = [ - f"-e git+ssh://git@{ssh_project_url}@{ssh_project_commit}#egg={ssh_project_name}" - ] - - sanitized, warnings = sanitize_freeze(req) - - expected_sanitized = [ - f"{ssh_project_name} @ git+https://{ssh_project_url}@{ssh_project_commit}" - ] - expected_warnings = [ - f"Normalize VCS requirement 'git+ssh://git@{ssh_project_url}@{ssh_project_commit}#egg={ssh_project_name}' " - f"to '{ssh_project_name} @ git+ssh://git@{ssh_project_url}@{ssh_project_commit}'", - f"Normalize requirement '{ssh_project_name} @ git+ssh://git@{ssh_project_url}@{ssh_project_commit}' " - f"to '{ssh_project_name} @ git+https://{ssh_project_url}@{ssh_project_commit}'", - ] - - assert sanitized == expected_sanitized - assert warnings == expected_warnings - - -@pytest.mark.parametrize("exists", [True, False]) -def test_editable_local_path_with_comment_replacement(tmp_path, exists): - path = tmp_path / "repo_name" - if exists: - path.mkdir() - - comment = "# Editable Git install with no remote (project_name==1.0.0)" - replacement = "project_name==1.0.0" - req = [comment, f"-e {path}"] - - sanitized, warnings = sanitize_freeze(req) - - assert sanitized == [replacement] - assert warnings == [f"Replaced editable install '{path}' with '{replacement}'."] - - -@pytest.mark.parametrize("exists", [True, False]) -def test_editable_local_path_without_comment_replacement(tmp_path, exists): - path = tmp_path / "repo_name" - if exists: - path.mkdir() - warning = f"Editable path exists locally: '{path}'" - else: - warning = f"Editable path does not exist locally: '{path}'" - - req = [f"-e {path}"] - - sanitized, warnings = sanitize_freeze(req) - - assert sanitized == [f"-e {path}"] - if exists: - assert warnings == [warning] - else: - assert warnings == [warning] - - -def test_branch_specified_requirement(): - project_name = "ewoksutils" - project_url = "github.com/ewoks-kit/ewoksutils.git" - project_branch = "main" - - req = [f"{project_name}@ git+https://{project_url}@{project_branch}"] - - sanitized, warnings = sanitize_freeze(req) - - # No warnings expected here (assuming valid format) - assert sanitized == [f"{project_name}@ git+https://{project_url}@{project_branch}"] - assert warnings == [] - - -def test_invalid_requirement_warning(): - project_url = "github.com/ewoks-kit/ewoksutils.git" - - req = [f"git+https://{project_url}"] - - sanitized, warnings = sanitize_freeze(req) - - assert sanitized == [f"git+https://{project_url}"] - assert any("Possibly invalid requirement format" in w for w in warnings) diff --git a/src/ewoks/tests/requirements/pip/test_install_cli.py b/src/ewoks/tests/requirements/pip/test_install_cli.py deleted file mode 100644 index a4bd326..0000000 --- a/src/ewoks/tests/requirements/pip/test_install_cli.py +++ /dev/null @@ -1,141 +0,0 @@ -import json -import subprocess -import sys - -import pytest - -from ...._requirements.utils.metadata import from_pip_freeze - - -def test_install_pip_with_freeze(venv): - with pytest.raises(Exception, match="package is not installed"): - _ = venv.get_version("ewoksdata") - - requirements = from_pip_freeze.pip_freeze_requirements(["ewoksdata"]) - - graph = { - "graph": { - "schema_version": "1.1", - "id": "test_install", - "requirements": requirements, - } - } - - argv = [ - sys.executable, - "-m", - "ewoks", - "install", - "--yes", - json.dumps(graph), - "--package-manager-name", - "pip", - "--package-manager-command", - f"{venv.python} -m pip", - ] - subprocess.check_call(argv) # noqa: S603 - Trusted test command; - - assert venv.get_version("ewoksdata") - - -def test_install_pip_without_freeze(venv): - with pytest.raises(Exception, match="package is not installed"): - _ = venv.get_version("ewoksdata") - - requirements = from_pip_freeze.pip_freeze_requirements(["ewoksdata"]) - requirements["distributions"] = [ - {"name": "ewoksdata", "version": ""}, - ] - - graph = { - "graph": { - "schema_version": "1.1", - "id": "test_install", - "requirements": requirements, - } - } - - argv = [ - sys.executable, - "-m", - "ewoks", - "install", - "--yes", - json.dumps(graph), - "--package-manager-name", - "pip", - "--package-manager-command", - f"{venv.python} -m pip", - ] - subprocess.check_call(argv) # noqa: S603 - Trusted test command; - - assert venv.get_version("ewoksdata") - - -def test_install_legacy_pip_freeze(venv): - with pytest.raises(Exception, match="package is not installed"): - _ = venv.get_version("ewoksdata") - - requirements = ["ewoksdata"] - graph = { - "graph": { - "schema_version": "1.1", - "id": "test_install", - "requirements": requirements, - } - } - - argv = [ - sys.executable, - "-m", - "ewoks", - "install", - "--yes", - json.dumps(graph), - "--package-manager-name", - "pip", - "--package-manager-command", - f"{venv.python} -m pip", - ] - subprocess.check_call(argv) # noqa: S603 - Trusted test command; - - assert venv.get_version("ewoksdata") - - -def test_install_without_requirements(venv): - with pytest.raises(Exception, match="package is not installed"): - _ = venv.get_version("ewoksdata") - - nodes = [ - { - "id": 1, - "task_identifier": 'ewoksdata.tasks.normalization.Normalization"', - "task_type": "class", - }, - { - "id": 2, - "task_identifier": "path/to/my/script", - "task_type": "script", - }, # Check that unsupported task type goes through without error - ] - - graph = { - "graph": {"schema_version": "1.1", "id": "test_install"}, - "nodes": nodes, - } - - argv = [ - sys.executable, - "-m", - "ewoks", - "install", - "--yes", - json.dumps(graph), - "--package-manager-name", - "pip", - "--package-manager-command", - f"{venv.python} -m pip", - ] - subprocess.check_call(argv) # noqa: S603 - Trusted test command; - - assert venv.get_version("ewoksdata") diff --git a/src/ewoks/tests/requirements/test_environment.py b/src/ewoks/tests/requirements/test_environment.py new file mode 100644 index 0000000..b2abfa0 --- /dev/null +++ b/src/ewoks/tests/requirements/test_environment.py @@ -0,0 +1,38 @@ +"""Tests of the python environment layouts.""" + +import sys +from pathlib import Path + +import pytest + +from ..._requirements.utils.environment import ENVIRONMENT_SUBDIRS +from ..._requirements.utils.environment import Environment + +_INTERPRETERS = [ + ("linux", Path("bin", "python")), + ("win32", Path("Scripts", "python.exe")), + ("win32", Path("python.exe")), +] +"""Interpreter locations inside an environment prefix, per platform.""" + + +@pytest.mark.parametrize("subdir", ENVIRONMENT_SUBDIRS) +@pytest.mark.parametrize("platform, relative", _INTERPRETERS) +def test_environment_layout(platform, relative, subdir, monkeypatch, tmp_path): + """Every interpreter location is found in every environment layout.""" + monkeypatch.setattr(sys, "platform", platform) + prefix = tmp_path / subdir if subdir else tmp_path + python = prefix / relative + python.parent.mkdir(parents=True, exist_ok=True) + python.touch() + + environment = Environment.from_location(tmp_path) + + assert environment.location == tmp_path + assert environment.prefix == prefix + assert environment.python == python + + +def test_no_environment(tmp_path): + with pytest.raises(ValueError, match="No python environment found"): + Environment.from_location(tmp_path) diff --git a/src/ewoks/tests/requirements/test_install_cli.py b/src/ewoks/tests/requirements/test_install_cli.py new file mode 100644 index 0000000..a0b4342 --- /dev/null +++ b/src/ewoks/tests/requirements/test_install_cli.py @@ -0,0 +1,240 @@ +"""Tests of `ewoks install`, which does not depend on the package manager.""" + +import json +import shutil +import subprocess +import sys +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator +from typing import List +from typing import Optional + +import pytest + +from ..._requirements import environment_name +from ..._requirements.utils.environment import Environment +from ...bindings import load_graph +from .managers import ManagerCase +from .utils import DISTRIBUTIONS +from .utils import PYTHON_VERSION +from .utils import REQUIREMENTS +from .utils import assert_installed +from .utils import manager_requirements + +_GRAPH_ID = "test_install" +"""Identifier of the workflows to install, which is also the name of the +environment created for them.""" + + +def test_install_with_files(fast_manager_case, env_root): + files = fast_manager_case.native_files(DISTRIBUTIONS, PYTHON_VERSION) + if not files: + pytest.skip(f"{fast_manager_case.NAME} cannot generate its requirement files") + requirements = manager_requirements( + fast_manager_case.NAME, + files=files, + distributions=DISTRIBUTIONS, + python_version=PYTHON_VERSION, + ) + + with _install(_graph(requirements), fast_manager_case, env_root) as location: + assert_installed(Environment.from_location(location), DISTRIBUTIONS) + + +def test_install_without_files(fast_manager_case, env_root): + requirements = manager_requirements( + fast_manager_case.NAME, distributions=DISTRIBUTIONS + ) + + with _install(_graph(requirements), fast_manager_case, env_root) as location: + assert_installed(Environment.from_location(location), DISTRIBUTIONS) + + +def test_install_legacy_requirements_list(fast_manager_case, env_root): + requirements = REQUIREMENTS + + with _install(_graph(requirements), fast_manager_case, env_root) as location: + assert_installed(Environment.from_location(location), DISTRIBUTIONS) + + +def test_install_without_requirements(fast_manager_case, env_root): + """Requirements are guessed from the workflow nodes.""" + nodes = [ + { + "id": 1, + # The distribution that provides this module is installed + "task_identifier": "six.moves.range", + "task_type": "method", + }, + { + "id": 2, + "task_identifier": "path/to/my/script", + "task_type": "script", + }, # Check that unsupported task type goes through without error + ] + graph = { + "graph": {"schema_version": "1.1", "id": _GRAPH_ID}, + "nodes": nodes, + } + + with _install(graph, fast_manager_case, env_root) as location: + assert Environment.from_location(location).distribution_version("six") + + +def test_install_env_name(fast_manager_case, env_root): + """The environment name replaces the workflow identifier.""" + graph = _graph(REQUIREMENTS) + + with _install(graph, fast_manager_case, env_root, env_name="myenv") as location: + assert location == env_root / "myenv" + assert_installed(Environment.from_location(location), DISTRIBUTIONS) + + +def test_install_default_location(fast_manager_case, isolated_home): + """Without a root directory the package manager provides the location.""" + graph = _graph(REQUIREMENTS, graph_id=f"{_GRAPH_ID}_{fast_manager_case.NAME}") + + with _install(graph, fast_manager_case) as location: + assert isolated_home in location.parents + assert_installed(Environment.from_location(location), DISTRIBUTIONS) + + +def test_default_location_without_workflow_id(fast_manager_case, isolated_home): + """Workflows without an id do not share a location.""" + first = _location({"graph": {"label": "first"}}, fast_manager_case) + second = _location({"graph": {"label": "second"}}, fast_manager_case) + + assert first != second + assert first.parent == second.parent + + +def test_install_existing_environment(fast_manager_case, env_root): + """An existing environment is installed in, unless it is cleaned first.""" + graph = _graph(REQUIREMENTS) + + with _install(graph, fast_manager_case, env_root) as location: + marker = location / "marker.txt" + marker.touch() + + _run_install(graph, fast_manager_case, env_root) + assert marker.exists() + + _run_install(graph, fast_manager_case, env_root, clean=True) + assert not marker.exists() + assert_installed(Environment.from_location(location), DISTRIBUTIONS) + + +def test_clean_without_environment(fast_manager_case, env_root): + """Only a python environment is removed.""" + graph = _graph(REQUIREMENTS) + location = _location(graph, fast_manager_case, env_root) + location.mkdir(parents=True) + argv = _argv(fast_manager_case, env_root, clean=True) + [json.dumps(graph)] + + result = subprocess.run( # noqa: S603 - Trusted test command + argv, capture_output=True, text=True, check=False + ) + + assert "is not a python environment" in result.stdout + result.stderr + assert not list(location.iterdir()) + + +def test_install_single_workflow_per_name(fast_manager_case, env_root): + graph = json.dumps(_graph(REQUIREMENTS)) + argv = _argv(fast_manager_case, env_root, env_name="myenv") + [graph, graph] + + result = subprocess.run( # noqa: S603 - Trusted test command + argv, capture_output=True, text=True, check=False + ) + + assert result.returncode != 0 + assert "'--env-name' requires a single workflow" in result.stderr + + +def test_install_in_place_without_environment(fast_manager_case, env_root): + """Installing in the current python environment does not create one.""" + argv = _argv(fast_manager_case, env_root) + ["--in-place", json.dumps(_graph([]))] + + result = subprocess.run( # noqa: S603 - Trusted test command + argv, capture_output=True, text=True, check=False + ) + + assert result.returncode != 0 + assert "'--in-place' cannot be combined" in result.stderr + + +def _graph(requirements, graph_id: str = _GRAPH_ID) -> dict: + return { + "graph": { + "schema_version": "1.1", + "id": graph_id, + "requirements": requirements, + } + } + + +def _location( + graph: dict, + manager_case: ManagerCase, + env_root: Optional[Path] = None, + env_name: Optional[str] = None, +) -> Path: + """Location of the environment that `ewoks install` creates for this workflow.""" + name = env_name or environment_name(load_graph(graph)) + return manager_case.manager().environment_location(name, env_root) + + +def _argv( + manager_case: ManagerCase, + env_root: Optional[Path] = None, + env_name: Optional[str] = None, + clean: bool = False, +) -> List[str]: + argv = [ + sys.executable, + "-m", + "ewoks", + "install", + "--yes", + "--package-manager-name", + manager_case.NAME, + ] + if manager_case.command(): + argv += ["--package-manager-command", manager_case.cli_command()] + if env_root: + argv += ["--env-root", str(env_root)] + if env_name: + argv += ["--env-name", env_name] + if clean: + argv += ["--clean"] + return argv + + +@contextmanager +def _install( + graph: dict, + manager_case: ManagerCase, + env_root: Optional[Path] = None, + env_name: Optional[str] = None, + clean: bool = False, +) -> Iterator[Path]: + """Install a workflow and remove the environment it created, also when it was + not created inside `env_root`.""" + location = _location(graph, manager_case, env_root, env_name) + try: + _run_install(graph, manager_case, env_root, env_name, clean) + yield location + finally: + shutil.rmtree(location, ignore_errors=True) + + +def _run_install( + graph: dict, + manager_case: ManagerCase, + env_root: Optional[Path] = None, + env_name: Optional[str] = None, + clean: bool = False, +) -> None: + argv = _argv(manager_case, env_root, env_name, clean) + [json.dumps(graph)] + subprocess.check_call(argv) # noqa: S603 - Trusted test command diff --git a/src/ewoks/tests/requirements/test_managers.py b/src/ewoks/tests/requirements/test_managers.py new file mode 100644 index 0000000..43c9f4b --- /dev/null +++ b/src/ewoks/tests/requirements/test_managers.py @@ -0,0 +1,249 @@ +"""Tests that apply to all package managers.""" + +from typing import Iterator + +import pytest + +from ..._requirements import install_requirements +from ..._requirements.utils import base_manager +from ..._requirements.utils import detect +from ..._requirements.utils._supported import get_supported_managers +from ..._requirements.utils.detect import get_manager +from ..._requirements.utils.environment import Environment +from ..._requirements.utils.parse import parse_requirements +from ...errors import AbortException +from .managers import ManagerCase +from .utils import DISTRIBUTIONS +from .utils import EWOKS_DEPENDENCIES +from .utils import PYTHON_VERSION +from .utils import assert_installed +from .utils import assert_not_installed +from .utils import current_metadata +from .utils import manager_requirements + + +def test_available(manager): + version = manager.version() + assert version + assert version[0].isdigit() + + +def test_not_available(manager_case, monkeypatch): + monkeypatch.setattr(manager_case.MANAGER_CLS, "version", lambda self: None) + with pytest.raises(ValueError, match="not available"): + get_manager(manager_name=manager_case.NAME) + + +def test_select_manager(manager_case): + manager = get_manager( + manager_name=manager_case.NAME, manager_command=manager_case.command() + ) + assert isinstance(manager, manager_case.MANAGER_CLS) + + +def test_is_active(manager): + assert isinstance(manager.is_active(), bool) + + +def test_detect_manager(manager_case, uncached_distribution_count, monkeypatch): + """The package manager that installed the distributions is detected when no + package manager is active.""" + for manager_cls in get_supported_managers().values(): + monkeypatch.setattr(manager_cls, "is_active", lambda self: False) + distributions = [ + dist.model_copy(update={"installer": manager_case.INSTALLER}) + for dist in DISTRIBUTIONS + ] + monkeypatch.setattr( + detect, + "current_requirements", + lambda: current_metadata(distributions, PYTHON_VERSION), + ) + + manager = get_manager() + + assert isinstance(manager, manager_case.MANAGER_CLS) + + +@pytest.fixture +def uncached_distribution_count() -> Iterator[None]: + """The distributions of the current python environment are counted once.""" + detect._manager_distribution_count.cache_clear() + yield + detect._manager_distribution_count.cache_clear() + + +def test_gather_requirements(manager, manager_case, monkeypatch): + """Requirements are gathered from the current python environment.""" + metadata = current_metadata(DISTRIBUTIONS, PYTHON_VERSION) + monkeypatch.setattr(base_manager, "current_requirements", lambda: metadata) + + requirements = manager.gather_requirements() + + assert isinstance(requirements, manager_case.MANAGER_CLS.REQUIREMENTS_MODEL) + assert requirements.manager.name == manager_case.NAME + assert requirements.manager.version + assert requirements.python.version == PYTHON_VERSION + assert requirements.distributions == DISTRIBUTIONS + assert isinstance(requirements.manager.files, dict) + + +def test_create_environment(environment): + assert environment.exists() + assert environment.python_version() == PYTHON_VERSION + assert Environment.from_location(environment.location) == environment + assert_not_installed(environment, DISTRIBUTIONS) + + +def test_install_native(manager, manager_case, environment): + """Install with the files generated by the package manager itself.""" + requirements = _requirements(manager_case, native=True) + assert manager.is_native(requirements, environment) + + manager.install_files(requirements, environment) + + assert_installed(environment, DISTRIBUTIONS) + + +def test_install_distributions(manager, manager_case, environment): + """Without files, the python distributions are installed.""" + requirements = _requirements(manager_case, native=False) + assert not manager.is_native(requirements, environment) + + manager.install_distributions(requirements, environment) + + assert_installed(environment, DISTRIBUTIONS) + + +def test_install_nothing(manager, manager_case, tmp_path): + """Nothing is installed so the environment is not needed.""" + requirements = parse_requirements(manager_requirements(manager_case.NAME)) + environment = Environment.at_location(tmp_path) + + with pytest.raises(ValueError, match="No distributions provided to install"): + manager.install_distributions(requirements, environment) + + +def test_install_files_failure(fast_manager_case, env_root, monkeypatch): + """The python distributions are installed when the files of the package manager + cannot be installed. Both installations are confirmed.""" + requirements = _requirements(fast_manager_case, native=True) + + def install_files(self, requirements, environment) -> None: + raise RuntimeError("cannot install the files") + + monkeypatch.setattr(fast_manager_case.MANAGER_CLS, "install_files", install_files) + confirmations = [] + + environment = install_requirements( + requirements, + env_name="environment", + env_root=env_root, + manager_name=fast_manager_case.NAME, + manager_command=fast_manager_case.command(), + confirm=confirmations.append, + ) + + assert_installed(environment, DISTRIBUTIONS) + assert len(confirmations) == 2 + assert "Manager:" in confirmations[0] + assert "Distributions:" in confirmations[1] + assert "package manager failed" in confirmations[1] + + +def test_install_not_confirmed(fast_manager_case, env_root): + """Nothing is created when an installation is not confirmed. The confirmation + mentions why the python distributions are installed.""" + requirements = _requirements(fast_manager_case, native=False) + confirmations = [] + + def confirm(description: str) -> None: + confirmations.append(description) + raise AbortException() + + with pytest.raises(AbortException): + install_requirements( + requirements, + env_name="environment", + env_root=env_root, + manager_name=fast_manager_case.NAME, + manager_command=fast_manager_case.command(), + confirm=confirm, + ) + + assert not (env_root / "environment").exists() + assert "do not provide files" in confirmations[0] + + +def test_install_without_name(fast_manager_case): + """A root directory and cleaning do not define an environment on their own.""" + requirements = parse_requirements(manager_requirements(fast_manager_case.NAME)) + + with pytest.raises(ValueError, match="requires an environment name"): + install_requirements(requirements, env_root="/root/of/the/environments") + + with pytest.raises(ValueError, match="requires an environment name"): + install_requirements(requirements, clean=True) + + +def test_environment_location(manager, tmp_path): + """The package manager provides the location of a named environment.""" + default = manager.environment_location("myenv") + assert default.is_absolute() + assert default.name == "myenv" + assert default.parent == manager.environments_root().expanduser() + + in_root = manager.environment_location("myenv", tmp_path) + assert in_root == tmp_path / "myenv" + + +def test_install_in_place_not_supported(manager_case, monkeypatch): + """A package manager that is explicitly requested is not replaced.""" + monkeypatch.setattr(manager_case.MANAGER_CLS, "CAN_INSTALL_IN_PLACE", False) + requirements = parse_requirements(manager_requirements(manager_case.NAME)) + + with pytest.raises(ValueError, match="cannot install in the current"): + install_requirements(requirements, manager_name=manager_case.NAME) + + +def test_ewoks_already_installed(manager, manager_case): + """Ewoks is installed in the current python environment.""" + requirements = parse_requirements(manager_requirements(manager_case.NAME)) + + manager.ensure_ewoks(requirements, Environment.current()) + + +def test_add_ewoks(fast_manager, fast_manager_case, fast_environment): + """Ewoks is added without changing the versions of the requirements it depends + on.""" + requirements = parse_requirements( + manager_requirements( + fast_manager_case.NAME, + distributions=EWOKS_DEPENDENCIES, + python_version=PYTHON_VERSION, + ) + ) + fast_manager.install_distributions(requirements, fast_environment) + + fast_manager.ensure_ewoks(requirements, fast_environment) + + assert fast_environment.distribution_version("ewoks") + assert_installed(fast_environment, EWOKS_DEPENDENCIES) + + +def _requirements(manager_case: ManagerCase, native: bool): + python_version = PYTHON_VERSION + if native: + files = manager_case.native_files(DISTRIBUTIONS, python_version) + if not files: + pytest.skip(f"{manager_case.NAME} cannot generate its requirement files") + else: + files = None + return parse_requirements( + manager_requirements( + manager_case.NAME, + files=files, + distributions=DISTRIBUTIONS, + python_version=python_version, + ) + ) diff --git a/src/ewoks/tests/requirements/test_pip_venv.py b/src/ewoks/tests/requirements/test_pip_venv.py new file mode 100644 index 0000000..09a0528 --- /dev/null +++ b/src/ewoks/tests/requirements/test_pip_venv.py @@ -0,0 +1,56 @@ +"""Tests specific to the pip-venv package manager.""" + +import importlib.metadata +import sys + +import pytest + +from ..._requirements.pip_venv import PipVenvManager +from ..._requirements.utils.requirements_txt import REQUIREMENTS_FILENAME +from .utils import PYTHON_VERSION + +pytestmark = pytest.mark.skipif( + PipVenvManager().version() is None, reason="pip is not installed" +) + + +def test_default_command(): + manager = PipVenvManager() + assert manager._cmd_args == (sys.executable,) + assert repr(manager) == f"PipVenvManager({sys.executable})" + + +def test_version(): + assert PipVenvManager().version() == importlib.metadata.version("pip") + + +def test_version_not_available(monkeypatch): + def not_found(name: str) -> str: + raise importlib.metadata.PackageNotFoundError(name) + + monkeypatch.setattr(importlib.metadata, "version", not_found) + assert PipVenvManager().version() is None + + +def test_never_active(): + """Pip is used when no other package manager is detected.""" + assert PipVenvManager().is_active() is False + + +def test_gather_requirements_txt(): + """The current environment is described in `requirements.txt` format.""" + requirements = PipVenvManager().gather_requirements() + + lines = requirements.manager.files[REQUIREMENTS_FILENAME].splitlines() + assert lines + assert any("ewoks" in requirement for requirement in lines) + + +def test_create_environment_python_version(env_root, caplog): + """`venv` cannot provide another python version than the one it runs on.""" + location = env_root / "environment" + + environment = PipVenvManager().create_environment(location, "1.2.3") + + assert "cannot provide python 1.2.3" in caplog.text + assert environment.python_version() == PYTHON_VERSION diff --git a/src/ewoks/tests/requirements/test_requirements_txt.py b/src/ewoks/tests/requirements/test_requirements_txt.py new file mode 100644 index 0000000..4efca7b --- /dev/null +++ b/src/ewoks/tests/requirements/test_requirements_txt.py @@ -0,0 +1,218 @@ +"""Tests of the `requirements.txt` format.""" + +import pytest + +from ..._requirements.utils import requirements_txt +from ..._requirements.utils.metadata import models +from .utils import DISTRIBUTIONS +from .utils import REQUIREMENTS + + +def test_normal_requirement(caplog): + assert requirements_txt.sanitize(["ewoks==1.1.0"]) == ["ewoks==1.1.0"] + assert not caplog.text + + +def test_editable_ssh_vcs_url_normalized(caplog): + ssh_project_name = "querypool" + ssh_project_url = "gitlab.esrf.fr/dau/querypool.git" + ssh_project_commit = "ab6acc7e140ed33eb896b1336a5a5aac6b60cc0f" + requirement = f"-e git+ssh://git@{ssh_project_url}@{ssh_project_commit}#egg={ssh_project_name}" + + sanitized = requirements_txt.sanitize([requirement]) + + assert sanitized == [ + f"{ssh_project_name} @ git+https://{ssh_project_url}@{ssh_project_commit}" + ] + assert ( + f"Normalize VCS requirement 'git+ssh://git@{ssh_project_url}@{ssh_project_commit}" + f"#egg={ssh_project_name}'" in caplog.text + ) + assert "to 'querypool @ git+https" in caplog.text + + +@pytest.mark.parametrize("exists", [True, False]) +def test_editable_local_path_with_comment_replacement(tmp_path, caplog, exists): + path = tmp_path / "repo_name" + if exists: + path.mkdir() + + comment = "# Editable Git install with no remote (project_name==1.0.0)" + replacement = "project_name==1.0.0" + + sanitized = requirements_txt.sanitize([comment, f"-e {path}"]) + + assert sanitized == [replacement] + assert f"Replaced editable install '{path}' with '{replacement}'." in caplog.text + + +@pytest.mark.parametrize("exists", [True, False]) +def test_editable_local_path_without_comment_replacement(tmp_path, caplog, exists): + path = tmp_path / "repo_name" + if exists: + path.mkdir() + warning = f"Editable path exists locally: '{path}'" + else: + warning = f"Editable path does not exist locally: '{path}'" + + sanitized = requirements_txt.sanitize([f"-e {path}"]) + + assert sanitized == [f"-e {path}"] + assert warning in caplog.text + + +def test_branch_specified_requirement(caplog): + project_name = "ewoksutils" + project_url = "github.com/ewoks-kit/ewoksutils.git" + requirement = f"{project_name}@ git+https://{project_url}@main" + + assert requirements_txt.sanitize([requirement]) == [requirement] + assert not caplog.text + + +def test_invalid_requirement_warning(caplog): + requirement = "git+https://github.com/ewoks-kit/ewoksutils.git" + + assert requirements_txt.sanitize([requirement]) == [requirement] + assert "Possibly invalid requirement format" in caplog.text + + +def test_distributions_requirements(): + distributions = [ + *DISTRIBUTIONS, + models.Distribution( + name="mypackage1", + version="1.0", + git=models.GitInfo(commit="123", remote="https://host/group/repo.git"), + ), + models.Distribution( + name="mypackage2", + version="1.0", + archive=models.ArchiveInfo( + url="https://host/mypackage2-1.0-py3-none-any.whl", + hashes={"sha256": "123"}, + ), + ), + ] + + assert requirements_txt.distributions_requirements(distributions) == [ + *REQUIREMENTS, + "mypackage1 @ git+https://host/group/repo.git@123", + # The hash is a URL fragment: pip requires a hash for every requirement as + # soon as one is provided as a '--hash' option + "mypackage2 @ https://host/mypackage2-1.0-py3-none-any.whl#sha256=123", + ] + assert requirements_txt.manifest_requirements( + distributions + ) == requirements_txt.distributions_requirements(distributions) + assert requirements_txt.version_constraints(distributions) == [ + *REQUIREMENTS, + "mypackage1==1.0", + "mypackage2==1.0", + ] + + +def test_manifest_requirements_unique(): + """A package manager manifest cannot contain the same dependency twice.""" + # The same distributions, but named differently and with another version + duplicates = [ + dist.model_copy(update={"name": dist.name.upper(), "version": "0.0.1"}) + for dist in DISTRIBUTIONS + ] + distributions = DISTRIBUTIONS + duplicates + + assert requirements_txt.manifest_requirements(distributions) == REQUIREMENTS + + +def test_manifest_requirements_without_comments(): + """Warnings that `pip freeze` adds as comments are not requirements.""" + distributions = [ + models.Distribution( + name="mypackage", version="1.0", git=models.GitInfo(commit="123") + ) + ] + + requirements = requirements_txt.distributions_requirements(distributions) + assert requirements[0].startswith("# ") + + assert requirements_txt.manifest_requirements(distributions) == ["mypackage==1.0"] + + +@pytest.mark.parametrize( + "requirement,expected", + [ + *( + ( + f"{dist.name}=={dist.version}", + {"name": dist.name, "version": dist.version}, + ) + for dist in DISTRIBUTIONS + ), + ("mypackage", {"name": "mypackage", "version": ""}), + ("mypackage>=1.0", {"name": "mypackage", "version": ""}), + ( + "mypackage @ git+https://host/repo.git@123", + { + "name": "mypackage", + "version": "", + "git": { + "commit": "123", + "remote": "https://host/repo.git", + "uncommitted_changes": False, + }, + }, + ), + ( + "mypackage @ https://host/mypackage-1.0.tar.gz", + { + "name": "mypackage", + "version": "", + "archive": {"url": "https://host/mypackage-1.0.tar.gz", "hashes": {}}, + }, + ), + ( + "mypackage @ https://host/mypackage-1.0.tar.gz#sha256=123", + { + "name": "mypackage", + "version": "", + "archive": { + "url": "https://host/mypackage-1.0.tar.gz", + "hashes": {"sha256": "123"}, + }, + }, + ), + ( + # Only the hashes are taken out of the fragment + "mypackage @ https://host/repo.tar.gz#sha256=123&subdirectory=sub", + { + "name": "mypackage", + "version": "", + "archive": { + "url": "https://host/repo.tar.gz#subdirectory=sub", + "hashes": {"sha256": "123"}, + }, + }, + ), + ( + # A git URL without revision cannot be split + "mypackage @ git+https://host/repo.git", + { + "name": "mypackage", + "version": "", + "archive": {"url": "git+https://host/repo.git", "hashes": {}}, + }, + ), + ], +) +def test_distribution_from_requirement(requirement, expected): + distribution = requirements_txt.distribution_from_requirement(requirement) + + assert distribution is not None + assert distribution.model_dump(exclude_none=True) == expected + + +def test_distribution_from_invalid_requirement(caplog): + assert ( + requirements_txt.distribution_from_requirement("git+https://host/repo") is None + ) + assert "Skip invalid requirement" in caplog.text diff --git a/src/ewoks/tests/requirements/utils.py b/src/ewoks/tests/requirements/utils.py index 651d016..d8b62ee 100644 --- a/src/ewoks/tests/requirements/utils.py +++ b/src/ewoks/tests/requirements/utils.py @@ -1,7 +1,79 @@ +import platform +import sys +from typing import Any +from typing import Dict +from typing import List +from typing import Mapping +from typing import Optional +from typing import Tuple + from ewokscore.graph import TaskGraph +from ..._requirements.utils.environment import Environment +from ..._requirements.utils.metadata import models from ..._requirements.utils.parse import parse_requirements +PYTHON_VERSION = platform.python_version() +"""Python version of the environments created by the tests. The version at +runtime is used because a package manager can only provide a python version that +is available on the machine.""" + + +def _pinned(name: str, versions: Mapping[Tuple[int, int], str]) -> models.Distribution: + """Distribution pinned to a version the running python supports. The versions + are keyed by the lowest python version they support.""" + version = next( + version + for python_version, version in sorted(versions.items(), reverse=True) + if python_version <= sys.version_info[:2] + ) + return models.Distribution(name=name, version=version) + + +DISTRIBUTIONS = [ + _pinned("six", {(3, 8): "1.17.0"}), + _pinned("iniconfig", {(3, 8): "2.0.0"}), +] +"""Small distributions used to verify that an installation took place. Unlike the +python version, the versions cannot be derived at runtime: they must be exact +versions that a package manager would not install by itself.""" + +REQUIREMENTS = [f"{dist.name}=={dist.version}" for dist in DISTRIBUTIONS] +"""`DISTRIBUTIONS` as requirements.""" + +EWOKS_DEPENDENCIES = [ + _pinned("networkx", {(3, 8): "3.1", (3, 10): "3.4.2"}), + _pinned("packaging", {(3, 8): "24.1"}), +] +"""Older versions of distributions that ewoks depends on, to verify that adding +ewoks to an environment does not change the versions of the requirements.""" + + +def assert_installed( + environment: Environment, distributions: List[models.Distribution] +) -> None: + """Every distribution of `distributions` is installed with its exact version.""" + assert _installed_versions(environment, distributions) == { + dist.name: dist.version for dist in distributions + } + + +def assert_not_installed( + environment: Environment, distributions: List[models.Distribution] +) -> None: + """No distribution of `distributions` is installed.""" + assert _installed_versions(environment, distributions) == { + dist.name: None for dist in distributions + } + + +def _installed_versions( + environment: Environment, distributions: List[models.Distribution] +) -> Dict[str, Optional[str]]: + return { + dist.name: environment.distribution_version(dist.name) for dist in distributions + } + def assert_in_graph_requirements(graph: TaskGraph, *distribution_names) -> None: assert distribution_names, "no names provides" @@ -11,3 +83,59 @@ def assert_in_graph_requirements(graph: TaskGraph, *distribution_names) -> None: existing = {distribution.name for distribution in requirements.distributions} not_existing = set(distribution_names) - existing assert not not_existing, f"{sorted(not_existing)} not in requirements" + + +def current_metadata( + distributions: List[models.Distribution], + python_version: str, +) -> Dict[str, Any]: + """Metadata of a python environment with few distributions: the current + environment can have hundreds, which some package managers take minutes to + resolve. + """ + return { + "system": { + "system": "", + "release": "", + "version": "", + "machine": "", + "processor": "", + }, + "python": { + "version": python_version, + "implementation": "", + "compiler": "", + "build": "", + }, + "distributions": distributions, + } + + +def manager_requirements( + manager_name: str, + files: Optional[Dict[str, str]] = None, + distributions: Optional[List[models.Distribution]] = None, + python_version: str = "", +) -> Dict[str, Any]: + """Workflow requirements as generated by a package manager.""" + return { + "system": { + "system": "", + "release": "", + "version": "", + "machine": "", + "processor": "", + }, + "python": { + "version": python_version, + "implementation": "", + "compiler": "", + "build": "", + }, + "distributions": [dist.model_dump() for dist in distributions or list()], + "manager": { + "name": manager_name, + "version": "", + "files": files or dict(), + }, + } diff --git a/src/ewoks/tests/test_convert_cli.py b/src/ewoks/tests/test_convert_cli.py index 82e1113..dc814d1 100644 --- a/src/ewoks/tests/test_convert_cli.py +++ b/src/ewoks/tests/test_convert_cli.py @@ -9,6 +9,7 @@ from ewoksutils.import_utils import import_qualname from orangewidget.widget import OWBaseWidget +from .. import _requirements from ..__main__ import main from .requirements.utils import assert_in_graph_requirements from .utils import has_default_input @@ -81,6 +82,27 @@ def test_convert_with_taskid_inputs(tmpdir): assert has_default_input(node, "value", "test") +def test_convert_requirements_failure(tmpdir, monkeypatch): + def add_requirements(*args, **kwargs) -> None: + raise RuntimeError("no package manager") + + monkeypatch.setattr(_requirements, "add_requirements", add_requirements) + + destination = str(tmpdir / "demo.json") + argv = [ + sys.executable, + "convert", + "demo", + destination, + "--test", + ] + + with pytest.raises(RuntimeError, match="no package manager"): + main(argv=argv, shell=False) + + assert not os.path.exists(destination) + + @pytest.mark.parametrize("graph_name", graph_names()) def test_convert_to_ows(graph_name, tmpdir): destination = str(tmpdir / f"{graph_name}.ows") diff --git a/src/ewoks/tests/test_execute_cli.py b/src/ewoks/tests/test_execute_cli.py index b3b0832..c123936 100644 --- a/src/ewoks/tests/test_execute_cli.py +++ b/src/ewoks/tests/test_execute_cli.py @@ -95,3 +95,21 @@ def test_execute_with_convert_destination_inputs_all(tmpdir): assert has_default_input(node, "b", 42) assert_in_graph_requirements(graph, "ewokscore") + + +def test_execute_in_environment(tmpdir): + """The workflow is executed by the python interpreter of another environment.""" + destination = str(tmpdir / "convert.json") + argv = [ + sys.executable, + "execute", + "demo", + "--test", + "--env", + sys.prefix, + "-o", + f"convert_destination={destination}", + ] + + assert main(argv=argv, shell=False) == 0 + assert os.path.exists(destination)